From 127d33e4b5cf7c9a4f9202483deeb4c5d0e72ebb Mon Sep 17 00:00:00 2001 From: Yasuyuki Tanaka Date: Wed, 29 May 2019 23:21:56 +0200 Subject: [PATCH 1/9] Add missing close() calls to serial.Serial() instances --- otbox.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/otbox.py b/otbox.py index 1a61bb7..8e80c92 100644 --- a/otbox.py +++ b/otbox.py @@ -483,6 +483,7 @@ def _mqtt_handler_discovermotes(self, deviceType, deviceId, payload): break else: self.logger.debug("No EUI64 found in '%s'", line) + ser.close() for e in self.motesinfo: if 'EUI64' in e: @@ -593,6 +594,7 @@ def _mqtt_handler_tomoteserialbytes(self, deviceType, deviceId, payload): mote = self._eui64_to_moteinfoelem(deviceId) serialHandler = serial.Serial(mote['serialport'], baudrate=self.tb.baudrate, xonxoff=True) serialHandler.write(bytearray(payload['serialbytes'])) + serialHandler.close() self.SerialportHandlers[mote['serialport']].connectSerialPort() def _mqtt_handler_reset(self, deviceType, deviceId, payload): @@ -611,6 +613,7 @@ def _mqtt_handler_reset(self, deviceType, deviceId, payload): pyserialHandler.setRTS(False) time.sleep(0.2) pyserialHandler.setDTR(False) + pyserialHandler.close() ## start serial self.SerialportHandlers[mote['serialport']].connectSerialPort() From f41af10e862d50ef396c0189d4d90b27944ed0c3 Mon Sep 17 00:00:00 2001 From: Yasuyuki Tanaka Date: Fri, 31 May 2019 11:51:20 +0200 Subject: [PATCH 2/9] Better error info back to MQTT client on a bootload_mote() failure Currently, a client has no clue why firmware programming fails by seeing a response, which says only AssertionError happened by "assert mote['bootload_success'] ==True". With this change, the client could have raw outputs, stdout and stderr, by cc2538-bsl.py, which may help for troubleshooting. --- otbox.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/otbox.py b/otbox.py index 8e80c92..ca7f85e 100644 --- a/otbox.py +++ b/otbox.py @@ -678,6 +678,8 @@ def _bootload_motes(self, serialports, firmware_file): # record whether bootload worked or not returnVal += [ongoing_bootloads[ongoing_bootload].returncode== 0] + if not returnVal[-1]: + raise RuntimeError(stdout, stderr) self.logger.debug("Finished bootload_motes with returnVal %s", str(returnVal)) return returnVal From 45ba0f6d7aed406791a4d34cb922754c6529b9c6 Mon Sep 17 00:00:00 2001 From: Yasuyuki Tanaka Date: Fri, 31 May 2019 14:48:23 +0200 Subject: [PATCH 3/9] #102. use a separate firmware file (firmware_temp) for each mote This change is applied to only OpenTestbed --- otbox.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/otbox.py b/otbox.py index ca7f85e..9b6522f 100644 --- a/otbox.py +++ b/otbox.py @@ -84,7 +84,7 @@ def __init__(self, otbox): self.firmware_eui64_retrieval = os.path.join(os.path.dirname(__file__), 'bootloaders', 'opentestbed', '01bsp_eui64_prog.ihex') - self.firmware_temp = os.path.join(os.path.dirname(__file__), 'bootloaders', 'opentestbed', 'firmware_mote.ihex') + self.firmware_temp_dir = os.path.join(os.path.dirname(__file__), 'bootloaders', 'opentestbed') def bootload_mote(self, serialport, firmware_file): return subprocess.Popen( @@ -552,13 +552,24 @@ def _mqtt_handler_program(self, deviceType, deviceId, payload): self.SerialportHandlers[mote['serialport']].disconnectSerialPort() time.sleep(2) # wait 2 seconds to release the serial ports + if isinstance(self.tb, OpenTestbed): + # For OpenTestbed, we will use a separate firmware_temp + # for each deviceId. Although other testbeds may need to + # do the same, we keep the original implementation for + # them for now. + firmware_name = '{0}_{1}'.format(deviceId, payload['description']) + firmware_temp = os.path.join(self.tb.firmware_temp_dir, + firmware_name) + else: + firmware_temp = self.tb.firmware_temp + if 'url' in payload and payload['url'].startswith("ftp://"): # use urllib to get firmware from ftp server (requests doesn't support for ftp) - urllib.urlretrieve(payload['url'],self.tb.firmware_temp) + urllib.urlretrieve(payload['url'],firmware_temp) urllib.urlcleanup() else: # store the firmware to load into a temporary file - with open(self.tb.firmware_temp, 'wb') as f: + with open(firmware_temp, 'wb') as f: if 'url' in payload: # download file from url if present file = requests.get(payload['url'], allow_redirects=True) f.write(file.content) @@ -571,7 +582,7 @@ def _mqtt_handler_program(self, deviceType, deviceId, payload): # bootload the mote bootload_success = self._bootload_motes( serialports = [mote['serialport']], - firmware_file = self.tb.firmware_temp, + firmware_file = firmware_temp, ) assert len(bootload_success)==1 From 787c067791929cce48976933b592870244dc5b83 Mon Sep 17 00:00:00 2001 From: Yasuyuki Tanaka Date: Mon, 3 Jun 2019 16:38:52 +0200 Subject: [PATCH 4/9] #102. Reset the mote before programming With this reset, firmware programming succeeds even without time.sleep(2), which is removed by this commit. --- otbox.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/otbox.py b/otbox.py index 9b6522f..37d923a 100644 --- a/otbox.py +++ b/otbox.py @@ -548,9 +548,11 @@ def _mqtt_handler_program(self, deviceType, deviceId, payload): payload = json.loads(payload) # shorthand mote = self._eui64_to_moteinfoelem(deviceId) + # reset the mote first + self._mqtt_handler_reset(deviceType, deviceId, payload=None) + # disconnect from the serialports self.SerialportHandlers[mote['serialport']].disconnectSerialPort() - time.sleep(2) # wait 2 seconds to release the serial ports if isinstance(self.tb, OpenTestbed): # For OpenTestbed, we will use a separate firmware_temp From 3b3c32a1a4880ce063a7dbb897e4167ec43bdfa8 Mon Sep 17 00:00:00 2001 From: Yasuyuki Tanaka Date: Mon, 3 Jun 2019 19:17:12 +0200 Subject: [PATCH 5/9] #102. reset the mote in _bootload_motes() --- otbox.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/otbox.py b/otbox.py index 37d923a..b047259 100644 --- a/otbox.py +++ b/otbox.py @@ -548,9 +548,6 @@ def _mqtt_handler_program(self, deviceType, deviceId, payload): payload = json.loads(payload) # shorthand mote = self._eui64_to_moteinfoelem(deviceId) - # reset the mote first - self._mqtt_handler_reset(deviceType, deviceId, payload=None) - # disconnect from the serialports self.SerialportHandlers[mote['serialport']].disconnectSerialPort() @@ -618,15 +615,7 @@ def _mqtt_handler_reset(self, deviceType, deviceId, payload): mote = self._eui64_to_moteinfoelem(deviceId) self.SerialportHandlers[mote['serialport']].disconnectSerialPort() - pyserialHandler = serial.Serial(mote['serialport'], baudrate=self.tb.baudrate) - pyserialHandler.setDTR(False) - pyserialHandler.setRTS(True) - time.sleep(0.2) - pyserialHandler.setDTR(True) - pyserialHandler.setRTS(False) - time.sleep(0.2) - pyserialHandler.setDTR(False) - pyserialHandler.close() + self._reset_mote(mote['serialport']) ## start serial self.SerialportHandlers[mote['serialport']].connectSerialPort() @@ -677,6 +666,9 @@ def _bootload_motes(self, serialports, firmware_file): # simply the name port = serialport.split('/')[-1] + # reset the mote first + self._reset_mote(serialport) + # stop serial reader ongoing_bootloads[port] = self.tb.bootload_mote(serialport, firmware_file) @@ -713,7 +705,16 @@ def _reboot_function(self): time.sleep(3) subprocess.call(["sudo","reboot"]) - + def _reset_mote(self, serial_port): + pyserialHandler = serial.Serial(serial_port, baudrate=self.tb.baudrate) + pyserialHandler.setDTR(False) + pyserialHandler.setRTS(True) + time.sleep(0.2) + pyserialHandler.setDTR(True) + pyserialHandler.setRTS(False) + time.sleep(0.2) + pyserialHandler.setDTR(False) + pyserialHandler.close() def _discover_serialports_availables(self): serialports_available = [] From 057e457b6d6a8becd89457d1106ff081fa914705 Mon Sep 17 00:00:00 2001 From: Yasuyuki Tanaka Date: Mon, 3 Jun 2019 16:51:07 +0200 Subject: [PATCH 6/9] Wait after publishing heartbeat This makes listeners get aware of its liveness sooner. --- otbox.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/otbox.py b/otbox.py index b047259..19ecffd 100644 --- a/otbox.py +++ b/otbox.py @@ -643,13 +643,13 @@ def _mqtt_handler_disable(self, deviceType, deviceId, payload): def _heartbeatthread_func(self): while True: - # wait a bit - time.sleep(OtBox.HEARTBEAT_PERIOD) # publish a heartbeat message self.mqttclient.publish( topic = '{0}/heartbeat'.format(self.mqtttopic_box_notif_prefix), payload = json.dumps({'software_version': OTBOX_VERSION}), ) + # wait a bit + time.sleep(OtBox.HEARTBEAT_PERIOD) #=== helpers From 5aae19e3d1e0d6a57266c7ef340798e7e4ec29b6 Mon Sep 17 00:00:00 2001 From: Yasuyuki Tanaka Date: Mon, 3 Jun 2019 17:05:45 +0200 Subject: [PATCH 7/9] Discover motes on startup --- otbox.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/otbox.py b/otbox.py index 19ecffd..46bf7e5 100644 --- a/otbox.py +++ b/otbox.py @@ -171,11 +171,7 @@ def bootload_mote(self, serialport, firmware_file): stderr=subprocess.PIPE) def on_mqtt_connect(self): - # in case of IoT-LAB mote discovery is started immediately upon `otbox.py` startup - payload_status = { - 'token': 123 - } - self._otbox._mqtt_handler_discovermotes('box', 'all', json.dumps(payload_status)) + pass class WilabTestbed(Testbed): @@ -202,11 +198,7 @@ def bootload_mote(self, serialport, firmware_file): return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) def on_mqtt_connect(self): - # in case of WiLab mote discovery is started immediately upon `otbox.py` startup - payload_status = { - 'token': 123 - } - self._otbox._mqtt_handler_discovermotes('box', 'all', json.dumps(payload_status)) + pass AVAILABLE_TESTBEDS = { @@ -307,6 +299,8 @@ def _on_mqtt_connect(self, client, userdata, flags, rc): self.tb.on_mqtt_connect() + # discover motes + self._excecute_command_safely('box', 'all', json.dumps({'token': 123}), 'discovermotes') def _on_mqtt_message(self, client, userdata, message): From feba023c8f394c4c6e1d09d825c4a474a558a100 Mon Sep 17 00:00:00 2001 From: Yasuyuki Tanaka Date: Mon, 3 Jun 2019 17:18:54 +0200 Subject: [PATCH 8/9] Fix typo --- otbox.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/otbox.py b/otbox.py index 46bf7e5..57a4530 100644 --- a/otbox.py +++ b/otbox.py @@ -300,14 +300,14 @@ def _on_mqtt_connect(self, client, userdata, flags, rc): self.tb.on_mqtt_connect() # discover motes - self._excecute_command_safely('box', 'all', json.dumps({'token': 123}), 'discovermotes') + self._execute_command_safely('box', 'all', json.dumps({'token': 123}), 'discovermotes') def _on_mqtt_message(self, client, userdata, message): # call the handler - self._excecute_commands(message.topic, message.payload) + self._execute_commands(message.topic, message.payload) - def _excecute_commands(self, topic, payload): + def _execute_commands(self, topic, payload): # parse the topic to extract deviceType, deviceId and cmd ([0-9\-]+) try: m = re.search('{0}/deviceType/([a-z]+)/deviceId/([\w,\-]+)/cmd/([a-z]+)'.format(self.testbed), topic) @@ -336,7 +336,7 @@ def _excecute_commands(self, topic, payload): for d in device_to_comand: commands_handlers += [threading.Thread( name = '{0}_command_{1}'.format(d, cmd), - target = self._excecute_command_safely, + target = self._execute_command_safely, args = (deviceType, d, payload, cmd)) ] for handler in commands_handlers: @@ -344,7 +344,7 @@ def _excecute_commands(self, topic, payload): except: self.logger.exception("Could not parse command with topic %s", topic) - def _excecute_command_safely(self, deviceType, deviceId, payload, cmd): + def _execute_command_safely(self, deviceType, deviceId, payload, cmd): ''' Executes the handler of a command in a try/except environment so exception doesn't crash server. ''' From 09c91347698137e517777e06f92af9f6c60c9013 Mon Sep 17 00:00:00 2001 From: Yasuyuki Tanaka Date: Mon, 3 Jun 2019 18:49:21 +0200 Subject: [PATCH 9/9] Handle the case when an image is unavailable --- otbox.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/otbox.py b/otbox.py index 57a4530..831cd76 100644 --- a/otbox.py +++ b/otbox.py @@ -18,6 +18,7 @@ import base64 import argparse import logging +import sys try: from PIL import Image @@ -126,10 +127,19 @@ def _mqtt_handler_picturetoscreen(self, deviceType, deviceId, payload): {{TESTBED}}/deviceType/box/deviceId/box1/cmd/picturetoscreen ''' assert deviceType==DEVICETYPE_BOX - image = Image.open(requests.get(json.loads(payload)['url'], stream=True).raw) - image.thumbnail((480,320),Image.ANTIALIAS) - self._otbox.change_image_queue.put(image) - self._otbox.change_image_queue.join() + try: + image_url = json.loads(payload)['url'] + response = requests.get(image_url, stream=True) + image = Image.open(response.raw) + except IOError: + # the image is not available now; skip this one + print('image at {0} is not available, '.format(image_url) + + 'HTTP response code {0}'.format(response.status_code)) + sys.stdout.flush() + else: + image.thumbnail((480,320),Image.ANTIALIAS) + self._otbox.change_image_queue.put(image) + self._otbox.change_image_queue.join() return {} def _mqtt_handler_colortoscreen(self, deviceType, deviceId, payload):