From e17b3e4a3e8659571b317ef782678bc3f003a869 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 1 Nov 2023 00:39:56 +0000 Subject: [PATCH 01/27] otaclient: implement new OTAServicer to replace OTAClientWrapper --- otaclient/app/ota_client.py | 209 ++++++++++++++++++++++-------------- 1 file changed, 129 insertions(+), 80 deletions(-) diff --git a/otaclient/app/ota_client.py b/otaclient/app/ota_client.py index 330537a97..7b84bcac1 100644 --- a/otaclient/app/ota_client.py +++ b/otaclient/app/ota_client.py @@ -475,15 +475,14 @@ class OTAClient(OTAClientProtocol): create_standby_cls: type of create standby slot mechanism to use """ - DEFAULT_FIRMWARE_VERSION = "unknown" OTACLIENT_VERSION = __version__ def __init__( self, *, - boot_control_cls: Type[BootControllerProtocol], + boot_controller: BootControllerProtocol, create_standby_cls: Type[StandbySlotCreatorProtocol], - my_ecu_id: str = "", + my_ecu_id: str, control_flags: OTAClientControlFlags, proxy: Optional[str] = None, ): @@ -491,15 +490,13 @@ def __init__( # ensure only one update/rollback session is running self._lock = threading.Lock() - self.boot_controller = boot_control_cls() + self.boot_controller = boot_controller self.create_standby_cls = create_standby_cls self.live_ota_status = LiveOTAStatus( self.boot_controller.get_booted_ota_status() ) - self.current_version = ( - self.boot_controller.load_version() or self.DEFAULT_FIRMWARE_VERSION - ) + self.current_version = self.boot_controller.load_version() self.proxy = proxy self.control_flags = control_flags @@ -589,115 +586,167 @@ def status(self) -> wrapper.StatusResponseEcuV2: return status_report -class OTAClientBusy(Exception): - """Raised when otaclient receive another request when doing update/rollback.""" - - -class OTAClientWrapper: - """OTAClient stub implementation that wraps OTAClient, manage update/rollback session, - and exposes async API for OTAClientServiceStub. - - All actual API request handlings are wrapped and dispatched into threadpool for execution. - Only one ongoing update/rollback is allowed, with the help of exclusive update_rollback lock. - - If this ECU enables otaproxy and has child ECUs depend on it, control_flags is used to prevent - this ECU's reboot before the otaproxy dependency is resolved. - - OTAClient itself is self-contained, its error handling is done internally, no exception will be raised - from calling API methods of OTAClient. - """ - +class OTAServicer: def __init__( self, *, - ecu_info: ECUInfo, - executor: ThreadPoolExecutor, control_flags: OTAClientControlFlags, + ecu_info: ECUInfo, + executor: Optional[ThreadPoolExecutor] = None, + otaclient_version: str = __version__, proxy: Optional[str] = None, ) -> None: - # only one update/rollback is allowed at a time - self.update_rollback_lock = asyncio.Lock() - self.my_ecu_id = ecu_info.ecu_id + self.ecu_id = ecu_info.ecu_id + self.otaclient_version = otaclient_version + + # default boot startup failure if boot controller crashed without + # raising specific error + self._otaclient_startup_failed_status = wrapper.StatusResponseEcuV2( + ecu_id=ecu_info.ecu_id, + otaclient_version=otaclient_version, + ota_status=wrapper.StatusOta.FAILURE, + ) + self._update_rollback_lock = asyncio.Lock() self._run_in_executor = partial( asyncio.get_running_loop().run_in_executor, executor ) + self._last_operation: Optional[wrapper.StatusOta] = None - # create otaclient instance and control flags - self._otaclient = OTAClient( - boot_control_cls=get_boot_controller(ecu_info.get_bootloader()), - create_standby_cls=get_standby_slot_creator(cfg.STANDBY_CREATION_MODE), - my_ecu_id=self.my_ecu_id, - control_flags=control_flags, - proxy=proxy, - ) + # + # ------ compose otaclient ------ + # + self._otaclient_inst: Optional[OTAClient] = None - # proxy used by local otaclient - # NOTE: it can be an upper otaproxy, local otaproxy, or no proxy - self.local_used_proxy_url = proxy - self.last_operation = None # update/rollback/None + # select boot_controller and standby_slot implementations + _bootctrl_cls = get_boot_controller(ecu_info.get_bootloader()) + _standby_slot_creator = get_standby_slot_creator(cfg.STANDBY_CREATION_MODE) + + # boot controller starts up + try: + _bootctrl_inst = _bootctrl_cls() + except Exception as e: + self._otaclient_startup_failed_status = wrapper.StatusResponseEcuV2( + ecu_id=ecu_info.ecu_id, + otaclient_version=otaclient_version, + ota_status=wrapper.StatusOta.FAILURE, + failure_type=wrapper.FailureType.UNRECOVERABLE, + failure_reason=f"{e!r}", + failure_traceback=f"{e!r}", + ) + return - # property + # compose otaclient + try: + self._otaclient_inst = OTAClient( + boot_controller=_bootctrl_inst, + create_standby_cls=_standby_slot_creator, + my_ecu_id=ecu_info.ecu_id, + control_flags=control_flags, + proxy=proxy, + ) + except Exception as e: + return @property def is_busy(self) -> bool: - return self.update_rollback_lock.locked() + return self._update_rollback_lock.locked() + + async def dispatch_update( + self, request: wrapper.UpdateRequestEcu + ) -> wrapper.UpdateResponseEcu: + # prevent update operation if otaclient is not started + if self._otaclient_inst is None: + return wrapper.UpdateResponseEcu( + ecu_id=self.ecu_id, + result=wrapper.FailureType.UNRECOVERABLE, + ) - # API + # check and acquire lock + if self._update_rollback_lock.locked(): + logger.warning( + f"ongoing operation: {self.last_operation=}, ignore incoming {request=}" + ) + return wrapper.UpdateResponseEcu( + ecu_id=self.ecu_id, + result=wrapper.FailureType.RECOVERABLE, + ) - async def dispatch_update(self, request: wrapper.UpdateRequestEcu): - """Dispatch OTA update execution to background thread. + # immediately take the lock if not locked + await self._update_rollback_lock.acquire() + self.last_operation = wrapper.StatusOta.UPDATING + + async def _update_task(): + if self._otaclient_inst is None: + return - Raises: - OTAClientBusy if otaclient is already executing update/rollback. - """ - if self.update_rollback_lock.locked(): - raise OTAClientBusy(f"ongoing operation: {self.last_operation=}") - else: # immediately take the lock if not locked - await self.update_rollback_lock.acquire() - self.last_operation = wrapper.StatusOta.UPDATING - - async def _update(): - """Background waiting for update to be finished and - release update_rollback lock.""" try: await self._run_in_executor( partial( - self._otaclient.update, + self._otaclient_inst.update, request.version, request.url, request.cookies, ) ) + except Exception: + pass # error should be collected by otaclient, not us finally: self.last_operation = None - self.update_rollback_lock.release() + self._update_rollback_lock.release() # dispatch update to background - asyncio.create_task(_update()) + asyncio.create_task(_update_task()) - async def dispatch_rollback(self, _: wrapper.RollbackRequestEcu): - """Dispatch OTA rollback execution to background thread. + return wrapper.UpdateResponseEcu( + ecu_id=self.ecu_id, result=wrapper.FailureType.NO_FAILURE + ) + + async def dispatch_rollback( + self, request: wrapper.RollbackRequestEcu + ) -> wrapper.RollbackResponseEcu: + # prevent rollback operation if otaclient is not started + if self._otaclient_inst is None: + return wrapper.RollbackResponseEcu( + ecu_id=self.ecu_id, + result=wrapper.FailureType.UNRECOVERABLE, + ) + + # check and acquire lock + if self._update_rollback_lock.locked(): + logger.warning( + f"ongoing operation: {self.last_operation=}, ignore incoming {request=}" + ) + return wrapper.RollbackResponseEcu( + ecu_id=self.ecu_id, + result=wrapper.FailureType.RECOVERABLE, + ) + + # immediately take the lock if not locked + await self._update_rollback_lock.acquire() + self.last_operation = wrapper.StatusOta.ROLLBACKING + + async def _rollback_task(): + if self._otaclient_inst is None: + return - Raises: - OTAClientBusy if otaclient is already executing update/rollback. - """ - if self.update_rollback_lock.locked(): - raise OTAClientBusy(f"ongoing operation: {self.last_operation=}") - else: # immediately take the lock if not locked - await self.update_rollback_lock.acquire() - self.last_operation = wrapper.StatusOta.ROLLBACKING - - async def _rollback(): - """Background waiting for rollback to be finished and - release update_rollback lock.""" try: - await self._run_in_executor(self._otaclient.rollback) + await self._run_in_executor(self._otaclient_inst.rollback) + except Exception: + pass # error should be collected by otaclient, not us finally: self.last_operation = None - self.update_rollback_lock.release() + self._update_rollback_lock.release() # dispatch to background - asyncio.create_task(_rollback()) + asyncio.create_task(_rollback_task()) + + return wrapper.RollbackResponseEcu( + ecu_id=self.ecu_id, result=wrapper.FailureType.NO_FAILURE + ) async def get_status(self) -> wrapper.StatusResponseEcuV2: - return await self._run_in_executor(self._otaclient.status) + # otaclient is not started due to boot control startup failed + if self._otaclient_inst is None: + return self._otaclient_startup_failed_status + # otaclient core started, query status from it + return await self._run_in_executor(self._otaclient_inst.status) From 38633450d833fa15cc03927f427ad172df61c567 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 1 Nov 2023 00:46:49 +0000 Subject: [PATCH 02/27] ota_service_stub: adapt according to otaclient.OTAServicer --- otaclient/app/ota_client_stub.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/otaclient/app/ota_client_stub.py b/otaclient/app/ota_client_stub.py index 855f3b151..d9affd1c6 100644 --- a/otaclient/app/ota_client_stub.py +++ b/otaclient/app/ota_client_stub.py @@ -30,7 +30,7 @@ from .common import ensure_otaproxy_start from .boot_control._common import CMDHelperFuncs from .ecu_info import ECUContact, ECUInfo -from .ota_client import OTAClientBusy, OTAClientControlFlags, OTAClientWrapper +from .ota_client import OTAClientControlFlags, OTAServicer from .ota_client_call import ECUNoResponse, OtaClientCall from .proto import wrapper from .proxy_info import proxy_cfg @@ -675,7 +675,7 @@ def __init__( ecu_status_storage: ECUStatusStorage, *, ecu_info: ECUInfo, - otaclient_wrapper: OTAClientWrapper, + otaclient_wrapper: OTAServicer, ) -> None: self._otaclient_wrapper = otaclient_wrapper # for local ECU status polling self._ecu_status_storage = ecu_status_storage @@ -736,7 +736,7 @@ def __init__(self, *, ecu_info: ECUInfo, _proxy_cfg=proxy_cfg): self.my_ecu_id = ecu_info.ecu_id self._otaclient_control_flags = OTAClientControlFlags() - self._otaclient_wrapper = OTAClientWrapper( + self._otaclient_wrapper = OTAServicer( ecu_info=ecu_info, executor=self._executor, control_flags=self._otaclient_control_flags, @@ -878,13 +878,10 @@ async def update(self, request: wrapper.UpdateRequest) -> wrapper.UpdateResponse # second: dispatch update request to local if required by incoming request if update_req_ecu := request.find_ecu(self.my_ecu_id): - _resp_ecu = wrapper.UpdateResponseEcu(ecu_id=self.my_ecu_id) - try: - await self._otaclient_wrapper.dispatch_update(update_req_ecu) + _resp_ecu = await self._otaclient_wrapper.dispatch_update(update_req_ecu) + # local otaclient accepts the update request + if _resp_ecu.result == wrapper.FailureType.NO_FAILURE: update_acked_ecus.add(self.my_ecu_id) - except OTAClientBusy as e: - logger.warning(f"self ECU is busy, ignore local update request: {e!r}") - _resp_ecu.result = wrapper.FailureType.RECOVERABLE response.add_ecu(_resp_ecu) # finally, trigger ecu_status_storage entering active mode if needed @@ -895,6 +892,7 @@ async def update(self, request: wrapper.UpdateRequest) -> wrapper.UpdateResponse update_acked_ecus ) ) + return response async def rollback( @@ -943,13 +941,10 @@ async def rollback( # second: dispatch rollback request to local if required if rollback_req := request.find_ecu(self.my_ecu_id): - _resp_ecu = wrapper.RollbackResponseEcu(ecu_id=self.my_ecu_id) - try: + response.add_ecu( await self._otaclient_wrapper.dispatch_rollback(rollback_req) - except OTAClientBusy as e: - logger.warning(f"self ECU is busy, ignore local rollback: {e!r}") - _resp_ecu.result = wrapper.FailureType.RECOVERABLE - response.add_ecu(_resp_ecu) + ) + return response async def status(self, _=None) -> wrapper.StatusResponse: From ab8a2a91e96a96f1207f79a1b4136331141688f4 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 1 Nov 2023 03:32:35 +0000 Subject: [PATCH 03/27] errors: redefine OTA errors --- otaclient/app/errors.py | 346 +++++++++++++++------------------------- 1 file changed, 130 insertions(+), 216 deletions(-) diff --git a/otaclient/app/errors.py b/otaclient/app/errors.py index 990edaad9..ca64303f6 100644 --- a/otaclient/app/errors.py +++ b/otaclient/app/errors.py @@ -11,75 +11,54 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +"""OTA error code definition""" -"""OTA error code definition""" import traceback from enum import Enum, unique +from typing import ClassVar from .proto import wrapper @unique -class OTAErrorCode(Enum): +class OTAErrorCode(int, Enum): E_UNSPECIFIC = 0 + # + # ------ network related errors ------ + # E_NETWORK = 100 E_OTAMETA_DOWNLOAD_FAILED = 101 + # + # ------ recoverable errors ------ + # E_OTA_ERR_RECOVERABLE = 200 - E_OTAUPDATE_BUSY = 201 - E_INVALID_STATUS_FOR_OTAUPDATE = 202 - E_INVALID_OTAUPDATE_REQUEST = 203 - E_INVALID_STATUS_FOR_OTAROLLBACK = 204 - E_OTAMETA_VERIFICATION_FAILED = 205 - E_UPDATEDELTA_GENERATION_FAILED = 206 - E_APPLY_OTAUPDATE_FAILED = 207 - E_METADATA_JWT_VERIFICATION_FAILED = 208 - E_OTAPROXY_FAILED_TO_START = 209 - E_INVALID_METADATAJWT = 210 + E_OTA_BUSY = 201 + E_INVALID_STATUS_FOR_OTAROLLBACK = 202 + # + # ------ unrecoverable errors ------ + # E_OTA_ERR_UNRECOVERABLE = 300 E_BOOTCONTROL_PLATFORM_UNSUPPORTED = 301 - E_BOOTCONTROL_INIT_ERR = 302 + E_BOOTCONTROL_STARTUP_ERR = 302 E_BOOTCONTROL_PREUPDATE_FAILED = 303 E_BOOTCONTROL_POSTUPDATE_FAILED = 304 - E_BOOTCONTROL_POSTROLLBACK_FAILED = 305 - E_STANDBY_SLOT_SPACE_NOT_ENOUGH_ERROR = 306 - E_BOOTCONTROL_PREROLLBACK_FAILED = 307 - - def to_str(self) -> str: + E_BOOTCONTROL_PREROLLBACK_FAILED = 305 + E_BOOTCONTROL_POSTROLLBACK_FAILED = 306 + E_STANDBY_SLOT_INSUFFICIENT_SPACE = 307 + E_INVALID_OTAUPDATE_REQUEST = 308 + E_METADATAJWT_CERT_VERIFICATION_FAILED = 309 + E_METADATAJWT_INVALID = 310 + E_OTAPROXY_FAILED_TO_START = 311 + E_UPDATEDELTA_GENERATION_FAILED = 312 + E_APPLY_OTAUPDATE_FAILED = 313 + + def to_errcode_str(self) -> str: return f"{self.value:0>3}" - def get_errcode(self) -> int: - return self.value - - def get_errname(self) -> str: - return self.name - - -@unique -class OTAModules(Enum): - General = 0 - BootController = 1 - StandbySlotCreater = 2 - Downloader = 3 - API = 4 - OTASERVICE = 5 - - def to_str(self) -> str: - return f"{self.value:0>2}" - - -@unique -class OTAAPI(Enum): - Unspecific = 0 - Update = 1 - Rollback = 2 - - def to_str(self) -> str: - return f"{self.value:0>2}" - class OTAError(Exception): """Errors that happen during otaclient code executing. @@ -88,240 +67,175 @@ class OTAError(Exception): It should always be captured by the OTAError at otaclient.py. """ - failure_type: wrapper.FailureType = wrapper.FailureType.RECOVERABLE - module: OTAModules = OTAModules.General - errcode: OTAErrorCode = OTAErrorCode.E_UNSPECIFIC - desc: str = "no description available for this error" - - -class OTA_APIError(Exception): - """Errors that happen during processing API request. + ERROR_PREFIX: ClassVar[str] = "E" - This exception class should be the top level exception for each API entry. - This exception must be created by wrapping an OTAClientError. - """ - - api: OTAAPI = OTAAPI.Unspecific - _err_prefix = "E" + failure_type: wrapper.FailureType = wrapper.FailureType.RECOVERABLE + failure_errcode: OTAErrorCode = OTAErrorCode.E_UNSPECIFIC + failure_description: str = "no description available for this error" - def __init__(self, ota_err: OTAError, *args: object) -> None: + def __init__(self, *args: object, module: str) -> None: + self.module = module super().__init__(*args) - self.otaclient_err = ota_err - self.errcode = ota_err.errcode - self.module = ota_err.module - self.failure_type = ota_err.failure_type - self.errdesc = ota_err.desc - - def get_errcode(self) -> "OTAErrorCode": - return self.errcode - - def get_errcode_str(self) -> str: - return f"{self._err_prefix}{self.errcode.to_str()}" - - def get_err_type(self) -> wrapper.FailureType: - return self.failure_type - - def get_err_reason(self, *, append_desc=True, append_detail=False) -> str: - r"""Return a failure_reason str. - - Format: Ec-aabb-dee[: [\n]] - c: FAILURE_TYPE_1digit - aa: API_2digits - bb: MODULE_2digits - dee: ERRCODE_3digits - """ - _errdesc = "" - if append_desc: - _errdesc = f": {self.errdesc}." - - _detail = "" - if append_detail: - _detail = f"\n{self.otaclient_err.__cause__!r}" - return ( - f"{self._err_prefix}" - f"{self.failure_type.to_str()}" - "-" - f"{self.api.to_str()}{self.module.to_str()}" - "-" - f"{self.errcode.to_str()}" - f"{_errdesc}{_detail}" - ) + @property + def failure_errcode_str(self) -> str: + return f"{self.ERROR_PREFIX}{self.failure_errcode.to_errcode_str()}" - def get_traceback(self, *, splitter="") -> str: - """Format the traceback into a str with splitter as .""" + def get_failure_traceback(self, *, splitter="") -> str: return splitter.join( traceback.format_exception(type(self), self, self.__traceback__) ) + def failure_reason(self, *, append_traceback=False) -> str: + """Return failure_reason str.""" + _failure_info = { + "module": self.module, + "exec_args": self.args, + } + if append_traceback: + _failure_info["failure_traceback"] = self.get_failure_traceback() -class OTAUpdateError(OTA_APIError): - api = OTAAPI.Update - + return ( + f"{self.failure_errcode_str}: {self.failure_description}" + f"\n{_failure_info}" + ) -class OTARollbackError(OTA_APIError): - api = OTAAPI.Rollback +# +# ------ Network related error ------ +# -###### error exception classes ###### _NETWORK_ERR_DEFAULT_DESC = ( - "error related to network connection detected, " - "please check the Internet connection and try again" + "network connection unstable, please check the connection and try again" ) -### network error ### class NetworkError(OTAError): """Generic network error""" failure_type: wrapper.FailureType = wrapper.FailureType.RECOVERABLE - module: OTAModules = OTAModules.Downloader - errcode: OTAErrorCode = OTAErrorCode.E_NETWORK - desc: str = _NETWORK_ERR_DEFAULT_DESC + failure_errcode: OTAErrorCode = OTAErrorCode.E_NETWORK + failure_description: str = _NETWORK_ERR_DEFAULT_DESC class OTAMetaDownloadFailed(NetworkError): - errcode: OTAErrorCode = OTAErrorCode.E_OTAMETA_DOWNLOAD_FAILED - desc: str = f"{_NETWORK_ERR_DEFAULT_DESC}: failed to download ota image meta" + failure_errcode: OTAErrorCode = OTAErrorCode.E_OTAMETA_DOWNLOAD_FAILED + failure_description: str = ( + f"failed to download OTA meta due to {_NETWORK_ERR_DEFAULT_DESC}" + ) + +# +# ------ recoverable error ------ +# -### recoverable error ### _RECOVERABLE_DEFAULT_DESC = ( - "recoverable ota error(unrelated to network) detected, " - "please resend the request or retry after a restart of ota-client/ECU" + "recoverable OTA error(unrelated to network) detected, " + "please retry after reboot device or restart otaclient" ) class OTAErrorRecoverable(OTAError): failure_type: wrapper.FailureType = wrapper.FailureType.RECOVERABLE - # followings are default values - module: OTAModules = OTAModules.General - errcode: OTAErrorCode = OTAErrorCode.E_OTA_ERR_RECOVERABLE - desc: str = _RECOVERABLE_DEFAULT_DESC - - -class OTAUpdateBusy(OTAErrorRecoverable): - module: OTAModules = OTAModules.API - errcode: OTAErrorCode = OTAErrorCode.E_OTAUPDATE_BUSY - desc: str = f"{_RECOVERABLE_DEFAULT_DESC}: on-going ota update detected, this request has been ignored" + failure_errcode: OTAErrorCode = OTAErrorCode.E_OTA_ERR_RECOVERABLE + failure_description: str = _RECOVERABLE_DEFAULT_DESC -class OTARollBusy(OTAErrorRecoverable): - module: OTAModules = OTAModules.API - errcode: OTAErrorCode = OTAErrorCode.E_OTAUPDATE_BUSY - desc: str = f"{_RECOVERABLE_DEFAULT_DESC}: on-going ota update detected, this request has been ignored" - - -class InvalidStatusForOTAUpdate(OTAErrorRecoverable): - module: OTAModules = OTAModules.API - errcode: OTAErrorCode = OTAErrorCode.E_INVALID_STATUS_FOR_OTAUPDATE - desc: str = f"{_RECOVERABLE_DEFAULT_DESC}: current ota-status indicates it should not accept ota update" - - -class InvalidUpdateRequest(OTAErrorRecoverable): - module: OTAModules = OTAModules.API - errcode: OTAErrorCode = OTAErrorCode.E_INVALID_OTAUPDATE_REQUEST - desc: str = ( - f"{_RECOVERABLE_DEFAULT_DESC}: incoming ota update request's content is invalid" - ) +class OTABusy(OTAErrorRecoverable): + failure_errcode: OTAErrorCode = OTAErrorCode.E_OTA_BUSY + failure_description: str = "on-going OTA operation(update or rollback) detected, this request has been ignored" class InvalidStatusForOTARollback(OTAErrorRecoverable): - module: OTAModules = OTAModules.API - errcode: OTAErrorCode = OTAErrorCode.E_INVALID_STATUS_FOR_OTAROLLBACK - desc: str = f"{_RECOVERABLE_DEFAULT_DESC}: current ota-status indicates it should not accept ota rollback" + failure_errcode: OTAErrorCode = OTAErrorCode.E_INVALID_STATUS_FOR_OTAROLLBACK + failure_description: str = "previous OTA is not succeeded, reject OTA rollback" -class OTAMetaVerificationFailed(OTAErrorRecoverable): - module: OTAModules = OTAModules.StandbySlotCreater - errcode: OTAErrorCode = OTAErrorCode.E_OTAMETA_VERIFICATION_FAILED - desc: str = ( - f"{_RECOVERABLE_DEFAULT_DESC}: hash verification failed for ota meta files" - ) +# +# ------ recoverable error ------ +# +_UNRECOVERABLE_DEFAULT_DESC = ( + "unrecoverable OTA error detected, please contact technical support" +) -class UpdateDeltaGenerationFailed(OTAErrorRecoverable): - module: OTAModules = OTAModules.StandbySlotCreater - errcode: OTAErrorCode = OTAErrorCode.E_UPDATEDELTA_GENERATION_FAILED - desc: str = f"{_RECOVERABLE_DEFAULT_DESC}: (rebuild_mode) failed to calculate and/or prepare update delta" + +class OTAErrorUnRecoverable(OTAError): + failure_type: wrapper.FailureType = wrapper.FailureType.RECOVERABLE + failure_errcode: OTAErrorCode = OTAErrorCode.E_OTA_ERR_UNRECOVERABLE + failure_description: str = _UNRECOVERABLE_DEFAULT_DESC -class ApplyOTAUpdateFailed(OTAErrorRecoverable): - module: OTAModules = OTAModules.StandbySlotCreater - errcode: OTAErrorCode = OTAErrorCode.E_APPLY_OTAUPDATE_FAILED - desc: str = f"{_RECOVERABLE_DEFAULT_DESC}: failed to apply ota update" +class BootControlPlatformUnsupported(OTAErrorUnRecoverable): + failure_errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_PLATFORM_UNSUPPORTED + failure_description: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: current ECU platform is not supported by the boot controller module" -class MetadataJWTVerficationFailed(OTAErrorRecoverable): - module: OTAModules = OTAModules.API - errcode: OTAErrorCode = OTAErrorCode.E_METADATA_JWT_VERIFICATION_FAILED - desc: str = f"{_RECOVERABLE_DEFAULT_DESC}: verification failed for metadata.jwt" +class BootControlStartupFailed(OTAErrorUnRecoverable): + failure_errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_STARTUP_ERR + failure_description: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: failed to start boot controller module for this device" -class MetadataJWTInvalid(OTAErrorRecoverable): - module: OTAModules = OTAModules.API - errcode: OTAErrorCode = OTAErrorCode.E_INVALID_METADATAJWT - desc: str = f"{_RECOVERABLE_DEFAULT_DESC}: invalid metadata.jwt" +class BootControlPreUpdateFailed(OTAErrorUnRecoverable): + failure_errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_PREUPDATE_FAILED + failure_description: str = ( + f"{_UNRECOVERABLE_DEFAULT_DESC}: boot control pre_update process failed" + ) -class OTAProxyFailedToStart(OTAErrorRecoverable): - module: OTAModules = OTAModules.OTASERVICE - errcode: OTAErrorCode = OTAErrorCode.E_OTAPROXY_FAILED_TO_START - desc: str = ( - f"{_RECOVERABLE_DEFAULT_DESC}: ota_proxy is required but failed to start" +class BootControlPostUpdateFailed(OTAErrorUnRecoverable): + failure_errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_POSTUPDATE_FAILED + failure_description: str = ( + f"{_UNRECOVERABLE_DEFAULT_DESC}: boot control post_update process failed" ) -### unrecoverable error ### -_UNRECOVERABLE_DEFAULT_DESC = ( - "unrecoverable ota error detected, please contact technical support" -) +class BootControlPreRollbackFailed(OTAErrorUnRecoverable): + failure_errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_PREROLLBACK_FAILED + failure_description: str = ( + f"{_UNRECOVERABLE_DEFAULT_DESC}: pre_rollback process failed" + ) -class OTAErrorUnRecoverable(OTAError): - failure_type: wrapper.FailureType = wrapper.FailureType.RECOVERABLE - module: OTAModules = OTAModules.General - errcode: OTAErrorCode = OTAErrorCode.E_OTA_ERR_UNRECOVERABLE - desc: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: unspecific unrecoverable ota error, please contact technical support" +class BootControlPostRollbackFailed(OTAErrorUnRecoverable): + failure_errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_POSTROLLBACK_FAILED + failure_description: str = ( + f"{_UNRECOVERABLE_DEFAULT_DESC}: post_rollback process failed" + ) -class BootControlPlatformUnsupported(OTAErrorUnRecoverable): - module: OTAModules = OTAModules.BootController - errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_PLATFORM_UNSUPPORTED - desc: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: current ECU platform is not supported by the boot controller module" +class StandbySlotInsufficientSpace(OTAErrorUnRecoverable): + failure_errcode: OTAErrorCode = OTAErrorCode.E_STANDBY_SLOT_INSUFFICIENT_SPACE + failure_description: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: standby slot has insufficient space to apply update" -class BootControlInitError(OTAErrorUnRecoverable): - module: OTAModules = OTAModules.BootController - errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_INIT_ERR - desc: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: failed to init boot controller module" +class InvalidUpdateRequest(OTAErrorUnRecoverable): + failure_errcode: OTAErrorCode = OTAErrorCode.E_INVALID_OTAUPDATE_REQUEST + failure_description: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: incoming OTA update request's content is invalid" -class BootControlPreUpdateFailed(OTAErrorUnRecoverable): - module: OTAModules = OTAModules.BootController - errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_PREUPDATE_FAILED - desc: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: pre_update process failed" +class MetadataJWTInvalid(OTAErrorUnRecoverable): + failure_errcode: OTAErrorCode = OTAErrorCode.E_METADATAJWT_INVALID + failure_description: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: verfication for metadata.jwt is OK but metadata.jwt's content is invalid" -class BootControlPostUpdateFailed(OTAErrorUnRecoverable): - module: OTAModules = OTAModules.BootController - errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_POSTUPDATE_FAILED - desc: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: post_update process failed, switch boot is not finished" +class MetadataJWTVerficationFailed(OTAErrorUnRecoverable): + failure_errcode: OTAErrorCode = OTAErrorCode.E_METADATAJWT_CERT_VERIFICATION_FAILED + failure_description: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: certificate verification failed for OTA metadata.jwt" -class BootControlPostRollbackFailed(OTAErrorUnRecoverable): - module: OTAModules = OTAModules.BootController - errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_POSTUPDATE_FAILED - desc: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: post_rollback process failed, switch boot is not finished" +class OTAProxyFailedToStart(OTAErrorUnRecoverable): + failure_errcode: OTAErrorCode = OTAErrorCode.E_OTAPROXY_FAILED_TO_START + failure_description: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: otaproxy is required for multiple ECU update but otaproxy failed to start" -class StandbySlotSpaceNotEnoughError(OTAErrorUnRecoverable): - module: OTAModules = OTAModules.StandbySlotCreater - errcode: OTAErrorCode = OTAErrorCode.E_STANDBY_SLOT_SPACE_NOT_ENOUGH_ERROR - desc: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: standby slot has insufficient space to apply update, abort" +class UpdateDeltaGenerationFailed(OTAErrorUnRecoverable): + failure_errcode: OTAErrorCode = OTAErrorCode.E_UPDATEDELTA_GENERATION_FAILED + failure_description: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: failed to calculate and/or prepare update delta" -class BootControlPreRollbackFailed(OTAErrorUnRecoverable): - module: OTAModules = OTAModules.BootController - errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_PREROLLBACK_FAILED - desc: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: pre_rollback process failed" +class ApplyOTAUpdateFailed(OTAErrorUnRecoverable): + failure_errcode: OTAErrorCode = OTAErrorCode.E_APPLY_OTAUPDATE_FAILED + failure_description: str = ( + f"{_UNRECOVERABLE_DEFAULT_DESC}: failed to apply OTA update to standby slot" + ) From f70df699d9f22535b8218ac9c89004d4610cbb86 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 1 Nov 2023 06:14:11 +0000 Subject: [PATCH 04/27] grub: adjust according to errors' change --- otaclient/app/boot_control/_grub.py | 31 +++++++++++++++++------------ 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/otaclient/app/boot_control/_grub.py b/otaclient/app/boot_control/_grub.py index 421901897..3cdd73153 100644 --- a/otaclient/app/boot_control/_grub.py +++ b/otaclient/app/boot_control/_grub.py @@ -48,7 +48,7 @@ write_str_to_file_sync, ) from ..errors import ( - BootControlInitError, + BootControlStartupFailed, BootControlPostRollbackFailed, BootControlPostUpdateFailed, BootControlPreRollbackFailed, @@ -683,8 +683,8 @@ def prepare_standby_dev(self, *, erase_standby: bool): # TODO: check the standby file system status # if not erase the standby slot except Exception as e: - _err_msg = f"failed to prepare standby dev: {e!r}" - raise BootControlPreUpdateFailed(_err_msg) from e + logger.error(f"failed to prepare standby dev: {e!r}") + raise def finalize_update_switch_boot(self): """Finalize switch boot and use boot files from current booted slot.""" @@ -743,8 +743,9 @@ def __init__(self) -> None: force_initialize=self._boot_control.initialized, ) except Exception as e: - logger.error(f"failed on init boot controller: {e!r}") - raise BootControlInitError from e + _err_msg = f"failed on start grub boot controller: {e!r}" + logger.error(_err_msg) + raise BootControlStartupFailed(_err_msg, module=__name__) from e def _update_fstab(self, *, active_slot_fstab: Path, standby_slot_fstab: Path): """Update standby fstab based on active slot's fstab and just installed new stanby fstab. @@ -861,8 +862,9 @@ def pre_update(self, version: str, *, standby_as_ref: bool, erase_standby=False) # remove old files under standby ota_partition folder self._cleanup_standby_ota_partition_folder() except Exception as e: - logger.error(f"failed on pre_update: {e!r}") - raise BootControlPreUpdateFailed from e + _err_msg = f"failed on pre_update: {e!r}" + logger.error(_err_msg) + raise BootControlPreUpdateFailed(_err_msg, module=__name__) from e def post_update(self) -> Generator[None, None, None]: try: @@ -889,8 +891,9 @@ def post_update(self) -> Generator[None, None, None]: yield # hand over control to otaclient CMDHelperFuncs.reboot() except Exception as e: - logger.error(f"failed on post_update: {e!r}") - raise BootControlPostUpdateFailed from e + _err_msg = f"failed on post_update: {e!r}" + logger.error(_err_msg) + raise BootControlPostUpdateFailed(_err_msg, module=__name__) from e def pre_rollback(self): try: @@ -899,8 +902,9 @@ def pre_rollback(self): self._mp_control.mount_standby() self._ota_status_control.pre_rollback_standby() except Exception as e: - logger.error(f"failed on pre_rollback: {e!r}") - raise BootControlPreRollbackFailed from e + _err_msg = f"failed on pre_rollback: {e!r}" + logger.error(_err_msg) + raise BootControlPreRollbackFailed(_err_msg, module=__name__) from e def post_rollback(self): try: @@ -909,5 +913,6 @@ def post_rollback(self): self._mp_control.umount_all(ignore_error=True) CMDHelperFuncs.reboot() except Exception as e: - logger.error(f"failed on pre_rollback: {e!r}") - raise BootControlPostRollbackFailed from e + _err_msg = f"failed on pre_rollback: {e!r}" + logger.error(_err_msg) + raise BootControlPostRollbackFailed(_err_msg, module=__name__) from e From fe5e5a45c0d21427ab0501eb1aaea36a9457cf31 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 1 Nov 2023 06:14:56 +0000 Subject: [PATCH 05/27] cboot: adjust according to errors' change --- otaclient/app/boot_control/_cboot.py | 126 ++++++++++++++------------- 1 file changed, 66 insertions(+), 60 deletions(-) diff --git a/otaclient/app/boot_control/_cboot.py b/otaclient/app/boot_control/_cboot.py index 9beacef05..e640a09a1 100644 --- a/otaclient/app/boot_control/_cboot.py +++ b/otaclient/app/boot_control/_cboot.py @@ -31,7 +31,7 @@ ) from ..errors import ( - BootControlInitError, + BootControlStartupFailed, BootControlPlatformUnsupported, BootControlPostRollbackFailed, BootControlPostUpdateFailed, @@ -167,26 +167,21 @@ def is_slot_marked_successful(cls, slot: str) -> bool: class _CBootControl: def __init__(self): - try: - # NOTE: only support rqx-580, rqx-58g platform right now! - # detect the chip id - self.chip_id = read_str_from_file(cfg.TEGRA_CHIP_ID_PATH) - if not self.chip_id or int(self.chip_id) not in cfg.CHIP_ID_MODEL_MAP: - raise NotImplementedError( - f"unsupported platform found (chip_id: {self.chip_id}), abort" - ) + # NOTE: only support rqx-580, rqx-58g platform right now! + # detect the chip id + self.chip_id = read_str_from_file(cfg.TEGRA_CHIP_ID_PATH) + if not self.chip_id or int(self.chip_id) not in cfg.CHIP_ID_MODEL_MAP: + raise NotImplementedError( + f"unsupported platform found (chip_id: {self.chip_id}), abort" + ) - self.chip_id = int(self.chip_id) - self.model = cfg.CHIP_ID_MODEL_MAP[self.chip_id] - logger.info(f"{self.model=}, (chip_id={hex(self.chip_id)})") + self.chip_id = int(self.chip_id) + self.model = cfg.CHIP_ID_MODEL_MAP[self.chip_id] + logger.info(f"{self.model=}, (chip_id={hex(self.chip_id)})") - # initializing dev info - self._init_dev_info() - logger.info(f"finished cboot control init: {Nvbootctrl.dump_slots_info()=}") - except NotImplementedError as e: - raise BootControlPlatformUnsupported from e - except Exception as e: - raise BootControlInitError from e + # initializing dev info + self._init_dev_info() + logger.info(f"finished cboot control init: {Nvbootctrl.dump_slots_info()=}") def _init_dev_info(self): self.current_slot: str = Nvbootctrl.get_current_slot() @@ -311,38 +306,45 @@ class CBootController( BootControllerProtocol, ): def __init__(self) -> None: - self._cboot_control: _CBootControl = _CBootControl() - - # load paths - ## first try to unmount standby dev if possible - self.standby_slot_dev = self._cboot_control.get_standby_rootfs_dev() - CMDHelperFuncs.umount(self.standby_slot_dev) - - self.standby_slot_mount_point = Path(cfg.MOUNT_POINT) - self.standby_slot_mount_point.mkdir(exist_ok=True) - - ## refroot mount point - _refroot_mount_point = cfg.ACTIVE_ROOT_MOUNT_POINT - # first try to umount refroot mount point - CMDHelperFuncs.umount(_refroot_mount_point) - if not os.path.isdir(_refroot_mount_point): - os.mkdir(_refroot_mount_point) - self.ref_slot_mount_point = Path(_refroot_mount_point) - - ## ota-status dir - ### current slot - self.current_ota_status_dir = Path(cfg.ACTIVE_ROOTFS_PATH) / Path( - cfg.OTA_STATUS_DIR - ).relative_to("/") - self.current_ota_status_dir.mkdir(parents=True, exist_ok=True) - ### standby slot - # NOTE: might not yet be populated before OTA update applied! - self.standby_ota_status_dir = self.standby_slot_mount_point / Path( - cfg.OTA_STATUS_DIR - ).relative_to("/") - - # init ota-status - self._init_boot_control() + try: + self._cboot_control: _CBootControl = _CBootControl() + + # load paths + ## first try to unmount standby dev if possible + self.standby_slot_dev = self._cboot_control.get_standby_rootfs_dev() + CMDHelperFuncs.umount(self.standby_slot_dev) + + self.standby_slot_mount_point = Path(cfg.MOUNT_POINT) + self.standby_slot_mount_point.mkdir(exist_ok=True) + + ## refroot mount point + _refroot_mount_point = cfg.ACTIVE_ROOT_MOUNT_POINT + # first try to umount refroot mount point + CMDHelperFuncs.umount(_refroot_mount_point) + if not os.path.isdir(_refroot_mount_point): + os.mkdir(_refroot_mount_point) + self.ref_slot_mount_point = Path(_refroot_mount_point) + + ## ota-status dir + ### current slot + self.current_ota_status_dir = Path(cfg.ACTIVE_ROOTFS_PATH) / Path( + cfg.OTA_STATUS_DIR + ).relative_to("/") + self.current_ota_status_dir.mkdir(parents=True, exist_ok=True) + ### standby slot + # NOTE: might not yet be populated before OTA update applied! + self.standby_ota_status_dir = self.standby_slot_mount_point / Path( + cfg.OTA_STATUS_DIR + ).relative_to("/") + + # init ota-status + self._init_boot_control() + except NotImplementedError as e: + raise BootControlPlatformUnsupported(module=__name__) from e + except Exception as e: + raise BootControlStartupFailed( + f"unspecific boot controller startup failure: {e!r}", module=__name__ + ) from e ###### private methods ###### @@ -436,7 +438,7 @@ def _populate_boot_folder_to_separate_bootdev(self): except _errors.MountError as e: _msg = f"failed to mount standby boot dev: {e!r}" logger.error(_msg) - raise _errors.BootControlError(_msg) from e + raise try: dst = _boot_dir_mount_point / "boot" @@ -511,8 +513,9 @@ def pre_update(self, version: str, *, standby_as_ref: bool, erase_standby=False) logger.info("pre-update setting finished") except Exception as e: - logger.exception(f"failed on pre_update: {e!r}") - raise BootControlPreUpdateFailed from e + _err_msg = f"failed on pre_update: {e!r}" + logger.exception(_err_msg) + raise BootControlPreUpdateFailed(f"{e!r}", module=__name__) from e def post_update(self) -> Generator[None, None, None]: try: @@ -550,8 +553,9 @@ def post_update(self) -> Generator[None, None, None]: yield # hand over control back to otaclient CMDHelperFuncs.reboot() except Exception as e: - logger.exception(f"failed on post_update: {e!r}") - raise BootControlPostUpdateFailed from e + _err_msg = f"failed on post_update: {e!r}" + logger.exception(_err_msg) + raise BootControlPostUpdateFailed(_err_msg, module=__name__) from e def pre_rollback(self): try: @@ -563,13 +567,15 @@ def pre_rollback(self): # store ROLLBACKING status to standby self._store_standby_ota_status(wrapper.StatusOta.ROLLBACKING) except Exception as e: - logger.exception(f"failed on pre_rollback: {e!r}") - raise BootControlPreRollbackFailed from e + _err_msg = f"failed on pre_rollback: {e!r}" + logger.exception(_err_msg) + raise BootControlPreRollbackFailed(_err_msg, module=__name__) from e def post_rollback(self): try: self._cboot_control.switch_boot() CMDHelperFuncs.reboot() except Exception as e: - logger.exception(f"failed on post_rollback: {e!r}") - raise BootControlPostRollbackFailed from e + _err_msg = f"failed on post_rollback: {e!r}" + logger.exception(_err_msg) + raise BootControlPostRollbackFailed(_err_msg, module=__name__) from e From d3b220af9a04907172c9871ef16ac24e14d9deba Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 1 Nov 2023 07:23:24 +0000 Subject: [PATCH 06/27] rpi_boot: adjust according to errors's change --- otaclient/app/boot_control/_rpi_boot.py | 127 +++++++++++++----------- 1 file changed, 68 insertions(+), 59 deletions(-) diff --git a/otaclient/app/boot_control/_rpi_boot.py b/otaclient/app/boot_control/_rpi_boot.py index 2f4e5abea..1a14e2710 100644 --- a/otaclient/app/boot_control/_rpi_boot.py +++ b/otaclient/app/boot_control/_rpi_boot.py @@ -11,9 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +"""Boot control support for Raspberry pi 4 Model B.""" -r"""Boot control support for Raspberry pi 4 Model B.""" import os import re from string import Template @@ -22,12 +22,11 @@ from .. import log_setting from ..errors import ( - BootControlInitError, + BootControlStartupFailed, BootControlPostRollbackFailed, BootControlPostUpdateFailed, BootControlPreRollbackFailed, BootControlPreUpdateFailed, - OTAError, ) from ..proto import wrapper from ..common import replace_atomic, subprocess_call @@ -51,6 +50,10 @@ ) +class _RPIBootControllerError(Exception): + """rpi_boot module internal used exception.""" + + class _RPIBootControl: """Boot control helper for rpi4 support. @@ -88,7 +91,7 @@ def __init__(self) -> None: ): _err_msg = "system-boot is not presented or not mounted!" logger.error(_err_msg) - raise BootControlInitError(_err_msg) + raise ValueError(_err_msg) self._init_slots_info() self._init_boot_files() @@ -137,7 +140,7 @@ def _init_slots_info(self): except Exception: raise ValueError( f"unexpected partition layout: {_raw_child_partitions}" - ) + ) from None # it is OK if standby_slot dev doesn't have fslabel or fslabel != standby_slot_id # we will always set the fslabel self._standby_slot = self.AB_FLIPS[self._active_slot] @@ -149,7 +152,7 @@ def _init_slots_info(self): except Exception as e: _err_msg = f"failed to detect AB partition: {e!r}" logger.error(_err_msg) - raise BootControlInitError(_err_msg) from None + raise _RPIBootControllerError(_err_msg) from e def _init_boot_files(self): """Check the availability of boot files. @@ -175,7 +178,7 @@ def _init_boot_files(self): if not self.config_txt_active_slot.is_file(): _err_msg = f"missing {self.config_txt_active_slot=}" logger.error(_err_msg) - raise BootControlInitError(_err_msg) + raise _RPIBootControllerError(_err_msg) self.cmdline_txt_active_slot = ( self.system_boot_path / f"{cfg.CMDLINE_TXT}{self.SEP_CHAR}{self.active_slot}" @@ -183,7 +186,7 @@ def _init_boot_files(self): if not self.cmdline_txt_active_slot.is_file(): _err_msg = f"missing {self.cmdline_txt_active_slot=}" logger.error(_err_msg) - raise BootControlInitError(_err_msg) + raise _RPIBootControllerError(_err_msg) self.vmlinuz_active_slot = ( self.system_boot_path / f"{cfg.VMLINUZ}{self.SEP_CHAR}{self.active_slot}" ) @@ -198,7 +201,7 @@ def _init_boot_files(self): if not self.config_txt_standby_slot.is_file(): _err_msg = f"missing {self.config_txt_standby_slot=}" logger.error(_err_msg) - raise BootControlInitError(_err_msg) + raise _RPIBootControllerError(_err_msg) self.cmdline_txt_standby_slot = ( self.system_boot_path / f"{cfg.CMDLINE_TXT}{self.SEP_CHAR}{self.standby_slot}" @@ -206,7 +209,7 @@ def _init_boot_files(self): if not self.cmdline_txt_standby_slot.is_file(): _err_msg = f"missing {self.cmdline_txt_standby_slot=}" logger.error(_err_msg) - raise BootControlInitError(_err_msg) + raise _RPIBootControllerError(_err_msg) self.vmlinuz_standby_slot = ( self.system_boot_path / f"{cfg.VMLINUZ}{self.SEP_CHAR}{self.standby_slot}" ) @@ -224,8 +227,9 @@ def _update_firmware(self): subprocess_call("flash-kernel", raise_exception=True) os.sync() except Exception as e: - logger.error(f"flash-kernel failed: {e!r}") - raise + _err_msg = f"flash-kernel failed: {e!r}" + logger.error(_err_msg) + raise _RPIBootControllerError(_err_msg) try: # check if the vmlinuz and initrd.img presented in /boot/firmware(system-boot), @@ -238,9 +242,10 @@ def _update_firmware(self): ).is_file(): os.replace(_initrd_img, self.initrd_img_active_slot) os.sync() - except Exception: - logger.error(f"apply new kernel,initrd.img for {self.active_slot} failed") - raise + except Exception as e: + _err_msg = f"apply new kernel,initrd.img for {self.active_slot} failed" + logger.error(_err_msg) + raise _RPIBootControllerError(_err_msg) # exposed API methods/properties @property @@ -345,7 +350,8 @@ def prepare_standby_dev(self, *, erase_standby: bool): CMDHelperFuncs.set_dev_fslabel(self.active_slot_dev, self.standby_slot) except Exception as e: _err_msg = f"failed to prepare standby dev: {e!r}" - raise BootControlPreUpdateFailed(_err_msg) from e + logger.error(_err_msg) + raise _RPIBootControllerError(_err_msg) from e def prepare_tryboot_txt(self): """Copy the standby slot's config.txt as tryboot.txt.""" @@ -358,7 +364,7 @@ def prepare_tryboot_txt(self): except Exception as e: _err_msg = f"failed to prepare tryboot.txt for {self._standby_slot}" logger.error(_err_msg) - raise BootControlPostUpdateFailed(_err_msg) from e + raise _RPIBootControllerError(_err_msg) from e def reboot_tryboot(self): """Reboot with tryboot flag.""" @@ -369,44 +375,49 @@ def reboot_tryboot(self): except Exception as e: _err_msg = "failed to reboot" logger.exception(_err_msg) - raise BootControlPostUpdateFailed(_err_msg) from e + raise _RPIBootControllerError(_err_msg) from e class RPIBootController(BootControllerProtocol): """BootControllerProtocol implementation for rpi4 support.""" def __init__(self) -> None: - self._rpiboot_control = _RPIBootControl() - # mount point prepare - self._mp_control = SlotMountHelper( - standby_slot_dev=self._rpiboot_control.standby_slot_dev, - standby_slot_mount_point=cfg.MOUNT_POINT, - active_slot_dev=self._rpiboot_control.active_slot_dev, - active_slot_mount_point=cfg.ACTIVE_ROOT_MOUNT_POINT, - ) - # init ota-status files - self._ota_status_control = OTAStatusFilesControl( - active_slot=self._rpiboot_control.active_slot, - standby_slot=self._rpiboot_control.standby_slot, - current_ota_status_dir=Path(cfg.ACTIVE_ROOTFS_PATH) - / Path(cfg.OTA_STATUS_DIR).relative_to("/"), - # NOTE: might not yet be populated before OTA update applied! - standby_ota_status_dir=Path(cfg.MOUNT_POINT) - / Path(cfg.OTA_STATUS_DIR).relative_to("/"), - finalize_switching_boot=self._rpiboot_control.finalize_switching_boot, - ) - - # 20230613: remove any leftover flag file if ota_status is not UPDATING/ROLLBACKING - if self._ota_status_control.booted_ota_status not in ( - wrapper.StatusOta.UPDATING, - wrapper.StatusOta.ROLLBACKING, - ): - _flag_file = ( - self._rpiboot_control.system_boot_path / cfg.SWITCH_BOOT_FLAG_FILE + try: + self._rpiboot_control = _RPIBootControl() + # mount point prepare + self._mp_control = SlotMountHelper( + standby_slot_dev=self._rpiboot_control.standby_slot_dev, + standby_slot_mount_point=cfg.MOUNT_POINT, + active_slot_dev=self._rpiboot_control.active_slot_dev, + active_slot_mount_point=cfg.ACTIVE_ROOT_MOUNT_POINT, ) - _flag_file.unlink(missing_ok=True) + # init ota-status files + self._ota_status_control = OTAStatusFilesControl( + active_slot=self._rpiboot_control.active_slot, + standby_slot=self._rpiboot_control.standby_slot, + current_ota_status_dir=Path(cfg.ACTIVE_ROOTFS_PATH) + / Path(cfg.OTA_STATUS_DIR).relative_to("/"), + # NOTE: might not yet be populated before OTA update applied! + standby_ota_status_dir=Path(cfg.MOUNT_POINT) + / Path(cfg.OTA_STATUS_DIR).relative_to("/"), + finalize_switching_boot=self._rpiboot_control.finalize_switching_boot, + ) + + # 20230613: remove any leftover flag file if ota_status is not UPDATING/ROLLBACKING + if self._ota_status_control.booted_ota_status not in ( + wrapper.StatusOta.UPDATING, + wrapper.StatusOta.ROLLBACKING, + ): + _flag_file = ( + self._rpiboot_control.system_boot_path / cfg.SWITCH_BOOT_FLAG_FILE + ) + _flag_file.unlink(missing_ok=True) - logger.debug("rpi_boot initialization finished") + logger.debug("rpi_boot initialization finished") + except Exception as e: + _err_msg = f"failed to start rpi boot controller: {e!r}" + logger.error(_err_msg) + raise BootControlStartupFailed(_err_msg, module=__name__) from e def _copy_kernel_for_standby_slot(self): """Copy the kernel and initrd_img files from current slot /boot @@ -452,7 +463,7 @@ def _copy_kernel_for_standby_slot(self): except Exception as e: _err_msg = "failed to copy kernel/initrd_img for standby slot" logger.error(_err_msg) - raise BootControlPostUpdateFailed(f"{e!r}") + raise _RPIBootControllerError(f"{e!r}") def _write_standby_fstab(self): """Override the standby's fstab file. @@ -474,7 +485,7 @@ def _write_standby_fstab(self): except Exception as e: _err_msg = f"failed to update fstab file for standby slot: {e!r}" logger.error(_err_msg) - raise BootControlPostUpdateFailed(_err_msg) from e + raise _RPIBootControllerError(_err_msg) from e # APIs @@ -503,12 +514,10 @@ def pre_update(self, version: str, *, standby_as_ref: bool, erase_standby: bool) self._rpiboot_control.system_boot_path / cfg.SWITCH_BOOT_FLAG_FILE ) _flag_file.unlink(missing_ok=True) - except OTAError: - raise except Exception as e: _err_msg = f"failed on pre_update: {e!r}" logger.error(_err_msg) - raise BootControlPreUpdateFailed(_err_msg) from e + raise BootControlPreUpdateFailed(_err_msg, module=__name__) from e def pre_rollback(self): try: @@ -519,7 +528,7 @@ def pre_rollback(self): except Exception as e: _err_msg = f"failed on pre_rollback: {e!r}" logger.error(_err_msg) - raise BootControlPreRollbackFailed(_err_msg) from e + raise BootControlPreRollbackFailed(_err_msg, module=__name__) from e def post_rollback(self): try: @@ -527,10 +536,10 @@ def post_rollback(self): self._rpiboot_control.prepare_tryboot_txt() self._mp_control.umount_all(ignore_error=True) self._rpiboot_control.reboot_tryboot() - except OTAError: - raise except Exception as e: - raise BootControlPostRollbackFailed from e + _err_msg = f"failed on post_rollback: {e!r}" + logger.error(_err_msg) + raise BootControlPostRollbackFailed(_err_msg, module=__name__) from e def post_update(self) -> Generator[None, None, None]: try: @@ -542,10 +551,10 @@ def post_update(self) -> Generator[None, None, None]: self._mp_control.umount_all(ignore_error=True) yield # hand over control back to otaclient self._rpiboot_control.reboot_tryboot() - except OTAError: - raise except Exception as e: - raise BootControlPostUpdateFailed from e + _err_msg = f"failed on post_update: {e!r}" + logger.error(_err_msg) + raise BootControlPostUpdateFailed(_err_msg, module=__name__) from e def on_operation_failure(self): """Failure registering and cleanup at failure.""" From 37d3bd06c2ae5e9985b2e1e30bc6a2195ab377d6 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 1 Nov 2023 07:47:00 +0000 Subject: [PATCH 07/27] grub: adjust again according to errors' change --- otaclient/app/boot_control/_grub.py | 97 ++++++++++++++++++----------- 1 file changed, 61 insertions(+), 36 deletions(-) diff --git a/otaclient/app/boot_control/_grub.py b/otaclient/app/boot_control/_grub.py index 3cdd73153..d42de4e6e 100644 --- a/otaclient/app/boot_control/_grub.py +++ b/otaclient/app/boot_control/_grub.py @@ -56,7 +56,6 @@ ) from ..proto import wrapper -from . import _errors from ._common import ( CMDHelperFuncs, OTAStatusFilesControl, @@ -72,6 +71,10 @@ ) +class _GrubBootControllerError(Exception): + """Grub boot controller internal used exception.""" + + @dataclass class _GrubMenuEntry: """ @@ -327,7 +330,9 @@ def _get_sibling_dev(self, active_dev: str) -> str: parent = CMDHelperFuncs.get_parent_dev(active_dev) boot_dev = CMDHelperFuncs.get_dev_by_mount_point("/boot") if not boot_dev: - raise _errors.ABPartitionError("/boot is not mounted") + _err_msg = "/boot is not mounted" + logger.error(_err_msg) + raise ValueError(_err_msg) # list children device file from parent device cmd = f"-Pp -o NAME,FSTYPE {parent}" @@ -344,9 +349,9 @@ def _get_sibling_dev(self, active_dev: str) -> str: ): return m.group(1) - raise _errors.ABPartitionError( - f"{parent=} has unexpected partition layout: {output=}" - ) + _err_msg = f"{parent=} has unexpected partition layout: {output=}" + logger.error(_err_msg) + raise ValueError(_err_msg) def _detect_active_slot(self) -> Tuple[str, str]: """Get active slot's slot_id. @@ -455,7 +460,7 @@ def _check_active_slot_ota_partition_file(self): GrubHelper.INITRD_OTA, GrubHelper.INITRD_OTA_STANDBY, ) - except ValueError as e: + except Exception as e: logger.error( f"failed to get current booted kernel and initrd.image: {e!r}, " "try to use active slot ota-partition files" @@ -500,7 +505,8 @@ def _check_active_slot_ota_partition_file(self): logger.info(f"ota-partition files for {self.active_slot} are ready") - def _get_current_booted_files(self) -> Tuple[str, str]: + @staticmethod + def _get_current_booted_files() -> Tuple[str, str]: """Return the name of booted kernel and initrd. Expected booted kernel and initrd are located under /boot. @@ -571,7 +577,7 @@ def _grub_update_on_booted_slot(self, *, abort_on_standby_missed=True): "refuse to do grub-update" ) logger.error(msg) - raise ValueError(msg) + raise _GrubBootControllerError(msg) # step1: update grub_default file _in = grub_default_file.read_text() @@ -586,7 +592,9 @@ def _grub_update_on_booted_slot(self, *, abort_on_standby_missed=True): ): active_slot_entry_idx, _ = res else: - raise ValueError("boot entry for ACTIVE slot not found, abort") + _err_msg = "boot entry for ACTIVE slot not found, abort" + logger.error(_err_msg) + raise _GrubBootControllerError(_err_msg) # step3: update grub_default again, setting default to # ensure the active slot to be the default @@ -620,7 +628,8 @@ def _grub_update_on_booted_slot(self, *, abort_on_standby_missed=True): "only current active slot's entry is populated." ) if abort_on_standby_missed: - raise ValueError(msg) + logger.error(msg) + raise _GrubBootControllerError(msg) logger.warning(msg) logger.info(f"generated grub_cfg: {pformat(grub_cfg_content)}") @@ -683,43 +692,59 @@ def prepare_standby_dev(self, *, erase_standby: bool): # TODO: check the standby file system status # if not erase the standby slot except Exception as e: - logger.error(f"failed to prepare standby dev: {e!r}") - raise + _err_msg = f"failed to prepare standby dev: {e!r}" + logger.error(_err_msg) + raise _GrubBootControllerError(_err_msg) from e def finalize_update_switch_boot(self): """Finalize switch boot and use boot files from current booted slot.""" # NOTE: since we have not yet switched boot, the active/standby relationship is # reversed here corresponding to booted slot. - self._prepare_kernel_initrd_links(self.standby_ota_partition_folder) - self._ensure_ota_partition_symlinks(active_slot=self.standby_slot) - self._ensure_standby_slot_boot_files_symlinks(standby_slot=self.active_slot) + try: + self._prepare_kernel_initrd_links(self.standby_ota_partition_folder) + self._ensure_ota_partition_symlinks(active_slot=self.standby_slot) + self._ensure_standby_slot_boot_files_symlinks(standby_slot=self.active_slot) - self._grub_update_on_booted_slot(abort_on_standby_missed=True) + self._grub_update_on_booted_slot(abort_on_standby_missed=True) - # switch ota-partition symlink to current booted slot - self._ensure_ota_partition_symlinks(active_slot=self.active_slot) - self._ensure_standby_slot_boot_files_symlinks(standby_slot=self.standby_slot) - return True + # switch ota-partition symlink to current booted slot + self._ensure_ota_partition_symlinks(active_slot=self.active_slot) + self._ensure_standby_slot_boot_files_symlinks( + standby_slot=self.standby_slot + ) + return True + except Exception as e: + _err_msg = f"grub: failed to finalize switch boot: {e!r}" + logger.error(_err_msg) + raise _GrubBootControllerError(_err_msg) from e def grub_reboot_to_standby(self): """Temporarily boot to standby slot after OTA applied to standby slot.""" # ensure all required symlinks for standby slot are presented and valid - self._prepare_kernel_initrd_links(self.standby_ota_partition_folder) - self._ensure_standby_slot_boot_files_symlinks(standby_slot=self.standby_slot) - - # ensure all required symlinks for active slot are presented and valid - # NOTE: reboot after post-update is still using the current active slot's - # ota-partition symlinks(not yet switch boot). - self._prepare_kernel_initrd_links(self.active_ota_partition_folder) - self._ensure_ota_partition_symlinks(active_slot=self.active_slot) - self._grub_update_on_booted_slot(abort_on_standby_missed=True) - - idx, _ = GrubHelper.get_entry( - read_str_from_file(self.grub_file), - kernel_ver=GrubHelper.SUFFIX_OTA_STANDBY, - ) - GrubHelper.grub_reboot(idx) - logger.info(f"system will reboot to {self.standby_slot=}: boot entry {idx}") + try: + self._prepare_kernel_initrd_links(self.standby_ota_partition_folder) + self._ensure_standby_slot_boot_files_symlinks( + standby_slot=self.standby_slot + ) + + # ensure all required symlinks for active slot are presented and valid + # NOTE: reboot after post-update is still using the current active slot's + # ota-partition symlinks(not yet switch boot). + self._prepare_kernel_initrd_links(self.active_ota_partition_folder) + self._ensure_ota_partition_symlinks(active_slot=self.active_slot) + self._grub_update_on_booted_slot(abort_on_standby_missed=True) + + idx, _ = GrubHelper.get_entry( + read_str_from_file(self.grub_file), + kernel_ver=GrubHelper.SUFFIX_OTA_STANDBY, + ) + GrubHelper.grub_reboot(idx) + logger.info(f"system will reboot to {self.standby_slot=}: boot entry {idx}") + + except Exception as e: + _err_msg = f"grub: failed to grub_reboot to standby: {e!r}" + logger.error(_err_msg) + raise _GrubBootControllerError(_err_msg) from e class GrubController(BootControllerProtocol): From 4429288185c8a6d8ea5d74ecb94c50d67d1f183c Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 1 Nov 2023 07:55:29 +0000 Subject: [PATCH 08/27] cboot: adjust again according to errors' change --- otaclient/app/boot_control/_cboot.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/otaclient/app/boot_control/_cboot.py b/otaclient/app/boot_control/_cboot.py index e640a09a1..7a2041eed 100644 --- a/otaclient/app/boot_control/_cboot.py +++ b/otaclient/app/boot_control/_cboot.py @@ -40,7 +40,6 @@ ) from ..proto import wrapper -from . import _errors from ._common import ( OTAStatusMixin, PrepareMountMixin, @@ -58,7 +57,7 @@ ) -class NvbootctrlError(_errors.BootControlError): +class NvbootctrlError(Exception): """Specific internal errors related to nvbootctrl cmd.""" @@ -216,12 +215,12 @@ def _init_dev_info(self): # ensure rootfs is as expected if not Nvbootctrl.check_rootdev(self.current_rootfs_dev): msg = f"rootfs mismatch, expect {self.current_rootfs_dev} as rootfs" - raise ValueError(msg) + raise NvbootctrlError(msg) elif Nvbootctrl.check_rootdev(self.standby_rootfs_dev): msg = ( f"rootfs mismatch, expect {self.standby_rootfs_dev} as standby slot dev" ) - raise ValueError(msg) + raise NvbootctrlError(msg) logger.info("dev info initializing completed") logger.info( @@ -435,10 +434,10 @@ def _populate_boot_folder_to_separate_bootdev(self): self._cboot_control.get_standby_boot_dev(), _boot_dir_mount_point, ) - except _errors.MountError as e: + except Exception as e: _msg = f"failed to mount standby boot dev: {e!r}" logger.error(_msg) - raise + raise NvbootctrlError(_msg) from e try: dst = _boot_dir_mount_point / "boot" @@ -450,14 +449,14 @@ def _populate_boot_folder_to_separate_bootdev(self): except Exception as e: _msg = f"failed to populate boot folder to separate bootdev: {e!r}" logger.error(_msg) - raise _errors.BootControlError(_msg) from e + raise NvbootctrlError(_msg) from e finally: # unmount standby emmc boot dev on finish/failure try: CMDHelperFuncs.umount(_boot_dir_mount_point) - except _errors.MountError as e: + except Exception as e: _failure_msg = f"failed to umount boot dev: {e!r}" - logger.error(_failure_msg) + logger.warning(_failure_msg) # no need to raise to the caller ###### public methods ###### From da0b1601223eff8cd0e8f9447556db3dfe3adde4 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 2 Nov 2023 01:08:01 +0000 Subject: [PATCH 09/27] otaclient: adjust according to ota_errors' change --- otaclient/app/ota_client.py | 218 +++++++++++++++++++----------------- 1 file changed, 113 insertions(+), 105 deletions(-) diff --git a/otaclient/app/ota_client.py b/otaclient/app/ota_client.py index 7b84bcac1..e79247e54 100644 --- a/otaclient/app/ota_client.py +++ b/otaclient/app/ota_client.py @@ -25,35 +25,12 @@ from typing import Optional, Type, Iterator from urllib.parse import urlparse -from . import ota_metadata +from . import downloader, ota_metadata, errors as ota_errors from .boot_control import BootControllerProtocol, get_boot_controller from .common import RetryTaskMap, get_backoff, ensure_otaproxy_start from .configs import config as cfg from .create_standby import StandbySlotCreatorProtocol, get_standby_slot_creator -from .downloader import ( - EMPTY_FILE_SHA256, - DestinationNotAvailableError, - DownloadFailedSpaceNotEnough, - Downloader, - HashVerificaitonError, -) from .ecu_info import ECUInfo -from .errors import ( - MetadataJWTInvalid, - MetadataJWTVerficationFailed, - NetworkError, - ApplyOTAUpdateFailed, - InvalidUpdateRequest, - OTAMetaVerificationFailed, - OTAErrorUnRecoverable, - OTAMetaDownloadFailed, - StandbySlotSpaceNotEnoughError, - UpdateDeltaGenerationFailed, - OTA_APIError, - OTAError, - OTARollbackError, - OTAUpdateError, -) from .interface import OTAClientProtocol from .ota_status import LiveOTAStatus from .proto import wrapper @@ -130,7 +107,7 @@ def __init__( self.total_remove_files_num = 0 # init downloader - self._downloader = Downloader() + self._downloader = downloader.Downloader() self.proxy = proxy # init ota update statics collector @@ -155,7 +132,7 @@ def _download_file(entry: wrapper.RegularInf) -> RegInfProcessedStats: _fhash_str = entry.get_hash() # special treatment to empty file - if _fhash_str == EMPTY_FILE_SHA256: + if _fhash_str == downloader.EMPTY_FILE_SHA256: return cur_stat _local_copy = self._ota_tmp_on_standby / _fhash_str @@ -172,7 +149,7 @@ def _download_file(entry: wrapper.RegularInf) -> RegInfProcessedStats: logger.debug("download neede OTA image files...") # special treatment to empty file, create it first - _empty_file = self._ota_tmp_on_standby / EMPTY_FILE_SHA256 + _empty_file = self._ota_tmp_on_standby / downloader.EMPTY_FILE_SHA256 _empty_file.touch() last_active_timestamp = int(time.time()) @@ -257,8 +234,11 @@ def _update_standby_slot(self): self.total_download_fiies_size = _delta_bundle.total_download_files_size self.total_remove_files_num = len(_delta_bundle.rm_delta) except Exception as e: - logger.error(f"failed to generate delta: {e!r}") - raise UpdateDeltaGenerationFailed from e + _err_msg = f"failed to generate delta: {e!r}" + logger.error(_err_msg) + raise ota_errors.UpdateDeltaGenerationFailed( + _err_msg, module=__name__ + ) from e # --- download needed files --- # logger.info( @@ -268,12 +248,16 @@ def _update_standby_slot(self): self.update_phase = wrapper.UpdatePhase.DOWNLOADING_OTA_FILES try: self._download_files(_delta_bundle.get_download_list()) - except DownloadFailedSpaceNotEnough: - logger.critical("not enough space is left on standby slot") - raise StandbySlotSpaceNotEnoughError from None + except downloader.DownloadFailedSpaceNotEnough: + _err_msg = "not enough space is left on standby slot" + logger.error(_err_msg) + raise ota_errors.StandbySlotInsufficientSpace( + _err_msg, module=__name__ + ) from None except Exception as e: - logger.error(f"failed to finish downloading files: {e!r}") - raise NetworkError from e + _err_msg = f"unspecific error, failed to finish downloading files: {e!r}" + logger.error(_err_msg) + raise ota_errors.NetworkError(_err_msg, module=__name__) from e # shutdown downloader on download finished self._downloader.shutdown() @@ -284,8 +268,10 @@ def _update_standby_slot(self): try: self._standby_slot_creator.create_standby_slot() except Exception as e: - logger.error(f"failed to apply update to standby slot: {e!r}") - raise ApplyOTAUpdateFailed from e + _err_msg = f"failed to apply update to standby slot: {e!r}" + logger.error(_err_msg) + raise ota_errors.ApplyOTAUpdateFailed(_err_msg, module=__name__) from e + logger.info("finished updating standby slot") def _execute_update( @@ -329,7 +315,9 @@ def _execute_update( ), f"invalid cookies, expecting json object: {cookies_json}" self._downloader.configure_cookies(_cookies) except (JSONDecodeError, AssertionError) as e: - raise InvalidUpdateRequest from e + _err_msg = f"cookie is invalid: {cookies_json=}" + logger.error(_err_msg) + raise ota_errors.InvalidUpdateRequest(_err_msg, module=__name__) from e # configure proxy logger.debug("configure proxy setting...") @@ -354,30 +342,41 @@ def _execute_update( self.total_files_size_uncompressed = ( self._otameta.total_files_size_uncompressed ) - except HashVerificaitonError as e: - logger.error("failed to verify ota metafiles hash") - raise OTAMetaVerificationFailed from e - except DestinationNotAvailableError as e: - logger.error("failed to save ota metafiles") - raise OTAErrorUnRecoverable from e + except downloader.HashVerificaitonError as e: + _err_msg = f"downloader: keep failing to verify ota metafiles' hash: {e!r}" + logger.error(_err_msg) + raise ota_errors.MetadataJWTVerficationFailed( + _err_msg, module=__name__ + ) from e + except downloader.DestinationNotAvailableError as e: + _err_msg = f"downloader: failed to save ota metafiles: {e!r}" + logger.error(_err_msg) + raise ota_errors.OTAErrorUnRecoverable(_err_msg, module=__name__) from e except ota_metadata.MetadataJWTVerificationFailed as e: - logger.error(f"failed to verify metadata.jwt: {e!r}") - raise MetadataJWTVerficationFailed from e + _err_msg = f"failed to verify metadata.jwt: {e!r}" + logger.error(_err_msg) + raise ota_errors.MetadataJWTVerficationFailed( + _err_msg, module=__name__ + ) from e except ota_metadata.MetadataJWTPayloadInvalid as e: - logger.error(f"metadata.jwt is invalid: {e!r}") - raise MetadataJWTInvalid from e + _err_msg = f"metadata.jwt is invalid: {e!r}" + logger.error(_err_msg) + raise ota_errors.MetadataJWTInvalid(_err_msg, module=__name__) from e except Exception as e: - logger.error(f"failed to prepare ota metafiles: {e!r}") - raise OTAMetaDownloadFailed from e + _err_msg = f"unspecific error, failed to prepare ota metafiles: {e!r}" + logger.error(_err_msg) + raise ota_errors.OTAMetaDownloadFailed(_err_msg, module=__name__) from e # ------ execute local update ------ # logger.info("enter local OTA update...") try: self._update_standby_slot() - except OTAError: - raise + except ota_errors.OTAError: + raise # no need to wrap an OTAError again except Exception as e: - raise ApplyOTAUpdateFailed(f"unspecific applying OTA update failure: {e!r}") + raise ota_errors.ApplyOTAUpdateFailed( + f"unspecific applying OTA update failure: {e!r}", module=__name__ + ) # ------ post update ------ # logger.info("local update finished, wait on all subecs...") @@ -440,14 +439,14 @@ def execute( """ try: self._execute_update(version, raw_url_base, cookies_json) - except OTAError as e: + except ota_errors.OTAError as e: logger.error(f"update failed: {e!r}") self._boot_controller.on_operation_failure() - raise OTAUpdateError(e) from e + raise # do not cover the OTA error again except Exception as e: - raise OTAUpdateError( - ApplyOTAUpdateFailed(f"unspecific OTA failure: {e!r}") - ) from e + _err_msg = f"unspecific error, update failed: {e!r}" + self._boot_controller.on_operation_failure() + raise ota_errors.ApplyOTAUpdateFailed(_err_msg, module=__name__) from e finally: self.shutdown() @@ -458,21 +457,22 @@ def __init__(self, boot_controller: BootControllerProtocol) -> None: def execute(self): try: - # enter rollback self._boot_controller.pre_rollback() - # leave rollback self._boot_controller.post_rollback() - except OTAError as e: + except ota_errors.OTAError as e: logger.error(f"rollback failed: {e!r}") self._boot_controller.on_operation_failure() - raise OTARollbackError(e) from e + raise class OTAClient(OTAClientProtocol): """ Init params: - boot_control_cls: type of boot control mechanism to use + boot_controller: boot control instance create_standby_cls: type of create standby slot mechanism to use + my_ecu_id: ECU id of the device running this otaclient instance + control_flags: flags used by otaclient and ota_service stub for synchronization + proxy: upper otaproxy URL """ OTACLIENT_VERSION = __version__ @@ -486,35 +486,40 @@ def __init__( control_flags: OTAClientControlFlags, proxy: Optional[str] = None, ): - self.my_ecu_id = my_ecu_id - # ensure only one update/rollback session is running - self._lock = threading.Lock() - - self.boot_controller = boot_controller - self.create_standby_cls = create_standby_cls - self.live_ota_status = LiveOTAStatus( - self.boot_controller.get_booted_ota_status() - ) + try: + self.my_ecu_id = my_ecu_id + # ensure only one update/rollback session is running + self._lock = threading.Lock() + + self.boot_controller = boot_controller + self.create_standby_cls = create_standby_cls + self.live_ota_status = LiveOTAStatus( + self.boot_controller.get_booted_ota_status() + ) - self.current_version = self.boot_controller.load_version() - self.proxy = proxy - self.control_flags = control_flags + self.current_version = self.boot_controller.load_version() + self.proxy = proxy + self.control_flags = control_flags - # executors for update/rollback - self._update_executor: _OTAUpdater = None # type: ignore - self._rollback_executor: _OTARollbacker = None # type: ignore + # executors for update/rollback + self._update_executor: _OTAUpdater = None # type: ignore + self._rollback_executor: _OTARollbacker = None # type: ignore - # err record - self.last_failure_type = wrapper.FailureType.NO_FAILURE - self.last_failure_reason = "" - self.last_failure_traceback = "" + # err record + self.last_failure_type = wrapper.FailureType.NO_FAILURE + self.last_failure_reason = "" + self.last_failure_traceback = "" + except Exception as e: + _err_msg = f"failed to start otaclient core: {e!r}" + logger.error(_err_msg) + raise ota_errors.OTAClientStartupFailed(_err_msg, module=__name__) from e - def _on_failure(self, exc: OTA_APIError, ota_status: wrapper.StatusOta): + def _on_failure(self, exc: ota_errors.OTAError, ota_status: wrapper.StatusOta): self.live_ota_status.set_ota_status(ota_status) try: - self.last_failure_type = exc.get_err_type() - self.last_failure_reason = exc.get_err_reason() - self.last_failure_traceback = exc.get_traceback() + self.last_failure_type = exc.failure_type + self.last_failure_reason = exc.get_failure_reason() + self.last_failure_traceback = exc.get_failure_traceback() logger.error(f"on {ota_status=}: {self.last_failure_traceback=}") finally: exc = None # type: ignore , prevent ref cycle @@ -536,7 +541,7 @@ def update(self, version: str, url_base: str, cookies_json: str): self.last_failure_traceback = "" self.live_ota_status.set_ota_status(wrapper.StatusOta.UPDATING) self._update_executor.execute(version, url_base, cookies_json) - except OTAUpdateError as e: + except ota_errors.OTAError as e: self._on_failure(e, wrapper.StatusOta.FAILURE) finally: self._update_executor = None # type: ignore @@ -560,7 +565,7 @@ def rollback(self): self.live_ota_status.set_ota_status(wrapper.StatusOta.ROLLBACKING) self._rollback_executor.execute() # silently ignore overlapping request - except OTARollbackError as e: + except ota_errors.OTAError as e: self._on_failure(e, wrapper.StatusOta.ROLLBACK_FAILURE) finally: self._rollback_executor = None # type: ignore @@ -599,12 +604,14 @@ def __init__( self.ecu_id = ecu_info.ecu_id self.otaclient_version = otaclient_version - # default boot startup failure if boot controller crashed without + # default boot startup failure if boot_controller/otaclient_core crashed without # raising specific error self._otaclient_startup_failed_status = wrapper.StatusResponseEcuV2( ecu_id=ecu_info.ecu_id, otaclient_version=otaclient_version, ota_status=wrapper.StatusOta.FAILURE, + failure_type=wrapper.FailureType.UNRECOVERABLE, + failure_reason="unspecific error", ) self._update_rollback_lock = asyncio.Lock() self._run_in_executor = partial( @@ -624,18 +631,18 @@ def __init__( # boot controller starts up try: _bootctrl_inst = _bootctrl_cls() - except Exception as e: + except ota_errors.OTAError as e: self._otaclient_startup_failed_status = wrapper.StatusResponseEcuV2( ecu_id=ecu_info.ecu_id, otaclient_version=otaclient_version, ota_status=wrapper.StatusOta.FAILURE, failure_type=wrapper.FailureType.UNRECOVERABLE, - failure_reason=f"{e!r}", - failure_traceback=f"{e!r}", + failure_reason=e.get_failure_reason(), + failure_traceback=e.get_failure_traceback(), ) return - # compose otaclient + # otaclient core starts up try: self._otaclient_inst = OTAClient( boot_controller=_bootctrl_inst, @@ -644,21 +651,24 @@ def __init__( control_flags=control_flags, proxy=proxy, ) - except Exception as e: + except ota_errors.OTAError as e: + self._otaclient_startup_failed_status = wrapper.StatusResponseEcuV2( + ecu_id=ecu_info.ecu_id, + otaclient_version=otaclient_version, + ota_status=wrapper.StatusOta.FAILURE, + failure_type=wrapper.FailureType.UNRECOVERABLE, + failure_reason=e.get_failure_reason(), + failure_traceback=e.get_failure_traceback(), + ) return - @property - def is_busy(self) -> bool: - return self._update_rollback_lock.locked() - async def dispatch_update( self, request: wrapper.UpdateRequestEcu ) -> wrapper.UpdateResponseEcu: # prevent update operation if otaclient is not started if self._otaclient_inst is None: return wrapper.UpdateResponseEcu( - ecu_id=self.ecu_id, - result=wrapper.FailureType.UNRECOVERABLE, + ecu_id=self.ecu_id, result=wrapper.FailureType.UNRECOVERABLE ) # check and acquire lock @@ -667,8 +677,7 @@ async def dispatch_update( f"ongoing operation: {self.last_operation=}, ignore incoming {request=}" ) return wrapper.UpdateResponseEcu( - ecu_id=self.ecu_id, - result=wrapper.FailureType.RECOVERABLE, + ecu_id=self.ecu_id, result=wrapper.FailureType.RECOVERABLE ) # immediately take the lock if not locked @@ -707,8 +716,7 @@ async def dispatch_rollback( # prevent rollback operation if otaclient is not started if self._otaclient_inst is None: return wrapper.RollbackResponseEcu( - ecu_id=self.ecu_id, - result=wrapper.FailureType.UNRECOVERABLE, + ecu_id=self.ecu_id, result=wrapper.FailureType.UNRECOVERABLE ) # check and acquire lock @@ -717,8 +725,7 @@ async def dispatch_rollback( f"ongoing operation: {self.last_operation=}, ignore incoming {request=}" ) return wrapper.RollbackResponseEcu( - ecu_id=self.ecu_id, - result=wrapper.FailureType.RECOVERABLE, + ecu_id=self.ecu_id, result=wrapper.FailureType.RECOVERABLE ) # immediately take the lock if not locked @@ -748,5 +755,6 @@ async def get_status(self) -> wrapper.StatusResponseEcuV2: # otaclient is not started due to boot control startup failed if self._otaclient_inst is None: return self._otaclient_startup_failed_status + # otaclient core started, query status from it return await self._run_in_executor(self._otaclient_inst.status) From 3dd6e11e697cbf7f7d6df7f7e1e82b3923130ccf Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 2 Nov 2023 01:08:38 +0000 Subject: [PATCH 10/27] errors: define OTAClientStartupFailed error --- otaclient/app/errors.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/otaclient/app/errors.py b/otaclient/app/errors.py index ca64303f6..7b7c4cbeb 100644 --- a/otaclient/app/errors.py +++ b/otaclient/app/errors.py @@ -55,6 +55,7 @@ class OTAErrorCode(int, Enum): E_OTAPROXY_FAILED_TO_START = 311 E_UPDATEDELTA_GENERATION_FAILED = 312 E_APPLY_OTAUPDATE_FAILED = 313 + E_OTACLIENT_STARTUP_FAILED = 314 def to_errcode_str(self) -> str: return f"{self.value:0>3}" @@ -86,7 +87,7 @@ def get_failure_traceback(self, *, splitter="") -> str: traceback.format_exception(type(self), self, self.__traceback__) ) - def failure_reason(self, *, append_traceback=False) -> str: + def get_failure_reason(self, *, append_traceback=False) -> str: """Return failure_reason str.""" _failure_info = { "module": self.module, @@ -239,3 +240,10 @@ class ApplyOTAUpdateFailed(OTAErrorUnRecoverable): failure_description: str = ( f"{_UNRECOVERABLE_DEFAULT_DESC}: failed to apply OTA update to standby slot" ) + + +class OTAClientStartupFailed(OTAErrorUnRecoverable): + failure_errcode: OTAErrorCode = OTAErrorCode.E_OTACLIENT_STARTUP_FAILED + failure_description: str = ( + f"{_UNRECOVERABLE_DEFAULT_DESC}: failed to start otaclient instance" + ) From c9595be0b44e9996b253ae344fc2b8ae158c8e44 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 2 Nov 2023 01:20:10 +0000 Subject: [PATCH 11/27] otaclient.OTAServicer: re-add missing is_busy and local_used_proxy attributes --- otaclient/app/ota_client.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/otaclient/app/ota_client.py b/otaclient/app/ota_client.py index e79247e54..b0a9c0223 100644 --- a/otaclient/app/ota_client.py +++ b/otaclient/app/ota_client.py @@ -603,6 +603,7 @@ def __init__( ) -> None: self.ecu_id = ecu_info.ecu_id self.otaclient_version = otaclient_version + self.local_used_proxy_url = proxy # default boot startup failure if boot_controller/otaclient_core crashed without # raising specific error @@ -662,6 +663,10 @@ def __init__( ) return + @property + def is_busy(self) -> bool: + return self._update_rollback_lock.locked() + async def dispatch_update( self, request: wrapper.UpdateRequestEcu ) -> wrapper.UpdateResponseEcu: From a4aed9d095409d309e3cf211a332db1c1110ce35 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 2 Nov 2023 02:13:08 +0000 Subject: [PATCH 12/27] test_otaclient_stub: update accordingly to make it work again --- tests/test_ota_client_stub.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/test_ota_client_stub.py b/tests/test_ota_client_stub.py index 448c175e5..a741aeb43 100644 --- a/tests/test_ota_client_stub.py +++ b/tests/test_ota_client_stub.py @@ -16,14 +16,13 @@ import asyncio import pytest from concurrent.futures import ThreadPoolExecutor -from functools import partial from pathlib import Path from typing import Any, Dict, List, Set import pytest_mock from otaclient.app.ecu_info import ECUInfo -from otaclient.app.ota_client import OTAClientBusy, OTAClientWrapper +from otaclient.app.ota_client import OTAServicer from otaclient.app.ota_client_call import OtaClientCall from otaclient.app.ota_client_stub import ( ECUStatusStorage, @@ -716,7 +715,7 @@ async def setup_test(self, tmp_path: Path, mocker: pytest_mock.MockerFixture): await asyncio.sleep(self.ENSURE_NEXT_CHECKING_ROUND) # ensure the task stopping # --- mocker --- # - self.otaclient_wrapper = mocker.MagicMock(spec=OTAClientWrapper) + self.otaclient_wrapper = mocker.MagicMock(spec=OTAServicer) self.ecu_status_tracker = mocker.MagicMock() self.otaproxy_launcher = mocker.MagicMock(spec=OTAProxyLauncher) # mock OTAClientCall, make update_call return success on any update dispatches to subECUs @@ -736,7 +735,7 @@ async def setup_test(self, tmp_path: Path, mocker: pytest_mock.MockerFixture): mocker.MagicMock(return_value=self.ecu_storage), ) mocker.patch( - f"{cfg.OTACLIENT_STUB_MODULE_PATH}.OTAClientWrapper", + f"{cfg.OTACLIENT_STUB_MODULE_PATH}.OTAServicer", mocker.MagicMock(return_value=self.otaclient_wrapper), ) mocker.patch( @@ -886,6 +885,11 @@ async def test_update_normal( update_target_ids: Set[str], expected: wrapper.UpdateResponse, ): + # --- setup --- # + self.otaclient_wrapper.dispatch_update.return_value = wrapper.UpdateResponseEcu( + ecu_id=self.ecu_info.ecu_id, result=wrapper.FailureType.NO_FAILURE + ) + # --- execution --- # resp = await self.otaclient_service_stub.update(update_request) @@ -898,7 +902,9 @@ async def test_update_normal( async def test_update_local_ecu_busy(self): # --- preparation --- # - self.otaclient_wrapper.dispatch_update.side_effect = OTAClientBusy() + self.otaclient_wrapper.dispatch_update.return_value = wrapper.UpdateResponseEcu( + ecu_id="autoware", result=wrapper.FailureType.RECOVERABLE + ) update_request_ecu = wrapper.UpdateRequestEcu( ecu_id="autoware", version="version", url="url", cookies="cookies" ) From 073706e58d3825044798fb84b6e398dbf246fdc0 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 2 Nov 2023 03:01:44 +0000 Subject: [PATCH 13/27] test_otaclient: update accordinlgy to make it work again --- otaclient/app/ota_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/otaclient/app/ota_client.py b/otaclient/app/ota_client.py index b0a9c0223..495ae6063 100644 --- a/otaclient/app/ota_client.py +++ b/otaclient/app/ota_client.py @@ -604,6 +604,7 @@ def __init__( self.ecu_id = ecu_info.ecu_id self.otaclient_version = otaclient_version self.local_used_proxy_url = proxy + self.last_operation: Optional[wrapper.StatusOta] = None # default boot startup failure if boot_controller/otaclient_core crashed without # raising specific error @@ -618,7 +619,6 @@ def __init__( self._run_in_executor = partial( asyncio.get_running_loop().run_in_executor, executor ) - self._last_operation: Optional[wrapper.StatusOta] = None # # ------ compose otaclient ------ From 31a250015e5cc78f03394ce291ebd3cd3d9b1e3b Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 2 Nov 2023 03:46:10 +0000 Subject: [PATCH 14/27] errors: provide a error report generate API --- otaclient/app/errors.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/otaclient/app/errors.py b/otaclient/app/errors.py index 7b7c4cbeb..04503590e 100644 --- a/otaclient/app/errors.py +++ b/otaclient/app/errors.py @@ -101,6 +101,18 @@ def get_failure_reason(self, *, append_traceback=False) -> str: f"\n{_failure_info}" ) + def get_error_report(self, title: str) -> str: + _traceback = self.get_failure_traceback(splitter="\n") + return ( + f"\n{title}\n" + "\n------ failure_reason ------\n" + f"{self.get_failure_reason()}" + "\n------ end of failure_reason ------\n" + "\n------ failure traceback ------\n" + f"failure_traceback: {_traceback}" + "\n------ end of failure traceback ------\n" + ) + # # ------ Network related error ------ From da31b1266fa97eeb87bddd8d35ca7d639d95612c Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 2 Nov 2023 03:46:38 +0000 Subject: [PATCH 15/27] otaclient: fix error logging not properly formatted --- otaclient/app/ota_client.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/otaclient/app/ota_client.py b/otaclient/app/ota_client.py index 495ae6063..c4f7f4095 100644 --- a/otaclient/app/ota_client.py +++ b/otaclient/app/ota_client.py @@ -520,7 +520,7 @@ def _on_failure(self, exc: ota_errors.OTAError, ota_status: wrapper.StatusOta): self.last_failure_type = exc.failure_type self.last_failure_reason = exc.get_failure_reason() self.last_failure_traceback = exc.get_failure_traceback() - logger.error(f"on {ota_status=}: {self.last_failure_traceback=}") + logger.error(f"{ota_status.name}, traceback: {self.last_failure_traceback}") finally: exc = None # type: ignore , prevent ref cycle @@ -633,13 +633,16 @@ def __init__( try: _bootctrl_inst = _bootctrl_cls() except ota_errors.OTAError as e: + logger.error( + e.get_error_report(title=f"boot controller startup failed: {e!r}") + ) self._otaclient_startup_failed_status = wrapper.StatusResponseEcuV2( ecu_id=ecu_info.ecu_id, otaclient_version=otaclient_version, ota_status=wrapper.StatusOta.FAILURE, failure_type=wrapper.FailureType.UNRECOVERABLE, failure_reason=e.get_failure_reason(), - failure_traceback=e.get_failure_traceback(), + failure_traceback=e.get_failure_traceback(splitter="\n"), ) return @@ -653,13 +656,16 @@ def __init__( proxy=proxy, ) except ota_errors.OTAError as e: + logger.error( + e.get_error_report(title=f"otaclient core startup failed: {e!r}") + ) self._otaclient_startup_failed_status = wrapper.StatusResponseEcuV2( ecu_id=ecu_info.ecu_id, otaclient_version=otaclient_version, ota_status=wrapper.StatusOta.FAILURE, failure_type=wrapper.FailureType.UNRECOVERABLE, failure_reason=e.get_failure_reason(), - failure_traceback=e.get_failure_traceback(), + failure_traceback=e.get_failure_traceback(splitter="\n"), ) return From 36c1ae3c86108b6a54c98ca9170b9f070dc3e2a5 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 2 Nov 2023 03:47:04 +0000 Subject: [PATCH 16/27] test_otaclient: update accordingly to make it work again --- tests/test_ota_client.py | 43 +++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/tests/test_ota_client.py b/tests/test_ota_client.py index 541fead4a..5990e5295 100644 --- a/tests/test_ota_client.py +++ b/tests/test_ota_client.py @@ -30,13 +30,12 @@ from otaclient.app.create_standby.common import DeltaBundle, RegularDelta from otaclient.app.configs import config as otaclient_cfg from otaclient.app.ecu_info import ECUInfo -from otaclient.app.errors import OTAErrorRecoverable, OTAUpdateError +from otaclient.app.errors import OTAError, OTAErrorRecoverable from otaclient.app.ota_client import ( OTAClient, _OTAUpdater, - OTAClientBusy, OTAClientControlFlags, - OTAClientWrapper, + OTAServicer, ) from otaclient.app.ota_metadata import parse_regulars_from_txt, parse_dirs_from_txt from otaclient.app.proto.wrapper import RegularInf, DirectoryInf @@ -236,7 +235,7 @@ def mock_setup(self, mocker: pytest_mock.MockerFixture): ) self.ota_client = OTAClient( - boot_control_cls=mocker.MagicMock(return_value=self.boot_controller), + boot_controller=self.boot_controller, create_standby_cls=mocker.MagicMock(), my_ecu_id=self.MY_ECU_ID, control_flags=self.control_flags, @@ -274,7 +273,7 @@ def test_update_normal_finished(self): def test_update_interrupted(self): # inject exception - _error = OTAUpdateError(OTAErrorRecoverable("network disconnected")) + _error = OTAErrorRecoverable("network disconnected", module=__name__) self.ota_updater.execute.side_effect = _error # --- execution --- # @@ -356,7 +355,7 @@ def test_status_in_update(self): ) -class TestOTAClientWrapper: +class TestOTAServicer: BOOTLOADER_TYPE = BootloaderType.GRUB ECU_INFO = ECUInfo( format_version=1, @@ -372,23 +371,27 @@ async def mock_setup(self, mocker: pytest_mock.MockerFixture): self.otaclient = mocker.MagicMock(spec=OTAClient) self.otaclient_cls = mocker.MagicMock(return_value=self.otaclient) self.standby_slot_creator_cls = mocker.MagicMock() - self.boot_control_cls = mocker.MagicMock() + self.boot_controller = mocker.MagicMock(spec=BootControllerProtocol) self.control_flags = mocker.MagicMock(spec=OTAClientControlFlags) - # patch OTAClient + # + # ------ patching ------ + # mocker.patch(f"{cfg.OTACLIENT_MODULE_PATH}.OTAClient", self.otaclient_cls) mocker.patch( f"{cfg.OTACLIENT_MODULE_PATH}.get_boot_controller", - return_value=self.boot_control_cls, + return_value=mocker.MagicMock(return_value=self.boot_controller), ) mocker.patch( f"{cfg.OTACLIENT_MODULE_PATH}.get_standby_slot_creator", return_value=self.standby_slot_creator_cls, ) - # start the stub + # + # ------ start OTAServicer instance ------ + # self.local_use_proxy = "" - self.otaclient_stub = OTAClientWrapper( + self.otaclient_stub = OTAServicer( ecu_info=self.ECU_INFO, executor=self._executor, control_flags=self.control_flags, @@ -401,13 +404,19 @@ async def mock_setup(self, mocker: pytest_mock.MockerFixture): self._executor.shutdown(wait=False) def test_stub_initializing(self): + # + # ------ assertion ------ + # + + # ensure the OTAServicer properly compose otaclient core self.otaclient_cls.assert_called_once_with( - boot_control_cls=self.boot_control_cls, + boot_controller=self.boot_controller, create_standby_cls=self.standby_slot_creator_cls, my_ecu_id=self.ECU_INFO.ecu_id, control_flags=self.control_flags, proxy=self.local_use_proxy, ) + assert self.otaclient_stub.last_operation is None assert self.otaclient_stub.local_used_proxy_url is self.local_use_proxy @@ -434,9 +443,8 @@ def _updating(*args, **kwargs): assert self.otaclient_stub.last_operation is wrapper.StatusOta.UPDATING assert self.otaclient_stub.is_busy # test ota update/rollback exclusive lock, - # OTAClientBusy error should be raised on another request - with pytest.raises(OTAClientBusy): - await self.otaclient_stub.dispatch_update(update_request_ecu) + resp = await self.otaclient_stub.dispatch_update(update_request_ecu) + assert resp.result == wrapper.FailureType.RECOVERABLE # finish up update _updating_event.set() @@ -465,9 +473,8 @@ def _rollbacking(*args, **kwargs): assert self.otaclient_stub.last_operation is wrapper.StatusOta.ROLLBACKING assert self.otaclient_stub.is_busy # test ota update/rollback exclusive lock, - # OTAClientBusy error should be raised on another request - with pytest.raises(OTAClientBusy): - await self.otaclient_stub.dispatch_rollback(wrapper.RollbackRequestEcu()) + resp = await self.otaclient_stub.dispatch_rollback(wrapper.RollbackRequestEcu()) + assert resp.result == wrapper.FailureType.RECOVERABLE # finish up rollback _rollbacking_event.set() From 62cf3390070a834c865b8aba7cba25ffbd9c7e9d Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 2 Nov 2023 03:59:30 +0000 Subject: [PATCH 17/27] otaclient@L262: properly capture downloading group failure --- otaclient/app/ota_client.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/otaclient/app/ota_client.py b/otaclient/app/ota_client.py index c4f7f4095..c7fb840e0 100644 --- a/otaclient/app/ota_client.py +++ b/otaclient/app/ota_client.py @@ -27,7 +27,12 @@ from . import downloader, ota_metadata, errors as ota_errors from .boot_control import BootControllerProtocol, get_boot_controller -from .common import RetryTaskMap, get_backoff, ensure_otaproxy_start +from .common import ( + get_backoff, + ensure_otaproxy_start, + RetryTaskMap, + RetryTaskMapInterrupted, +) from .configs import config as cfg from .create_standby import StandbySlotCreatorProtocol, get_standby_slot_creator from .ecu_info import ECUInfo @@ -254,6 +259,13 @@ def _update_standby_slot(self): raise ota_errors.StandbySlotInsufficientSpace( _err_msg, module=__name__ ) from None + except RetryTaskMapInterrupted as e: + _err_msg = ( + f"downloading group keeps failing for {cfg.DOWNLOAD_GROUP_INACTIVE_TIMEOUT}s, " + f"last_error: {e!r}" + ) + logger.error(_err_msg) + raise ota_errors.NetworkError(_err_msg, module=__name__) from e except Exception as e: _err_msg = f"unspecific error, failed to finish downloading files: {e!r}" logger.error(_err_msg) From 5de37ad1e49e3cf15316378fe3565a04729c3eba Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 2 Nov 2023 04:03:37 +0000 Subject: [PATCH 18/27] otaclient: on_failure uses OTAError.get_error_report API --- otaclient/app/ota_client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/otaclient/app/ota_client.py b/otaclient/app/ota_client.py index c7fb840e0..22ad659a5 100644 --- a/otaclient/app/ota_client.py +++ b/otaclient/app/ota_client.py @@ -532,7 +532,9 @@ def _on_failure(self, exc: ota_errors.OTAError, ota_status: wrapper.StatusOta): self.last_failure_type = exc.failure_type self.last_failure_reason = exc.get_failure_reason() self.last_failure_traceback = exc.get_failure_traceback() - logger.error(f"{ota_status.name}, traceback: {self.last_failure_traceback}") + logger.error( + exc.get_error_report(f"OTA failed with {ota_status.name}: {exc!r}") + ) finally: exc = None # type: ignore , prevent ref cycle From 9fd339b6c6b828f914ed61026f640e8f9345c039 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 2 Nov 2023 05:46:31 +0000 Subject: [PATCH 19/27] errors: get_failure_reason now only returns failure_code + failure_description --- otaclient/app/errors.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/otaclient/app/errors.py b/otaclient/app/errors.py index 04503590e..c22a903f1 100644 --- a/otaclient/app/errors.py +++ b/otaclient/app/errors.py @@ -87,19 +87,9 @@ def get_failure_traceback(self, *, splitter="") -> str: traceback.format_exception(type(self), self, self.__traceback__) ) - def get_failure_reason(self, *, append_traceback=False) -> str: + def get_failure_reason(self) -> str: """Return failure_reason str.""" - _failure_info = { - "module": self.module, - "exec_args": self.args, - } - if append_traceback: - _failure_info["failure_traceback"] = self.get_failure_traceback() - - return ( - f"{self.failure_errcode_str}: {self.failure_description}" - f"\n{_failure_info}" - ) + return f"{self.failure_errcode_str}: {self.failure_description}" def get_error_report(self, title: str) -> str: _traceback = self.get_failure_traceback(splitter="\n") From b72be80ad480c7fac8c7405f31b24326ee1d92af Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 2 Nov 2023 05:47:20 +0000 Subject: [PATCH 20/27] errors: get_error_report has new format, get_failure_traceback now by default use \n as splitter --- otaclient/app/errors.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/otaclient/app/errors.py b/otaclient/app/errors.py index c22a903f1..87aa06d0c 100644 --- a/otaclient/app/errors.py +++ b/otaclient/app/errors.py @@ -82,7 +82,7 @@ def __init__(self, *args: object, module: str) -> None: def failure_errcode_str(self) -> str: return f"{self.ERROR_PREFIX}{self.failure_errcode.to_errcode_str()}" - def get_failure_traceback(self, *, splitter="") -> str: + def get_failure_traceback(self, *, splitter="\n") -> str: return splitter.join( traceback.format_exception(type(self), self, self.__traceback__) ) @@ -91,16 +91,19 @@ def get_failure_reason(self) -> str: """Return failure_reason str.""" return f"{self.failure_errcode_str}: {self.failure_description}" - def get_error_report(self, title: str) -> str: - _traceback = self.get_failure_traceback(splitter="\n") + def get_error_report(self, title: str = "") -> str: + """The detailed failure report for debug use.""" return ( f"\n{title}\n" "\n------ failure_reason ------\n" f"{self.get_failure_reason()}" "\n------ end of failure_reason ------\n" - "\n------ failure traceback ------\n" - f"failure_traceback: {_traceback}" - "\n------ end of failure traceback ------\n" + "\n------ exception informaton ------\n" + f"{self!r}" + "\n------ end of exception informaton ------\n" + "\n------ exception traceback ------\n" + f"{self.get_failure_traceback()}" + "\n------ end of exception traceback ------\n" ) From bb91ae0aedb114dc6c7b19595b18164760478c5c Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 2 Nov 2023 05:47:49 +0000 Subject: [PATCH 21/27] errors: refine the failure descriptions --- otaclient/app/errors.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/otaclient/app/errors.py b/otaclient/app/errors.py index 87aa06d0c..db21bbf40 100644 --- a/otaclient/app/errors.py +++ b/otaclient/app/errors.py @@ -111,9 +111,7 @@ def get_error_report(self, title: str = "") -> str: # ------ Network related error ------ # -_NETWORK_ERR_DEFAULT_DESC = ( - "network connection unstable, please check the connection and try again" -) +_NETWORK_ERR_DEFAULT_DESC = "network unstable, please check the network connection" class NetworkError(OTAError): @@ -162,7 +160,7 @@ class InvalidStatusForOTARollback(OTAErrorRecoverable): # _UNRECOVERABLE_DEFAULT_DESC = ( - "unrecoverable OTA error detected, please contact technical support" + "unrecoverable OTA error, please contact technical support" ) @@ -174,50 +172,58 @@ class OTAErrorUnRecoverable(OTAError): class BootControlPlatformUnsupported(OTAErrorUnRecoverable): failure_errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_PLATFORM_UNSUPPORTED - failure_description: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: current ECU platform is not supported by the boot controller module" + failure_description: str = ( + f"{_UNRECOVERABLE_DEFAULT_DESC}: bootloader for this ECU is not supported" + ) class BootControlStartupFailed(OTAErrorUnRecoverable): failure_errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_STARTUP_ERR - failure_description: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: failed to start boot controller module for this device" + failure_description: str = ( + f"{_UNRECOVERABLE_DEFAULT_DESC}: boot controller startup failed" + ) class BootControlPreUpdateFailed(OTAErrorUnRecoverable): failure_errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_PREUPDATE_FAILED failure_description: str = ( - f"{_UNRECOVERABLE_DEFAULT_DESC}: boot control pre_update process failed" + f"{_UNRECOVERABLE_DEFAULT_DESC}: boot_control pre_update process failed" ) class BootControlPostUpdateFailed(OTAErrorUnRecoverable): failure_errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_POSTUPDATE_FAILED failure_description: str = ( - f"{_UNRECOVERABLE_DEFAULT_DESC}: boot control post_update process failed" + f"{_UNRECOVERABLE_DEFAULT_DESC}: boot_control post_update process failed" ) class BootControlPreRollbackFailed(OTAErrorUnRecoverable): failure_errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_PREROLLBACK_FAILED failure_description: str = ( - f"{_UNRECOVERABLE_DEFAULT_DESC}: pre_rollback process failed" + f"{_UNRECOVERABLE_DEFAULT_DESC}: boot_control pre_rollback process failed" ) class BootControlPostRollbackFailed(OTAErrorUnRecoverable): failure_errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_POSTROLLBACK_FAILED failure_description: str = ( - f"{_UNRECOVERABLE_DEFAULT_DESC}: post_rollback process failed" + f"{_UNRECOVERABLE_DEFAULT_DESC}: boot_control post_rollback process failed" ) class StandbySlotInsufficientSpace(OTAErrorUnRecoverable): failure_errcode: OTAErrorCode = OTAErrorCode.E_STANDBY_SLOT_INSUFFICIENT_SPACE - failure_description: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: standby slot has insufficient space to apply update" + failure_description: str = ( + f"{_UNRECOVERABLE_DEFAULT_DESC}: insufficient space at standby slot" + ) class InvalidUpdateRequest(OTAErrorUnRecoverable): failure_errcode: OTAErrorCode = OTAErrorCode.E_INVALID_OTAUPDATE_REQUEST - failure_description: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: incoming OTA update request's content is invalid" + failure_description: str = ( + f"{_UNRECOVERABLE_DEFAULT_DESC}: incoming OTA update request is invalid" + ) class MetadataJWTInvalid(OTAErrorUnRecoverable): From 05a64be012c9582177191803db51044bb284ded4 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 2 Nov 2023 05:53:50 +0000 Subject: [PATCH 22/27] configs: new config DEBUG_MODE(default is False) --- otaclient/app/configs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/otaclient/app/configs.py b/otaclient/app/configs.py index 2f8bc771f..533915f6c 100644 --- a/otaclient/app/configs.py +++ b/otaclient/app/configs.py @@ -184,6 +184,8 @@ class BaseConfig(_InternalSettings): # default version string to be reported in status API response DEFAULT_VERSION_STR = "" + DEBUG_MODE = False + # init cfgs server_cfg = OtaClientServerConfig() From a7882206641e324e838a74126a26d27e9de76626 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 2 Nov 2023 05:54:30 +0000 Subject: [PATCH 23/27] otaclient: only include traceback info in status API resp when DEBUG_MODE is on --- otaclient/app/ota_client.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/otaclient/app/ota_client.py b/otaclient/app/ota_client.py index 22ad659a5..585477385 100644 --- a/otaclient/app/ota_client.py +++ b/otaclient/app/ota_client.py @@ -531,7 +531,9 @@ def _on_failure(self, exc: ota_errors.OTAError, ota_status: wrapper.StatusOta): try: self.last_failure_type = exc.failure_type self.last_failure_reason = exc.get_failure_reason() - self.last_failure_traceback = exc.get_failure_traceback() + if cfg.DEBUG_MODE: + self.last_failure_traceback = exc.get_failure_traceback() + logger.error( exc.get_error_report(f"OTA failed with {ota_status.name}: {exc!r}") ) @@ -550,9 +552,13 @@ def update(self, version: str, url_base: str, cookies_json: str): control_flags=self.control_flags, proxy=self.proxy, ) + + # reset failure information on handling new update request self.last_failure_type = wrapper.FailureType.NO_FAILURE self.last_failure_reason = "" self.last_failure_traceback = "" + + # enter update self.live_ota_status.set_ota_status(wrapper.StatusOta.UPDATING) self._update_executor.execute(version, url_base, cookies_json) except ota_errors.OTAError as e: @@ -573,9 +579,13 @@ def rollback(self): self._rollback_executor = _OTARollbacker( boot_controller=self.boot_controller ) + + # clear failure information on handling new rollback request self.last_failure_type = wrapper.FailureType.NO_FAILURE self.last_failure_reason = "" self.last_failure_traceback = "" + + # entering rollback self.live_ota_status.set_ota_status(wrapper.StatusOta.ROLLBACKING) self._rollback_executor.execute() # silently ignore overlapping request @@ -656,8 +666,12 @@ def __init__( ota_status=wrapper.StatusOta.FAILURE, failure_type=wrapper.FailureType.UNRECOVERABLE, failure_reason=e.get_failure_reason(), - failure_traceback=e.get_failure_traceback(splitter="\n"), ) + + if cfg.DEBUG_MODE: + self._otaclient_startup_failed_status.failure_traceback = ( + e.get_failure_traceback() + ) return # otaclient core starts up @@ -679,8 +693,12 @@ def __init__( ota_status=wrapper.StatusOta.FAILURE, failure_type=wrapper.FailureType.UNRECOVERABLE, failure_reason=e.get_failure_reason(), - failure_traceback=e.get_failure_traceback(splitter="\n"), ) + + if cfg.DEBUG_MODE: + self._otaclient_startup_failed_status.failure_traceback = ( + e.get_failure_traceback() + ) return @property From 9d2d0de917f707a7f974d02c2546042cf8cd9dfd Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 2 Nov 2023 06:38:40 +0000 Subject: [PATCH 24/27] rpi_boot: minor fix to make linter happy --- otaclient/app/boot_control/_rpi_boot.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/otaclient/app/boot_control/_rpi_boot.py b/otaclient/app/boot_control/_rpi_boot.py index 1a14e2710..f9f4c536e 100644 --- a/otaclient/app/boot_control/_rpi_boot.py +++ b/otaclient/app/boot_control/_rpi_boot.py @@ -243,7 +243,9 @@ def _update_firmware(self): os.replace(_initrd_img, self.initrd_img_active_slot) os.sync() except Exception as e: - _err_msg = f"apply new kernel,initrd.img for {self.active_slot} failed" + _err_msg = ( + f"apply new kernel,initrd.img for {self.active_slot} failed: {e!r}" + ) logger.error(_err_msg) raise _RPIBootControllerError(_err_msg) From a912a83567c8de43983c6a4db5e1701d390b7499 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 2 Nov 2023 07:06:14 +0000 Subject: [PATCH 25/27] boot_controllers: do not directly import errors from app.errors --- otaclient/app/boot_control/_cboot.py | 30 ++++++++++++------------- otaclient/app/boot_control/_grub.py | 27 +++++++++++----------- otaclient/app/boot_control/_rpi_boot.py | 27 +++++++++++----------- 3 files changed, 43 insertions(+), 41 deletions(-) diff --git a/otaclient/app/boot_control/_cboot.py b/otaclient/app/boot_control/_cboot.py index 7a2041eed..512789b3b 100644 --- a/otaclient/app/boot_control/_cboot.py +++ b/otaclient/app/boot_control/_cboot.py @@ -21,7 +21,7 @@ from typing import Generator, Optional -from .. import log_setting +from .. import log_setting, errors as ota_errors from ..common import ( copytree_identical, read_str_from_file, @@ -30,14 +30,6 @@ write_str_to_file_sync, ) -from ..errors import ( - BootControlStartupFailed, - BootControlPlatformUnsupported, - BootControlPostRollbackFailed, - BootControlPostUpdateFailed, - BootControlPreRollbackFailed, - BootControlPreUpdateFailed, -) from ..proto import wrapper from ._common import ( @@ -339,9 +331,9 @@ def __init__(self) -> None: # init ota-status self._init_boot_control() except NotImplementedError as e: - raise BootControlPlatformUnsupported(module=__name__) from e + raise ota_errors.BootControlPlatformUnsupported(module=__name__) from e except Exception as e: - raise BootControlStartupFailed( + raise ota_errors.BootControlStartupFailed( f"unspecific boot controller startup failure: {e!r}", module=__name__ ) from e @@ -514,7 +506,9 @@ def pre_update(self, version: str, *, standby_as_ref: bool, erase_standby=False) except Exception as e: _err_msg = f"failed on pre_update: {e!r}" logger.exception(_err_msg) - raise BootControlPreUpdateFailed(f"{e!r}", module=__name__) from e + raise ota_errors.BootControlPreUpdateFailed( + f"{e!r}", module=__name__ + ) from e def post_update(self) -> Generator[None, None, None]: try: @@ -554,7 +548,9 @@ def post_update(self) -> Generator[None, None, None]: except Exception as e: _err_msg = f"failed on post_update: {e!r}" logger.exception(_err_msg) - raise BootControlPostUpdateFailed(_err_msg, module=__name__) from e + raise ota_errors.BootControlPostUpdateFailed( + _err_msg, module=__name__ + ) from e def pre_rollback(self): try: @@ -568,7 +564,9 @@ def pre_rollback(self): except Exception as e: _err_msg = f"failed on pre_rollback: {e!r}" logger.exception(_err_msg) - raise BootControlPreRollbackFailed(_err_msg, module=__name__) from e + raise ota_errors.BootControlPreRollbackFailed( + _err_msg, module=__name__ + ) from e def post_rollback(self): try: @@ -577,4 +575,6 @@ def post_rollback(self): except Exception as e: _err_msg = f"failed on post_rollback: {e!r}" logger.exception(_err_msg) - raise BootControlPostRollbackFailed(_err_msg, module=__name__) from e + raise ota_errors.BootControlPostRollbackFailed( + _err_msg, module=__name__ + ) from e diff --git a/otaclient/app/boot_control/_grub.py b/otaclient/app/boot_control/_grub.py index d42de4e6e..8c3bf8971 100644 --- a/otaclient/app/boot_control/_grub.py +++ b/otaclient/app/boot_control/_grub.py @@ -39,7 +39,7 @@ from pathlib import Path from pprint import pformat -from .. import log_setting +from .. import log_setting, errors as ota_errors from ..common import ( re_symlink_atomic, read_str_from_file, @@ -47,13 +47,6 @@ subprocess_check_output, write_str_to_file_sync, ) -from ..errors import ( - BootControlStartupFailed, - BootControlPostRollbackFailed, - BootControlPostUpdateFailed, - BootControlPreRollbackFailed, - BootControlPreUpdateFailed, -) from ..proto import wrapper from ._common import ( @@ -770,7 +763,7 @@ def __init__(self) -> None: except Exception as e: _err_msg = f"failed on start grub boot controller: {e!r}" logger.error(_err_msg) - raise BootControlStartupFailed(_err_msg, module=__name__) from e + raise ota_errors.BootControlStartupFailed(_err_msg, module=__name__) from e def _update_fstab(self, *, active_slot_fstab: Path, standby_slot_fstab: Path): """Update standby fstab based on active slot's fstab and just installed new stanby fstab. @@ -889,7 +882,9 @@ def pre_update(self, version: str, *, standby_as_ref: bool, erase_standby=False) except Exception as e: _err_msg = f"failed on pre_update: {e!r}" logger.error(_err_msg) - raise BootControlPreUpdateFailed(_err_msg, module=__name__) from e + raise ota_errors.BootControlPreUpdateFailed( + _err_msg, module=__name__ + ) from e def post_update(self) -> Generator[None, None, None]: try: @@ -918,7 +913,9 @@ def post_update(self) -> Generator[None, None, None]: except Exception as e: _err_msg = f"failed on post_update: {e!r}" logger.error(_err_msg) - raise BootControlPostUpdateFailed(_err_msg, module=__name__) from e + raise ota_errors.BootControlPostUpdateFailed( + _err_msg, module=__name__ + ) from e def pre_rollback(self): try: @@ -929,7 +926,9 @@ def pre_rollback(self): except Exception as e: _err_msg = f"failed on pre_rollback: {e!r}" logger.error(_err_msg) - raise BootControlPreRollbackFailed(_err_msg, module=__name__) from e + raise ota_errors.BootControlPreRollbackFailed( + _err_msg, module=__name__ + ) from e def post_rollback(self): try: @@ -940,4 +939,6 @@ def post_rollback(self): except Exception as e: _err_msg = f"failed on pre_rollback: {e!r}" logger.error(_err_msg) - raise BootControlPostRollbackFailed(_err_msg, module=__name__) from e + raise ota_errors.BootControlPostRollbackFailed( + _err_msg, module=__name__ + ) from e diff --git a/otaclient/app/boot_control/_rpi_boot.py b/otaclient/app/boot_control/_rpi_boot.py index f9f4c536e..7317a48de 100644 --- a/otaclient/app/boot_control/_rpi_boot.py +++ b/otaclient/app/boot_control/_rpi_boot.py @@ -20,14 +20,7 @@ from pathlib import Path from typing import Generator -from .. import log_setting -from ..errors import ( - BootControlStartupFailed, - BootControlPostRollbackFailed, - BootControlPostUpdateFailed, - BootControlPreRollbackFailed, - BootControlPreUpdateFailed, -) +from .. import log_setting, errors as ota_errors from ..proto import wrapper from ..common import replace_atomic, subprocess_call @@ -419,7 +412,7 @@ def __init__(self) -> None: except Exception as e: _err_msg = f"failed to start rpi boot controller: {e!r}" logger.error(_err_msg) - raise BootControlStartupFailed(_err_msg, module=__name__) from e + raise ota_errors.BootControlStartupFailed(_err_msg, module=__name__) from e def _copy_kernel_for_standby_slot(self): """Copy the kernel and initrd_img files from current slot /boot @@ -519,7 +512,9 @@ def pre_update(self, version: str, *, standby_as_ref: bool, erase_standby: bool) except Exception as e: _err_msg = f"failed on pre_update: {e!r}" logger.error(_err_msg) - raise BootControlPreUpdateFailed(_err_msg, module=__name__) from e + raise ota_errors.BootControlPreUpdateFailed( + _err_msg, module=__name__ + ) from e def pre_rollback(self): try: @@ -530,7 +525,9 @@ def pre_rollback(self): except Exception as e: _err_msg = f"failed on pre_rollback: {e!r}" logger.error(_err_msg) - raise BootControlPreRollbackFailed(_err_msg, module=__name__) from e + raise ota_errors.BootControlPreRollbackFailed( + _err_msg, module=__name__ + ) from e def post_rollback(self): try: @@ -541,7 +538,9 @@ def post_rollback(self): except Exception as e: _err_msg = f"failed on post_rollback: {e!r}" logger.error(_err_msg) - raise BootControlPostRollbackFailed(_err_msg, module=__name__) from e + raise ota_errors.BootControlPostRollbackFailed( + _err_msg, module=__name__ + ) from e def post_update(self) -> Generator[None, None, None]: try: @@ -556,7 +555,9 @@ def post_update(self) -> Generator[None, None, None]: except Exception as e: _err_msg = f"failed on post_update: {e!r}" logger.error(_err_msg) - raise BootControlPostUpdateFailed(_err_msg, module=__name__) from e + raise ota_errors.BootControlPostUpdateFailed( + _err_msg, module=__name__ + ) from e def on_operation_failure(self): """Failure registering and cleanup at failure.""" From 9f1b41f621ea26dae485f442c146daccdb2053cd Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 2 Nov 2023 07:13:26 +0000 Subject: [PATCH 26/27] errors: fix get_error_report missing module name info --- otaclient/app/errors.py | 1 + 1 file changed, 1 insertion(+) diff --git a/otaclient/app/errors.py b/otaclient/app/errors.py index db21bbf40..81f901c9a 100644 --- a/otaclient/app/errors.py +++ b/otaclient/app/errors.py @@ -95,6 +95,7 @@ def get_error_report(self, title: str = "") -> str: """The detailed failure report for debug use.""" return ( f"\n{title}\n" + f"@module: {self.module}" "\n------ failure_reason ------\n" f"{self.get_failure_reason()}" "\n------ end of failure_reason ------\n" From 16cc689223d26e5adce9534acb09df8c64b8cc8d Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Tue, 7 Nov 2023 07:58:04 +0000 Subject: [PATCH 27/27] errors: rename OTAErrorUnRecoverable to OTAErrorUnrecoverable --- otaclient/app/errors.py | 30 +++++++++++++++--------------- otaclient/app/ota_client.py | 2 +- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/otaclient/app/errors.py b/otaclient/app/errors.py index 81f901c9a..837e99425 100644 --- a/otaclient/app/errors.py +++ b/otaclient/app/errors.py @@ -165,96 +165,96 @@ class InvalidStatusForOTARollback(OTAErrorRecoverable): ) -class OTAErrorUnRecoverable(OTAError): +class OTAErrorUnrecoverable(OTAError): failure_type: wrapper.FailureType = wrapper.FailureType.RECOVERABLE failure_errcode: OTAErrorCode = OTAErrorCode.E_OTA_ERR_UNRECOVERABLE failure_description: str = _UNRECOVERABLE_DEFAULT_DESC -class BootControlPlatformUnsupported(OTAErrorUnRecoverable): +class BootControlPlatformUnsupported(OTAErrorUnrecoverable): failure_errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_PLATFORM_UNSUPPORTED failure_description: str = ( f"{_UNRECOVERABLE_DEFAULT_DESC}: bootloader for this ECU is not supported" ) -class BootControlStartupFailed(OTAErrorUnRecoverable): +class BootControlStartupFailed(OTAErrorUnrecoverable): failure_errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_STARTUP_ERR failure_description: str = ( f"{_UNRECOVERABLE_DEFAULT_DESC}: boot controller startup failed" ) -class BootControlPreUpdateFailed(OTAErrorUnRecoverable): +class BootControlPreUpdateFailed(OTAErrorUnrecoverable): failure_errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_PREUPDATE_FAILED failure_description: str = ( f"{_UNRECOVERABLE_DEFAULT_DESC}: boot_control pre_update process failed" ) -class BootControlPostUpdateFailed(OTAErrorUnRecoverable): +class BootControlPostUpdateFailed(OTAErrorUnrecoverable): failure_errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_POSTUPDATE_FAILED failure_description: str = ( f"{_UNRECOVERABLE_DEFAULT_DESC}: boot_control post_update process failed" ) -class BootControlPreRollbackFailed(OTAErrorUnRecoverable): +class BootControlPreRollbackFailed(OTAErrorUnrecoverable): failure_errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_PREROLLBACK_FAILED failure_description: str = ( f"{_UNRECOVERABLE_DEFAULT_DESC}: boot_control pre_rollback process failed" ) -class BootControlPostRollbackFailed(OTAErrorUnRecoverable): +class BootControlPostRollbackFailed(OTAErrorUnrecoverable): failure_errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_POSTROLLBACK_FAILED failure_description: str = ( f"{_UNRECOVERABLE_DEFAULT_DESC}: boot_control post_rollback process failed" ) -class StandbySlotInsufficientSpace(OTAErrorUnRecoverable): +class StandbySlotInsufficientSpace(OTAErrorUnrecoverable): failure_errcode: OTAErrorCode = OTAErrorCode.E_STANDBY_SLOT_INSUFFICIENT_SPACE failure_description: str = ( f"{_UNRECOVERABLE_DEFAULT_DESC}: insufficient space at standby slot" ) -class InvalidUpdateRequest(OTAErrorUnRecoverable): +class InvalidUpdateRequest(OTAErrorUnrecoverable): failure_errcode: OTAErrorCode = OTAErrorCode.E_INVALID_OTAUPDATE_REQUEST failure_description: str = ( f"{_UNRECOVERABLE_DEFAULT_DESC}: incoming OTA update request is invalid" ) -class MetadataJWTInvalid(OTAErrorUnRecoverable): +class MetadataJWTInvalid(OTAErrorUnrecoverable): failure_errcode: OTAErrorCode = OTAErrorCode.E_METADATAJWT_INVALID failure_description: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: verfication for metadata.jwt is OK but metadata.jwt's content is invalid" -class MetadataJWTVerficationFailed(OTAErrorUnRecoverable): +class MetadataJWTVerficationFailed(OTAErrorUnrecoverable): failure_errcode: OTAErrorCode = OTAErrorCode.E_METADATAJWT_CERT_VERIFICATION_FAILED failure_description: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: certificate verification failed for OTA metadata.jwt" -class OTAProxyFailedToStart(OTAErrorUnRecoverable): +class OTAProxyFailedToStart(OTAErrorUnrecoverable): failure_errcode: OTAErrorCode = OTAErrorCode.E_OTAPROXY_FAILED_TO_START failure_description: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: otaproxy is required for multiple ECU update but otaproxy failed to start" -class UpdateDeltaGenerationFailed(OTAErrorUnRecoverable): +class UpdateDeltaGenerationFailed(OTAErrorUnrecoverable): failure_errcode: OTAErrorCode = OTAErrorCode.E_UPDATEDELTA_GENERATION_FAILED failure_description: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: failed to calculate and/or prepare update delta" -class ApplyOTAUpdateFailed(OTAErrorUnRecoverable): +class ApplyOTAUpdateFailed(OTAErrorUnrecoverable): failure_errcode: OTAErrorCode = OTAErrorCode.E_APPLY_OTAUPDATE_FAILED failure_description: str = ( f"{_UNRECOVERABLE_DEFAULT_DESC}: failed to apply OTA update to standby slot" ) -class OTAClientStartupFailed(OTAErrorUnRecoverable): +class OTAClientStartupFailed(OTAErrorUnrecoverable): failure_errcode: OTAErrorCode = OTAErrorCode.E_OTACLIENT_STARTUP_FAILED failure_description: str = ( f"{_UNRECOVERABLE_DEFAULT_DESC}: failed to start otaclient instance" diff --git a/otaclient/app/ota_client.py b/otaclient/app/ota_client.py index 585477385..c244044fc 100644 --- a/otaclient/app/ota_client.py +++ b/otaclient/app/ota_client.py @@ -363,7 +363,7 @@ def _execute_update( except downloader.DestinationNotAvailableError as e: _err_msg = f"downloader: failed to save ota metafiles: {e!r}" logger.error(_err_msg) - raise ota_errors.OTAErrorUnRecoverable(_err_msg, module=__name__) from e + raise ota_errors.OTAErrorUnrecoverable(_err_msg, module=__name__) from e except ota_metadata.MetadataJWTVerificationFailed as e: _err_msg = f"failed to verify metadata.jwt: {e!r}" logger.error(_err_msg)