diff --git a/otaclient/app/boot_control/_cboot.py b/otaclient/app/boot_control/_cboot.py index 9beacef05..512789b3b 100644 --- a/otaclient/app/boot_control/_cboot.py +++ b/otaclient/app/boot_control/_cboot.py @@ -21,7 +21,7 @@ from typing import Generator, Optional -from .. import log_setting +from .. import log_setting, errors as ota_errors from ..common import ( copytree_identical, read_str_from_file, @@ -30,17 +30,8 @@ write_str_to_file_sync, ) -from ..errors import ( - BootControlInitError, - BootControlPlatformUnsupported, - BootControlPostRollbackFailed, - BootControlPostUpdateFailed, - BootControlPreRollbackFailed, - BootControlPreUpdateFailed, -) from ..proto import wrapper -from . import _errors from ._common import ( OTAStatusMixin, PrepareMountMixin, @@ -58,7 +49,7 @@ ) -class NvbootctrlError(_errors.BootControlError): +class NvbootctrlError(Exception): """Specific internal errors related to nvbootctrl cmd.""" @@ -167,26 +158,21 @@ def is_slot_marked_successful(cls, slot: str) -> bool: class _CBootControl: def __init__(self): - try: - # NOTE: only support rqx-580, rqx-58g platform right now! - # detect the chip id - self.chip_id = read_str_from_file(cfg.TEGRA_CHIP_ID_PATH) - if not self.chip_id or int(self.chip_id) not in cfg.CHIP_ID_MODEL_MAP: - raise NotImplementedError( - f"unsupported platform found (chip_id: {self.chip_id}), abort" - ) + # NOTE: only support rqx-580, rqx-58g platform right now! + # detect the chip id + self.chip_id = read_str_from_file(cfg.TEGRA_CHIP_ID_PATH) + if not self.chip_id or int(self.chip_id) not in cfg.CHIP_ID_MODEL_MAP: + raise NotImplementedError( + f"unsupported platform found (chip_id: {self.chip_id}), abort" + ) - self.chip_id = int(self.chip_id) - self.model = cfg.CHIP_ID_MODEL_MAP[self.chip_id] - logger.info(f"{self.model=}, (chip_id={hex(self.chip_id)})") + self.chip_id = int(self.chip_id) + self.model = cfg.CHIP_ID_MODEL_MAP[self.chip_id] + logger.info(f"{self.model=}, (chip_id={hex(self.chip_id)})") - # initializing dev info - self._init_dev_info() - logger.info(f"finished cboot control init: {Nvbootctrl.dump_slots_info()=}") - except NotImplementedError as e: - raise BootControlPlatformUnsupported from e - except Exception as e: - raise BootControlInitError from e + # initializing dev info + self._init_dev_info() + logger.info(f"finished cboot control init: {Nvbootctrl.dump_slots_info()=}") def _init_dev_info(self): self.current_slot: str = Nvbootctrl.get_current_slot() @@ -221,12 +207,12 @@ def _init_dev_info(self): # ensure rootfs is as expected if not Nvbootctrl.check_rootdev(self.current_rootfs_dev): msg = f"rootfs mismatch, expect {self.current_rootfs_dev} as rootfs" - raise ValueError(msg) + raise NvbootctrlError(msg) elif Nvbootctrl.check_rootdev(self.standby_rootfs_dev): msg = ( f"rootfs mismatch, expect {self.standby_rootfs_dev} as standby slot dev" ) - raise ValueError(msg) + raise NvbootctrlError(msg) logger.info("dev info initializing completed") logger.info( @@ -311,38 +297,45 @@ class CBootController( BootControllerProtocol, ): def __init__(self) -> None: - self._cboot_control: _CBootControl = _CBootControl() - - # load paths - ## first try to unmount standby dev if possible - self.standby_slot_dev = self._cboot_control.get_standby_rootfs_dev() - CMDHelperFuncs.umount(self.standby_slot_dev) - - self.standby_slot_mount_point = Path(cfg.MOUNT_POINT) - self.standby_slot_mount_point.mkdir(exist_ok=True) - - ## refroot mount point - _refroot_mount_point = cfg.ACTIVE_ROOT_MOUNT_POINT - # first try to umount refroot mount point - CMDHelperFuncs.umount(_refroot_mount_point) - if not os.path.isdir(_refroot_mount_point): - os.mkdir(_refroot_mount_point) - self.ref_slot_mount_point = Path(_refroot_mount_point) - - ## ota-status dir - ### current slot - self.current_ota_status_dir = Path(cfg.ACTIVE_ROOTFS_PATH) / Path( - cfg.OTA_STATUS_DIR - ).relative_to("/") - self.current_ota_status_dir.mkdir(parents=True, exist_ok=True) - ### standby slot - # NOTE: might not yet be populated before OTA update applied! - self.standby_ota_status_dir = self.standby_slot_mount_point / Path( - cfg.OTA_STATUS_DIR - ).relative_to("/") - - # init ota-status - self._init_boot_control() + try: + self._cboot_control: _CBootControl = _CBootControl() + + # load paths + ## first try to unmount standby dev if possible + self.standby_slot_dev = self._cboot_control.get_standby_rootfs_dev() + CMDHelperFuncs.umount(self.standby_slot_dev) + + self.standby_slot_mount_point = Path(cfg.MOUNT_POINT) + self.standby_slot_mount_point.mkdir(exist_ok=True) + + ## refroot mount point + _refroot_mount_point = cfg.ACTIVE_ROOT_MOUNT_POINT + # first try to umount refroot mount point + CMDHelperFuncs.umount(_refroot_mount_point) + if not os.path.isdir(_refroot_mount_point): + os.mkdir(_refroot_mount_point) + self.ref_slot_mount_point = Path(_refroot_mount_point) + + ## ota-status dir + ### current slot + self.current_ota_status_dir = Path(cfg.ACTIVE_ROOTFS_PATH) / Path( + cfg.OTA_STATUS_DIR + ).relative_to("/") + self.current_ota_status_dir.mkdir(parents=True, exist_ok=True) + ### standby slot + # NOTE: might not yet be populated before OTA update applied! + self.standby_ota_status_dir = self.standby_slot_mount_point / Path( + cfg.OTA_STATUS_DIR + ).relative_to("/") + + # init ota-status + self._init_boot_control() + except NotImplementedError as e: + raise ota_errors.BootControlPlatformUnsupported(module=__name__) from e + except Exception as e: + raise ota_errors.BootControlStartupFailed( + f"unspecific boot controller startup failure: {e!r}", module=__name__ + ) from e ###### private methods ###### @@ -433,10 +426,10 @@ def _populate_boot_folder_to_separate_bootdev(self): self._cboot_control.get_standby_boot_dev(), _boot_dir_mount_point, ) - except _errors.MountError as e: + except Exception as e: _msg = f"failed to mount standby boot dev: {e!r}" logger.error(_msg) - raise _errors.BootControlError(_msg) from e + raise NvbootctrlError(_msg) from e try: dst = _boot_dir_mount_point / "boot" @@ -448,14 +441,14 @@ def _populate_boot_folder_to_separate_bootdev(self): except Exception as e: _msg = f"failed to populate boot folder to separate bootdev: {e!r}" logger.error(_msg) - raise _errors.BootControlError(_msg) from e + raise NvbootctrlError(_msg) from e finally: # unmount standby emmc boot dev on finish/failure try: CMDHelperFuncs.umount(_boot_dir_mount_point) - except _errors.MountError as e: + except Exception as e: _failure_msg = f"failed to umount boot dev: {e!r}" - logger.error(_failure_msg) + logger.warning(_failure_msg) # no need to raise to the caller ###### public methods ###### @@ -511,8 +504,11 @@ def pre_update(self, version: str, *, standby_as_ref: bool, erase_standby=False) logger.info("pre-update setting finished") except Exception as e: - logger.exception(f"failed on pre_update: {e!r}") - raise BootControlPreUpdateFailed from e + _err_msg = f"failed on pre_update: {e!r}" + logger.exception(_err_msg) + raise ota_errors.BootControlPreUpdateFailed( + f"{e!r}", module=__name__ + ) from e def post_update(self) -> Generator[None, None, None]: try: @@ -550,8 +546,11 @@ def post_update(self) -> Generator[None, None, None]: yield # hand over control back to otaclient CMDHelperFuncs.reboot() except Exception as e: - logger.exception(f"failed on post_update: {e!r}") - raise BootControlPostUpdateFailed from e + _err_msg = f"failed on post_update: {e!r}" + logger.exception(_err_msg) + raise ota_errors.BootControlPostUpdateFailed( + _err_msg, module=__name__ + ) from e def pre_rollback(self): try: @@ -563,13 +562,19 @@ def pre_rollback(self): # store ROLLBACKING status to standby self._store_standby_ota_status(wrapper.StatusOta.ROLLBACKING) except Exception as e: - logger.exception(f"failed on pre_rollback: {e!r}") - raise BootControlPreRollbackFailed from e + _err_msg = f"failed on pre_rollback: {e!r}" + logger.exception(_err_msg) + raise ota_errors.BootControlPreRollbackFailed( + _err_msg, module=__name__ + ) from e def post_rollback(self): try: self._cboot_control.switch_boot() CMDHelperFuncs.reboot() except Exception as e: - logger.exception(f"failed on post_rollback: {e!r}") - raise BootControlPostRollbackFailed from e + _err_msg = f"failed on post_rollback: {e!r}" + logger.exception(_err_msg) + raise ota_errors.BootControlPostRollbackFailed( + _err_msg, module=__name__ + ) from e diff --git a/otaclient/app/boot_control/_grub.py b/otaclient/app/boot_control/_grub.py index 421901897..8c3bf8971 100644 --- a/otaclient/app/boot_control/_grub.py +++ b/otaclient/app/boot_control/_grub.py @@ -39,7 +39,7 @@ from pathlib import Path from pprint import pformat -from .. import log_setting +from .. import log_setting, errors as ota_errors from ..common import ( re_symlink_atomic, read_str_from_file, @@ -47,16 +47,8 @@ subprocess_check_output, write_str_to_file_sync, ) -from ..errors import ( - BootControlInitError, - BootControlPostRollbackFailed, - BootControlPostUpdateFailed, - BootControlPreRollbackFailed, - BootControlPreUpdateFailed, -) from ..proto import wrapper -from . import _errors from ._common import ( CMDHelperFuncs, OTAStatusFilesControl, @@ -72,6 +64,10 @@ ) +class _GrubBootControllerError(Exception): + """Grub boot controller internal used exception.""" + + @dataclass class _GrubMenuEntry: """ @@ -327,7 +323,9 @@ def _get_sibling_dev(self, active_dev: str) -> str: parent = CMDHelperFuncs.get_parent_dev(active_dev) boot_dev = CMDHelperFuncs.get_dev_by_mount_point("/boot") if not boot_dev: - raise _errors.ABPartitionError("/boot is not mounted") + _err_msg = "/boot is not mounted" + logger.error(_err_msg) + raise ValueError(_err_msg) # list children device file from parent device cmd = f"-Pp -o NAME,FSTYPE {parent}" @@ -344,9 +342,9 @@ def _get_sibling_dev(self, active_dev: str) -> str: ): return m.group(1) - raise _errors.ABPartitionError( - f"{parent=} has unexpected partition layout: {output=}" - ) + _err_msg = f"{parent=} has unexpected partition layout: {output=}" + logger.error(_err_msg) + raise ValueError(_err_msg) def _detect_active_slot(self) -> Tuple[str, str]: """Get active slot's slot_id. @@ -455,7 +453,7 @@ def _check_active_slot_ota_partition_file(self): GrubHelper.INITRD_OTA, GrubHelper.INITRD_OTA_STANDBY, ) - except ValueError as e: + except Exception as e: logger.error( f"failed to get current booted kernel and initrd.image: {e!r}, " "try to use active slot ota-partition files" @@ -500,7 +498,8 @@ def _check_active_slot_ota_partition_file(self): logger.info(f"ota-partition files for {self.active_slot} are ready") - def _get_current_booted_files(self) -> Tuple[str, str]: + @staticmethod + def _get_current_booted_files() -> Tuple[str, str]: """Return the name of booted kernel and initrd. Expected booted kernel and initrd are located under /boot. @@ -571,7 +570,7 @@ def _grub_update_on_booted_slot(self, *, abort_on_standby_missed=True): "refuse to do grub-update" ) logger.error(msg) - raise ValueError(msg) + raise _GrubBootControllerError(msg) # step1: update grub_default file _in = grub_default_file.read_text() @@ -586,7 +585,9 @@ def _grub_update_on_booted_slot(self, *, abort_on_standby_missed=True): ): active_slot_entry_idx, _ = res else: - raise ValueError("boot entry for ACTIVE slot not found, abort") + _err_msg = "boot entry for ACTIVE slot not found, abort" + logger.error(_err_msg) + raise _GrubBootControllerError(_err_msg) # step3: update grub_default again, setting default to # ensure the active slot to be the default @@ -620,7 +621,8 @@ def _grub_update_on_booted_slot(self, *, abort_on_standby_missed=True): "only current active slot's entry is populated." ) if abort_on_standby_missed: - raise ValueError(msg) + logger.error(msg) + raise _GrubBootControllerError(msg) logger.warning(msg) logger.info(f"generated grub_cfg: {pformat(grub_cfg_content)}") @@ -684,42 +686,58 @@ def prepare_standby_dev(self, *, erase_standby: bool): # if not erase the standby slot except Exception as e: _err_msg = f"failed to prepare standby dev: {e!r}" - raise BootControlPreUpdateFailed(_err_msg) from e + logger.error(_err_msg) + raise _GrubBootControllerError(_err_msg) from e def finalize_update_switch_boot(self): """Finalize switch boot and use boot files from current booted slot.""" # NOTE: since we have not yet switched boot, the active/standby relationship is # reversed here corresponding to booted slot. - self._prepare_kernel_initrd_links(self.standby_ota_partition_folder) - self._ensure_ota_partition_symlinks(active_slot=self.standby_slot) - self._ensure_standby_slot_boot_files_symlinks(standby_slot=self.active_slot) + try: + self._prepare_kernel_initrd_links(self.standby_ota_partition_folder) + self._ensure_ota_partition_symlinks(active_slot=self.standby_slot) + self._ensure_standby_slot_boot_files_symlinks(standby_slot=self.active_slot) - self._grub_update_on_booted_slot(abort_on_standby_missed=True) + self._grub_update_on_booted_slot(abort_on_standby_missed=True) - # switch ota-partition symlink to current booted slot - self._ensure_ota_partition_symlinks(active_slot=self.active_slot) - self._ensure_standby_slot_boot_files_symlinks(standby_slot=self.standby_slot) - return True + # switch ota-partition symlink to current booted slot + self._ensure_ota_partition_symlinks(active_slot=self.active_slot) + self._ensure_standby_slot_boot_files_symlinks( + standby_slot=self.standby_slot + ) + return True + except Exception as e: + _err_msg = f"grub: failed to finalize switch boot: {e!r}" + logger.error(_err_msg) + raise _GrubBootControllerError(_err_msg) from e def grub_reboot_to_standby(self): """Temporarily boot to standby slot after OTA applied to standby slot.""" # ensure all required symlinks for standby slot are presented and valid - self._prepare_kernel_initrd_links(self.standby_ota_partition_folder) - self._ensure_standby_slot_boot_files_symlinks(standby_slot=self.standby_slot) - - # ensure all required symlinks for active slot are presented and valid - # NOTE: reboot after post-update is still using the current active slot's - # ota-partition symlinks(not yet switch boot). - self._prepare_kernel_initrd_links(self.active_ota_partition_folder) - self._ensure_ota_partition_symlinks(active_slot=self.active_slot) - self._grub_update_on_booted_slot(abort_on_standby_missed=True) - - idx, _ = GrubHelper.get_entry( - read_str_from_file(self.grub_file), - kernel_ver=GrubHelper.SUFFIX_OTA_STANDBY, - ) - GrubHelper.grub_reboot(idx) - logger.info(f"system will reboot to {self.standby_slot=}: boot entry {idx}") + try: + self._prepare_kernel_initrd_links(self.standby_ota_partition_folder) + self._ensure_standby_slot_boot_files_symlinks( + standby_slot=self.standby_slot + ) + + # ensure all required symlinks for active slot are presented and valid + # NOTE: reboot after post-update is still using the current active slot's + # ota-partition symlinks(not yet switch boot). + self._prepare_kernel_initrd_links(self.active_ota_partition_folder) + self._ensure_ota_partition_symlinks(active_slot=self.active_slot) + self._grub_update_on_booted_slot(abort_on_standby_missed=True) + + idx, _ = GrubHelper.get_entry( + read_str_from_file(self.grub_file), + kernel_ver=GrubHelper.SUFFIX_OTA_STANDBY, + ) + GrubHelper.grub_reboot(idx) + logger.info(f"system will reboot to {self.standby_slot=}: boot entry {idx}") + + except Exception as e: + _err_msg = f"grub: failed to grub_reboot to standby: {e!r}" + logger.error(_err_msg) + raise _GrubBootControllerError(_err_msg) from e class GrubController(BootControllerProtocol): @@ -743,8 +761,9 @@ def __init__(self) -> None: force_initialize=self._boot_control.initialized, ) except Exception as e: - logger.error(f"failed on init boot controller: {e!r}") - raise BootControlInitError from e + _err_msg = f"failed on start grub boot controller: {e!r}" + logger.error(_err_msg) + raise ota_errors.BootControlStartupFailed(_err_msg, module=__name__) from e def _update_fstab(self, *, active_slot_fstab: Path, standby_slot_fstab: Path): """Update standby fstab based on active slot's fstab and just installed new stanby fstab. @@ -861,8 +880,11 @@ def pre_update(self, version: str, *, standby_as_ref: bool, erase_standby=False) # remove old files under standby ota_partition folder self._cleanup_standby_ota_partition_folder() except Exception as e: - logger.error(f"failed on pre_update: {e!r}") - raise BootControlPreUpdateFailed from e + _err_msg = f"failed on pre_update: {e!r}" + logger.error(_err_msg) + raise ota_errors.BootControlPreUpdateFailed( + _err_msg, module=__name__ + ) from e def post_update(self) -> Generator[None, None, None]: try: @@ -889,8 +911,11 @@ def post_update(self) -> Generator[None, None, None]: yield # hand over control to otaclient CMDHelperFuncs.reboot() except Exception as e: - logger.error(f"failed on post_update: {e!r}") - raise BootControlPostUpdateFailed from e + _err_msg = f"failed on post_update: {e!r}" + logger.error(_err_msg) + raise ota_errors.BootControlPostUpdateFailed( + _err_msg, module=__name__ + ) from e def pre_rollback(self): try: @@ -899,8 +924,11 @@ def pre_rollback(self): self._mp_control.mount_standby() self._ota_status_control.pre_rollback_standby() except Exception as e: - logger.error(f"failed on pre_rollback: {e!r}") - raise BootControlPreRollbackFailed from e + _err_msg = f"failed on pre_rollback: {e!r}" + logger.error(_err_msg) + raise ota_errors.BootControlPreRollbackFailed( + _err_msg, module=__name__ + ) from e def post_rollback(self): try: @@ -909,5 +937,8 @@ def post_rollback(self): self._mp_control.umount_all(ignore_error=True) CMDHelperFuncs.reboot() except Exception as e: - logger.error(f"failed on pre_rollback: {e!r}") - raise BootControlPostRollbackFailed from e + _err_msg = f"failed on pre_rollback: {e!r}" + logger.error(_err_msg) + raise ota_errors.BootControlPostRollbackFailed( + _err_msg, module=__name__ + ) from e diff --git a/otaclient/app/boot_control/_rpi_boot.py b/otaclient/app/boot_control/_rpi_boot.py index 2f4e5abea..7317a48de 100644 --- a/otaclient/app/boot_control/_rpi_boot.py +++ b/otaclient/app/boot_control/_rpi_boot.py @@ -11,24 +11,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +"""Boot control support for Raspberry pi 4 Model B.""" -r"""Boot control support for Raspberry pi 4 Model B.""" import os import re from string import Template from pathlib import Path from typing import Generator -from .. import log_setting -from ..errors import ( - BootControlInitError, - BootControlPostRollbackFailed, - BootControlPostUpdateFailed, - BootControlPreRollbackFailed, - BootControlPreUpdateFailed, - OTAError, -) +from .. import log_setting, errors as ota_errors from ..proto import wrapper from ..common import replace_atomic, subprocess_call @@ -51,6 +43,10 @@ ) +class _RPIBootControllerError(Exception): + """rpi_boot module internal used exception.""" + + class _RPIBootControl: """Boot control helper for rpi4 support. @@ -88,7 +84,7 @@ def __init__(self) -> None: ): _err_msg = "system-boot is not presented or not mounted!" logger.error(_err_msg) - raise BootControlInitError(_err_msg) + raise ValueError(_err_msg) self._init_slots_info() self._init_boot_files() @@ -137,7 +133,7 @@ def _init_slots_info(self): except Exception: raise ValueError( f"unexpected partition layout: {_raw_child_partitions}" - ) + ) from None # it is OK if standby_slot dev doesn't have fslabel or fslabel != standby_slot_id # we will always set the fslabel self._standby_slot = self.AB_FLIPS[self._active_slot] @@ -149,7 +145,7 @@ def _init_slots_info(self): except Exception as e: _err_msg = f"failed to detect AB partition: {e!r}" logger.error(_err_msg) - raise BootControlInitError(_err_msg) from None + raise _RPIBootControllerError(_err_msg) from e def _init_boot_files(self): """Check the availability of boot files. @@ -175,7 +171,7 @@ def _init_boot_files(self): if not self.config_txt_active_slot.is_file(): _err_msg = f"missing {self.config_txt_active_slot=}" logger.error(_err_msg) - raise BootControlInitError(_err_msg) + raise _RPIBootControllerError(_err_msg) self.cmdline_txt_active_slot = ( self.system_boot_path / f"{cfg.CMDLINE_TXT}{self.SEP_CHAR}{self.active_slot}" @@ -183,7 +179,7 @@ def _init_boot_files(self): if not self.cmdline_txt_active_slot.is_file(): _err_msg = f"missing {self.cmdline_txt_active_slot=}" logger.error(_err_msg) - raise BootControlInitError(_err_msg) + raise _RPIBootControllerError(_err_msg) self.vmlinuz_active_slot = ( self.system_boot_path / f"{cfg.VMLINUZ}{self.SEP_CHAR}{self.active_slot}" ) @@ -198,7 +194,7 @@ def _init_boot_files(self): if not self.config_txt_standby_slot.is_file(): _err_msg = f"missing {self.config_txt_standby_slot=}" logger.error(_err_msg) - raise BootControlInitError(_err_msg) + raise _RPIBootControllerError(_err_msg) self.cmdline_txt_standby_slot = ( self.system_boot_path / f"{cfg.CMDLINE_TXT}{self.SEP_CHAR}{self.standby_slot}" @@ -206,7 +202,7 @@ def _init_boot_files(self): if not self.cmdline_txt_standby_slot.is_file(): _err_msg = f"missing {self.cmdline_txt_standby_slot=}" logger.error(_err_msg) - raise BootControlInitError(_err_msg) + raise _RPIBootControllerError(_err_msg) self.vmlinuz_standby_slot = ( self.system_boot_path / f"{cfg.VMLINUZ}{self.SEP_CHAR}{self.standby_slot}" ) @@ -224,8 +220,9 @@ def _update_firmware(self): subprocess_call("flash-kernel", raise_exception=True) os.sync() except Exception as e: - logger.error(f"flash-kernel failed: {e!r}") - raise + _err_msg = f"flash-kernel failed: {e!r}" + logger.error(_err_msg) + raise _RPIBootControllerError(_err_msg) try: # check if the vmlinuz and initrd.img presented in /boot/firmware(system-boot), @@ -238,9 +235,12 @@ def _update_firmware(self): ).is_file(): os.replace(_initrd_img, self.initrd_img_active_slot) os.sync() - except Exception: - logger.error(f"apply new kernel,initrd.img for {self.active_slot} failed") - raise + except Exception as e: + _err_msg = ( + f"apply new kernel,initrd.img for {self.active_slot} failed: {e!r}" + ) + logger.error(_err_msg) + raise _RPIBootControllerError(_err_msg) # exposed API methods/properties @property @@ -345,7 +345,8 @@ def prepare_standby_dev(self, *, erase_standby: bool): CMDHelperFuncs.set_dev_fslabel(self.active_slot_dev, self.standby_slot) except Exception as e: _err_msg = f"failed to prepare standby dev: {e!r}" - raise BootControlPreUpdateFailed(_err_msg) from e + logger.error(_err_msg) + raise _RPIBootControllerError(_err_msg) from e def prepare_tryboot_txt(self): """Copy the standby slot's config.txt as tryboot.txt.""" @@ -358,7 +359,7 @@ def prepare_tryboot_txt(self): except Exception as e: _err_msg = f"failed to prepare tryboot.txt for {self._standby_slot}" logger.error(_err_msg) - raise BootControlPostUpdateFailed(_err_msg) from e + raise _RPIBootControllerError(_err_msg) from e def reboot_tryboot(self): """Reboot with tryboot flag.""" @@ -369,44 +370,49 @@ def reboot_tryboot(self): except Exception as e: _err_msg = "failed to reboot" logger.exception(_err_msg) - raise BootControlPostUpdateFailed(_err_msg) from e + raise _RPIBootControllerError(_err_msg) from e class RPIBootController(BootControllerProtocol): """BootControllerProtocol implementation for rpi4 support.""" def __init__(self) -> None: - self._rpiboot_control = _RPIBootControl() - # mount point prepare - self._mp_control = SlotMountHelper( - standby_slot_dev=self._rpiboot_control.standby_slot_dev, - standby_slot_mount_point=cfg.MOUNT_POINT, - active_slot_dev=self._rpiboot_control.active_slot_dev, - active_slot_mount_point=cfg.ACTIVE_ROOT_MOUNT_POINT, - ) - # init ota-status files - self._ota_status_control = OTAStatusFilesControl( - active_slot=self._rpiboot_control.active_slot, - standby_slot=self._rpiboot_control.standby_slot, - current_ota_status_dir=Path(cfg.ACTIVE_ROOTFS_PATH) - / Path(cfg.OTA_STATUS_DIR).relative_to("/"), - # NOTE: might not yet be populated before OTA update applied! - standby_ota_status_dir=Path(cfg.MOUNT_POINT) - / Path(cfg.OTA_STATUS_DIR).relative_to("/"), - finalize_switching_boot=self._rpiboot_control.finalize_switching_boot, - ) - - # 20230613: remove any leftover flag file if ota_status is not UPDATING/ROLLBACKING - if self._ota_status_control.booted_ota_status not in ( - wrapper.StatusOta.UPDATING, - wrapper.StatusOta.ROLLBACKING, - ): - _flag_file = ( - self._rpiboot_control.system_boot_path / cfg.SWITCH_BOOT_FLAG_FILE + try: + self._rpiboot_control = _RPIBootControl() + # mount point prepare + self._mp_control = SlotMountHelper( + standby_slot_dev=self._rpiboot_control.standby_slot_dev, + standby_slot_mount_point=cfg.MOUNT_POINT, + active_slot_dev=self._rpiboot_control.active_slot_dev, + active_slot_mount_point=cfg.ACTIVE_ROOT_MOUNT_POINT, + ) + # init ota-status files + self._ota_status_control = OTAStatusFilesControl( + active_slot=self._rpiboot_control.active_slot, + standby_slot=self._rpiboot_control.standby_slot, + current_ota_status_dir=Path(cfg.ACTIVE_ROOTFS_PATH) + / Path(cfg.OTA_STATUS_DIR).relative_to("/"), + # NOTE: might not yet be populated before OTA update applied! + standby_ota_status_dir=Path(cfg.MOUNT_POINT) + / Path(cfg.OTA_STATUS_DIR).relative_to("/"), + finalize_switching_boot=self._rpiboot_control.finalize_switching_boot, ) - _flag_file.unlink(missing_ok=True) - logger.debug("rpi_boot initialization finished") + # 20230613: remove any leftover flag file if ota_status is not UPDATING/ROLLBACKING + if self._ota_status_control.booted_ota_status not in ( + wrapper.StatusOta.UPDATING, + wrapper.StatusOta.ROLLBACKING, + ): + _flag_file = ( + self._rpiboot_control.system_boot_path / cfg.SWITCH_BOOT_FLAG_FILE + ) + _flag_file.unlink(missing_ok=True) + + logger.debug("rpi_boot initialization finished") + except Exception as e: + _err_msg = f"failed to start rpi boot controller: {e!r}" + logger.error(_err_msg) + raise ota_errors.BootControlStartupFailed(_err_msg, module=__name__) from e def _copy_kernel_for_standby_slot(self): """Copy the kernel and initrd_img files from current slot /boot @@ -452,7 +458,7 @@ def _copy_kernel_for_standby_slot(self): except Exception as e: _err_msg = "failed to copy kernel/initrd_img for standby slot" logger.error(_err_msg) - raise BootControlPostUpdateFailed(f"{e!r}") + raise _RPIBootControllerError(f"{e!r}") def _write_standby_fstab(self): """Override the standby's fstab file. @@ -474,7 +480,7 @@ def _write_standby_fstab(self): except Exception as e: _err_msg = f"failed to update fstab file for standby slot: {e!r}" logger.error(_err_msg) - raise BootControlPostUpdateFailed(_err_msg) from e + raise _RPIBootControllerError(_err_msg) from e # APIs @@ -503,12 +509,12 @@ def pre_update(self, version: str, *, standby_as_ref: bool, erase_standby: bool) self._rpiboot_control.system_boot_path / cfg.SWITCH_BOOT_FLAG_FILE ) _flag_file.unlink(missing_ok=True) - except OTAError: - raise except Exception as e: _err_msg = f"failed on pre_update: {e!r}" logger.error(_err_msg) - raise BootControlPreUpdateFailed(_err_msg) from e + raise ota_errors.BootControlPreUpdateFailed( + _err_msg, module=__name__ + ) from e def pre_rollback(self): try: @@ -519,7 +525,9 @@ def pre_rollback(self): except Exception as e: _err_msg = f"failed on pre_rollback: {e!r}" logger.error(_err_msg) - raise BootControlPreRollbackFailed(_err_msg) from e + raise ota_errors.BootControlPreRollbackFailed( + _err_msg, module=__name__ + ) from e def post_rollback(self): try: @@ -527,10 +535,12 @@ def post_rollback(self): self._rpiboot_control.prepare_tryboot_txt() self._mp_control.umount_all(ignore_error=True) self._rpiboot_control.reboot_tryboot() - except OTAError: - raise except Exception as e: - raise BootControlPostRollbackFailed from e + _err_msg = f"failed on post_rollback: {e!r}" + logger.error(_err_msg) + raise ota_errors.BootControlPostRollbackFailed( + _err_msg, module=__name__ + ) from e def post_update(self) -> Generator[None, None, None]: try: @@ -542,10 +552,12 @@ def post_update(self) -> Generator[None, None, None]: self._mp_control.umount_all(ignore_error=True) yield # hand over control back to otaclient self._rpiboot_control.reboot_tryboot() - except OTAError: - raise except Exception as e: - raise BootControlPostUpdateFailed from e + _err_msg = f"failed on post_update: {e!r}" + logger.error(_err_msg) + raise ota_errors.BootControlPostUpdateFailed( + _err_msg, module=__name__ + ) from e def on_operation_failure(self): """Failure registering and cleanup at failure.""" diff --git a/otaclient/app/configs.py b/otaclient/app/configs.py index 2f8bc771f..533915f6c 100644 --- a/otaclient/app/configs.py +++ b/otaclient/app/configs.py @@ -184,6 +184,8 @@ class BaseConfig(_InternalSettings): # default version string to be reported in status API response DEFAULT_VERSION_STR = "" + DEBUG_MODE = False + # init cfgs server_cfg = OtaClientServerConfig() diff --git a/otaclient/app/errors.py b/otaclient/app/errors.py index 990edaad9..837e99425 100644 --- a/otaclient/app/errors.py +++ b/otaclient/app/errors.py @@ -11,75 +11,55 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +"""OTA error code definition""" -"""OTA error code definition""" import traceback from enum import Enum, unique +from typing import ClassVar from .proto import wrapper @unique -class OTAErrorCode(Enum): +class OTAErrorCode(int, Enum): E_UNSPECIFIC = 0 + # + # ------ network related errors ------ + # E_NETWORK = 100 E_OTAMETA_DOWNLOAD_FAILED = 101 + # + # ------ recoverable errors ------ + # E_OTA_ERR_RECOVERABLE = 200 - E_OTAUPDATE_BUSY = 201 - E_INVALID_STATUS_FOR_OTAUPDATE = 202 - E_INVALID_OTAUPDATE_REQUEST = 203 - E_INVALID_STATUS_FOR_OTAROLLBACK = 204 - E_OTAMETA_VERIFICATION_FAILED = 205 - E_UPDATEDELTA_GENERATION_FAILED = 206 - E_APPLY_OTAUPDATE_FAILED = 207 - E_METADATA_JWT_VERIFICATION_FAILED = 208 - E_OTAPROXY_FAILED_TO_START = 209 - E_INVALID_METADATAJWT = 210 + E_OTA_BUSY = 201 + E_INVALID_STATUS_FOR_OTAROLLBACK = 202 + # + # ------ unrecoverable errors ------ + # E_OTA_ERR_UNRECOVERABLE = 300 E_BOOTCONTROL_PLATFORM_UNSUPPORTED = 301 - E_BOOTCONTROL_INIT_ERR = 302 + E_BOOTCONTROL_STARTUP_ERR = 302 E_BOOTCONTROL_PREUPDATE_FAILED = 303 E_BOOTCONTROL_POSTUPDATE_FAILED = 304 - E_BOOTCONTROL_POSTROLLBACK_FAILED = 305 - E_STANDBY_SLOT_SPACE_NOT_ENOUGH_ERROR = 306 - E_BOOTCONTROL_PREROLLBACK_FAILED = 307 - - def to_str(self) -> str: + E_BOOTCONTROL_PREROLLBACK_FAILED = 305 + E_BOOTCONTROL_POSTROLLBACK_FAILED = 306 + E_STANDBY_SLOT_INSUFFICIENT_SPACE = 307 + E_INVALID_OTAUPDATE_REQUEST = 308 + E_METADATAJWT_CERT_VERIFICATION_FAILED = 309 + E_METADATAJWT_INVALID = 310 + E_OTAPROXY_FAILED_TO_START = 311 + E_UPDATEDELTA_GENERATION_FAILED = 312 + E_APPLY_OTAUPDATE_FAILED = 313 + E_OTACLIENT_STARTUP_FAILED = 314 + + def to_errcode_str(self) -> str: return f"{self.value:0>3}" - def get_errcode(self) -> int: - return self.value - - def get_errname(self) -> str: - return self.name - - -@unique -class OTAModules(Enum): - General = 0 - BootController = 1 - StandbySlotCreater = 2 - Downloader = 3 - API = 4 - OTASERVICE = 5 - - def to_str(self) -> str: - return f"{self.value:0>2}" - - -@unique -class OTAAPI(Enum): - Unspecific = 0 - Update = 1 - Rollback = 2 - - def to_str(self) -> str: - return f"{self.value:0>2}" - class OTAError(Exception): """Errors that happen during otaclient code executing. @@ -88,240 +68,194 @@ class OTAError(Exception): It should always be captured by the OTAError at otaclient.py. """ - failure_type: wrapper.FailureType = wrapper.FailureType.RECOVERABLE - module: OTAModules = OTAModules.General - errcode: OTAErrorCode = OTAErrorCode.E_UNSPECIFIC - desc: str = "no description available for this error" + ERROR_PREFIX: ClassVar[str] = "E" + failure_type: wrapper.FailureType = wrapper.FailureType.RECOVERABLE + failure_errcode: OTAErrorCode = OTAErrorCode.E_UNSPECIFIC + failure_description: str = "no description available for this error" -class OTA_APIError(Exception): - """Errors that happen during processing API request. - - This exception class should be the top level exception for each API entry. - This exception must be created by wrapping an OTAClientError. - """ - - api: OTAAPI = OTAAPI.Unspecific - _err_prefix = "E" - - def __init__(self, ota_err: OTAError, *args: object) -> None: + def __init__(self, *args: object, module: str) -> None: + self.module = module super().__init__(*args) - self.otaclient_err = ota_err - self.errcode = ota_err.errcode - self.module = ota_err.module - self.failure_type = ota_err.failure_type - self.errdesc = ota_err.desc - - def get_errcode(self) -> "OTAErrorCode": - return self.errcode - - def get_errcode_str(self) -> str: - return f"{self._err_prefix}{self.errcode.to_str()}" - - def get_err_type(self) -> wrapper.FailureType: - return self.failure_type - - def get_err_reason(self, *, append_desc=True, append_detail=False) -> str: - r"""Return a failure_reason str. - - Format: Ec-aabb-dee[: [\n]] - c: FAILURE_TYPE_1digit - aa: API_2digits - bb: MODULE_2digits - dee: ERRCODE_3digits - """ - _errdesc = "" - if append_desc: - _errdesc = f": {self.errdesc}." - - _detail = "" - if append_detail: - _detail = f"\n{self.otaclient_err.__cause__!r}" - return ( - f"{self._err_prefix}" - f"{self.failure_type.to_str()}" - "-" - f"{self.api.to_str()}{self.module.to_str()}" - "-" - f"{self.errcode.to_str()}" - f"{_errdesc}{_detail}" - ) + @property + def failure_errcode_str(self) -> str: + return f"{self.ERROR_PREFIX}{self.failure_errcode.to_errcode_str()}" - def get_traceback(self, *, splitter="") -> str: - """Format the traceback into a str with splitter as .""" + def get_failure_traceback(self, *, splitter="\n") -> str: return splitter.join( traceback.format_exception(type(self), self, self.__traceback__) ) + def get_failure_reason(self) -> str: + """Return failure_reason str.""" + return f"{self.failure_errcode_str}: {self.failure_description}" -class OTAUpdateError(OTA_APIError): - api = OTAAPI.Update - + def get_error_report(self, title: str = "") -> str: + """The detailed failure report for debug use.""" + return ( + f"\n{title}\n" + f"@module: {self.module}" + "\n------ failure_reason ------\n" + f"{self.get_failure_reason()}" + "\n------ end of failure_reason ------\n" + "\n------ exception informaton ------\n" + f"{self!r}" + "\n------ end of exception informaton ------\n" + "\n------ exception traceback ------\n" + f"{self.get_failure_traceback()}" + "\n------ end of exception traceback ------\n" + ) -class OTARollbackError(OTA_APIError): - api = OTAAPI.Rollback +# +# ------ Network related error ------ +# -###### error exception classes ###### -_NETWORK_ERR_DEFAULT_DESC = ( - "error related to network connection detected, " - "please check the Internet connection and try again" -) +_NETWORK_ERR_DEFAULT_DESC = "network unstable, please check the network connection" -### network error ### class NetworkError(OTAError): """Generic network error""" failure_type: wrapper.FailureType = wrapper.FailureType.RECOVERABLE - module: OTAModules = OTAModules.Downloader - errcode: OTAErrorCode = OTAErrorCode.E_NETWORK - desc: str = _NETWORK_ERR_DEFAULT_DESC + failure_errcode: OTAErrorCode = OTAErrorCode.E_NETWORK + failure_description: str = _NETWORK_ERR_DEFAULT_DESC class OTAMetaDownloadFailed(NetworkError): - errcode: OTAErrorCode = OTAErrorCode.E_OTAMETA_DOWNLOAD_FAILED - desc: str = f"{_NETWORK_ERR_DEFAULT_DESC}: failed to download ota image meta" + failure_errcode: OTAErrorCode = OTAErrorCode.E_OTAMETA_DOWNLOAD_FAILED + failure_description: str = ( + f"failed to download OTA meta due to {_NETWORK_ERR_DEFAULT_DESC}" + ) + +# +# ------ recoverable error ------ +# -### recoverable error ### _RECOVERABLE_DEFAULT_DESC = ( - "recoverable ota error(unrelated to network) detected, " - "please resend the request or retry after a restart of ota-client/ECU" + "recoverable OTA error(unrelated to network) detected, " + "please retry after reboot device or restart otaclient" ) class OTAErrorRecoverable(OTAError): failure_type: wrapper.FailureType = wrapper.FailureType.RECOVERABLE - # followings are default values - module: OTAModules = OTAModules.General - errcode: OTAErrorCode = OTAErrorCode.E_OTA_ERR_RECOVERABLE - desc: str = _RECOVERABLE_DEFAULT_DESC + failure_errcode: OTAErrorCode = OTAErrorCode.E_OTA_ERR_RECOVERABLE + failure_description: str = _RECOVERABLE_DEFAULT_DESC -class OTAUpdateBusy(OTAErrorRecoverable): - module: OTAModules = OTAModules.API - errcode: OTAErrorCode = OTAErrorCode.E_OTAUPDATE_BUSY - desc: str = f"{_RECOVERABLE_DEFAULT_DESC}: on-going ota update detected, this request has been ignored" +class OTABusy(OTAErrorRecoverable): + failure_errcode: OTAErrorCode = OTAErrorCode.E_OTA_BUSY + failure_description: str = "on-going OTA operation(update or rollback) detected, this request has been ignored" -class OTARollBusy(OTAErrorRecoverable): - module: OTAModules = OTAModules.API - errcode: OTAErrorCode = OTAErrorCode.E_OTAUPDATE_BUSY - desc: str = f"{_RECOVERABLE_DEFAULT_DESC}: on-going ota update detected, this request has been ignored" - +class InvalidStatusForOTARollback(OTAErrorRecoverable): + failure_errcode: OTAErrorCode = OTAErrorCode.E_INVALID_STATUS_FOR_OTAROLLBACK + failure_description: str = "previous OTA is not succeeded, reject OTA rollback" -class InvalidStatusForOTAUpdate(OTAErrorRecoverable): - module: OTAModules = OTAModules.API - errcode: OTAErrorCode = OTAErrorCode.E_INVALID_STATUS_FOR_OTAUPDATE - desc: str = f"{_RECOVERABLE_DEFAULT_DESC}: current ota-status indicates it should not accept ota update" +# +# ------ recoverable error ------ +# -class InvalidUpdateRequest(OTAErrorRecoverable): - module: OTAModules = OTAModules.API - errcode: OTAErrorCode = OTAErrorCode.E_INVALID_OTAUPDATE_REQUEST - desc: str = ( - f"{_RECOVERABLE_DEFAULT_DESC}: incoming ota update request's content is invalid" - ) +_UNRECOVERABLE_DEFAULT_DESC = ( + "unrecoverable OTA error, please contact technical support" +) -class InvalidStatusForOTARollback(OTAErrorRecoverable): - module: OTAModules = OTAModules.API - errcode: OTAErrorCode = OTAErrorCode.E_INVALID_STATUS_FOR_OTAROLLBACK - desc: str = f"{_RECOVERABLE_DEFAULT_DESC}: current ota-status indicates it should not accept ota rollback" +class OTAErrorUnrecoverable(OTAError): + failure_type: wrapper.FailureType = wrapper.FailureType.RECOVERABLE + failure_errcode: OTAErrorCode = OTAErrorCode.E_OTA_ERR_UNRECOVERABLE + failure_description: str = _UNRECOVERABLE_DEFAULT_DESC -class OTAMetaVerificationFailed(OTAErrorRecoverable): - module: OTAModules = OTAModules.StandbySlotCreater - errcode: OTAErrorCode = OTAErrorCode.E_OTAMETA_VERIFICATION_FAILED - desc: str = ( - f"{_RECOVERABLE_DEFAULT_DESC}: hash verification failed for ota meta files" +class BootControlPlatformUnsupported(OTAErrorUnrecoverable): + failure_errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_PLATFORM_UNSUPPORTED + failure_description: str = ( + f"{_UNRECOVERABLE_DEFAULT_DESC}: bootloader for this ECU is not supported" ) -class UpdateDeltaGenerationFailed(OTAErrorRecoverable): - module: OTAModules = OTAModules.StandbySlotCreater - errcode: OTAErrorCode = OTAErrorCode.E_UPDATEDELTA_GENERATION_FAILED - desc: str = f"{_RECOVERABLE_DEFAULT_DESC}: (rebuild_mode) failed to calculate and/or prepare update delta" - - -class ApplyOTAUpdateFailed(OTAErrorRecoverable): - module: OTAModules = OTAModules.StandbySlotCreater - errcode: OTAErrorCode = OTAErrorCode.E_APPLY_OTAUPDATE_FAILED - desc: str = f"{_RECOVERABLE_DEFAULT_DESC}: failed to apply ota update" +class BootControlStartupFailed(OTAErrorUnrecoverable): + failure_errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_STARTUP_ERR + failure_description: str = ( + f"{_UNRECOVERABLE_DEFAULT_DESC}: boot controller startup failed" + ) -class MetadataJWTVerficationFailed(OTAErrorRecoverable): - module: OTAModules = OTAModules.API - errcode: OTAErrorCode = OTAErrorCode.E_METADATA_JWT_VERIFICATION_FAILED - desc: str = f"{_RECOVERABLE_DEFAULT_DESC}: verification failed for metadata.jwt" +class BootControlPreUpdateFailed(OTAErrorUnrecoverable): + failure_errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_PREUPDATE_FAILED + failure_description: str = ( + f"{_UNRECOVERABLE_DEFAULT_DESC}: boot_control pre_update process failed" + ) -class MetadataJWTInvalid(OTAErrorRecoverable): - module: OTAModules = OTAModules.API - errcode: OTAErrorCode = OTAErrorCode.E_INVALID_METADATAJWT - desc: str = f"{_RECOVERABLE_DEFAULT_DESC}: invalid metadata.jwt" +class BootControlPostUpdateFailed(OTAErrorUnrecoverable): + failure_errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_POSTUPDATE_FAILED + failure_description: str = ( + f"{_UNRECOVERABLE_DEFAULT_DESC}: boot_control post_update process failed" + ) -class OTAProxyFailedToStart(OTAErrorRecoverable): - module: OTAModules = OTAModules.OTASERVICE - errcode: OTAErrorCode = OTAErrorCode.E_OTAPROXY_FAILED_TO_START - desc: str = ( - f"{_RECOVERABLE_DEFAULT_DESC}: ota_proxy is required but failed to start" +class BootControlPreRollbackFailed(OTAErrorUnrecoverable): + failure_errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_PREROLLBACK_FAILED + failure_description: str = ( + f"{_UNRECOVERABLE_DEFAULT_DESC}: boot_control pre_rollback process failed" ) -### unrecoverable error ### -_UNRECOVERABLE_DEFAULT_DESC = ( - "unrecoverable ota error detected, please contact technical support" -) +class BootControlPostRollbackFailed(OTAErrorUnrecoverable): + failure_errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_POSTROLLBACK_FAILED + failure_description: str = ( + f"{_UNRECOVERABLE_DEFAULT_DESC}: boot_control post_rollback process failed" + ) -class OTAErrorUnRecoverable(OTAError): - failure_type: wrapper.FailureType = wrapper.FailureType.RECOVERABLE - module: OTAModules = OTAModules.General - errcode: OTAErrorCode = OTAErrorCode.E_OTA_ERR_UNRECOVERABLE - desc: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: unspecific unrecoverable ota error, please contact technical support" +class StandbySlotInsufficientSpace(OTAErrorUnrecoverable): + failure_errcode: OTAErrorCode = OTAErrorCode.E_STANDBY_SLOT_INSUFFICIENT_SPACE + failure_description: str = ( + f"{_UNRECOVERABLE_DEFAULT_DESC}: insufficient space at standby slot" + ) -class BootControlPlatformUnsupported(OTAErrorUnRecoverable): - module: OTAModules = OTAModules.BootController - errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_PLATFORM_UNSUPPORTED - desc: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: current ECU platform is not supported by the boot controller module" +class InvalidUpdateRequest(OTAErrorUnrecoverable): + failure_errcode: OTAErrorCode = OTAErrorCode.E_INVALID_OTAUPDATE_REQUEST + failure_description: str = ( + f"{_UNRECOVERABLE_DEFAULT_DESC}: incoming OTA update request is invalid" + ) -class BootControlInitError(OTAErrorUnRecoverable): - module: OTAModules = OTAModules.BootController - errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_INIT_ERR - desc: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: failed to init boot controller module" +class MetadataJWTInvalid(OTAErrorUnrecoverable): + failure_errcode: OTAErrorCode = OTAErrorCode.E_METADATAJWT_INVALID + failure_description: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: verfication for metadata.jwt is OK but metadata.jwt's content is invalid" -class BootControlPreUpdateFailed(OTAErrorUnRecoverable): - module: OTAModules = OTAModules.BootController - errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_PREUPDATE_FAILED - desc: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: pre_update process failed" +class MetadataJWTVerficationFailed(OTAErrorUnrecoverable): + failure_errcode: OTAErrorCode = OTAErrorCode.E_METADATAJWT_CERT_VERIFICATION_FAILED + failure_description: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: certificate verification failed for OTA metadata.jwt" -class BootControlPostUpdateFailed(OTAErrorUnRecoverable): - module: OTAModules = OTAModules.BootController - errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_POSTUPDATE_FAILED - desc: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: post_update process failed, switch boot is not finished" +class OTAProxyFailedToStart(OTAErrorUnrecoverable): + failure_errcode: OTAErrorCode = OTAErrorCode.E_OTAPROXY_FAILED_TO_START + failure_description: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: otaproxy is required for multiple ECU update but otaproxy failed to start" -class BootControlPostRollbackFailed(OTAErrorUnRecoverable): - module: OTAModules = OTAModules.BootController - errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_POSTUPDATE_FAILED - desc: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: post_rollback process failed, switch boot is not finished" +class UpdateDeltaGenerationFailed(OTAErrorUnrecoverable): + failure_errcode: OTAErrorCode = OTAErrorCode.E_UPDATEDELTA_GENERATION_FAILED + failure_description: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: failed to calculate and/or prepare update delta" -class StandbySlotSpaceNotEnoughError(OTAErrorUnRecoverable): - module: OTAModules = OTAModules.StandbySlotCreater - errcode: OTAErrorCode = OTAErrorCode.E_STANDBY_SLOT_SPACE_NOT_ENOUGH_ERROR - desc: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: standby slot has insufficient space to apply update, abort" +class ApplyOTAUpdateFailed(OTAErrorUnrecoverable): + failure_errcode: OTAErrorCode = OTAErrorCode.E_APPLY_OTAUPDATE_FAILED + failure_description: str = ( + f"{_UNRECOVERABLE_DEFAULT_DESC}: failed to apply OTA update to standby slot" + ) -class BootControlPreRollbackFailed(OTAErrorUnRecoverable): - module: OTAModules = OTAModules.BootController - errcode: OTAErrorCode = OTAErrorCode.E_BOOTCONTROL_PREROLLBACK_FAILED - desc: str = f"{_UNRECOVERABLE_DEFAULT_DESC}: pre_rollback process failed" +class OTAClientStartupFailed(OTAErrorUnrecoverable): + failure_errcode: OTAErrorCode = OTAErrorCode.E_OTACLIENT_STARTUP_FAILED + failure_description: str = ( + f"{_UNRECOVERABLE_DEFAULT_DESC}: failed to start otaclient instance" + ) diff --git a/otaclient/app/ota_client.py b/otaclient/app/ota_client.py index 330537a97..c244044fc 100644 --- a/otaclient/app/ota_client.py +++ b/otaclient/app/ota_client.py @@ -25,35 +25,17 @@ from typing import Optional, Type, Iterator from urllib.parse import urlparse -from . import ota_metadata +from . import downloader, ota_metadata, errors as ota_errors from .boot_control import BootControllerProtocol, get_boot_controller -from .common import RetryTaskMap, get_backoff, ensure_otaproxy_start +from .common import ( + get_backoff, + ensure_otaproxy_start, + RetryTaskMap, + RetryTaskMapInterrupted, +) from .configs import config as cfg from .create_standby import StandbySlotCreatorProtocol, get_standby_slot_creator -from .downloader import ( - EMPTY_FILE_SHA256, - DestinationNotAvailableError, - DownloadFailedSpaceNotEnough, - Downloader, - HashVerificaitonError, -) from .ecu_info import ECUInfo -from .errors import ( - MetadataJWTInvalid, - MetadataJWTVerficationFailed, - NetworkError, - ApplyOTAUpdateFailed, - InvalidUpdateRequest, - OTAMetaVerificationFailed, - OTAErrorUnRecoverable, - OTAMetaDownloadFailed, - StandbySlotSpaceNotEnoughError, - UpdateDeltaGenerationFailed, - OTA_APIError, - OTAError, - OTARollbackError, - OTAUpdateError, -) from .interface import OTAClientProtocol from .ota_status import LiveOTAStatus from .proto import wrapper @@ -130,7 +112,7 @@ def __init__( self.total_remove_files_num = 0 # init downloader - self._downloader = Downloader() + self._downloader = downloader.Downloader() self.proxy = proxy # init ota update statics collector @@ -155,7 +137,7 @@ def _download_file(entry: wrapper.RegularInf) -> RegInfProcessedStats: _fhash_str = entry.get_hash() # special treatment to empty file - if _fhash_str == EMPTY_FILE_SHA256: + if _fhash_str == downloader.EMPTY_FILE_SHA256: return cur_stat _local_copy = self._ota_tmp_on_standby / _fhash_str @@ -172,7 +154,7 @@ def _download_file(entry: wrapper.RegularInf) -> RegInfProcessedStats: logger.debug("download neede OTA image files...") # special treatment to empty file, create it first - _empty_file = self._ota_tmp_on_standby / EMPTY_FILE_SHA256 + _empty_file = self._ota_tmp_on_standby / downloader.EMPTY_FILE_SHA256 _empty_file.touch() last_active_timestamp = int(time.time()) @@ -257,8 +239,11 @@ def _update_standby_slot(self): self.total_download_fiies_size = _delta_bundle.total_download_files_size self.total_remove_files_num = len(_delta_bundle.rm_delta) except Exception as e: - logger.error(f"failed to generate delta: {e!r}") - raise UpdateDeltaGenerationFailed from e + _err_msg = f"failed to generate delta: {e!r}" + logger.error(_err_msg) + raise ota_errors.UpdateDeltaGenerationFailed( + _err_msg, module=__name__ + ) from e # --- download needed files --- # logger.info( @@ -268,12 +253,23 @@ def _update_standby_slot(self): self.update_phase = wrapper.UpdatePhase.DOWNLOADING_OTA_FILES try: self._download_files(_delta_bundle.get_download_list()) - except DownloadFailedSpaceNotEnough: - logger.critical("not enough space is left on standby slot") - raise StandbySlotSpaceNotEnoughError from None + except downloader.DownloadFailedSpaceNotEnough: + _err_msg = "not enough space is left on standby slot" + logger.error(_err_msg) + raise ota_errors.StandbySlotInsufficientSpace( + _err_msg, module=__name__ + ) from None + except RetryTaskMapInterrupted as e: + _err_msg = ( + f"downloading group keeps failing for {cfg.DOWNLOAD_GROUP_INACTIVE_TIMEOUT}s, " + f"last_error: {e!r}" + ) + logger.error(_err_msg) + raise ota_errors.NetworkError(_err_msg, module=__name__) from e except Exception as e: - logger.error(f"failed to finish downloading files: {e!r}") - raise NetworkError from e + _err_msg = f"unspecific error, failed to finish downloading files: {e!r}" + logger.error(_err_msg) + raise ota_errors.NetworkError(_err_msg, module=__name__) from e # shutdown downloader on download finished self._downloader.shutdown() @@ -284,8 +280,10 @@ def _update_standby_slot(self): try: self._standby_slot_creator.create_standby_slot() except Exception as e: - logger.error(f"failed to apply update to standby slot: {e!r}") - raise ApplyOTAUpdateFailed from e + _err_msg = f"failed to apply update to standby slot: {e!r}" + logger.error(_err_msg) + raise ota_errors.ApplyOTAUpdateFailed(_err_msg, module=__name__) from e + logger.info("finished updating standby slot") def _execute_update( @@ -329,7 +327,9 @@ def _execute_update( ), f"invalid cookies, expecting json object: {cookies_json}" self._downloader.configure_cookies(_cookies) except (JSONDecodeError, AssertionError) as e: - raise InvalidUpdateRequest from e + _err_msg = f"cookie is invalid: {cookies_json=}" + logger.error(_err_msg) + raise ota_errors.InvalidUpdateRequest(_err_msg, module=__name__) from e # configure proxy logger.debug("configure proxy setting...") @@ -354,30 +354,41 @@ def _execute_update( self.total_files_size_uncompressed = ( self._otameta.total_files_size_uncompressed ) - except HashVerificaitonError as e: - logger.error("failed to verify ota metafiles hash") - raise OTAMetaVerificationFailed from e - except DestinationNotAvailableError as e: - logger.error("failed to save ota metafiles") - raise OTAErrorUnRecoverable from e + except downloader.HashVerificaitonError as e: + _err_msg = f"downloader: keep failing to verify ota metafiles' hash: {e!r}" + logger.error(_err_msg) + raise ota_errors.MetadataJWTVerficationFailed( + _err_msg, module=__name__ + ) from e + except downloader.DestinationNotAvailableError as e: + _err_msg = f"downloader: failed to save ota metafiles: {e!r}" + logger.error(_err_msg) + raise ota_errors.OTAErrorUnrecoverable(_err_msg, module=__name__) from e except ota_metadata.MetadataJWTVerificationFailed as e: - logger.error(f"failed to verify metadata.jwt: {e!r}") - raise MetadataJWTVerficationFailed from e + _err_msg = f"failed to verify metadata.jwt: {e!r}" + logger.error(_err_msg) + raise ota_errors.MetadataJWTVerficationFailed( + _err_msg, module=__name__ + ) from e except ota_metadata.MetadataJWTPayloadInvalid as e: - logger.error(f"metadata.jwt is invalid: {e!r}") - raise MetadataJWTInvalid from e + _err_msg = f"metadata.jwt is invalid: {e!r}" + logger.error(_err_msg) + raise ota_errors.MetadataJWTInvalid(_err_msg, module=__name__) from e except Exception as e: - logger.error(f"failed to prepare ota metafiles: {e!r}") - raise OTAMetaDownloadFailed from e + _err_msg = f"unspecific error, failed to prepare ota metafiles: {e!r}" + logger.error(_err_msg) + raise ota_errors.OTAMetaDownloadFailed(_err_msg, module=__name__) from e # ------ execute local update ------ # logger.info("enter local OTA update...") try: self._update_standby_slot() - except OTAError: - raise + except ota_errors.OTAError: + raise # no need to wrap an OTAError again except Exception as e: - raise ApplyOTAUpdateFailed(f"unspecific applying OTA update failure: {e!r}") + raise ota_errors.ApplyOTAUpdateFailed( + f"unspecific applying OTA update failure: {e!r}", module=__name__ + ) # ------ post update ------ # logger.info("local update finished, wait on all subecs...") @@ -440,14 +451,14 @@ def execute( """ try: self._execute_update(version, raw_url_base, cookies_json) - except OTAError as e: + except ota_errors.OTAError as e: logger.error(f"update failed: {e!r}") self._boot_controller.on_operation_failure() - raise OTAUpdateError(e) from e + raise # do not cover the OTA error again except Exception as e: - raise OTAUpdateError( - ApplyOTAUpdateFailed(f"unspecific OTA failure: {e!r}") - ) from e + _err_msg = f"unspecific error, update failed: {e!r}" + self._boot_controller.on_operation_failure() + raise ota_errors.ApplyOTAUpdateFailed(_err_msg, module=__name__) from e finally: self.shutdown() @@ -458,67 +469,74 @@ def __init__(self, boot_controller: BootControllerProtocol) -> None: def execute(self): try: - # enter rollback self._boot_controller.pre_rollback() - # leave rollback self._boot_controller.post_rollback() - except OTAError as e: + except ota_errors.OTAError as e: logger.error(f"rollback failed: {e!r}") self._boot_controller.on_operation_failure() - raise OTARollbackError(e) from e + raise class OTAClient(OTAClientProtocol): """ Init params: - boot_control_cls: type of boot control mechanism to use + boot_controller: boot control instance create_standby_cls: type of create standby slot mechanism to use + my_ecu_id: ECU id of the device running this otaclient instance + control_flags: flags used by otaclient and ota_service stub for synchronization + proxy: upper otaproxy URL """ - DEFAULT_FIRMWARE_VERSION = "unknown" OTACLIENT_VERSION = __version__ def __init__( self, *, - boot_control_cls: Type[BootControllerProtocol], + boot_controller: BootControllerProtocol, create_standby_cls: Type[StandbySlotCreatorProtocol], - my_ecu_id: str = "", + my_ecu_id: str, control_flags: OTAClientControlFlags, proxy: Optional[str] = None, ): - self.my_ecu_id = my_ecu_id - # ensure only one update/rollback session is running - self._lock = threading.Lock() - - self.boot_controller = boot_control_cls() - self.create_standby_cls = create_standby_cls - self.live_ota_status = LiveOTAStatus( - self.boot_controller.get_booted_ota_status() - ) + try: + self.my_ecu_id = my_ecu_id + # ensure only one update/rollback session is running + self._lock = threading.Lock() + + self.boot_controller = boot_controller + self.create_standby_cls = create_standby_cls + self.live_ota_status = LiveOTAStatus( + self.boot_controller.get_booted_ota_status() + ) - self.current_version = ( - self.boot_controller.load_version() or self.DEFAULT_FIRMWARE_VERSION - ) - self.proxy = proxy - self.control_flags = control_flags + self.current_version = self.boot_controller.load_version() + self.proxy = proxy + self.control_flags = control_flags - # executors for update/rollback - self._update_executor: _OTAUpdater = None # type: ignore - self._rollback_executor: _OTARollbacker = None # type: ignore + # executors for update/rollback + self._update_executor: _OTAUpdater = None # type: ignore + self._rollback_executor: _OTARollbacker = None # type: ignore - # err record - self.last_failure_type = wrapper.FailureType.NO_FAILURE - self.last_failure_reason = "" - self.last_failure_traceback = "" + # err record + self.last_failure_type = wrapper.FailureType.NO_FAILURE + self.last_failure_reason = "" + self.last_failure_traceback = "" + except Exception as e: + _err_msg = f"failed to start otaclient core: {e!r}" + logger.error(_err_msg) + raise ota_errors.OTAClientStartupFailed(_err_msg, module=__name__) from e - def _on_failure(self, exc: OTA_APIError, ota_status: wrapper.StatusOta): + def _on_failure(self, exc: ota_errors.OTAError, ota_status: wrapper.StatusOta): self.live_ota_status.set_ota_status(ota_status) try: - self.last_failure_type = exc.get_err_type() - self.last_failure_reason = exc.get_err_reason() - self.last_failure_traceback = exc.get_traceback() - logger.error(f"on {ota_status=}: {self.last_failure_traceback=}") + self.last_failure_type = exc.failure_type + self.last_failure_reason = exc.get_failure_reason() + if cfg.DEBUG_MODE: + self.last_failure_traceback = exc.get_failure_traceback() + + logger.error( + exc.get_error_report(f"OTA failed with {ota_status.name}: {exc!r}") + ) finally: exc = None # type: ignore , prevent ref cycle @@ -534,12 +552,16 @@ def update(self, version: str, url_base: str, cookies_json: str): control_flags=self.control_flags, proxy=self.proxy, ) + + # reset failure information on handling new update request self.last_failure_type = wrapper.FailureType.NO_FAILURE self.last_failure_reason = "" self.last_failure_traceback = "" + + # enter update self.live_ota_status.set_ota_status(wrapper.StatusOta.UPDATING) self._update_executor.execute(version, url_base, cookies_json) - except OTAUpdateError as e: + except ota_errors.OTAError as e: self._on_failure(e, wrapper.StatusOta.FAILURE) finally: self._update_executor = None # type: ignore @@ -557,13 +579,17 @@ def rollback(self): self._rollback_executor = _OTARollbacker( boot_controller=self.boot_controller ) + + # clear failure information on handling new rollback request self.last_failure_type = wrapper.FailureType.NO_FAILURE self.last_failure_reason = "" self.last_failure_traceback = "" + + # entering rollback self.live_ota_status.set_ota_status(wrapper.StatusOta.ROLLBACKING) self._rollback_executor.execute() # silently ignore overlapping request - except OTARollbackError as e: + except ota_errors.OTAError as e: self._on_failure(e, wrapper.StatusOta.ROLLBACK_FAILURE) finally: self._rollback_executor = None # type: ignore @@ -589,115 +615,189 @@ def status(self) -> wrapper.StatusResponseEcuV2: return status_report -class OTAClientBusy(Exception): - """Raised when otaclient receive another request when doing update/rollback.""" - - -class OTAClientWrapper: - """OTAClient stub implementation that wraps OTAClient, manage update/rollback session, - and exposes async API for OTAClientServiceStub. - - All actual API request handlings are wrapped and dispatched into threadpool for execution. - Only one ongoing update/rollback is allowed, with the help of exclusive update_rollback lock. - - If this ECU enables otaproxy and has child ECUs depend on it, control_flags is used to prevent - this ECU's reboot before the otaproxy dependency is resolved. - - OTAClient itself is self-contained, its error handling is done internally, no exception will be raised - from calling API methods of OTAClient. - """ - +class OTAServicer: def __init__( self, *, - ecu_info: ECUInfo, - executor: ThreadPoolExecutor, control_flags: OTAClientControlFlags, + ecu_info: ECUInfo, + executor: Optional[ThreadPoolExecutor] = None, + otaclient_version: str = __version__, proxy: Optional[str] = None, ) -> None: - # only one update/rollback is allowed at a time - self.update_rollback_lock = asyncio.Lock() - self.my_ecu_id = ecu_info.ecu_id + self.ecu_id = ecu_info.ecu_id + self.otaclient_version = otaclient_version + self.local_used_proxy_url = proxy + self.last_operation: Optional[wrapper.StatusOta] = None + + # default boot startup failure if boot_controller/otaclient_core crashed without + # raising specific error + self._otaclient_startup_failed_status = wrapper.StatusResponseEcuV2( + ecu_id=ecu_info.ecu_id, + otaclient_version=otaclient_version, + ota_status=wrapper.StatusOta.FAILURE, + failure_type=wrapper.FailureType.UNRECOVERABLE, + failure_reason="unspecific error", + ) + self._update_rollback_lock = asyncio.Lock() self._run_in_executor = partial( asyncio.get_running_loop().run_in_executor, executor ) - # create otaclient instance and control flags - self._otaclient = OTAClient( - boot_control_cls=get_boot_controller(ecu_info.get_bootloader()), - create_standby_cls=get_standby_slot_creator(cfg.STANDBY_CREATION_MODE), - my_ecu_id=self.my_ecu_id, - control_flags=control_flags, - proxy=proxy, - ) + # + # ------ compose otaclient ------ + # + self._otaclient_inst: Optional[OTAClient] = None - # proxy used by local otaclient - # NOTE: it can be an upper otaproxy, local otaproxy, or no proxy - self.local_used_proxy_url = proxy - self.last_operation = None # update/rollback/None + # select boot_controller and standby_slot implementations + _bootctrl_cls = get_boot_controller(ecu_info.get_bootloader()) + _standby_slot_creator = get_standby_slot_creator(cfg.STANDBY_CREATION_MODE) + + # boot controller starts up + try: + _bootctrl_inst = _bootctrl_cls() + except ota_errors.OTAError as e: + logger.error( + e.get_error_report(title=f"boot controller startup failed: {e!r}") + ) + self._otaclient_startup_failed_status = wrapper.StatusResponseEcuV2( + ecu_id=ecu_info.ecu_id, + otaclient_version=otaclient_version, + ota_status=wrapper.StatusOta.FAILURE, + failure_type=wrapper.FailureType.UNRECOVERABLE, + failure_reason=e.get_failure_reason(), + ) + + if cfg.DEBUG_MODE: + self._otaclient_startup_failed_status.failure_traceback = ( + e.get_failure_traceback() + ) + return - # property + # otaclient core starts up + try: + self._otaclient_inst = OTAClient( + boot_controller=_bootctrl_inst, + create_standby_cls=_standby_slot_creator, + my_ecu_id=ecu_info.ecu_id, + control_flags=control_flags, + proxy=proxy, + ) + except ota_errors.OTAError as e: + logger.error( + e.get_error_report(title=f"otaclient core startup failed: {e!r}") + ) + self._otaclient_startup_failed_status = wrapper.StatusResponseEcuV2( + ecu_id=ecu_info.ecu_id, + otaclient_version=otaclient_version, + ota_status=wrapper.StatusOta.FAILURE, + failure_type=wrapper.FailureType.UNRECOVERABLE, + failure_reason=e.get_failure_reason(), + ) + + if cfg.DEBUG_MODE: + self._otaclient_startup_failed_status.failure_traceback = ( + e.get_failure_traceback() + ) + return @property def is_busy(self) -> bool: - return self.update_rollback_lock.locked() + return self._update_rollback_lock.locked() + + async def dispatch_update( + self, request: wrapper.UpdateRequestEcu + ) -> wrapper.UpdateResponseEcu: + # prevent update operation if otaclient is not started + if self._otaclient_inst is None: + return wrapper.UpdateResponseEcu( + ecu_id=self.ecu_id, result=wrapper.FailureType.UNRECOVERABLE + ) - # API + # check and acquire lock + if self._update_rollback_lock.locked(): + logger.warning( + f"ongoing operation: {self.last_operation=}, ignore incoming {request=}" + ) + return wrapper.UpdateResponseEcu( + ecu_id=self.ecu_id, result=wrapper.FailureType.RECOVERABLE + ) - async def dispatch_update(self, request: wrapper.UpdateRequestEcu): - """Dispatch OTA update execution to background thread. + # immediately take the lock if not locked + await self._update_rollback_lock.acquire() + self.last_operation = wrapper.StatusOta.UPDATING + + async def _update_task(): + if self._otaclient_inst is None: + return - Raises: - OTAClientBusy if otaclient is already executing update/rollback. - """ - if self.update_rollback_lock.locked(): - raise OTAClientBusy(f"ongoing operation: {self.last_operation=}") - else: # immediately take the lock if not locked - await self.update_rollback_lock.acquire() - self.last_operation = wrapper.StatusOta.UPDATING - - async def _update(): - """Background waiting for update to be finished and - release update_rollback lock.""" try: await self._run_in_executor( partial( - self._otaclient.update, + self._otaclient_inst.update, request.version, request.url, request.cookies, ) ) + except Exception: + pass # error should be collected by otaclient, not us finally: self.last_operation = None - self.update_rollback_lock.release() + self._update_rollback_lock.release() # dispatch update to background - asyncio.create_task(_update()) + asyncio.create_task(_update_task()) - async def dispatch_rollback(self, _: wrapper.RollbackRequestEcu): - """Dispatch OTA rollback execution to background thread. + return wrapper.UpdateResponseEcu( + ecu_id=self.ecu_id, result=wrapper.FailureType.NO_FAILURE + ) + + async def dispatch_rollback( + self, request: wrapper.RollbackRequestEcu + ) -> wrapper.RollbackResponseEcu: + # prevent rollback operation if otaclient is not started + if self._otaclient_inst is None: + return wrapper.RollbackResponseEcu( + ecu_id=self.ecu_id, result=wrapper.FailureType.UNRECOVERABLE + ) + + # check and acquire lock + if self._update_rollback_lock.locked(): + logger.warning( + f"ongoing operation: {self.last_operation=}, ignore incoming {request=}" + ) + return wrapper.RollbackResponseEcu( + ecu_id=self.ecu_id, result=wrapper.FailureType.RECOVERABLE + ) + + # immediately take the lock if not locked + await self._update_rollback_lock.acquire() + self.last_operation = wrapper.StatusOta.ROLLBACKING + + async def _rollback_task(): + if self._otaclient_inst is None: + return - Raises: - OTAClientBusy if otaclient is already executing update/rollback. - """ - if self.update_rollback_lock.locked(): - raise OTAClientBusy(f"ongoing operation: {self.last_operation=}") - else: # immediately take the lock if not locked - await self.update_rollback_lock.acquire() - self.last_operation = wrapper.StatusOta.ROLLBACKING - - async def _rollback(): - """Background waiting for rollback to be finished and - release update_rollback lock.""" try: - await self._run_in_executor(self._otaclient.rollback) + await self._run_in_executor(self._otaclient_inst.rollback) + except Exception: + pass # error should be collected by otaclient, not us finally: self.last_operation = None - self.update_rollback_lock.release() + self._update_rollback_lock.release() # dispatch to background - asyncio.create_task(_rollback()) + asyncio.create_task(_rollback_task()) + + return wrapper.RollbackResponseEcu( + ecu_id=self.ecu_id, result=wrapper.FailureType.NO_FAILURE + ) async def get_status(self) -> wrapper.StatusResponseEcuV2: - return await self._run_in_executor(self._otaclient.status) + # otaclient is not started due to boot control startup failed + if self._otaclient_inst is None: + return self._otaclient_startup_failed_status + + # otaclient core started, query status from it + return await self._run_in_executor(self._otaclient_inst.status) diff --git a/otaclient/app/ota_client_stub.py b/otaclient/app/ota_client_stub.py index 855f3b151..d9affd1c6 100644 --- a/otaclient/app/ota_client_stub.py +++ b/otaclient/app/ota_client_stub.py @@ -30,7 +30,7 @@ from .common import ensure_otaproxy_start from .boot_control._common import CMDHelperFuncs from .ecu_info import ECUContact, ECUInfo -from .ota_client import OTAClientBusy, OTAClientControlFlags, OTAClientWrapper +from .ota_client import OTAClientControlFlags, OTAServicer from .ota_client_call import ECUNoResponse, OtaClientCall from .proto import wrapper from .proxy_info import proxy_cfg @@ -675,7 +675,7 @@ def __init__( ecu_status_storage: ECUStatusStorage, *, ecu_info: ECUInfo, - otaclient_wrapper: OTAClientWrapper, + otaclient_wrapper: OTAServicer, ) -> None: self._otaclient_wrapper = otaclient_wrapper # for local ECU status polling self._ecu_status_storage = ecu_status_storage @@ -736,7 +736,7 @@ def __init__(self, *, ecu_info: ECUInfo, _proxy_cfg=proxy_cfg): self.my_ecu_id = ecu_info.ecu_id self._otaclient_control_flags = OTAClientControlFlags() - self._otaclient_wrapper = OTAClientWrapper( + self._otaclient_wrapper = OTAServicer( ecu_info=ecu_info, executor=self._executor, control_flags=self._otaclient_control_flags, @@ -878,13 +878,10 @@ async def update(self, request: wrapper.UpdateRequest) -> wrapper.UpdateResponse # second: dispatch update request to local if required by incoming request if update_req_ecu := request.find_ecu(self.my_ecu_id): - _resp_ecu = wrapper.UpdateResponseEcu(ecu_id=self.my_ecu_id) - try: - await self._otaclient_wrapper.dispatch_update(update_req_ecu) + _resp_ecu = await self._otaclient_wrapper.dispatch_update(update_req_ecu) + # local otaclient accepts the update request + if _resp_ecu.result == wrapper.FailureType.NO_FAILURE: update_acked_ecus.add(self.my_ecu_id) - except OTAClientBusy as e: - logger.warning(f"self ECU is busy, ignore local update request: {e!r}") - _resp_ecu.result = wrapper.FailureType.RECOVERABLE response.add_ecu(_resp_ecu) # finally, trigger ecu_status_storage entering active mode if needed @@ -895,6 +892,7 @@ async def update(self, request: wrapper.UpdateRequest) -> wrapper.UpdateResponse update_acked_ecus ) ) + return response async def rollback( @@ -943,13 +941,10 @@ async def rollback( # second: dispatch rollback request to local if required if rollback_req := request.find_ecu(self.my_ecu_id): - _resp_ecu = wrapper.RollbackResponseEcu(ecu_id=self.my_ecu_id) - try: + response.add_ecu( await self._otaclient_wrapper.dispatch_rollback(rollback_req) - except OTAClientBusy as e: - logger.warning(f"self ECU is busy, ignore local rollback: {e!r}") - _resp_ecu.result = wrapper.FailureType.RECOVERABLE - response.add_ecu(_resp_ecu) + ) + return response async def status(self, _=None) -> wrapper.StatusResponse: diff --git a/tests/test_ota_client.py b/tests/test_ota_client.py index 541fead4a..5990e5295 100644 --- a/tests/test_ota_client.py +++ b/tests/test_ota_client.py @@ -30,13 +30,12 @@ from otaclient.app.create_standby.common import DeltaBundle, RegularDelta from otaclient.app.configs import config as otaclient_cfg from otaclient.app.ecu_info import ECUInfo -from otaclient.app.errors import OTAErrorRecoverable, OTAUpdateError +from otaclient.app.errors import OTAError, OTAErrorRecoverable from otaclient.app.ota_client import ( OTAClient, _OTAUpdater, - OTAClientBusy, OTAClientControlFlags, - OTAClientWrapper, + OTAServicer, ) from otaclient.app.ota_metadata import parse_regulars_from_txt, parse_dirs_from_txt from otaclient.app.proto.wrapper import RegularInf, DirectoryInf @@ -236,7 +235,7 @@ def mock_setup(self, mocker: pytest_mock.MockerFixture): ) self.ota_client = OTAClient( - boot_control_cls=mocker.MagicMock(return_value=self.boot_controller), + boot_controller=self.boot_controller, create_standby_cls=mocker.MagicMock(), my_ecu_id=self.MY_ECU_ID, control_flags=self.control_flags, @@ -274,7 +273,7 @@ def test_update_normal_finished(self): def test_update_interrupted(self): # inject exception - _error = OTAUpdateError(OTAErrorRecoverable("network disconnected")) + _error = OTAErrorRecoverable("network disconnected", module=__name__) self.ota_updater.execute.side_effect = _error # --- execution --- # @@ -356,7 +355,7 @@ def test_status_in_update(self): ) -class TestOTAClientWrapper: +class TestOTAServicer: BOOTLOADER_TYPE = BootloaderType.GRUB ECU_INFO = ECUInfo( format_version=1, @@ -372,23 +371,27 @@ async def mock_setup(self, mocker: pytest_mock.MockerFixture): self.otaclient = mocker.MagicMock(spec=OTAClient) self.otaclient_cls = mocker.MagicMock(return_value=self.otaclient) self.standby_slot_creator_cls = mocker.MagicMock() - self.boot_control_cls = mocker.MagicMock() + self.boot_controller = mocker.MagicMock(spec=BootControllerProtocol) self.control_flags = mocker.MagicMock(spec=OTAClientControlFlags) - # patch OTAClient + # + # ------ patching ------ + # mocker.patch(f"{cfg.OTACLIENT_MODULE_PATH}.OTAClient", self.otaclient_cls) mocker.patch( f"{cfg.OTACLIENT_MODULE_PATH}.get_boot_controller", - return_value=self.boot_control_cls, + return_value=mocker.MagicMock(return_value=self.boot_controller), ) mocker.patch( f"{cfg.OTACLIENT_MODULE_PATH}.get_standby_slot_creator", return_value=self.standby_slot_creator_cls, ) - # start the stub + # + # ------ start OTAServicer instance ------ + # self.local_use_proxy = "" - self.otaclient_stub = OTAClientWrapper( + self.otaclient_stub = OTAServicer( ecu_info=self.ECU_INFO, executor=self._executor, control_flags=self.control_flags, @@ -401,13 +404,19 @@ async def mock_setup(self, mocker: pytest_mock.MockerFixture): self._executor.shutdown(wait=False) def test_stub_initializing(self): + # + # ------ assertion ------ + # + + # ensure the OTAServicer properly compose otaclient core self.otaclient_cls.assert_called_once_with( - boot_control_cls=self.boot_control_cls, + boot_controller=self.boot_controller, create_standby_cls=self.standby_slot_creator_cls, my_ecu_id=self.ECU_INFO.ecu_id, control_flags=self.control_flags, proxy=self.local_use_proxy, ) + assert self.otaclient_stub.last_operation is None assert self.otaclient_stub.local_used_proxy_url is self.local_use_proxy @@ -434,9 +443,8 @@ def _updating(*args, **kwargs): assert self.otaclient_stub.last_operation is wrapper.StatusOta.UPDATING assert self.otaclient_stub.is_busy # test ota update/rollback exclusive lock, - # OTAClientBusy error should be raised on another request - with pytest.raises(OTAClientBusy): - await self.otaclient_stub.dispatch_update(update_request_ecu) + resp = await self.otaclient_stub.dispatch_update(update_request_ecu) + assert resp.result == wrapper.FailureType.RECOVERABLE # finish up update _updating_event.set() @@ -465,9 +473,8 @@ def _rollbacking(*args, **kwargs): assert self.otaclient_stub.last_operation is wrapper.StatusOta.ROLLBACKING assert self.otaclient_stub.is_busy # test ota update/rollback exclusive lock, - # OTAClientBusy error should be raised on another request - with pytest.raises(OTAClientBusy): - await self.otaclient_stub.dispatch_rollback(wrapper.RollbackRequestEcu()) + resp = await self.otaclient_stub.dispatch_rollback(wrapper.RollbackRequestEcu()) + assert resp.result == wrapper.FailureType.RECOVERABLE # finish up rollback _rollbacking_event.set() diff --git a/tests/test_ota_client_stub.py b/tests/test_ota_client_stub.py index 448c175e5..a741aeb43 100644 --- a/tests/test_ota_client_stub.py +++ b/tests/test_ota_client_stub.py @@ -16,14 +16,13 @@ import asyncio import pytest from concurrent.futures import ThreadPoolExecutor -from functools import partial from pathlib import Path from typing import Any, Dict, List, Set import pytest_mock from otaclient.app.ecu_info import ECUInfo -from otaclient.app.ota_client import OTAClientBusy, OTAClientWrapper +from otaclient.app.ota_client import OTAServicer from otaclient.app.ota_client_call import OtaClientCall from otaclient.app.ota_client_stub import ( ECUStatusStorage, @@ -716,7 +715,7 @@ async def setup_test(self, tmp_path: Path, mocker: pytest_mock.MockerFixture): await asyncio.sleep(self.ENSURE_NEXT_CHECKING_ROUND) # ensure the task stopping # --- mocker --- # - self.otaclient_wrapper = mocker.MagicMock(spec=OTAClientWrapper) + self.otaclient_wrapper = mocker.MagicMock(spec=OTAServicer) self.ecu_status_tracker = mocker.MagicMock() self.otaproxy_launcher = mocker.MagicMock(spec=OTAProxyLauncher) # mock OTAClientCall, make update_call return success on any update dispatches to subECUs @@ -736,7 +735,7 @@ async def setup_test(self, tmp_path: Path, mocker: pytest_mock.MockerFixture): mocker.MagicMock(return_value=self.ecu_storage), ) mocker.patch( - f"{cfg.OTACLIENT_STUB_MODULE_PATH}.OTAClientWrapper", + f"{cfg.OTACLIENT_STUB_MODULE_PATH}.OTAServicer", mocker.MagicMock(return_value=self.otaclient_wrapper), ) mocker.patch( @@ -886,6 +885,11 @@ async def test_update_normal( update_target_ids: Set[str], expected: wrapper.UpdateResponse, ): + # --- setup --- # + self.otaclient_wrapper.dispatch_update.return_value = wrapper.UpdateResponseEcu( + ecu_id=self.ecu_info.ecu_id, result=wrapper.FailureType.NO_FAILURE + ) + # --- execution --- # resp = await self.otaclient_service_stub.update(update_request) @@ -898,7 +902,9 @@ async def test_update_normal( async def test_update_local_ecu_busy(self): # --- preparation --- # - self.otaclient_wrapper.dispatch_update.side_effect = OTAClientBusy() + self.otaclient_wrapper.dispatch_update.return_value = wrapper.UpdateResponseEcu( + ecu_id="autoware", result=wrapper.FailureType.RECOVERABLE + ) update_request_ecu = wrapper.UpdateRequestEcu( ecu_id="autoware", version="version", url="url", cookies="cookies" )