diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 52d6e9cdf..000000000 --- a/.flake8 +++ /dev/null @@ -1,14 +0,0 @@ -[flake8] -max-line-length = 88 -# ignore the following Errors: -# E266(too many leading '#'): -# sometimes we use multiple # for separting sections -# E203(white space before ':'): -# this error conflicts with black linting -# E501(line is too long): -# TODO: deal with it in the future -# E701(multiple statements on one line) -extend-ignore = E266, E501, E203, E701 -extend-exclude = *_pb2.py*, *_pb2_grpc.py* -# TODO: ignore tools and tests for now -exclude = tools/**, tests/**, proto/** diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4d68e283d..29cf760e3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,6 +8,11 @@ repos: - id: check-toml - id: end-of-file-fixer - id: trailing-whitespace + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.6.2 + hooks: + - id: ruff + args: [--fix] # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - repo: https://github.com/psf/black-pre-commit-mirror rev: 24.4.2 @@ -18,24 +23,6 @@ repos: # pre-commit's default_language_version, see # https://pre-commit.com/#top_level-default_language_version language_version: python3.11 - # - repo: https://github.com/pycqa/isort - # rev: 5.13.2 - # hooks: - # - id: isort - - repo: https://github.com/pycqa/flake8 - rev: 7.1.1 - hooks: - - id: flake8 - additional_dependencies: - - flake8-bugbear==24.2.6 - - flake8-comprehensions - - flake8-simplify - - repo: https://github.com/tox-dev/pyproject-fmt - rev: "2.2.1" - hooks: - - id: pyproject-fmt - # https://pyproject-fmt.readthedocs.io/en/latest/#calculating-max-supported-python-version - additional_dependencies: ["tox>=4.9"] - repo: https://github.com/igorshubovych/markdownlint-cli rev: v0.41.0 hooks: diff --git a/pyproject.toml b/pyproject.toml index 765cb45f2..3c72601d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,12 +43,11 @@ dependencies = [ optional-dependencies.dev = [ "black", "coverage", - "flake8", - "isort", "pytest==7.1.2", "pytest-asyncio==0.23.8", "pytest-mock==3.14", "requests-mock", + "ruff", ] urls.Source = "https://github.com/tier4/ota-client" @@ -85,22 +84,6 @@ features = [ "dev", ] -[tool.black] -line-length = 88 -target-version = [ - 'py38', -] -extend-exclude = '''( - ^.*(_pb2.pyi?|_pb2_grpc.pyi?)$ -)''' - -[tool.isort] -profile = "black" -extend_skip_glob = [ - "*_pb2.py*", - "_pb2_grpc.py*", -] - [tool.pytest.ini_options] asyncio_mode = "auto" log_auto_indent = true @@ -111,6 +94,18 @@ testpaths = [ "./tests", ] +[tool.black] +line-length = 88 +target-version = [ + 'py38', + 'py39', + 'py310', + 'py311', +] +extend-exclude = '''( + ^.*(_pb2.pyi?|_pb2_grpc.pyi?)$ +)''' + [tool.coverage.run] branch = false relative_files = true @@ -142,7 +137,40 @@ exclude = [ "**/__pycache__", ] ignore = [ + "proto/**", "**/*_pb2.py*", "**/*_pb2_grpc.py*", ] pythonVersion = "3.8" + +[tool.ruff] +target-version = "py38" +# NOTE: not include tests and tools for now +include = [ + "tests/**/*.py", + "src/**/*.py", + "pyproject.toml", +] +extend-exclude = [ + "*_pb2.py*", + "*_pb2_grpc.py*", +] + +[tool.ruff.lint] +select = [ + "E4", + "E7", + "E9", + "F", # pyflakes + "Q", # flake8-quotes + "I", # isort + "B", # flake8-bugbear + "A", # flake8-builtins + "ICN", # flake8-import-conventions +] +ignore = [ + "E266", # (too many leading '#'): sometimes we use multiple # for separting sections + "E203", # (white space before ':'): this error conflicts with black linting + "E701", # (multiple statements on one line) + "S101", # (use of assert): mostly we use assert for typing +] diff --git a/src/ota_metadata/legacy/parser.py b/src/ota_metadata/legacy/parser.py index 7f994036c..d4180102b 100644 --- a/src/ota_metadata/legacy/parser.py +++ b/src/ota_metadata/legacy/parser.py @@ -197,7 +197,7 @@ def __set__(self, obj, value: Any) -> None: except KeyError: raise ValueError( f"invalid metafile field {self.field_name}: {value}" - ) + ) from None # normal key-value field else: try: @@ -208,7 +208,7 @@ def __set__(self, obj, value: Any) -> None: except KeyError: raise ValueError( f"invalid metafile field {self.field_name}: {value}" - ) + ) from None raise ValueError(f"attempt to assign invalid {value=} to {self.field_name=}") def __set_name__(self, owner: type, name: str): diff --git a/src/otaclient/app/ota_client_stub.py b/src/otaclient/app/ota_client_stub.py index 0a243f3fb..9371413ca 100644 --- a/src/otaclient/app/ota_client_stub.py +++ b/src/otaclient/app/ota_client_stub.py @@ -28,9 +28,8 @@ from typing_extensions import Self -from ota_proxy import OTAProxyContextProto +from ota_proxy import OTAProxyContextProto, subprocess_otaproxy_launcher from ota_proxy import config as local_otaproxy_cfg -from ota_proxy import subprocess_otaproxy_launcher from otaclient import log_setting from otaclient.boot_control._common import CMDHelperFuncs from otaclient.configs.ecu_info import ECUContact diff --git a/src/otaclient/boot_control/_common.py b/src/otaclient/boot_control/_common.py index 52b9ca2f2..3921a4754 100644 --- a/src/otaclient/boot_control/_common.py +++ b/src/otaclient/boot_control/_common.py @@ -764,7 +764,9 @@ def preserve_ota_folder_to_standby(self): _dst = self.standby_slot_mount_point / Path(cfg.OTA_DIR).relative_to("/") shutil.copytree(_src, _dst, dirs_exist_ok=True) except Exception as e: - raise ValueError(f"failed to copy /boot/ota from active to standby: {e!r}") + raise ValueError( + f"failed to copy /boot/ota from active to standby: {e!r}" + ) from e def prepare_standby_dev( self, diff --git a/src/otaclient/boot_control/_grub.py b/src/otaclient/boot_control/_grub.py index 4774a373c..86c09e8bb 100644 --- a/src/otaclient/boot_control/_grub.py +++ b/src/otaclient/boot_control/_grub.py @@ -267,7 +267,7 @@ def grub_mkconfig() -> str: except CalledProcessError as e: raise ValueError( f"grub-mkconfig failed: {e.returncode=}, {e.stderr=}, {e.stdout=}" - ) + ) from None @staticmethod def grub_reboot(idx: int): diff --git a/src/otaclient/boot_control/_jetson_cboot.py b/src/otaclient/boot_control/_jetson_cboot.py index e551ef121..867b0d389 100644 --- a/src/otaclient/boot_control/_jetson_cboot.py +++ b/src/otaclient/boot_control/_jetson_cboot.py @@ -99,7 +99,9 @@ def is_unified_enabled(cls) -> bool | None: return False elif e.returncode == 69: return - raise ValueError(f"{cmd} returns unexpected result: {e.returncode=}, {e!r}") + raise ValueError( + f"{cmd} returns unexpected result: {e.returncode=}, {e!r}" + ) from None class NVUpdateEngine: @@ -189,7 +191,7 @@ def __init__(self): except Exception as e: _err_msg = f"failed to detect BSP version: {e!r}" logger.error(_err_msg) - raise JetsonCBootContrlError(_err_msg) + raise JetsonCBootContrlError(_err_msg) from None logger.info(f"{bsp_version=}") # ------ sanity check, jetson-cboot is not used after BSP R34 ------ # @@ -218,7 +220,7 @@ def __init__(self): except subprocess.CalledProcessError: _err_msg = "rootfs A/B is not enabled!" logger.error(_err_msg) - raise JetsonCBootContrlError(_err_msg) + raise JetsonCBootContrlError(_err_msg) from None self.unified_ab_enabled = unified_ab_enabled if unified_ab_enabled: diff --git a/src/otaclient/boot_control/_rpi_boot.py b/src/otaclient/boot_control/_rpi_boot.py index 89e0f075e..767d84670 100644 --- a/src/otaclient/boot_control/_rpi_boot.py +++ b/src/otaclient/boot_control/_rpi_boot.py @@ -152,7 +152,7 @@ def __init__(self) -> None: except Exception as e: _err_msg = f"failed to detect partition layout: {e!r}" logger.error(_err_msg) - raise _RPIBootControllerError(_err_msg) + raise _RPIBootControllerError(_err_msg) from None # check system-boot partition mount system_boot_partition = device_tree[1] @@ -185,7 +185,7 @@ def __init__(self) -> None: except ValueError: raise _RPIBootControllerError( f"active slot dev not found: {active_slot_dev=}, {rootfs_partitions=}" - ) + ) from None if idx == 0: # slot_a self.active_slot = SLOT_A @@ -335,7 +335,7 @@ def update_firmware(self, *, target_slot: SlotID, target_slot_mp: StrOrPath): except subprocess.CalledProcessError as e: _err_msg = f"flash-kernel failed: {e!r}\nstderr: {e.stderr.decode()}\nstdout: {e.stdout.decode()}" logger.error(_err_msg) - raise _RPIBootControllerError(_err_msg) + raise _RPIBootControllerError(_err_msg) from None try: # flash-kernel will install the kernel and initrd.img files from /boot to /boot/firmware @@ -359,7 +359,7 @@ def update_firmware(self, *, target_slot: SlotID, target_slot_mp: StrOrPath): except Exception as e: _err_msg = f"failed to apply new kernel,initrd.img for {target_slot}: {e!r}" logger.error(_err_msg) - raise _RPIBootControllerError(_err_msg) + raise _RPIBootControllerError(_err_msg) from None # exposed API methods/properties diff --git a/src/otaclient_api/v2/api_caller.py b/src/otaclient_api/v2/api_caller.py index 72bc6d38f..bcf6031b6 100644 --- a/src/otaclient_api/v2/api_caller.py +++ b/src/otaclient_api/v2/api_caller.py @@ -44,7 +44,7 @@ async def status_call( return types.StatusResponse.convert(resp) except Exception as e: _msg = f"{ecu_id=} failed to respond to status request on-time: {e!r}" - raise ECUNoResponse(_msg) + raise ECUNoResponse(_msg) from e @staticmethod async def update_call( @@ -63,7 +63,7 @@ async def update_call( return types.UpdateResponse.convert(resp) except Exception as e: _msg = f"{ecu_id=} failed to respond to update request on-time: {e!r}" - raise ECUNoResponse(_msg) + raise ECUNoResponse(_msg) from e @staticmethod async def rollback_call( @@ -82,4 +82,4 @@ async def rollback_call( return types.RollbackResponse.convert(resp) except Exception as e: _msg = f"{ecu_id=} failed to respond to rollback request on-time: {e!r}" - raise ECUNoResponse(_msg) + raise ECUNoResponse(_msg) from e diff --git a/src/otaclient_common/__init__.py b/src/otaclient_common/__init__.py index a73c58270..593eee24f 100644 --- a/src/otaclient_common/__init__.py +++ b/src/otaclient_common/__init__.py @@ -72,4 +72,4 @@ def import_from_file(path: Path) -> tuple[str, ModuleType]: _spec.loader.exec_module(_module) # type: ignore return _module_name, _module except Exception: - raise ImportError(f"failed to import module from {path=}.") + raise ImportError(f"failed to import module from {path=}.") from None diff --git a/src/otaclient_common/common.py b/src/otaclient_common/common.py index 49faf21c8..f8136236d 100644 --- a/src/otaclient_common/common.py +++ b/src/otaclient_common/common.py @@ -91,13 +91,13 @@ def read_str_from_file(path: Union[Path, str], *, missing_ok=True, default="") - raise -def write_str_to_file(path: Path, input: str): - path.write_text(input) +def write_str_to_file(path: Path, _input: str): + path.write_text(_input) -def write_str_to_file_sync(path: Union[Path, str], input: str): +def write_str_to_file_sync(path: Union[Path, str], _input: str): with open(path, "w") as f: - f.write(input) + f.write(_input) f.flush() os.fsync(f.fileno()) diff --git a/src/otaclient_common/downloader.py b/src/otaclient_common/downloader.py index 23c1cdbad..d5c93c5cf 100644 --- a/src/otaclient_common/downloader.py +++ b/src/otaclient_common/downloader.py @@ -110,7 +110,7 @@ def iter_chunk(self, src_stream: IO[bytes] | ByteString) -> Iterator[bytes]: except zstandard.ZstdError as e: raise BrokenDecompressionError( f"failure during decompressing file stream: {e!r}" - ) + ) from e # ------ OTA-Cache-File-Control protocol implementation ------ # @@ -557,7 +557,7 @@ def shutdown(self) -> None: """Close all the downloader instances.""" # at final, trigger an update to the total_downloaded_bytes, in case # we still need the total_downloaded_bytes data after pool shutdown. - self.total_downloaded_bytes + _ = self.total_downloaded_bytes with self._instance_map_lock: for _instance in self._instances: diff --git a/src/otaclient_common/linux.py b/src/otaclient_common/linux.py index 0cc34df61..5d1e427f3 100644 --- a/src/otaclient_common/linux.py +++ b/src/otaclient_common/linux.py @@ -30,7 +30,7 @@ def create_swapfile( - swapfile_fpath: str | Path, size_in_MiB: int, *, timeout=900 + swapfile_fpath: str | Path, size_in_mibytes: int, *, timeout=900 ) -> Path: """Create swapfile at with MiB. @@ -62,7 +62,7 @@ def create_swapfile( "if=/dev/zero", f"of={str(swapfile_fpath)}", "bs=1M", - f"count={size_in_MiB}", + f"count={size_in_mibytes}", ], timeout=timeout, ) @@ -106,7 +106,7 @@ def __init__(self, passwd_fpath: str | Path) -> None: self._by_name[_name] = _uid self._by_uid = {v: k for k, v in self._by_name.items()} except Exception as e: - raise ValueError(f"invalid or missing {passwd_fpath=}: {e!r}") + raise ValueError(f"invalid or missing {passwd_fpath=}: {e!r}") from None class ParsedGroup: @@ -131,7 +131,7 @@ def __init__(self, group_fpath: str | Path) -> None: self._by_name[_raw_list[0]] = int(_raw_list[2]) self._by_gid = {v: k for k, v in self._by_name.items()} except Exception as e: - raise ValueError(f"invalid or missing {group_fpath=}: {e!r}") + raise ValueError(f"invalid or missing {group_fpath=}: {e!r}") from None def map_uid_by_pwnam(*, src_db: ParsedPasswd, dst_db: ParsedPasswd, uid: int) -> int: @@ -143,7 +143,7 @@ def map_uid_by_pwnam(*, src_db: ParsedPasswd, dst_db: ParsedPasswd, uid: int) -> try: return dst_db._by_name[src_db._by_uid[uid]] except KeyError: - raise ValueError(f"failed to find mapping for {uid}") + raise ValueError(f"failed to find mapping for {uid}") from None def map_gid_by_grpnam(*, src_db: ParsedGroup, dst_db: ParsedGroup, gid: int) -> int: @@ -155,7 +155,7 @@ def map_gid_by_grpnam(*, src_db: ParsedGroup, dst_db: ParsedGroup, gid: int) -> try: return dst_db._by_name[src_db._by_gid[gid]] except KeyError: - raise ValueError(f"failed to find mapping for {gid}") + raise ValueError(f"failed to find mapping for {gid}") from None # diff --git a/tests/test_otaclient/test_boot_control/test_grub.py b/tests/test_otaclient/test_boot_control/test_grub.py index 41dd4ec55..20ea6f5e4 100644 --- a/tests/test_otaclient/test_boot_control/test_grub.py +++ b/tests/test_otaclient/test_boot_control/test_grub.py @@ -408,7 +408,7 @@ def test_grub_normal_update(self, mocker: pytest_mock.MockerFixture): @pytest.mark.parametrize( - "input, default_entry, expected", + "_input, default_entry, expected", ( ( # test point: @@ -457,9 +457,9 @@ def test_grub_normal_update(self, mocker: pytest_mock.MockerFixture): ), ) def test_update_grub_default( - input: str, default_entry: typing.Optional[int], expected: str + _input: str, default_entry: typing.Optional[int], expected: str ): from otaclient.boot_control._grub import GrubHelper - updated = GrubHelper.update_grub_default(input, default_entry_idx=default_entry) + updated = GrubHelper.update_grub_default(_input, default_entry_idx=default_entry) assert updated == expected diff --git a/tests/test_otaclient/test_boot_control/test_jetson_cboot.py b/tests/test_otaclient/test_boot_control/test_jetson_cboot.py index 203e43f03..740a68b00 100644 --- a/tests/test_otaclient/test_boot_control/test_jetson_cboot.py +++ b/tests/test_otaclient/test_boot_control/test_jetson_cboot.py @@ -25,7 +25,6 @@ import pytest from otaclient.boot_control import _jetson_cboot -from otaclient.boot_control._jetson_cboot import _CBootControl from otaclient.boot_control._jetson_common import ( BSPVersion, FirmwareBSPVersion, diff --git a/tests/test_otaclient/test_boot_control/test_ota_status_control.py b/tests/test_otaclient/test_boot_control/test_ota_status_control.py index 2b906a4dc..069aa1716 100644 --- a/tests/test_otaclient/test_boot_control/test_ota_status_control.py +++ b/tests/test_otaclient/test_boot_control/test_ota_status_control.py @@ -17,7 +17,7 @@ import threading from functools import partial from pathlib import Path -from typing import Optional, Union +from typing import Optional import pytest diff --git a/tests/test_otaclient/test_ota_client_service.py b/tests/test_otaclient/test_ota_client_service.py index ee1ca15cc..76810d737 100644 --- a/tests/test_otaclient/test_ota_client_service.py +++ b/tests/test_otaclient/test_ota_client_service.py @@ -25,7 +25,6 @@ from otaclient.configs.ecu_info import ECUInfo from otaclient_api.v2 import types as api_types from otaclient_api.v2.api_caller import OTAClientCall -from tests.conftest import cfg from tests.utils import compare_message OTACLIENT_APP_MAIN = "otaclient.app.main" diff --git a/tests/test_otaclient_api/test_v2/test_types.py b/tests/test_otaclient_api/test_v2/test_types.py index b3d74f8dc..e51f03649 100644 --- a/tests/test_otaclient_api/test_v2/test_types.py +++ b/tests/test_otaclient_api/test_v2/test_types.py @@ -163,20 +163,20 @@ def test_direct_compare(self): def test_assign_to_protobuf_message(self): """wrapper enum can be directly assigned in protobuf message.""" - l, r = v2.StatusProgress(phase=v2.REGULAR), v2.StatusProgress( + left, r = v2.StatusProgress(phase=v2.REGULAR), v2.StatusProgress( phase=api_types.StatusProgressPhase.REGULAR.value, # type: ignore ) - compare_message(l, r) + compare_message(left, r) def test_used_in_message_wrapper(self): """wrapper enum can be exported.""" - l, r = ( + left, r = ( v2.StatusProgress(phase=v2.REGULAR), api_types.StatusProgress( phase=api_types.StatusProgressPhase.REGULAR ).export_pb(), ) - compare_message(l, r) + compare_message(left, r) def test_converted_from_protobuf_enum(self): """wrapper enum can be converted from and to protobuf enum.""" diff --git a/tests/test_otaclient_common/test_persist_file_handling.py b/tests/test_otaclient_common/test_persist_file_handling.py index 1b9823811..637ee8629 100644 --- a/tests/test_otaclient_common/test_persist_file_handling.py +++ b/tests/test_otaclient_common/test_persist_file_handling.py @@ -491,7 +491,6 @@ def test_copy_tree_B_exists(mocker, tmp_path): os.chown(dst_B, 1, 2, follow_symlinks=False) os.chmod(dst_A, 0o765) os.chmod(dst_B, 0o654) - st = os.stat(dst_A, follow_symlinks=False) ( src_passwd_file, diff --git a/tests/utils.py b/tests/utils.py index 8a4e5bd2a..e53ae511c 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -86,7 +86,7 @@ def _dummy_logger(*args, **kwargs): def compare_dir(left: Path, right: Path): _a_glob = set(map(lambda x: x.relative_to(left), left.glob("**/*"))) _b_glob = set(map(lambda x: x.relative_to(right), right.glob("**/*"))) - if not _a_glob == _b_glob: # first check paths are identical + if _a_glob != _b_glob: # first check paths are identical raise ValueError( f"left and right mismatch, diff: {_a_glob.symmetric_difference(_b_glob)}\n" f"{_a_glob=}\n" @@ -193,19 +193,19 @@ def zstd_compress_file(src: Union[str, Path], dst: Union[str, Path]): cctx.copy_stream(src_f, dst_f) -def compare_message(l, r): +def compare_message(left, r): """ NOTE: we don't directly compare two protobuf message by == due to the behavior difference between empty Duration and unset Duration. """ - if (_proto_class := type(l)) is not type(r): - raise TypeError(f"{type(l)=} != {type(r)=}") + if (_proto_class := type(left)) is not type(r): + raise TypeError(f"{type(left)=} != {type(r)=}") for _attrn in _proto_class.__slots__: - _attrv_l, _attrv_r = getattr(l, _attrn), getattr(r, _attrn) + _attrv_l, _attrv_r = getattr(left, _attrn), getattr(r, _attrn) # first check each corresponding attr has the same type, - assert type(_attrv_l) == type(_attrv_r), f"compare failed on {_attrn=}" + assert type(_attrv_l) is type(_attrv_r), f"compare failed on {_attrn=}" if isinstance(_attrv_l, _Message): compare_message(_attrv_l, _attrv_r) diff --git a/tools/offline_ota_image_builder/manifest.py b/tools/offline_ota_image_builder/manifest.py index 26e33a63d..ca9ef0067 100644 --- a/tools/offline_ota_image_builder/manifest.py +++ b/tools/offline_ota_image_builder/manifest.py @@ -22,7 +22,7 @@ import json from dataclasses import asdict, dataclass, field -from typing import Dict, List +from typing import List from .configs import cfg