From 3db93450852cefd34c47cce189db144d93b3d4b8 Mon Sep 17 00:00:00 2001 From: Bodong YANG Date: Fri, 22 Dec 2023 02:55:26 +0000 Subject: [PATCH 01/17] pick changes from old branch, implement the PersisteFilesHanlder to replace copy_tree.py --- otaclient/_utils/unix.py | 79 +++++++++++++++++++++ otaclient/app/common.py | 138 +++++++++++++++++++++++++++++++++++++ otaclient/app/copy_tree.py | 128 ---------------------------------- 3 files changed, 217 insertions(+), 128 deletions(-) create mode 100644 otaclient/_utils/unix.py delete mode 100644 otaclient/app/copy_tree.py diff --git a/otaclient/_utils/unix.py b/otaclient/_utils/unix.py new file mode 100644 index 000000000..7ac1adc39 --- /dev/null +++ b/otaclient/_utils/unix.py @@ -0,0 +1,79 @@ +from __future__ import annotations +from .typing import StrOrPath + +_SPLITTER = ":" + + +class ParsedPasswd: + """Parse passwd and store name/uid mapping. + + Example passwd entry line: + nobody:x:65534:65534:nobody:/nonexistent:/usr/sbin/nologin + + Attrs: + _by_name (dict[str, int]): name:uid mapping. + _by_uid (dict[int, str]): uid:name mapping. + """ + + __slots__ = ["_by_name", "_by_uid"] + + def __init__(self, passwd_fpath: StrOrPath) -> None: + self._by_name: dict[str, int] = {} + try: + with open(passwd_fpath, "r") as f: + for line in f: + _raw_list = line.strip().split(_SPLITTER) + _name, _uid = _raw_list[0], int(_raw_list[2]) + self._by_name[_name] = _uid + self._by_uid = {v: k for k, v in self._by_name.items()} + except Exception as e: + raise ValueError(f"invalid or missing {passwd_fpath=}: {e!r}") + + +class ParsedGroup: + """Parse group and store name/gid mapping. + + Example group entry line: + nogroup:x:65534: + + Attrs: + _by_name (dict[str, int]): name:gid mapping. + _by_gid (dict[int, str]): gid:name mapping. + """ + + __slots__ = ["_by_name", "_by_gid"] + + def __init__(self, group_fpath: StrOrPath) -> None: + self._by_name: dict[str, int] = {} + try: + with open(group_fpath, "r") as f: + for line in f: + _raw_list = line.strip().split(_SPLITTER) + self._by_name[_raw_list[0]] = int(_raw_list[2]) + self._by_gid = {v: k for k, v in self._by_name.items()} + except Exception as e: + raise ValueError(f"invalid or missing {group_fpath=}: {e!r}") + + +def map_uid_by_pwnam(*, src_db: ParsedPasswd, dst_db: ParsedPasswd, uid: int) -> int: + """Perform src_uid -> src_name -> dst_name -> dst_uid mapping. + + Raises: + ValueError on failed mapping. + """ + try: + return dst_db._by_name[src_db._by_uid[uid]] + except KeyError: + raise ValueError(f"failed to find mapping for {uid}") + + +def map_gid_by_grpnam(*, src_db: ParsedGroup, dst_db: ParsedGroup, gid: int) -> int: + """Perform src_gid -> src_name -> dst_name -> dst_gid mapping. + + Raises: + ValueError on failed mapping. + """ + try: + return dst_db._by_name[src_db._by_gid[gid]] + except KeyError: + raise ValueError(f"failed to find mapping for {gid}") diff --git a/otaclient/app/common.py b/otaclient/app/common.py index 15c6808fb..7d543b61c 100644 --- a/otaclient/app/common.py +++ b/otaclient/app/common.py @@ -14,6 +14,7 @@ r"""Utils that shared between modules are listed here.""" +from __future__ import annotations import itertools import os import shlex @@ -41,6 +42,13 @@ ) from urllib.parse import urljoin +from otaclient._utils.typing import StrOrPath +from otaclient._utils.unix import ( + ParsedPasswd, + ParsedGroup, + map_gid_by_grpnam, + map_uid_by_pwnam, +) from .log_setting import get_logger from .configs import config as cfg @@ -541,3 +549,133 @@ def ensure_otaproxy_start( raise ConnectionError( f"failed to ensure connection to {otaproxy_url} in {probing_timeout=}seconds" ) + + +class PersistFilesHandler: + """Preserving files in persist list from to . + + Files being copied will have mode bits preserved, + and uid/gid preserved with mapping as follow: + + src_uid -> src_name -> dst_name -> dst_uid + src_gid -> src_name -> dst_name -> dst_gid + """ + + def __init__( + self, + src_passwd_file: StrOrPath, + src_group_file: StrOrPath, + dst_passwd_file: StrOrPath, + dst_group_file: StrOrPath, + *, + src_root: StrOrPath, + dst_root: StrOrPath, + ): + self._src_pw = ParsedPasswd(src_passwd_file) + self._src_grp = ParsedGroup(src_group_file) + self._dst_pw = ParsedPasswd(dst_passwd_file) + self._dst_grp = ParsedGroup(dst_group_file) + + self._src_root = Path(src_root) + self._dst_root = Path(dst_root) + + def _prepare_path( + self, + _src_path: Path, + _dst_path: Path, + *, + skip_invalid: bool = True, + ): + """For input original path(from persistents.txt), preserve it from to .""" + + # ------ check src_path, src_path must be existed ------ # + if not _src_path.exists(): + if not skip_invalid: + raise FileNotFoundError(f"{_src_path=} doesn't exist") + logger.warning(f"{_src_path=} doesn't exist, skip...") + return + + # ------ preserve src_path to dst_path ------ # + _src_stat = _src_path.stat() + # NOTE: check is_symlink first, as is_file/dir also returns True if + # src_path is a symlink points to existed file/dir. + # NOTE: cleanup dst ONLY when src is available! + if _src_path.is_symlink(): + _dst_path.unlink(missing_ok=True) + shutil.copy(_src_path, _dst_path, follow_symlinks=False) + elif _src_path.is_file(): + _dst_path.unlink(missing_ok=True) + shutil.copy(_src_path, _dst_path, follow_symlinks=False) + shutil.copymode(_src_path, _dst_path, follow_symlinks=False) + elif _src_path.is_dir(): + shutil.rmtree(_dst_path, ignore_errors=True) + _dst_path.mkdir(mode=_src_stat.st_mode, exist_ok=True) + elif skip_invalid: + logger.warning(f"{_src_path=} is missing or not file/symlink or dir, skip") + return + else: + raise ValueError( + f"type of {_src_path=} must be presented and one of file/dir/symlink" + ) + + # preserve uid/gid with mapping + _src_uid, _src_gid = _src_stat.st_uid, _src_stat.st_gid + try: + _dst_uid = map_uid_by_pwnam( + src_db=self._src_pw, dst_db=self._dst_pw, uid=_src_uid + ) + except ValueError: + logger.warning(f"failed to find mapping for {_src_uid=}, keep unchanged") + _dst_uid = _src_uid + + try: + _dst_gid = map_gid_by_grpnam( + src_db=self._src_grp, dst_db=self._dst_grp, gid=_src_gid + ) + except ValueError: + logger.warning(f"failed to find mapping for {_src_gid=}, keep unchanged") + _dst_gid = _src_gid + + os.chown(_dst_path, uid=_dst_uid, gid=_dst_gid) + + # API + + def preserve_persists_files( + self, _origin_entry: StrOrPath, *, skip_invalid: bool = True + ): + origin_entry = Path(_origin_entry).relative_to(cfg.DEFAULT_ACTIVE_ROOTFS) + src_path = self._src_root / origin_entry + dst_path = self._dst_root / origin_entry + + # ------ prepare parents ------ # + for _idx, _parent in enumerate(reversed(origin_entry.parents)): + if _idx == 0: + continue # skip first parent dir + self._prepare_path(self._src_root / _parent, self._dst_root / _parent) + + # ------ prepare entry itself ------ # + # for normal file/symlink, directly prepare it + if src_path.is_symlink() or src_path.is_file(): + self._prepare_path(src_path, dst_path) + # for dir, dive into is and prepare everything under this dir + elif src_path.is_dir(): + for src_dirpath, _, fnames in os.walk(src_path, followlinks=False): + _src_dpath = Path(src_dirpath) + _origin_dpath = _src_dpath.relative_to(self._src_root) + _dst_dpath = self._dst_root / _origin_dpath + + self._prepare_path(_src_dpath, _dst_dpath) + for _fname in fnames: + self._prepare_path( + _src_dpath / _fname, + _dst_dpath / _fname, + skip_invalid=skip_invalid, + ) + elif skip_invalid: + logger.warning( + f"{src_path=} must be presented and either a file/symlink/dir, skip" + ) + else: + raise ValueError( + f"{src_path=} must be presented and either a file/symlink/dir" + ) diff --git a/otaclient/app/copy_tree.py b/otaclient/app/copy_tree.py deleted file mode 100644 index 8e32beb17..000000000 --- a/otaclient/app/copy_tree.py +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright 2022 TIER IV, INC. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import os -import stat -import shutil -from pathlib import Path - -from . import log_setting - -logger = log_setting.get_logger(__name__) - - -class CopyTree: - def __init__( - self, - src_passwd_file: Path, - src_group_file: Path, - dst_passwd_file: Path, - dst_group_file: Path, - ): - self._src_passwd = self._id_to_name_dict(src_passwd_file) - self._src_group = self._id_to_name_dict(src_group_file) - self._dst_passwd = self._name_to_id_dict(dst_passwd_file) - self._dst_group = self._name_to_id_dict(dst_group_file) - - def copy_with_parents(self, src: Path, dst_dir: Path): - dst_path = dst_dir - if not dst_path.is_dir() or dst_path.is_symlink(): - raise ValueError(f"{dst_path} should be plain directory") - - for parent in reversed(list(src.parents)): - self._copy_preserve(parent, dst_path) - dst_path = dst_path / parent.name - self._copy_recursive(src, dst_path) - - """ private functions from here """ - - def _id_to_name_dict(self, passwd_or_group_file: Path): - lines = open(passwd_or_group_file).readlines() - entries = {} - for line in lines: - e = line.split(":") - entries[e[2]] = e[0] # {uid: name} - return entries - - def _name_to_id_dict(self, passwd_or_group_file: Path): - lines = open(passwd_or_group_file).readlines() - entries = {} - for line in lines: - e = line.split(":") - entries[e[0]] = e[2] # {name: uid} - return entries - - # src_uid -> src_name -> dst_name -> dst_uid - # src_gid -> src_name -> dst_name -> dst_gid - def _convert_src_id_to_dst_id(self, src_uid, src_gid): - logger.info(f"src_uid: {src_uid}, src_gid: {src_gid}") - user = self._src_passwd[str(src_uid)] - uid = self._dst_passwd[user] - group = self._src_group[str(src_gid)] - gid = self._dst_group[group] - logger.info(f"dst_uid: {uid}, dst_gid: {gid}") - return int(uid), int(gid) - - def _copy_stat(self, src, dst): - st = os.stat(src, follow_symlinks=False) - try: - dst_uid, dst_gid = self._convert_src_id_to_dst_id( - st[stat.ST_UID], st[stat.ST_GID] - ) - except KeyError: # In case src UID/GID not found, keep UID/GID as is. - logger.warning(f"uid: {st[stat.ST_UID]}, gid: {st[stat.ST_GID]} not found") - dst_uid, dst_gid = st[stat.ST_UID], st[stat.ST_GID] - os.chown(dst, dst_uid, dst_gid, follow_symlinks=False) - if not dst.is_symlink(): # symlink always 777 - os.chmod(dst, st[stat.ST_MODE]) - - def _copy_preserve(self, src: Path, dst_dir: Path): - if not dst_dir.is_dir() or dst_dir.is_symlink(): - raise ValueError(f"{dst_dir} should be plain directory") - - dst_path = dst_dir / src.name - # src is plain directory? - if src.is_dir() and not src.is_symlink(): - # if plain file or symlink (which links to file or directory) - if dst_path.is_file() or dst_path.is_symlink(): - logger.info(f"{src}: {dst_path} exists but is not a directory") - dst_path.unlink() - if dst_path.is_dir(): # dst_path exists as a directory - return # keep it untouched - logger.info(f"creating directory {dst_path}") - dst_path.mkdir() - self._copy_stat(src, dst_path) - # src is plain file or symlink - elif src.is_file() or src.is_symlink(): # includes broken symlink - if dst_path.is_symlink() or dst_path.is_file(): - # When source is symlink, shutil.copy2 fails to overwrite destination file. - # When destination is symlink, shutil.copy2 copy src file under destination. - # To avoid these cases, remove destination file beforehand. - dst_path.unlink() - if dst_path.is_dir(): - logger.info(f"{src}: {dst_path} exists as a directory") - shutil.rmtree(dst_path, ignore_errors=True) - - logger.info(f"copying file {dst_dir / src.name}") - shutil.copy2(src, dst_path, follow_symlinks=False) - self._copy_stat(src, dst_path) - else: - raise ValueError(f"{src} unintended file type") - - def _copy_recursive(self, src: Path, dst_dir: Path): - self._copy_preserve(src, dst_dir) - if src.is_dir() and not src.is_symlink(): - for src_child in src.glob("*"): - self._copy_recursive(src_child, dst_dir / src.name) From 5f972b93e9347550f9c9bd4c05c227d453d5c657 Mon Sep 17 00:00:00 2001 From: Bodong YANG Date: Fri, 22 Dec 2023 02:58:25 +0000 Subject: [PATCH 02/17] move PersistFilesHandler to create_standby module --- otaclient/app/common.py | 137 ------------------------ otaclient/app/create_standby/common.py | 138 +++++++++++++++++++++++++ 2 files changed, 138 insertions(+), 137 deletions(-) diff --git a/otaclient/app/common.py b/otaclient/app/common.py index 7d543b61c..270f4e46b 100644 --- a/otaclient/app/common.py +++ b/otaclient/app/common.py @@ -42,13 +42,6 @@ ) from urllib.parse import urljoin -from otaclient._utils.typing import StrOrPath -from otaclient._utils.unix import ( - ParsedPasswd, - ParsedGroup, - map_gid_by_grpnam, - map_uid_by_pwnam, -) from .log_setting import get_logger from .configs import config as cfg @@ -549,133 +542,3 @@ def ensure_otaproxy_start( raise ConnectionError( f"failed to ensure connection to {otaproxy_url} in {probing_timeout=}seconds" ) - - -class PersistFilesHandler: - """Preserving files in persist list from to . - - Files being copied will have mode bits preserved, - and uid/gid preserved with mapping as follow: - - src_uid -> src_name -> dst_name -> dst_uid - src_gid -> src_name -> dst_name -> dst_gid - """ - - def __init__( - self, - src_passwd_file: StrOrPath, - src_group_file: StrOrPath, - dst_passwd_file: StrOrPath, - dst_group_file: StrOrPath, - *, - src_root: StrOrPath, - dst_root: StrOrPath, - ): - self._src_pw = ParsedPasswd(src_passwd_file) - self._src_grp = ParsedGroup(src_group_file) - self._dst_pw = ParsedPasswd(dst_passwd_file) - self._dst_grp = ParsedGroup(dst_group_file) - - self._src_root = Path(src_root) - self._dst_root = Path(dst_root) - - def _prepare_path( - self, - _src_path: Path, - _dst_path: Path, - *, - skip_invalid: bool = True, - ): - """For input original path(from persistents.txt), preserve it from to .""" - - # ------ check src_path, src_path must be existed ------ # - if not _src_path.exists(): - if not skip_invalid: - raise FileNotFoundError(f"{_src_path=} doesn't exist") - logger.warning(f"{_src_path=} doesn't exist, skip...") - return - - # ------ preserve src_path to dst_path ------ # - _src_stat = _src_path.stat() - # NOTE: check is_symlink first, as is_file/dir also returns True if - # src_path is a symlink points to existed file/dir. - # NOTE: cleanup dst ONLY when src is available! - if _src_path.is_symlink(): - _dst_path.unlink(missing_ok=True) - shutil.copy(_src_path, _dst_path, follow_symlinks=False) - elif _src_path.is_file(): - _dst_path.unlink(missing_ok=True) - shutil.copy(_src_path, _dst_path, follow_symlinks=False) - shutil.copymode(_src_path, _dst_path, follow_symlinks=False) - elif _src_path.is_dir(): - shutil.rmtree(_dst_path, ignore_errors=True) - _dst_path.mkdir(mode=_src_stat.st_mode, exist_ok=True) - elif skip_invalid: - logger.warning(f"{_src_path=} is missing or not file/symlink or dir, skip") - return - else: - raise ValueError( - f"type of {_src_path=} must be presented and one of file/dir/symlink" - ) - - # preserve uid/gid with mapping - _src_uid, _src_gid = _src_stat.st_uid, _src_stat.st_gid - try: - _dst_uid = map_uid_by_pwnam( - src_db=self._src_pw, dst_db=self._dst_pw, uid=_src_uid - ) - except ValueError: - logger.warning(f"failed to find mapping for {_src_uid=}, keep unchanged") - _dst_uid = _src_uid - - try: - _dst_gid = map_gid_by_grpnam( - src_db=self._src_grp, dst_db=self._dst_grp, gid=_src_gid - ) - except ValueError: - logger.warning(f"failed to find mapping for {_src_gid=}, keep unchanged") - _dst_gid = _src_gid - - os.chown(_dst_path, uid=_dst_uid, gid=_dst_gid) - - # API - - def preserve_persists_files( - self, _origin_entry: StrOrPath, *, skip_invalid: bool = True - ): - origin_entry = Path(_origin_entry).relative_to(cfg.DEFAULT_ACTIVE_ROOTFS) - src_path = self._src_root / origin_entry - dst_path = self._dst_root / origin_entry - - # ------ prepare parents ------ # - for _idx, _parent in enumerate(reversed(origin_entry.parents)): - if _idx == 0: - continue # skip first parent dir - self._prepare_path(self._src_root / _parent, self._dst_root / _parent) - - # ------ prepare entry itself ------ # - # for normal file/symlink, directly prepare it - if src_path.is_symlink() or src_path.is_file(): - self._prepare_path(src_path, dst_path) - # for dir, dive into is and prepare everything under this dir - elif src_path.is_dir(): - for src_dirpath, _, fnames in os.walk(src_path, followlinks=False): - _src_dpath = Path(src_dirpath) - _origin_dpath = _src_dpath.relative_to(self._src_root) - _dst_dpath = self._dst_root / _origin_dpath - - self._prepare_path(_src_dpath, _dst_dpath) - for _fname in fnames: - self._prepare_path( - _src_dpath / _fname, - _dst_dpath / _fname, - skip_invalid=skip_invalid, - ) - elif skip_invalid: - logger.warning( - f"{src_path=} must be presented and either a file/symlink/dir, skip" - ) - else: - raise ValueError( - f"{src_path=} must be presented and either a file/symlink/dir" - ) diff --git a/otaclient/app/create_standby/common.py b/otaclient/app/create_standby/common.py index b7adc89c5..2d50826a4 100644 --- a/otaclient/app/create_standby/common.py +++ b/otaclient/app/create_standby/common.py @@ -18,6 +18,7 @@ import os import random import time +import shutil from concurrent.futures import ( Future, ThreadPoolExecutor, @@ -40,6 +41,13 @@ ) from weakref import WeakKeyDictionary, WeakValueDictionary +from otaclient._utils.typing import StrOrPath +from otaclient._utils.unix import ( + ParsedPasswd, + ParsedGroup, + map_gid_by_grpnam, + map_uid_by_pwnam, +) from ..common import create_tmp_fname from ..configs import config as cfg from ..ota_metadata import OTAMetadata, MetafilesV1 @@ -480,3 +488,133 @@ def calculate_and_process_delta(self) -> DeltaBundle: total_regular_num=self.total_regulars_num, total_download_files_size=self.total_download_files_size, ) + + +class PersistFilesHandler: + """Preserving files in persist list from to . + + Files being copied will have mode bits preserved, + and uid/gid preserved with mapping as follow: + + src_uid -> src_name -> dst_name -> dst_uid + src_gid -> src_name -> dst_name -> dst_gid + """ + + def __init__( + self, + src_passwd_file: StrOrPath, + src_group_file: StrOrPath, + dst_passwd_file: StrOrPath, + dst_group_file: StrOrPath, + *, + src_root: StrOrPath, + dst_root: StrOrPath, + ): + self._src_pw = ParsedPasswd(src_passwd_file) + self._src_grp = ParsedGroup(src_group_file) + self._dst_pw = ParsedPasswd(dst_passwd_file) + self._dst_grp = ParsedGroup(dst_group_file) + + self._src_root = Path(src_root) + self._dst_root = Path(dst_root) + + def _prepare_path( + self, + _src_path: Path, + _dst_path: Path, + *, + skip_invalid: bool = True, + ): + """For input original path(from persistents.txt), preserve it from to .""" + + # ------ check src_path, src_path must be existed ------ # + if not _src_path.exists(): + if not skip_invalid: + raise FileNotFoundError(f"{_src_path=} doesn't exist") + logger.warning(f"{_src_path=} doesn't exist, skip...") + return + + # ------ preserve src_path to dst_path ------ # + _src_stat = _src_path.stat() + # NOTE: check is_symlink first, as is_file/dir also returns True if + # src_path is a symlink points to existed file/dir. + # NOTE: cleanup dst ONLY when src is available! + if _src_path.is_symlink(): + _dst_path.unlink(missing_ok=True) + shutil.copy(_src_path, _dst_path, follow_symlinks=False) + elif _src_path.is_file(): + _dst_path.unlink(missing_ok=True) + shutil.copy(_src_path, _dst_path, follow_symlinks=False) + shutil.copymode(_src_path, _dst_path, follow_symlinks=False) + elif _src_path.is_dir(): + shutil.rmtree(_dst_path, ignore_errors=True) + _dst_path.mkdir(mode=_src_stat.st_mode, exist_ok=True) + elif skip_invalid: + logger.warning(f"{_src_path=} is missing or not file/symlink or dir, skip") + return + else: + raise ValueError( + f"type of {_src_path=} must be presented and one of file/dir/symlink" + ) + + # preserve uid/gid with mapping + _src_uid, _src_gid = _src_stat.st_uid, _src_stat.st_gid + try: + _dst_uid = map_uid_by_pwnam( + src_db=self._src_pw, dst_db=self._dst_pw, uid=_src_uid + ) + except ValueError: + logger.warning(f"failed to find mapping for {_src_uid=}, keep unchanged") + _dst_uid = _src_uid + + try: + _dst_gid = map_gid_by_grpnam( + src_db=self._src_grp, dst_db=self._dst_grp, gid=_src_gid + ) + except ValueError: + logger.warning(f"failed to find mapping for {_src_gid=}, keep unchanged") + _dst_gid = _src_gid + + os.chown(_dst_path, uid=_dst_uid, gid=_dst_gid) + + # API + + def preserve_persists_files( + self, _origin_entry: StrOrPath, *, skip_invalid: bool = True + ): + origin_entry = Path(_origin_entry).relative_to(cfg.DEFAULT_ACTIVE_ROOTFS) + src_path = self._src_root / origin_entry + dst_path = self._dst_root / origin_entry + + # ------ prepare parents ------ # + for _idx, _parent in enumerate(reversed(origin_entry.parents)): + if _idx == 0: + continue # skip first parent dir + self._prepare_path(self._src_root / _parent, self._dst_root / _parent) + + # ------ prepare entry itself ------ # + # for normal file/symlink, directly prepare it + if src_path.is_symlink() or src_path.is_file(): + self._prepare_path(src_path, dst_path) + # for dir, dive into is and prepare everything under this dir + elif src_path.is_dir(): + for src_dirpath, _, fnames in os.walk(src_path, followlinks=False): + _src_dpath = Path(src_dirpath) + _origin_dpath = _src_dpath.relative_to(self._src_root) + _dst_dpath = self._dst_root / _origin_dpath + + self._prepare_path(_src_dpath, _dst_dpath) + for _fname in fnames: + self._prepare_path( + _src_dpath / _fname, + _dst_dpath / _fname, + skip_invalid=skip_invalid, + ) + elif skip_invalid: + logger.warning( + f"{src_path=} must be presented and either a file/symlink/dir, skip" + ) + else: + raise ValueError( + f"{src_path=} must be presented and either a file/symlink/dir" + ) From 8c99c7e30464968161297d9a9021e431c81289ad Mon Sep 17 00:00:00 2001 From: Bodong YANG Date: Fri, 22 Dec 2023 03:00:57 +0000 Subject: [PATCH 03/17] create_standby: drop-in replace the copy_tree --- otaclient/app/create_standby/common.py | 2 +- otaclient/app/create_standby/rebuild_mode.py | 17 +++++------------ 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/otaclient/app/create_standby/common.py b/otaclient/app/create_standby/common.py index 2d50826a4..1e94062c9 100644 --- a/otaclient/app/create_standby/common.py +++ b/otaclient/app/create_standby/common.py @@ -579,7 +579,7 @@ def _prepare_path( # API - def preserve_persists_files( + def preserve_persist_entry( self, _origin_entry: StrOrPath, *, skip_invalid: bool = True ): origin_entry = Path(_origin_entry).relative_to(cfg.DEFAULT_ACTIVE_ROOTFS) diff --git a/otaclient/app/create_standby/rebuild_mode.py b/otaclient/app/create_standby/rebuild_mode.py index 1decb47c1..0985e546f 100644 --- a/otaclient/app/create_standby/rebuild_mode.py +++ b/otaclient/app/create_standby/rebuild_mode.py @@ -32,7 +32,7 @@ from ..proto.wrapper import RegularInf from .. import log_setting -from .common import HardlinkRegister, DeltaGenerator, DeltaBundle +from .common import HardlinkRegister, DeltaGenerator, DeltaBundle, PersistFilesHandler from .interface import StandbySlotCreatorProtocol logger = log_setting.get_logger(__name__) @@ -78,10 +78,7 @@ def _process_dirs(self): entry.mkdir_relative_to_mount_point(self.standby_slot_mp) def _process_persistents(self): - """NOTE: just copy from legacy mode""" - from ..copy_tree import CopyTree - - _copy_tree = CopyTree( + _handler = PersistFilesHandler( src_passwd_file=Path(cfg.PASSWD_FPATH), src_group_file=Path(cfg.GROUP_FPATH), dst_passwd_file=Path( @@ -90,16 +87,12 @@ def _process_persistents(self): dst_group_file=Path( replace_root(cfg.GROUP_FPATH, cfg.ACTIVE_ROOTFS, cfg.STANDBY_SLOT_MP) ), + src_root=self.active_slot_mp, + dst_root=self.standby_slot_mp, ) for _perinf in self._ota_metadata.iter_metafile(MetafilesV1.PERSISTENT_FNAME): - _perinf_path = Path(_perinf.path) - if ( - _perinf_path.is_file() - or _perinf_path.is_dir() - or _perinf_path.is_symlink() - ): # NOTE: not equivalent to perinf.path.exists() - _copy_tree.copy_with_parents(_perinf_path, self.standby_slot_mp) + _handler.preserve_persist_entry(_perinf.path) def _process_symlinks(self): for _symlink in self._ota_metadata.iter_metafile( From 722fe3441bfa9cc54f07fb064c35f78f023580e3 Mon Sep 17 00:00:00 2001 From: Bodong YANG Date: Fri, 22 Dec 2023 04:14:15 +0000 Subject: [PATCH 04/17] create_standby.PersistFilesHanlder: fixing symlink related --- otaclient/app/create_standby/common.py | 27 +++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/otaclient/app/create_standby/common.py b/otaclient/app/create_standby/common.py index 1e94062c9..df56fb2ad 100644 --- a/otaclient/app/create_standby/common.py +++ b/otaclient/app/create_standby/common.py @@ -528,27 +528,37 @@ def _prepare_path( """For input original path(from persistents.txt), preserve it from to .""" # ------ check src_path, src_path must be existed ------ # - if not _src_path.exists(): + # NOTE: Path.exists does follow link! if this symlink is dead, + # Path.exists also returns False! + if not (_src_path.is_symlink() or _src_path.is_file() or _src_path.is_dir()): if not skip_invalid: - raise FileNotFoundError(f"{_src_path=} doesn't exist") - logger.warning(f"{_src_path=} doesn't exist, skip...") + raise FileNotFoundError( + f"{_src_path=} doesn't exist or not a file/symlink/dir" + ) + logger.warning( + f"{_src_path=} doesn't exist or not a file/symlink/dir, skip..." + ) return # ------ preserve src_path to dst_path ------ # - _src_stat = _src_path.stat() + # NOTE: Path.stat always follow symlink, use os.stat instead + _src_stat = os.stat(_src_path, follow_symlinks=False) + # NOTE: check is_symlink first, as is_file/dir also returns True if # src_path is a symlink points to existed file/dir. # NOTE: cleanup dst ONLY when src is available! if _src_path.is_symlink(): _dst_path.unlink(missing_ok=True) shutil.copy(_src_path, _dst_path, follow_symlinks=False) + # no need to change symlink's mode, it is always 777 elif _src_path.is_file(): _dst_path.unlink(missing_ok=True) shutil.copy(_src_path, _dst_path, follow_symlinks=False) - shutil.copymode(_src_path, _dst_path, follow_symlinks=False) + os.chmod(_dst_path, _src_stat.st_mode) elif _src_path.is_dir(): shutil.rmtree(_dst_path, ignore_errors=True) - _dst_path.mkdir(mode=_src_stat.st_mode, exist_ok=True) + _dst_path.mkdir(exist_ok=True) + os.chmod(_dst_path, _src_stat.st_mode) elif skip_invalid: logger.warning(f"{_src_path=} is missing or not file/symlink or dir, skip") return @@ -557,7 +567,7 @@ def _prepare_path( f"type of {_src_path=} must be presented and one of file/dir/symlink" ) - # preserve uid/gid with mapping + # change owner with mapping _src_uid, _src_gid = _src_stat.st_uid, _src_stat.st_gid try: _dst_uid = map_uid_by_pwnam( @@ -574,8 +584,7 @@ def _prepare_path( except ValueError: logger.warning(f"failed to find mapping for {_src_gid=}, keep unchanged") _dst_gid = _src_gid - - os.chown(_dst_path, uid=_dst_uid, gid=_dst_gid) + os.chown(_dst_path, uid=_dst_uid, gid=_dst_gid, follow_symlinks=False) # API From 8a087b5566b8679510ddb30bee2664282e706a38 Mon Sep 17 00:00:00 2001 From: Bodong YANG Date: Fri, 22 Dec 2023 04:35:53 +0000 Subject: [PATCH 05/17] create new tests_create_standby package to hold related test files, adjust test_copy.py(->test_persist_file_handling.py) --- tests/test_create_standby/__init__.py | 0 .../test_create_standby.py | 163 ++++ .../test_persist_files_handling.py | 803 ++++++++++++++++++ 3 files changed, 966 insertions(+) create mode 100644 tests/test_create_standby/__init__.py create mode 100644 tests/test_create_standby/test_create_standby.py create mode 100644 tests/test_create_standby/test_persist_files_handling.py diff --git a/tests/test_create_standby/__init__.py b/tests/test_create_standby/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_create_standby/test_create_standby.py b/tests/test_create_standby/test_create_standby.py new file mode 100644 index 000000000..67a267b33 --- /dev/null +++ b/tests/test_create_standby/test_create_standby.py @@ -0,0 +1,163 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import shutil +import time +import typing +import pytest +from pathlib import Path +from pytest_mock import MockerFixture + +from otaclient.app.boot_control import BootControllerProtocol +from otaclient.configs.app_cfg import Config as otaclient_Config + +from tests.conftest import TestConfiguration as test_cfg +from tests.utils import SlotMeta, compare_dir + +import logging + +logger = logging.getLogger(__name__) + + +class Test_OTAupdate_with_create_standby_RebuildMode: + """ + NOTE: the boot_control is mocked, only testing + create_standby and the logics directly implemented by OTAUpdater. + + NOTE: testing the system using separated boot dev for each slots(like cboot). + """ + + @pytest.fixture + def setup_test(self, tmp_path: Path, ab_slots: SlotMeta): + # + # ------ prepare ab slots ------ # + # + self.slot_a = Path(ab_slots.slot_a) + self.slot_b = Path(ab_slots.slot_b) + self.slot_a_boot_dir = Path(ab_slots.slot_a_boot_dev) / "boot" + self.slot_b_boot_dir = Path(ab_slots.slot_b_boot_dev) / "boot" + self.ota_image_dir = Path(test_cfg.OTA_IMAGE_DIR) + + self.otaclient_run_dir = tmp_path / "otaclient_run_dir" + self.otaclient_run_dir.mkdir(parents=True, exist_ok=True) + + self.slot_a_boot_dir.mkdir(exist_ok=True, parents=True) + self.slot_b_boot_dir.mkdir(exist_ok=True, parents=True) + + # + # ------ prepare config ------ # + # + _otaclient_cfg = otaclient_Config(ACTIVE_ROOTFS=str(self.slot_a)) + self.otaclient_cfg = _otaclient_cfg + + # ------ prepare otaclient run dir ------ # + Path(_otaclient_cfg.RUN_DPATH).mkdir(exist_ok=True, parents=True) + + # + # ------ prepare mount space ------ # + # + Path(_otaclient_cfg.OTACLIENT_MOUNT_SPACE_DPATH).mkdir( + exist_ok=True, parents=True + ) + # directly point standby slot mp to self.slot_b + _standby_slot_mp = Path(_otaclient_cfg.STANDBY_SLOT_MP) + _standby_slot_mp.symlink_to(self.slot_b) + + # some important paths + self.ota_metafiles_tmp_dir = Path(_otaclient_cfg.STANDBY_IMAGE_META_DPATH) + self.ota_tmp_dir = Path(_otaclient_cfg.STANDBY_OTA_TMP_DPATH) + + yield + # cleanup slot_b after test + shutil.rmtree(self.slot_b, ignore_errors=True) + + @pytest.fixture(autouse=True) + def mock_setup(self, mocker: MockerFixture, setup_test): + # ------ mock boot_controller ------ # + self._boot_control = typing.cast( + BootControllerProtocol, mocker.MagicMock(spec=BootControllerProtocol) + ) + self._boot_control.get_standby_boot_dir.return_value = self.slot_b_boot_dir + + # ------ mock otaclient cfg ------ # + mocker.patch(f"{test_cfg.OTACLIENT_MODULE_PATH}.cfg", self.otaclient_cfg) + mocker.patch( + f"{test_cfg.CREATE_STANDBY_MODULE_PATH}.rebuild_mode.cfg", + self.otaclient_cfg, + ) + mocker.patch(f"{test_cfg.OTAMETA_MODULE_PATH}.cfg", self.otaclient_cfg) + + def test_update_with_create_standby_RebuildMode(self, mocker: MockerFixture): + from otaclient.app.ota_client import _OTAUpdater, OTAClientControlFlags + from otaclient.app.create_standby.rebuild_mode import RebuildMode + + # TODO: not test process_persistent currently, + # as we currently directly compare the standby slot + # with the OTA image. + RebuildMode._process_persistents = mocker.MagicMock() + + # ------ execution ------ # + otaclient_control_flags = typing.cast( + OTAClientControlFlags, mocker.MagicMock(spec=OTAClientControlFlags) + ) + _updater = _OTAUpdater( + boot_controller=self._boot_control, + create_standby_cls=RebuildMode, + proxy=None, + control_flags=otaclient_control_flags, + ) + # NOTE: mock the shutdown method as we need to assert before the + # updater is closed. + _updater_shutdown = _updater.shutdown + _updater.shutdown = mocker.MagicMock() + + _updater.execute( + version=test_cfg.UPDATE_VERSION, + raw_url_base=test_cfg.OTA_IMAGE_URL, + cookies_json=r'{"test": "my-cookie"}', + ) + time.sleep(2) # wait for downloader to record stats + + # ------ assertions ------ # + # --- assert update finished + _updater.shutdown.assert_called_once() + otaclient_control_flags.wait_can_reboot_flag.assert_called_once() + # --- ensure the update stats are collected + _snapshot = _updater._update_stats_collector.get_snapshot() + assert _snapshot.processed_files_num + assert _snapshot.processed_files_size + assert _snapshot.downloaded_files_num + assert _snapshot.downloaded_files_size + # assert _snapshot.downloaded_bytes + # assert _snapshot.downloading_elapsed_time.export_pb().ToNanoseconds() + assert _snapshot.update_applying_elapsed_time.export_pb().ToNanoseconds() + + # --- check slot creating result, ensure slot_a and slot_b is the same --- # + # NOTE: merge contents from slot_b_boot_dir to slot_b + shutil.copytree(self.slot_b_boot_dir, self.slot_b / "boot", dirs_exist_ok=True) + # NOTE: for some reason tmp dir is created under OTA_IMAGE_DIR/data, but not listed + # in the regulars.txt, so we create one here to make the test passed + (self.slot_b / "tmp").mkdir(exist_ok=True) + + # NOTE: remove the ota-meta dir and ota-tmp dir to resolve the difference with OTA image + shutil.rmtree(self.ota_metafiles_tmp_dir, ignore_errors=True) + shutil.rmtree(self.ota_tmp_dir, ignore_errors=True) + shutil.rmtree(self.slot_b / "opt/ota", ignore_errors=True) + + # --- check standby slot, ensure it is correctly populated + compare_dir(Path(test_cfg.OTA_IMAGE_DIR) / "data", self.slot_b) + + # ------ finally close the updater ------ # + _updater_shutdown() diff --git a/tests/test_create_standby/test_persist_files_handling.py b/tests/test_create_standby/test_persist_files_handling.py new file mode 100644 index 000000000..88789b370 --- /dev/null +++ b/tests/test_create_standby/test_persist_files_handling.py @@ -0,0 +1,803 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import annotations +import os +import stat +from pathlib import Path + +from otaclient._utils.path import replace_root +from otaclient.app.create_standby.common import PersistFilesHandler + + +def create_files(tmp_path: Path): + """ + 20231222: this function create and folders as preserved src rootfs and save destination rootfs. + """ + + dst = tmp_path / "dst" + dst.mkdir() + + src = tmp_path / "src" + src.mkdir() + + """ + src/ + src/a + src/to_a -> a + src/to_broken_a -> broken_a + src/A + src/A/b + src/A/to_b -> b + src/A/to_broken_b -> broken_b + src/A/B/ + src/A/B/c + src/A/B/to_c -> c + src/A/B/to_broken_c -> broken_c + src/A/B/C/ + """ + + a = src / "a" + a.write_text("a") + to_a = src / "to_a" + to_a.symlink_to("a") + to_broken_a = src / "to_broken_a" + to_broken_a.symlink_to("broken_a") + A = src / "A" + A.mkdir() + + b = A / "b" + b.write_text("b") + to_b = A / "to_b" + to_b.symlink_to("b") + to_broken_b = A / "to_broken_b" + to_broken_b.symlink_to("broken_b") + B = A / "B" + B.mkdir() + + c = B / "c" + c.write_text("c") + to_c = B / "to_c" + to_c.symlink_to("c") + to_broken_c = B / "to_broken_c" + to_broken_c.symlink_to("broken_c") + C = B / "C" + C.mkdir() + + os.chown(src, 0, 1, follow_symlinks=False) + os.chown(a, 2, 3, follow_symlinks=False) + os.chown(to_a, 4, 5, follow_symlinks=False) + os.chown(to_broken_a, 6, 7, follow_symlinks=False) + os.chown(A, 8, 9, follow_symlinks=False) + os.chown(b, 10, 13, follow_symlinks=False) + os.chown(to_b, 33, 34, follow_symlinks=False) + os.chown(to_broken_b, 38, 39, follow_symlinks=False) + os.chown(B, 41, 65534, follow_symlinks=False) + os.chown(c, 100, 102, follow_symlinks=False) + os.chown(to_c, 12345678, 87654321, follow_symlinks=False) # id can't be converted + os.chown(to_broken_c, 104, 104, follow_symlinks=False) + os.chown(C, 105, 105, follow_symlinks=False) + + os.chmod(src, 0o111) + os.chmod(a, 0o112) + # os.chmod(to_a, 0o113) + # os.chmod(to_broken_a, 0o114) + os.chmod(A, 0o115) + os.chmod(b, 0o116) + # os.chmod(to_b, 0o117) + # os.chmod(to_broken_b, 0o121) + os.chmod(B, 0o122) + os.chmod(c, 0o123) + # os.chmod(to_c, 0o124) + # os.chmod(to_broken_c, 0o125) + os.chmod(C, 0o126) + + return ( + dst, + src, + a, + to_a, + to_broken_a, + A, + b, + to_b, + to_broken_b, + B, + c, + to_c, + to_broken_c, + C, + ) + + +def uid_gid_mode(path): + st = os.stat(path, follow_symlinks=False) + return st[stat.ST_UID], st[stat.ST_GID], stat.S_IMODE(st[stat.ST_MODE]) + + +def assert_uid_gid_mode(path, uid, gid, mode): + _uid, _gid, _mode = uid_gid_mode(path) + assert _uid == uid + assert _gid == gid + assert _mode == mode + + +def create_passwd_group_files(tmp_path): + src_passwd = """\ +root:x:0:0:root:/root:/bin/bash +daemon:x:1:1:daemon:/usr/sbin:/usr/sbin/nologin +bin:x:2:2:bin:/bin:/usr/sbin/nologin +sys:x:3:3:sys:/dev:/usr/sbin/nologin +sync:x:4:65534:sync:/bin:/bin/sync +games:x:5:60:games:/usr/games:/usr/sbin/nologin +man:x:6:12:man:/var/cache/man:/usr/sbin/nologin +lp:x:7:7:lp:/var/spool/lpd:/usr/sbin/nologin +mail:x:8:8:mail:/var/mail:/usr/sbin/nologin +news:x:9:9:news:/var/spool/news:/usr/sbin/nologin +uucp:x:10:10:uucp:/var/spool/uucp:/usr/sbin/nologin +proxy:x:13:13:proxy:/bin:/usr/sbin/nologin +www-data:x:33:33:www-data:/var/www:/usr/sbin/nologin +backup:x:34:34:backup:/var/backups:/usr/sbin/nologin +list:x:38:38:Mailing List Manager:/var/list:/usr/sbin/nologin +irc:x:39:39:ircd:/var/run/ircd:/usr/sbin/nologin +gnats:x:41:41:Gnats Bug-Reporting System (admin):/var/lib/gnats:/usr/sbin/nologin +nobody:x:65534:65534:nobody:/nonexistent:/usr/sbin/nologin +systemd-network:x:100:102:systemd Network Management,,,:/run/systemd:/usr/sbin/nologin +systemd-resolve:x:101:103:systemd Resolver,,,:/run/systemd:/usr/sbin/nologin +systemd-timesync:x:102:104:systemd Time Synchronization,,,:/run/systemd:/usr/sbin/nologin +messagebus:x:103:106::/nonexistent:/usr/sbin/nologin +syslog:x:104:110::/home/syslog:/usr/sbin/nologin +_apt:x:105:65534::/nonexistent:/usr/sbin/nologin +""" + + # dst_passwd uid is converted with f"1{src_passwd gid}" + dst_passwd = """\ +root:x:10:0:root:/root:/bin/bash +daemon:x:11:1:daemon:/usr/sbin:/usr/sbin/nologin +bin:x:12:2:bin:/bin:/usr/sbin/nologin +sys:x:13:3:sys:/dev:/usr/sbin/nologin +sync:x:14:65534:sync:/bin:/bin/sync +games:x:15:60:games:/usr/games:/usr/sbin/nologin +man:x:16:12:man:/var/cache/man:/usr/sbin/nologin +lp:x:17:7:lp:/var/spool/lpd:/usr/sbin/nologin +mail:x:18:8:mail:/var/mail:/usr/sbin/nologin +news:x:19:9:news:/var/spool/news:/usr/sbin/nologin +uucp:x:110:10:uucp:/var/spool/uucp:/usr/sbin/nologin +proxy:x:113:13:proxy:/bin:/usr/sbin/nologin +www-data:x:133:33:www-data:/var/www:/usr/sbin/nologin +backup:x:134:34:backup:/var/backups:/usr/sbin/nologin +list:x:138:38:Mailing List Manager:/var/list:/usr/sbin/nologin +irc:x:139:39:ircd:/var/run/ircd:/usr/sbin/nologin +gnats:x:141:41:Gnats Bug-Reporting System (admin):/var/lib/gnats:/usr/sbin/nologin +nobody:x:165534:65534:nobody:/nonexistent:/usr/sbin/nologin +systemd-network:x:1100:102:systemd Network Management,,,:/run/systemd:/usr/sbin/nologin +systemd-resolve:x:1101:103:systemd Resolver,,,:/run/systemd:/usr/sbin/nologin +systemd-timesync:x:1102:104:systemd Time Synchronization,,,:/run/systemd:/usr/sbin/nologin +messagebus:x:1103:106::/nonexistent:/usr/sbin/nologin +syslog:x:1104:110::/home/syslog:/usr/sbin/nologin +_apt:x:1105:65534::/nonexistent:/usr/sbin/nologin +""" + + src_group = """\ +root:x:0: +daemon:x:1: +bin:x:2: +sys:x:3: +adm:x:4:syslog +tty:x:5:syslog +disk:x:6: +lp:x:7: +mail:x:8: +news:x:9: +uucp:x:10: +man:x:12: +proxy:x:13: +kmem:x:15: +dialout:x:20: +fax:x:21: +voice:x:22: +cdrom:x:24: +floppy:x:25: +tape:x:26: +sudo:x:27: +audio:x:29:pulse +dip:x:30: +www-data:x:33: +backup:x:34: +operator:x:37: +list:x:38: +irc:x:39: +src:x:40: +gnats:x:41: +shadow:x:42: +utmp:x:43: +video:x:44: +sasl:x:45: +plugdev:x:46: +staff:x:50: +games:x:60: +users:x:100: +nogroup:x:65534: +systemd-journal:x:101: +systemd-network:x:102: +systemd-resolve:x:103: +systemd-timesync:x:104: +crontab:x:105: +""" + + # dst_group gid is converted with f"2{src_group gid}" + dst_group = """\ +root:x:20: +daemon:x:21: +bin:x:22: +sys:x:23: +adm:x:24:syslog +tty:x:25:syslog +disk:x:26: +lp:x:27: +mail:x:28: +news:x:29: +uucp:x:210: +man:x:212: +proxy:x:213: +kmem:x:215: +dialout:x:220: +fax:x:221: +voice:x:222: +cdrom:x:224: +floppy:x:225: +tape:x:226: +sudo:x:227: +audio:x:229:pulse +dip:x:230: +www-data:x:233: +backup:x:234: +operator:x:237: +list:x:238: +irc:x:239: +src:x:240: +gnats:x:241: +shadow:x:242: +utmp:x:243: +video:x:244: +sasl:x:245: +plugdev:x:246: +staff:x:250: +games:x:260: +users:x:2100: +nogroup:x:265534: +systemd-journal:x:2101: +systemd-network:x:2102: +systemd-resolve:x:2103: +systemd-timesync:x:2104: +crontab:x:2105: +""" + src_passwd_file = tmp_path / "etc" / "src_passwd" + dst_passwd_file = tmp_path / "etc" / "dst_passwd" + src_group_file = tmp_path / "etc" / "src_group" + dst_group_file = tmp_path / "etc" / "dst_group" + (tmp_path / "etc").mkdir() + + src_passwd_file.write_text(src_passwd) + dst_passwd_file.write_text(dst_passwd) + src_group_file.write_text(src_group) + dst_group_file.write_text(dst_group) + return src_passwd_file, dst_passwd_file, src_group_file, dst_group_file + + +def test_copy_tree_src_dir(mocker, tmp_path): + ( + dst, + src, + a, + to_a, + to_broken_a, + A, + b, + to_b, + to_broken_b, + B, + c, + to_c, + to_broken_c, + C, + ) = create_files(tmp_path) + + ( + src_passwd_file, + dst_passwd_file, + src_group_file, + dst_group_file, + ) = create_passwd_group_files(tmp_path) + + # NOTE: persist entry must be canonical and starts with / + persist_entry = replace_root(B, src, "/") + PersistFilesHandler( + src_passwd_file=src_passwd_file, + src_group_file=src_group_file, + dst_passwd_file=dst_passwd_file, + dst_group_file=dst_group_file, + src_root=src, + dst_root=dst, + ).preserve_persist_entry(persist_entry, skip_invalid=False) + + # src/A + assert (dst / A.relative_to(src)).is_dir() + assert not (dst / A.relative_to(src)).is_symlink() + assert_uid_gid_mode(dst / A.relative_to(src), 18, 29, 0o115) + + # src/A/B/ + assert (dst / B.relative_to(src)).is_dir() + assert not (dst / B.relative_to(src)).is_symlink() + assert_uid_gid_mode(dst / B.relative_to(src), 141, 265534, 0o122) + + # src/A/B/c + assert (dst / c.relative_to(src)).is_file() + assert not (dst / c.relative_to(src)).is_symlink() + assert_uid_gid_mode(dst / c.relative_to(src), 1100, 2102, 0o123) + + # src/A/B/to_c + assert (dst / to_c.relative_to(src)).is_file() + assert (dst / to_c.relative_to(src)).is_symlink() + # uid, gid can't be converted so original uid, gid is used. + assert_uid_gid_mode(dst / to_c.relative_to(src), 12345678, 87654321, 0o777) + + # src/A/B/to_broken_c + assert not (dst / to_broken_c.relative_to(src)).is_file() + assert (dst / to_broken_c.relative_to(src)).is_symlink() + assert_uid_gid_mode(dst / to_broken_c.relative_to(src), 1104, 2104, 0o777) + + # src/A/B/C/ + assert (dst / C.relative_to(src)).is_dir() + assert not (dst / C.relative_to(src)).is_symlink() + assert_uid_gid_mode(dst / C.relative_to(src), 1105, 2105, 0o126) + + # followings should not exist + # src/a + assert not (dst / a.relative_to(src)).exists() + assert not (dst / a.relative_to(src)).is_symlink() + # src/to_a + assert not (dst / to_a.relative_to(src)).exists() + assert not (dst / to_a.relative_to(src)).is_symlink() + # src/to_broken_a + assert not (dst / to_broken_a.relative_to(src)).exists() + assert not (dst / to_broken_a.relative_to(src)).is_symlink() + # src/A/b + assert not (dst / b.relative_to(src)).exists() + assert not (dst / b.relative_to(src)).is_symlink() + # src/A/to_b + assert not (dst / to_b.relative_to(src)).exists() + assert not (dst / to_b.relative_to(src)).is_symlink() + # src/A/to_broken_b + assert not (dst / to_broken_b.relative_to(src)).exists() + assert not (dst / to_broken_b.relative_to(src)).is_symlink() + + +def test_copy_tree_src_file(mocker, tmp_path): + ( + dst, + src, + a, + to_a, + to_broken_a, + A, + b, + to_b, + to_broken_b, + B, + c, + to_c, + to_broken_c, + C, + ) = create_files(tmp_path) + + ( + src_passwd_file, + dst_passwd_file, + src_group_file, + dst_group_file, + ) = create_passwd_group_files(tmp_path) + + PersistFilesHandler( + src_passwd_file=src_passwd_file, + src_group_file=src_group_file, + dst_passwd_file=dst_passwd_file, + dst_group_file=dst_group_file, + src_root=src, + dst_root=dst, + ).preserve_persist_entry(replace_root(to_b, src, "/")) + + # src/A + assert (dst / A.relative_to(src)).is_dir() + assert not (dst / A.relative_to(src)).is_symlink() + assert_uid_gid_mode(dst / A.relative_to(src), 18, 29, 0o115) + + # src/A/to_b (src/A/b is not copied, so to_b.is_file() is False) + assert not (dst / to_b.relative_to(src)).is_file() + assert (dst / to_b.relative_to(src)).is_symlink() + assert_uid_gid_mode(dst / to_b.relative_to(src), 133, 234, 0o777) + + # followings should not exist + # src/a + assert not (dst / a.relative_to(src)).exists() + assert not (dst / a.relative_to(src)).is_symlink() + # src/to_a + assert not (dst / to_a.relative_to(src)).exists() + assert not (dst / to_a.relative_to(src)).is_symlink() + # src/to_broken_a + assert not (dst / to_broken_a.relative_to(src)).exists() + assert not (dst / to_broken_a.relative_to(src)).is_symlink() + # src/A/b + assert not (dst / b.relative_to(src)).exists() + assert not (dst / b.relative_to(src)).is_symlink() + # src/A/to_broken_b + assert not (dst / to_broken_b.relative_to(src)).exists() + assert not (dst / to_broken_b.relative_to(src)).is_symlink() + # src/A/B + assert not (dst / B.relative_to(src)).exists() + assert not (dst / B.relative_to(src)).is_symlink() + # src/A/B/c + assert not (dst / c.relative_to(src)).exists() + assert not (dst / c.relative_to(src)).is_symlink() + # src/A/B/to_c + assert not (dst / to_c.relative_to(src)).exists() + assert not (dst / to_c.relative_to(src)).is_symlink() + # src/A/B/to_broken_c + assert not (dst / to_broken_c.relative_to(src)).exists() + assert not (dst / to_broken_c.relative_to(src)).is_symlink() + # src/A/B/C/ + assert not (dst / C.relative_to(src)).exists() + assert not (dst / C.relative_to(src)).is_symlink() + + +def test_copy_tree_B_exists(mocker, tmp_path): + ( + dst, + src, + a, + to_a, + to_broken_a, + A, + b, + to_b, + to_broken_b, + B, + c, + to_c, + to_broken_c, + C, + ) = create_files(tmp_path) + + dst_A = dst / "A" + dst_A.mkdir(parents=True) + print(f"dst_A {dst_A}") + dst_B = dst_A / "B" + dst_B.mkdir() + + os.chown(dst_A, 0, 1, follow_symlinks=False) + os.chown(dst_B, 1, 2, follow_symlinks=False) + os.chmod(dst_A, 0o765) + os.chmod(dst_B, 0o654) + st = os.stat(dst_A, follow_symlinks=False) + + ( + src_passwd_file, + dst_passwd_file, + src_group_file, + dst_group_file, + ) = create_passwd_group_files(tmp_path) + + PersistFilesHandler( + src_passwd_file=src_passwd_file, + src_group_file=src_group_file, + dst_passwd_file=dst_passwd_file, + dst_group_file=dst_group_file, + src_root=src, + dst_root=dst, + ).preserve_persist_entry(replace_root(C, src, "/")) + + # src/A + assert (dst / A.relative_to(src)).is_dir() + assert not (dst / A.relative_to(src)).is_symlink() + # 'A' is created by this function before hand + # NOTE(20231222): if src dir exists in the dest, we should + # remove the dst and preserve src to dest. + assert_uid_gid_mode(dst / A.relative_to(src), 18, 29, 0o115) + + # src/A/B + assert (dst / B.relative_to(src)).is_dir() + assert not (dst / B.relative_to(src)).is_symlink() + # 'B' is created by this function before hand + assert_uid_gid_mode(dst / B.relative_to(src), 141, 265534, 0o122) + + # src/A/B/C/ + assert (dst / C.relative_to(src)).is_dir() + assert not (dst / C.relative_to(src)).is_symlink() + assert_uid_gid_mode(dst / C.relative_to(src), 1105, 2105, 0o126) + + # followings should not exist + # src/a + assert not (dst / a.relative_to(src)).exists() + assert not (dst / a.relative_to(src)).is_symlink() + # src/to_a + assert not (dst / to_a.relative_to(src)).exists() + assert not (dst / to_a.relative_to(src)).is_symlink() + # src/to_broken_a + assert not (dst / to_broken_a.relative_to(src)).exists() + assert not (dst / to_broken_a.relative_to(src)).is_symlink() + # src/A/b + assert not (dst / b.relative_to(src)).exists() + assert not (dst / b.relative_to(src)).is_symlink() + # src/A/to_b (src/A/b is not copied, so to_b.is_file() is False) + assert not (dst / to_b.relative_to(src)).exists() + assert not (dst / to_b.relative_to(src)).is_symlink() + # src/A/to_broken_b + assert not (dst / to_broken_b.relative_to(src)).exists() + assert not (dst / to_broken_b.relative_to(src)).is_symlink() + # src/A/B/c + assert not (dst / c.relative_to(src)).exists() + assert not (dst / c.relative_to(src)).is_symlink() + # src/A/B/to_c + assert not (dst / to_c.relative_to(src)).exists() + assert not (dst / to_c.relative_to(src)).is_symlink() + # src/A/B/to_broken_c + assert not (dst / to_broken_c.relative_to(src)).exists() + assert not (dst / to_broken_c.relative_to(src)).is_symlink() + + +def test_copy_tree_with_symlink_overwrite(mocker, tmp_path): + ( + dst, + src, + a, + to_a, + to_broken_a, + A, + b, + to_b, + to_broken_b, + B, + c, + to_c, + to_broken_c, + C, + ) = create_files(tmp_path) + + ( + src_passwd_file, + dst_passwd_file, + src_group_file, + dst_group_file, + ) = create_passwd_group_files(tmp_path) + + ct = PersistFilesHandler( + src_passwd_file=src_passwd_file, + src_group_file=src_group_file, + dst_passwd_file=dst_passwd_file, + dst_group_file=dst_group_file, + src_root=src, + dst_root=dst, + ) + + ct.preserve_persist_entry(replace_root(to_a, src, "/")) + ct.preserve_persist_entry(replace_root(to_broken_a, src, "/")) + + # followings should exist + # src/to_a + assert (dst / to_a.relative_to(src)).is_symlink() + # src/to_broken_a + assert (dst / to_broken_a.relative_to(src)).is_symlink() + + # overwrite symlinks + ct.preserve_persist_entry(replace_root(to_a, src, "/")) + ct.preserve_persist_entry(replace_root(to_broken_a, src, "/")) + + # followings should exist + # src/to_a + assert (dst / to_a.relative_to(src)).is_symlink() + # src/to_broken_a + assert (dst / to_broken_a.relative_to(src)).is_symlink() + + +def test_copy_tree_src_dir_dst_file(mocker, tmp_path): + ( + dst, + src, + a, + to_a, + to_broken_a, + A, + b, + to_b, + to_broken_b, + B, + c, + to_c, + to_broken_c, + C, + ) = create_files(tmp_path) + + ( + src_passwd_file, + dst_passwd_file, + src_group_file, + dst_group_file, + ) = create_passwd_group_files(tmp_path) + + ct = PersistFilesHandler( + src_passwd_file=src_passwd_file, + src_group_file=src_group_file, + dst_passwd_file=dst_passwd_file, + dst_group_file=dst_group_file, + src_root=src, + dst_root=dst, + ) + + (dst / src.relative_to("/") / "A").mkdir(parents=True) + # NOTE: create {dst}/{src}/A/B as *file* before hand + (dst / src.relative_to("/") / "A" / "B").write_text("B") + + ct.preserve_persist_entry(replace_root(B, src, "/")) + + # src/A + assert (dst / A.relative_to(src)).is_dir() + assert not (dst / A.relative_to(src)).is_symlink() + # NOTE: {dst}/{src}/A exists before copy so uid, gid and mode are unchanged. + assert_uid_gid_mode( + dst / A.relative_to(src), *uid_gid_mode(dst / src.relative_to(src) / "A") + ) + + # src/A/B/ + assert (dst / B.relative_to(src)).is_dir() + assert not (dst / B.relative_to(src)).is_symlink() + assert_uid_gid_mode(dst / B.relative_to(src), 141, 265534, 0o122) + + # src/A/B/c + assert (dst / c.relative_to(src)).is_file() + assert not (dst / c.relative_to(src)).is_symlink() + assert_uid_gid_mode(dst / c.relative_to(src), 1100, 2102, 0o123) + + # src/A/B/to_c + assert (dst / to_c.relative_to(src)).is_file() + assert (dst / to_c.relative_to(src)).is_symlink() + # uid, gid can't be converted so original uid, gid is used. + assert_uid_gid_mode(dst / to_c.relative_to(src), 12345678, 87654321, 0o777) + + # src/A/B/to_broken_c + assert not (dst / to_broken_c.relative_to(src)).is_file() + assert (dst / to_broken_c.relative_to(src)).is_symlink() + assert_uid_gid_mode(dst / to_broken_c.relative_to(src), 1104, 2104, 0o777) + + # src/A/B/C/ + assert (dst / C.relative_to(src)).is_dir() + assert not (dst / C.relative_to(src)).is_symlink() + assert_uid_gid_mode(dst / C.relative_to(src), 1105, 2105, 0o126) + + # followings should not exist + # src/a + assert not (dst / a.relative_to(src)).exists() + assert not (dst / a.relative_to(src)).is_symlink() + # src/to_a + assert not (dst / to_a.relative_to(src)).exists() + assert not (dst / to_a.relative_to(src)).is_symlink() + # src/to_broken_a + assert not (dst / to_broken_a.relative_to(src)).exists() + assert not (dst / to_broken_a.relative_to(src)).is_symlink() + # src/A/b + assert not (dst / b.relative_to(src)).exists() + assert not (dst / b.relative_to(src)).is_symlink() + # src/A/to_b + assert not (dst / to_b.relative_to(src)).exists() + assert not (dst / to_b.relative_to(src)).is_symlink() + # src/A/to_broken_b + assert not (dst / to_broken_b.relative_to(src)).exists() + assert not (dst / to_broken_b.relative_to(src)).is_symlink() + + +def test_copy_tree_src_file_dst_dir(mocker, tmp_path): + ( + dst, + src, + a, + to_a, + to_broken_a, + A, + b, + to_b, + to_broken_b, + B, + c, + to_c, + to_broken_c, + C, + ) = create_files(tmp_path) + + ( + src_passwd_file, + dst_passwd_file, + src_group_file, + dst_group_file, + ) = create_passwd_group_files(tmp_path) + + ct = PersistFilesHandler( + src_passwd_file=src_passwd_file, + src_group_file=src_group_file, + dst_passwd_file=dst_passwd_file, + dst_group_file=dst_group_file, + src_root=src, + dst_root=dst, + ) + + # NOTE: create {dst}/{src}/A/B/c as *dir* before hand + (dst / src.relative_to("/") / "A" / "B" / "c").mkdir(parents=True) + + ct.preserve_persist_entry(replace_root(B, src, "/")) + + # src/A + assert (dst / A.relative_to(src)).is_dir() + assert not (dst / A.relative_to(src)).is_symlink() + # NOTE: {dst}/{src}/A exists before copy so uid, gid and mode are unchanged. + assert_uid_gid_mode( + dst / A.relative_to(src), *uid_gid_mode(dst / src.relative_to(src) / "A") + ) + + # src/A/B/ + assert (dst / B.relative_to(src)).is_dir() + assert not (dst / B.relative_to(src)).is_symlink() + # NOTE: {dst}/{src}/A/B exists before copy so uid, gid and mode are unchanged. + assert_uid_gid_mode( + dst / B.relative_to(src), *uid_gid_mode(dst / src.relative_to(src) / "A" / "B") + ) + + # src/A/B/c + assert (dst / c.relative_to(src)).is_file() + assert not (dst / c.relative_to(src)).is_symlink() + assert_uid_gid_mode(dst / c.relative_to(src), 1100, 2102, 0o123) + + # src/A/B/to_c + assert (dst / to_c.relative_to(src)).is_file() + assert (dst / to_c.relative_to(src)).is_symlink() + # uid, gid can't be converted so original uid, gid is used. + assert_uid_gid_mode(dst / to_c.relative_to(src), 12345678, 87654321, 0o777) + + # src/A/B/to_broken_c + assert not (dst / to_broken_c.relative_to(src)).is_file() + assert (dst / to_broken_c.relative_to(src)).is_symlink() + assert_uid_gid_mode(dst / to_broken_c.relative_to(src), 1104, 2104, 0o777) + + # src/A/B/C/ + assert (dst / C.relative_to(src)).is_dir() + assert not (dst / C.relative_to(src)).is_symlink() + assert_uid_gid_mode(dst / C.relative_to(src), 1105, 2105, 0o126) + + # followings should not exist + # src/a + assert not (dst / a.relative_to(src)).exists() + assert not (dst / a.relative_to(src)).is_symlink() + # src/to_a + assert not (dst / to_a.relative_to(src)).exists() + assert not (dst / to_a.relative_to(src)).is_symlink() + # src/to_broken_a + assert not (dst / to_broken_a.relative_to(src)).exists() + assert not (dst / to_broken_a.relative_to(src)).is_symlink() + # src/A/b + assert not (dst / b.relative_to(src)).exists() + assert not (dst / b.relative_to(src)).is_symlink() + # src/A/to_b + assert not (dst / to_b.relative_to(src)).exists() + assert not (dst / to_b.relative_to(src)).is_symlink() + # src/A/to_broken_b + assert not (dst / to_broken_b.relative_to(src)).exists() + assert not (dst / to_broken_b.relative_to(src)).is_symlink() From 0738df5bdddcc1dfc8619a015b2c95ccefd5062b Mon Sep 17 00:00:00 2001 From: Bodong YANG Date: Tue, 26 Dec 2023 03:01:55 +0000 Subject: [PATCH 06/17] refactor PersistFilesHandler again --- otaclient/app/create_standby/common.py | 196 +++++++++++++------------ 1 file changed, 105 insertions(+), 91 deletions(-) diff --git a/otaclient/app/create_standby/common.py b/otaclient/app/create_standby/common.py index df56fb2ad..8a76330d7 100644 --- a/otaclient/app/create_standby/common.py +++ b/otaclient/app/create_standby/common.py @@ -15,6 +15,7 @@ from __future__ import annotations +import functools import os import random import time @@ -510,120 +511,133 @@ def __init__( src_root: StrOrPath, dst_root: StrOrPath, ): - self._src_pw = ParsedPasswd(src_passwd_file) - self._src_grp = ParsedGroup(src_group_file) - self._dst_pw = ParsedPasswd(dst_passwd_file) - self._dst_grp = ParsedGroup(dst_group_file) - - self._src_root = Path(src_root) - self._dst_root = Path(dst_root) - - def _prepare_path( - self, - _src_path: Path, - _dst_path: Path, - *, - skip_invalid: bool = True, - ): - """For input original path(from persistents.txt), preserve it from to .""" - - # ------ check src_path, src_path must be existed ------ # - # NOTE: Path.exists does follow link! if this symlink is dead, - # Path.exists also returns False! - if not (_src_path.is_symlink() or _src_path.is_file() or _src_path.is_dir()): - if not skip_invalid: - raise FileNotFoundError( - f"{_src_path=} doesn't exist or not a file/symlink/dir" - ) - logger.warning( - f"{_src_path=} doesn't exist or not a file/symlink/dir, skip..." + self._uid_mapper = functools.lru_cache()( + functools.partial( + map_uid_by_pwnam, + src_db=ParsedPasswd(src_passwd_file), + dst_db=ParsedPasswd(dst_passwd_file), ) - return - - # ------ preserve src_path to dst_path ------ # - # NOTE: Path.stat always follow symlink, use os.stat instead - _src_stat = os.stat(_src_path, follow_symlinks=False) - - # NOTE: check is_symlink first, as is_file/dir also returns True if - # src_path is a symlink points to existed file/dir. - # NOTE: cleanup dst ONLY when src is available! - if _src_path.is_symlink(): - _dst_path.unlink(missing_ok=True) - shutil.copy(_src_path, _dst_path, follow_symlinks=False) - # no need to change symlink's mode, it is always 777 - elif _src_path.is_file(): - _dst_path.unlink(missing_ok=True) - shutil.copy(_src_path, _dst_path, follow_symlinks=False) - os.chmod(_dst_path, _src_stat.st_mode) - elif _src_path.is_dir(): - shutil.rmtree(_dst_path, ignore_errors=True) - _dst_path.mkdir(exist_ok=True) - os.chmod(_dst_path, _src_stat.st_mode) - elif skip_invalid: - logger.warning(f"{_src_path=} is missing or not file/symlink or dir, skip") - return - else: - raise ValueError( - f"type of {_src_path=} must be presented and one of file/dir/symlink" + ) + self._gid_mapper = functools.lru_cache()( + functools.partial( + map_gid_by_grpnam, + src_db=ParsedGroup(src_group_file), + dst_db=ParsedGroup(dst_group_file), ) + ) + self._src_root = Path(src_root) + self._dst_root = Path(dst_root) - # change owner with mapping + def _chown_with_mapping( + self, _src_stat: os.stat_result, _dst_path: StrOrPath + ) -> None: _src_uid, _src_gid = _src_stat.st_uid, _src_stat.st_gid try: - _dst_uid = map_uid_by_pwnam( - src_db=self._src_pw, dst_db=self._dst_pw, uid=_src_uid - ) + _dst_uid = self._uid_mapper(uid=_src_uid) except ValueError: logger.warning(f"failed to find mapping for {_src_uid=}, keep unchanged") _dst_uid = _src_uid try: - _dst_gid = map_gid_by_grpnam( - src_db=self._src_grp, dst_db=self._dst_grp, gid=_src_gid - ) + _dst_gid = self._gid_mapper(gid=_src_gid) except ValueError: logger.warning(f"failed to find mapping for {_src_gid=}, keep unchanged") _dst_gid = _src_gid os.chown(_dst_path, uid=_dst_uid, gid=_dst_gid, follow_symlinks=False) + @staticmethod + def _rm_target(_target: Path) -> None: + """Remove target with proper methods.""" + if _target.is_symlink() or _target.is_file(): + return _target.unlink(missing_ok=True) + elif _target.is_dir(): + return shutil.rmtree(_target, ignore_errors=True) + raise ValueError(f"{_target} is not normal file/symlink/dir, failed to remove") + + def _prepare_symlink(self, _src_path: Path, _dst_path: Path) -> None: + _dst_path.symlink_to(os.readlink(_src_path)) + self._chown_with_mapping(_src_path.stat(), _dst_path) + + def _prepare_dir(self, _src_path: Path, _dst_path: Path) -> None: + _dst_path.mkdir(exist_ok=True) + + _src_stat = os.stat(_src_path, follow_symlinks=False) + os.chmod(_dst_path, _src_stat.st_mode) + self._chown_with_mapping(_src_stat, _dst_path) + + def _prepare_file(self, _src_path: Path, _dst_path: Path) -> None: + shutil.copy(_src_path, _dst_path, follow_symlinks=False) + + _src_stat = os.stat(_src_path, follow_symlinks=False) + os.chmod(_dst_path, _src_stat.st_mode) + self._chown_with_mapping(_src_stat, _dst_path) + + def _prepare_parent(self, _src_path: Path, _dst_path: Path) -> None: + if _dst_path.is_dir(): # keep the origin parent on dst as it + return + if _dst_path.is_symlink() or _dst_path.is_file(): + _dst_path.unlink(missing_ok=True) + self._prepare_dir(_src_path, _dst_path) + return + if _dst_path.exists(): + raise ValueError( + f"{_dst_path=} is not a normal file/symlink/dir, cannot cleanup" + ) + self._prepare_dir(_src_path, _dst_path) + # API - def preserve_persist_entry( - self, _origin_entry: StrOrPath, *, skip_invalid: bool = True - ): - origin_entry = Path(_origin_entry).relative_to(cfg.DEFAULT_ACTIVE_ROOTFS) + def preserve_persist_entry(self, _persist_entry: StrOrPath): + logger.info( + f"preserving {_persist_entry} from {self._src_root} to {self._dst_root}" + ) + # persist_entry in persists.txt must be rooted at / + origin_entry = Path(_persist_entry).relative_to(cfg.DEFAULT_ACTIVE_ROOTFS) src_path = self._src_root / origin_entry dst_path = self._dst_root / origin_entry # ------ prepare parents ------ # - for _idx, _parent in enumerate(reversed(origin_entry.parents)): - if _idx == 0: - continue # skip first parent dir - self._prepare_path(self._src_root / _parent, self._dst_root / _parent) + for _parent in reversed(origin_entry.parents): + self._prepare_parent(self._src_root / _parent, self._dst_root / _parent) # ------ prepare entry itself ------ # - # for normal file/symlink, directly prepare it - if src_path.is_symlink() or src_path.is_file(): - self._prepare_path(src_path, dst_path) - # for dir, dive into is and prepare everything under this dir - elif src_path.is_dir(): - for src_dirpath, _, fnames in os.walk(src_path, followlinks=False): - _src_dpath = Path(src_dirpath) - _origin_dpath = _src_dpath.relative_to(self._src_root) - _dst_dpath = self._dst_root / _origin_dpath - - self._prepare_path(_src_dpath, _dst_dpath) - for _fname in fnames: - self._prepare_path( - _src_dpath / _fname, - _dst_dpath / _fname, - skip_invalid=skip_invalid, - ) - elif skip_invalid: - logger.warning( - f"{src_path=} must be presented and either a file/symlink/dir, skip" - ) - else: + # NOTE: always check if symlink first! + if src_path.is_symlink(): + self._rm_target(dst_path) + self._prepare_symlink(src_path, dst_path) + return + + if src_path.is_file(): + self._rm_target(dst_path) + self._prepare_file(src_path, dst_path) + return + + # we only process normal file/symlink/dir + if not src_path.is_dir(): raise ValueError( f"{src_path=} must be presented and either a file/symlink/dir" ) + + # for src as dir, cleanup dst_dirc, + # dive into src_dir and preserve everything under the src dir + for src_curdir, dnames, fnames in os.walk(src_path, followlinks=False): + src_cur_dpath = Path(src_curdir) + dst_cur_dpath = self._dst_root / src_cur_dpath.relative_to(self._src_root) + + # ------ prepare current dir itself ------ # + self._rm_target(dst_cur_dpath) + self._prepare_dir(src_cur_dpath, dst_cur_dpath) + + # ------ prepare entries in current dir ------ # + for _fname in fnames: + _src_fpath, _dst_fpath = src_cur_dpath / _fname, dst_cur_dpath / _fname + if _src_fpath.is_symlink(): + self._prepare_symlink(_src_fpath, _dst_fpath) + continue + self._prepare_file(_src_fpath, _dst_fpath) + + # symlinks to dirs also included in dnames, we must handle it + for _dname in dnames: + _src_dpath = src_cur_dpath / _dname + if _src_dpath.is_symlink(): + self._prepare_symlink(_src_dpath, dst_cur_dpath / _dname) From cc5c77a633afe676ae4130eb3d549f9aa7d73eee Mon Sep 17 00:00:00 2001 From: Bodong YANG Date: Tue, 26 Dec 2023 03:13:05 +0000 Subject: [PATCH 07/17] fix test files --- tests/test_copy_tree.py | 768 ------------------ .../test_persist_files_handling.py | 8 +- 2 files changed, 3 insertions(+), 773 deletions(-) delete mode 100644 tests/test_copy_tree.py diff --git a/tests/test_copy_tree.py b/tests/test_copy_tree.py deleted file mode 100644 index 2a115ff46..000000000 --- a/tests/test_copy_tree.py +++ /dev/null @@ -1,768 +0,0 @@ -# Copyright 2022 TIER IV, INC. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import os -import stat -import pytest -from pathlib import Path - - -def create_files(tmp_path: Path): - dst = tmp_path / "dst" - dst.mkdir() - - src = tmp_path / "src" - src.mkdir() - - """ - src/ - src/a - src/to_a -> a - src/to_broken_a -> broken_a - src/A - src/A/b - src/A/to_b -> b - src/A/to_broken_b -> broken_b - src/A/B/ - src/A/B/c - src/A/B/to_c -> c - src/A/B/to_broken_c -> broken_c - src/A/B/C/ - """ - - a = src / "a" - a.write_text("a") - to_a = src / "to_a" - to_a.symlink_to("a") - to_broken_a = src / "to_broken_a" - to_broken_a.symlink_to("broken_a") - A = src / "A" - A.mkdir() - - b = A / "b" - b.write_text("b") - to_b = A / "to_b" - to_b.symlink_to("b") - to_broken_b = A / "to_broken_b" - to_broken_b.symlink_to("broken_b") - B = A / "B" - B.mkdir() - - c = B / "c" - c.write_text("c") - to_c = B / "to_c" - to_c.symlink_to("c") - to_broken_c = B / "to_broken_c" - to_broken_c.symlink_to("broken_c") - C = B / "C" - C.mkdir() - - os.chown(src, 0, 1, follow_symlinks=False) - os.chown(a, 2, 3, follow_symlinks=False) - os.chown(to_a, 4, 5, follow_symlinks=False) - os.chown(to_broken_a, 6, 7, follow_symlinks=False) - os.chown(A, 8, 9, follow_symlinks=False) - os.chown(b, 10, 13, follow_symlinks=False) - os.chown(to_b, 33, 34, follow_symlinks=False) - os.chown(to_broken_b, 38, 39, follow_symlinks=False) - os.chown(B, 41, 65534, follow_symlinks=False) - os.chown(c, 100, 102, follow_symlinks=False) - os.chown(to_c, 12345678, 87654321, follow_symlinks=False) # id can't be converted - os.chown(to_broken_c, 104, 104, follow_symlinks=False) - os.chown(C, 105, 105, follow_symlinks=False) - - os.chmod(src, 0o111) - os.chmod(a, 0o112) - # os.chmod(to_a, 0o113) - # os.chmod(to_broken_a, 0o114) - os.chmod(A, 0o115) - os.chmod(b, 0o116) - # os.chmod(to_b, 0o117) - # os.chmod(to_broken_b, 0o121) - os.chmod(B, 0o122) - os.chmod(c, 0o123) - # os.chmod(to_c, 0o124) - # os.chmod(to_broken_c, 0o125) - os.chmod(C, 0o126) - - return ( - dst, - src, - a, - to_a, - to_broken_a, - A, - b, - to_b, - to_broken_b, - B, - c, - to_c, - to_broken_c, - C, - ) - - -def uid_gid_mode(path): - st = os.stat(path, follow_symlinks=False) - return st[stat.ST_UID], st[stat.ST_GID], stat.S_IMODE(st[stat.ST_MODE]) - - -def assert_uid_gid_mode(path, uid, gid, mode): - _uid, _gid, _mode = uid_gid_mode(path) - assert _uid == uid - assert _gid == gid - assert _mode == mode - - -def create_passwd_group_files(tmp_path): - src_passwd = """\ -root:x:0:0:root:/root:/bin/bash -daemon:x:1:1:daemon:/usr/sbin:/usr/sbin/nologin -bin:x:2:2:bin:/bin:/usr/sbin/nologin -sys:x:3:3:sys:/dev:/usr/sbin/nologin -sync:x:4:65534:sync:/bin:/bin/sync -games:x:5:60:games:/usr/games:/usr/sbin/nologin -man:x:6:12:man:/var/cache/man:/usr/sbin/nologin -lp:x:7:7:lp:/var/spool/lpd:/usr/sbin/nologin -mail:x:8:8:mail:/var/mail:/usr/sbin/nologin -news:x:9:9:news:/var/spool/news:/usr/sbin/nologin -uucp:x:10:10:uucp:/var/spool/uucp:/usr/sbin/nologin -proxy:x:13:13:proxy:/bin:/usr/sbin/nologin -www-data:x:33:33:www-data:/var/www:/usr/sbin/nologin -backup:x:34:34:backup:/var/backups:/usr/sbin/nologin -list:x:38:38:Mailing List Manager:/var/list:/usr/sbin/nologin -irc:x:39:39:ircd:/var/run/ircd:/usr/sbin/nologin -gnats:x:41:41:Gnats Bug-Reporting System (admin):/var/lib/gnats:/usr/sbin/nologin -nobody:x:65534:65534:nobody:/nonexistent:/usr/sbin/nologin -systemd-network:x:100:102:systemd Network Management,,,:/run/systemd:/usr/sbin/nologin -systemd-resolve:x:101:103:systemd Resolver,,,:/run/systemd:/usr/sbin/nologin -systemd-timesync:x:102:104:systemd Time Synchronization,,,:/run/systemd:/usr/sbin/nologin -messagebus:x:103:106::/nonexistent:/usr/sbin/nologin -syslog:x:104:110::/home/syslog:/usr/sbin/nologin -_apt:x:105:65534::/nonexistent:/usr/sbin/nologin -""" - - # dst_passwd uid is converted with f"1{src_passwd gid}" - dst_passwd = """\ -root:x:10:0:root:/root:/bin/bash -daemon:x:11:1:daemon:/usr/sbin:/usr/sbin/nologin -bin:x:12:2:bin:/bin:/usr/sbin/nologin -sys:x:13:3:sys:/dev:/usr/sbin/nologin -sync:x:14:65534:sync:/bin:/bin/sync -games:x:15:60:games:/usr/games:/usr/sbin/nologin -man:x:16:12:man:/var/cache/man:/usr/sbin/nologin -lp:x:17:7:lp:/var/spool/lpd:/usr/sbin/nologin -mail:x:18:8:mail:/var/mail:/usr/sbin/nologin -news:x:19:9:news:/var/spool/news:/usr/sbin/nologin -uucp:x:110:10:uucp:/var/spool/uucp:/usr/sbin/nologin -proxy:x:113:13:proxy:/bin:/usr/sbin/nologin -www-data:x:133:33:www-data:/var/www:/usr/sbin/nologin -backup:x:134:34:backup:/var/backups:/usr/sbin/nologin -list:x:138:38:Mailing List Manager:/var/list:/usr/sbin/nologin -irc:x:139:39:ircd:/var/run/ircd:/usr/sbin/nologin -gnats:x:141:41:Gnats Bug-Reporting System (admin):/var/lib/gnats:/usr/sbin/nologin -nobody:x:165534:65534:nobody:/nonexistent:/usr/sbin/nologin -systemd-network:x:1100:102:systemd Network Management,,,:/run/systemd:/usr/sbin/nologin -systemd-resolve:x:1101:103:systemd Resolver,,,:/run/systemd:/usr/sbin/nologin -systemd-timesync:x:1102:104:systemd Time Synchronization,,,:/run/systemd:/usr/sbin/nologin -messagebus:x:1103:106::/nonexistent:/usr/sbin/nologin -syslog:x:1104:110::/home/syslog:/usr/sbin/nologin -_apt:x:1105:65534::/nonexistent:/usr/sbin/nologin -""" - - src_group = """\ -root:x:0: -daemon:x:1: -bin:x:2: -sys:x:3: -adm:x:4:syslog -tty:x:5:syslog -disk:x:6: -lp:x:7: -mail:x:8: -news:x:9: -uucp:x:10: -man:x:12: -proxy:x:13: -kmem:x:15: -dialout:x:20: -fax:x:21: -voice:x:22: -cdrom:x:24: -floppy:x:25: -tape:x:26: -sudo:x:27: -audio:x:29:pulse -dip:x:30: -www-data:x:33: -backup:x:34: -operator:x:37: -list:x:38: -irc:x:39: -src:x:40: -gnats:x:41: -shadow:x:42: -utmp:x:43: -video:x:44: -sasl:x:45: -plugdev:x:46: -staff:x:50: -games:x:60: -users:x:100: -nogroup:x:65534: -systemd-journal:x:101: -systemd-network:x:102: -systemd-resolve:x:103: -systemd-timesync:x:104: -crontab:x:105: -""" - - # dst_group gid is converted with f"2{src_group gid}" - dst_group = """\ -root:x:20: -daemon:x:21: -bin:x:22: -sys:x:23: -adm:x:24:syslog -tty:x:25:syslog -disk:x:26: -lp:x:27: -mail:x:28: -news:x:29: -uucp:x:210: -man:x:212: -proxy:x:213: -kmem:x:215: -dialout:x:220: -fax:x:221: -voice:x:222: -cdrom:x:224: -floppy:x:225: -tape:x:226: -sudo:x:227: -audio:x:229:pulse -dip:x:230: -www-data:x:233: -backup:x:234: -operator:x:237: -list:x:238: -irc:x:239: -src:x:240: -gnats:x:241: -shadow:x:242: -utmp:x:243: -video:x:244: -sasl:x:245: -plugdev:x:246: -staff:x:250: -games:x:260: -users:x:2100: -nogroup:x:265534: -systemd-journal:x:2101: -systemd-network:x:2102: -systemd-resolve:x:2103: -systemd-timesync:x:2104: -crontab:x:2105: -""" - src_passwd_file = tmp_path / "etc" / "src_passwd" - dst_passwd_file = tmp_path / "etc" / "dst_passwd" - src_group_file = tmp_path / "etc" / "src_group" - dst_group_file = tmp_path / "etc" / "dst_group" - (tmp_path / "etc").mkdir() - - src_passwd_file.write_text(src_passwd) - dst_passwd_file.write_text(dst_passwd) - src_group_file.write_text(src_group) - dst_group_file.write_text(dst_group) - return src_passwd_file, dst_passwd_file, src_group_file, dst_group_file - - -def test_copy_tree_src_dir(mocker, tmp_path): - from otaclient.app.copy_tree import CopyTree - - ( - dst, - src, - a, - to_a, - to_broken_a, - A, - b, - to_b, - to_broken_b, - B, - c, - to_c, - to_broken_c, - C, - ) = create_files(tmp_path) - - ( - src_passwd_file, - dst_passwd_file, - src_group_file, - dst_group_file, - ) = create_passwd_group_files(tmp_path) - - CopyTree( - src_passwd_file, src_group_file, dst_passwd_file, dst_group_file - ).copy_with_parents(B, dst) - - # src/A - assert (dst / A.relative_to("/")).is_dir() - assert not (dst / A.relative_to("/")).is_symlink() - assert_uid_gid_mode(dst / A.relative_to("/"), 18, 29, 0o115) - - # src/A/B/ - assert (dst / B.relative_to("/")).is_dir() - assert not (dst / B.relative_to("/")).is_symlink() - assert_uid_gid_mode(dst / B.relative_to("/"), 141, 265534, 0o122) - - # src/A/B/c - assert (dst / c.relative_to("/")).is_file() - assert not (dst / c.relative_to("/")).is_symlink() - assert_uid_gid_mode(dst / c.relative_to("/"), 1100, 2102, 0o123) - - # src/A/B/to_c - assert (dst / to_c.relative_to("/")).is_file() - assert (dst / to_c.relative_to("/")).is_symlink() - # uid, gid can't be converted so original uid, gid is used. - assert_uid_gid_mode(dst / to_c.relative_to("/"), 12345678, 87654321, 0o777) - - # src/A/B/to_broken_c - assert not (dst / to_broken_c.relative_to("/")).is_file() - assert (dst / to_broken_c.relative_to("/")).is_symlink() - assert_uid_gid_mode(dst / to_broken_c.relative_to("/"), 1104, 2104, 0o777) - - # src/A/B/C/ - assert (dst / C.relative_to("/")).is_dir() - assert not (dst / C.relative_to("/")).is_symlink() - assert_uid_gid_mode(dst / C.relative_to("/"), 1105, 2105, 0o126) - - # followings should not exist - # src/a - assert not (dst / a.relative_to("/")).exists() - assert not (dst / a.relative_to("/")).is_symlink() - # src/to_a - assert not (dst / to_a.relative_to("/")).exists() - assert not (dst / to_a.relative_to("/")).is_symlink() - # src/to_broken_a - assert not (dst / to_broken_a.relative_to("/")).exists() - assert not (dst / to_broken_a.relative_to("/")).is_symlink() - # src/A/b - assert not (dst / b.relative_to("/")).exists() - assert not (dst / b.relative_to("/")).is_symlink() - # src/A/to_b - assert not (dst / to_b.relative_to("/")).exists() - assert not (dst / to_b.relative_to("/")).is_symlink() - # src/A/to_broken_b - assert not (dst / to_broken_b.relative_to("/")).exists() - assert not (dst / to_broken_b.relative_to("/")).is_symlink() - - -def test_copy_tree_src_file(mocker, tmp_path): - from otaclient.app.copy_tree import CopyTree - - ( - dst, - src, - a, - to_a, - to_broken_a, - A, - b, - to_b, - to_broken_b, - B, - c, - to_c, - to_broken_c, - C, - ) = create_files(tmp_path) - - ( - src_passwd_file, - dst_passwd_file, - src_group_file, - dst_group_file, - ) = create_passwd_group_files(tmp_path) - - CopyTree( - src_passwd_file, src_group_file, dst_passwd_file, dst_group_file - ).copy_with_parents(to_b, dst) - - # src/A - assert (dst / A.relative_to("/")).is_dir() - assert not (dst / A.relative_to("/")).is_symlink() - assert_uid_gid_mode(dst / A.relative_to("/"), 18, 29, 0o115) - - # src/A/to_b (src/A/b is not copied, so to_b.is_file() is False) - assert not (dst / to_b.relative_to("/")).is_file() - assert (dst / to_b.relative_to("/")).is_symlink() - assert_uid_gid_mode(dst / to_b.relative_to("/"), 133, 234, 0o777) - - # followings should not exist - # src/a - assert not (dst / a.relative_to("/")).exists() - assert not (dst / a.relative_to("/")).is_symlink() - # src/to_a - assert not (dst / to_a.relative_to("/")).exists() - assert not (dst / to_a.relative_to("/")).is_symlink() - # src/to_broken_a - assert not (dst / to_broken_a.relative_to("/")).exists() - assert not (dst / to_broken_a.relative_to("/")).is_symlink() - # src/A/b - assert not (dst / b.relative_to("/")).exists() - assert not (dst / b.relative_to("/")).is_symlink() - # src/A/to_broken_b - assert not (dst / to_broken_b.relative_to("/")).exists() - assert not (dst / to_broken_b.relative_to("/")).is_symlink() - # src/A/B - assert not (dst / B.relative_to("/")).exists() - assert not (dst / B.relative_to("/")).is_symlink() - # src/A/B/c - assert not (dst / c.relative_to("/")).exists() - assert not (dst / c.relative_to("/")).is_symlink() - # src/A/B/to_c - assert not (dst / to_c.relative_to("/")).exists() - assert not (dst / to_c.relative_to("/")).is_symlink() - # src/A/B/to_broken_c - assert not (dst / to_broken_c.relative_to("/")).exists() - assert not (dst / to_broken_c.relative_to("/")).is_symlink() - # src/A/B/C/ - assert not (dst / C.relative_to("/")).exists() - assert not (dst / C.relative_to("/")).is_symlink() - - -def test_copy_tree_B_exists(mocker, tmp_path): - from otaclient.app.copy_tree import CopyTree - - ( - dst, - src, - a, - to_a, - to_broken_a, - A, - b, - to_b, - to_broken_b, - B, - c, - to_c, - to_broken_c, - C, - ) = create_files(tmp_path) - - dst_A = dst / tmp_path.relative_to("/") / "src" / "A" - dst_A.mkdir(parents=True) - print(f"dst_A {dst_A}") - dst_B = dst_A / "B" - dst_B.mkdir() - - os.chown(dst_A, 0, 1, follow_symlinks=False) - os.chown(dst_B, 1, 2, follow_symlinks=False) - os.chmod(dst_A, 0o765) - os.chmod(dst_B, 0o654) - st = os.stat(dst_A, follow_symlinks=False) - - ( - src_passwd_file, - dst_passwd_file, - src_group_file, - dst_group_file, - ) = create_passwd_group_files(tmp_path) - - CopyTree( - src_passwd_file, src_group_file, dst_passwd_file, dst_group_file - ).copy_with_parents(C, dst) - - # src/A - assert (dst / A.relative_to("/")).is_dir() - assert not (dst / A.relative_to("/")).is_symlink() - # 'A' is created by this function before hand - assert_uid_gid_mode(dst / A.relative_to("/"), 0, 1, 0o765) - - # src/A/B - assert (dst / B.relative_to("/")).is_dir() - assert not (dst / B.relative_to("/")).is_symlink() - # 'B' is created by this function before hand - assert_uid_gid_mode(dst / B.relative_to("/"), 1, 2, 0o654) - - # src/A/B/C/ - assert (dst / C.relative_to("/")).is_dir() - assert not (dst / C.relative_to("/")).is_symlink() - assert_uid_gid_mode(dst / C.relative_to("/"), 1105, 2105, 0o126) - - # followings should not exist - # src/a - assert not (dst / a.relative_to("/")).exists() - assert not (dst / a.relative_to("/")).is_symlink() - # src/to_a - assert not (dst / to_a.relative_to("/")).exists() - assert not (dst / to_a.relative_to("/")).is_symlink() - # src/to_broken_a - assert not (dst / to_broken_a.relative_to("/")).exists() - assert not (dst / to_broken_a.relative_to("/")).is_symlink() - # src/A/b - assert not (dst / b.relative_to("/")).exists() - assert not (dst / b.relative_to("/")).is_symlink() - # src/A/to_b (src/A/b is not copied, so to_b.is_file() is False) - assert not (dst / to_b.relative_to("/")).exists() - assert not (dst / to_b.relative_to("/")).is_symlink() - # src/A/to_broken_b - assert not (dst / to_broken_b.relative_to("/")).exists() - assert not (dst / to_broken_b.relative_to("/")).is_symlink() - # src/A/B/c - assert not (dst / c.relative_to("/")).exists() - assert not (dst / c.relative_to("/")).is_symlink() - # src/A/B/to_c - assert not (dst / to_c.relative_to("/")).exists() - assert not (dst / to_c.relative_to("/")).is_symlink() - # src/A/B/to_broken_c - assert not (dst / to_broken_c.relative_to("/")).exists() - assert not (dst / to_broken_c.relative_to("/")).is_symlink() - - -def test_copy_tree_with_symlink_overwrite(mocker, tmp_path): - from otaclient.app.copy_tree import CopyTree - - ( - dst, - src, - a, - to_a, - to_broken_a, - A, - b, - to_b, - to_broken_b, - B, - c, - to_c, - to_broken_c, - C, - ) = create_files(tmp_path) - - ( - src_passwd_file, - dst_passwd_file, - src_group_file, - dst_group_file, - ) = create_passwd_group_files(tmp_path) - - ct = CopyTree(src_passwd_file, src_group_file, dst_passwd_file, dst_group_file) - - ct.copy_with_parents(to_a, dst) - ct.copy_with_parents(to_broken_a, dst) - - # followings should exist - # src/to_a - assert (dst / to_a.relative_to("/")).is_symlink() - # src/to_broken_a - assert (dst / to_broken_a.relative_to("/")).is_symlink() - - # overwrite symlinks - ct.copy_with_parents(to_a, dst) - ct.copy_with_parents(to_broken_a, dst) - - # followings should exist - # src/to_a - assert (dst / to_a.relative_to("/")).is_symlink() - # src/to_broken_a - assert (dst / to_broken_a.relative_to("/")).is_symlink() - - -def test_copy_tree_src_dir_dst_file(mocker, tmp_path): - from otaclient.app.copy_tree import CopyTree - - ( - dst, - src, - a, - to_a, - to_broken_a, - A, - b, - to_b, - to_broken_b, - B, - c, - to_c, - to_broken_c, - C, - ) = create_files(tmp_path) - - ( - src_passwd_file, - dst_passwd_file, - src_group_file, - dst_group_file, - ) = create_passwd_group_files(tmp_path) - - ct = CopyTree(src_passwd_file, src_group_file, dst_passwd_file, dst_group_file) - - (dst / src.relative_to("/") / "A").mkdir(parents=True) - # NOTE: create {dst}/{src}/A/B as *file* before hand - (dst / src.relative_to("/") / "A" / "B").write_text("B") - - ct.copy_with_parents(B, dst) - - # src/A - assert (dst / A.relative_to("/")).is_dir() - assert not (dst / A.relative_to("/")).is_symlink() - # NOTE: {dst}/{src}/A exists before copy so uid, gid and mode are unchanged. - assert_uid_gid_mode( - dst / A.relative_to("/"), *uid_gid_mode(dst / src.relative_to("/") / "A") - ) - - # src/A/B/ - assert (dst / B.relative_to("/")).is_dir() - assert not (dst / B.relative_to("/")).is_symlink() - assert_uid_gid_mode(dst / B.relative_to("/"), 141, 265534, 0o122) - - # src/A/B/c - assert (dst / c.relative_to("/")).is_file() - assert not (dst / c.relative_to("/")).is_symlink() - assert_uid_gid_mode(dst / c.relative_to("/"), 1100, 2102, 0o123) - - # src/A/B/to_c - assert (dst / to_c.relative_to("/")).is_file() - assert (dst / to_c.relative_to("/")).is_symlink() - # uid, gid can't be converted so original uid, gid is used. - assert_uid_gid_mode(dst / to_c.relative_to("/"), 12345678, 87654321, 0o777) - - # src/A/B/to_broken_c - assert not (dst / to_broken_c.relative_to("/")).is_file() - assert (dst / to_broken_c.relative_to("/")).is_symlink() - assert_uid_gid_mode(dst / to_broken_c.relative_to("/"), 1104, 2104, 0o777) - - # src/A/B/C/ - assert (dst / C.relative_to("/")).is_dir() - assert not (dst / C.relative_to("/")).is_symlink() - assert_uid_gid_mode(dst / C.relative_to("/"), 1105, 2105, 0o126) - - # followings should not exist - # src/a - assert not (dst / a.relative_to("/")).exists() - assert not (dst / a.relative_to("/")).is_symlink() - # src/to_a - assert not (dst / to_a.relative_to("/")).exists() - assert not (dst / to_a.relative_to("/")).is_symlink() - # src/to_broken_a - assert not (dst / to_broken_a.relative_to("/")).exists() - assert not (dst / to_broken_a.relative_to("/")).is_symlink() - # src/A/b - assert not (dst / b.relative_to("/")).exists() - assert not (dst / b.relative_to("/")).is_symlink() - # src/A/to_b - assert not (dst / to_b.relative_to("/")).exists() - assert not (dst / to_b.relative_to("/")).is_symlink() - # src/A/to_broken_b - assert not (dst / to_broken_b.relative_to("/")).exists() - assert not (dst / to_broken_b.relative_to("/")).is_symlink() - - -def test_copy_tree_src_file_dst_dir(mocker, tmp_path): - from otaclient.app.copy_tree import CopyTree - - ( - dst, - src, - a, - to_a, - to_broken_a, - A, - b, - to_b, - to_broken_b, - B, - c, - to_c, - to_broken_c, - C, - ) = create_files(tmp_path) - - ( - src_passwd_file, - dst_passwd_file, - src_group_file, - dst_group_file, - ) = create_passwd_group_files(tmp_path) - - ct = CopyTree(src_passwd_file, src_group_file, dst_passwd_file, dst_group_file) - - # NOTE: create {dst}/{src}/A/B/c as *dir* before hand - (dst / src.relative_to("/") / "A" / "B" / "c").mkdir(parents=True) - - ct.copy_with_parents(B, dst) - - # src/A - assert (dst / A.relative_to("/")).is_dir() - assert not (dst / A.relative_to("/")).is_symlink() - # NOTE: {dst}/{src}/A exists before copy so uid, gid and mode are unchanged. - assert_uid_gid_mode( - dst / A.relative_to("/"), *uid_gid_mode(dst / src.relative_to("/") / "A") - ) - - # src/A/B/ - assert (dst / B.relative_to("/")).is_dir() - assert not (dst / B.relative_to("/")).is_symlink() - # NOTE: {dst}/{src}/A/B exists before copy so uid, gid and mode are unchanged. - assert_uid_gid_mode( - dst / B.relative_to("/"), *uid_gid_mode(dst / src.relative_to("/") / "A" / "B") - ) - - # src/A/B/c - assert (dst / c.relative_to("/")).is_file() - assert not (dst / c.relative_to("/")).is_symlink() - assert_uid_gid_mode(dst / c.relative_to("/"), 1100, 2102, 0o123) - - # src/A/B/to_c - assert (dst / to_c.relative_to("/")).is_file() - assert (dst / to_c.relative_to("/")).is_symlink() - # uid, gid can't be converted so original uid, gid is used. - assert_uid_gid_mode(dst / to_c.relative_to("/"), 12345678, 87654321, 0o777) - - # src/A/B/to_broken_c - assert not (dst / to_broken_c.relative_to("/")).is_file() - assert (dst / to_broken_c.relative_to("/")).is_symlink() - assert_uid_gid_mode(dst / to_broken_c.relative_to("/"), 1104, 2104, 0o777) - - # src/A/B/C/ - assert (dst / C.relative_to("/")).is_dir() - assert not (dst / C.relative_to("/")).is_symlink() - assert_uid_gid_mode(dst / C.relative_to("/"), 1105, 2105, 0o126) - - # followings should not exist - # src/a - assert not (dst / a.relative_to("/")).exists() - assert not (dst / a.relative_to("/")).is_symlink() - # src/to_a - assert not (dst / to_a.relative_to("/")).exists() - assert not (dst / to_a.relative_to("/")).is_symlink() - # src/to_broken_a - assert not (dst / to_broken_a.relative_to("/")).exists() - assert not (dst / to_broken_a.relative_to("/")).is_symlink() - # src/A/b - assert not (dst / b.relative_to("/")).exists() - assert not (dst / b.relative_to("/")).is_symlink() - # src/A/to_b - assert not (dst / to_b.relative_to("/")).exists() - assert not (dst / to_b.relative_to("/")).is_symlink() - # src/A/to_broken_b - assert not (dst / to_broken_b.relative_to("/")).exists() - assert not (dst / to_broken_b.relative_to("/")).is_symlink() diff --git a/tests/test_create_standby/test_persist_files_handling.py b/tests/test_create_standby/test_persist_files_handling.py index 88789b370..6f6d16508 100644 --- a/tests/test_create_standby/test_persist_files_handling.py +++ b/tests/test_create_standby/test_persist_files_handling.py @@ -331,7 +331,7 @@ def test_copy_tree_src_dir(mocker, tmp_path): dst_group_file=dst_group_file, src_root=src, dst_root=dst, - ).preserve_persist_entry(persist_entry, skip_invalid=False) + ).preserve_persist_entry(persist_entry) # src/A assert (dst / A.relative_to(src)).is_dir() @@ -512,15 +512,13 @@ def test_copy_tree_B_exists(mocker, tmp_path): assert (dst / A.relative_to(src)).is_dir() assert not (dst / A.relative_to(src)).is_symlink() # 'A' is created by this function before hand - # NOTE(20231222): if src dir exists in the dest, we should - # remove the dst and preserve src to dest. - assert_uid_gid_mode(dst / A.relative_to(src), 18, 29, 0o115) + assert_uid_gid_mode(dst / A.relative_to(src), 0, 1, 0o765) # src/A/B assert (dst / B.relative_to(src)).is_dir() assert not (dst / B.relative_to(src)).is_symlink() # 'B' is created by this function before hand - assert_uid_gid_mode(dst / B.relative_to(src), 141, 265534, 0o122) + assert_uid_gid_mode(dst / B.relative_to(src), 1, 2, 0o654) # src/A/B/C/ assert (dst / C.relative_to(src)).is_dir() From 1e725f590422fa043f156e770b6842f264012ba3 Mon Sep 17 00:00:00 2001 From: Bodong YANG Date: Tue, 26 Dec 2023 03:13:29 +0000 Subject: [PATCH 08/17] fix PersisFilesHandler --- otaclient/app/create_standby/common.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/otaclient/app/create_standby/common.py b/otaclient/app/create_standby/common.py index 8a76330d7..f81fd13a6 100644 --- a/otaclient/app/create_standby/common.py +++ b/otaclient/app/create_standby/common.py @@ -552,11 +552,15 @@ def _rm_target(_target: Path) -> None: return _target.unlink(missing_ok=True) elif _target.is_dir(): return shutil.rmtree(_target, ignore_errors=True) - raise ValueError(f"{_target} is not normal file/symlink/dir, failed to remove") + elif _target.exists(): + raise ValueError( + f"{_target} is not normal file/symlink/dir, failed to remove" + ) def _prepare_symlink(self, _src_path: Path, _dst_path: Path) -> None: _dst_path.symlink_to(os.readlink(_src_path)) - self._chown_with_mapping(_src_path.stat(), _dst_path) + # NOTE: to get stat from symlink, using os.stat with follow_symlinks=False + self._chown_with_mapping(os.stat(_src_path, follow_symlinks=False), _dst_path) def _prepare_dir(self, _src_path: Path, _dst_path: Path) -> None: _dst_path.mkdir(exist_ok=True) From c211a7a52ca7fd62faba5a98a39159431a97e8b7 Mon Sep 17 00:00:00 2001 From: Bodong YANG Date: Tue, 26 Dec 2023 03:38:44 +0000 Subject: [PATCH 09/17] rename test_create_standby.py to test_rebuild_mode.py --- .../{test_create_standby.py => test_rebuild_mode.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/test_create_standby/{test_create_standby.py => test_rebuild_mode.py} (100%) diff --git a/tests/test_create_standby/test_create_standby.py b/tests/test_create_standby/test_rebuild_mode.py similarity index 100% rename from tests/test_create_standby/test_create_standby.py rename to tests/test_create_standby/test_rebuild_mode.py From 983662974f539d4524b6718d228fc51d251cf418 Mon Sep 17 00:00:00 2001 From: Bodong YANG Date: Tue, 26 Dec 2023 03:46:32 +0000 Subject: [PATCH 10/17] remove duplicate test_create_standby --- tests/test_create_standby.py | 163 ----------------------------------- 1 file changed, 163 deletions(-) delete mode 100644 tests/test_create_standby.py diff --git a/tests/test_create_standby.py b/tests/test_create_standby.py deleted file mode 100644 index 67a267b33..000000000 --- a/tests/test_create_standby.py +++ /dev/null @@ -1,163 +0,0 @@ -# Copyright 2022 TIER IV, INC. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import shutil -import time -import typing -import pytest -from pathlib import Path -from pytest_mock import MockerFixture - -from otaclient.app.boot_control import BootControllerProtocol -from otaclient.configs.app_cfg import Config as otaclient_Config - -from tests.conftest import TestConfiguration as test_cfg -from tests.utils import SlotMeta, compare_dir - -import logging - -logger = logging.getLogger(__name__) - - -class Test_OTAupdate_with_create_standby_RebuildMode: - """ - NOTE: the boot_control is mocked, only testing - create_standby and the logics directly implemented by OTAUpdater. - - NOTE: testing the system using separated boot dev for each slots(like cboot). - """ - - @pytest.fixture - def setup_test(self, tmp_path: Path, ab_slots: SlotMeta): - # - # ------ prepare ab slots ------ # - # - self.slot_a = Path(ab_slots.slot_a) - self.slot_b = Path(ab_slots.slot_b) - self.slot_a_boot_dir = Path(ab_slots.slot_a_boot_dev) / "boot" - self.slot_b_boot_dir = Path(ab_slots.slot_b_boot_dev) / "boot" - self.ota_image_dir = Path(test_cfg.OTA_IMAGE_DIR) - - self.otaclient_run_dir = tmp_path / "otaclient_run_dir" - self.otaclient_run_dir.mkdir(parents=True, exist_ok=True) - - self.slot_a_boot_dir.mkdir(exist_ok=True, parents=True) - self.slot_b_boot_dir.mkdir(exist_ok=True, parents=True) - - # - # ------ prepare config ------ # - # - _otaclient_cfg = otaclient_Config(ACTIVE_ROOTFS=str(self.slot_a)) - self.otaclient_cfg = _otaclient_cfg - - # ------ prepare otaclient run dir ------ # - Path(_otaclient_cfg.RUN_DPATH).mkdir(exist_ok=True, parents=True) - - # - # ------ prepare mount space ------ # - # - Path(_otaclient_cfg.OTACLIENT_MOUNT_SPACE_DPATH).mkdir( - exist_ok=True, parents=True - ) - # directly point standby slot mp to self.slot_b - _standby_slot_mp = Path(_otaclient_cfg.STANDBY_SLOT_MP) - _standby_slot_mp.symlink_to(self.slot_b) - - # some important paths - self.ota_metafiles_tmp_dir = Path(_otaclient_cfg.STANDBY_IMAGE_META_DPATH) - self.ota_tmp_dir = Path(_otaclient_cfg.STANDBY_OTA_TMP_DPATH) - - yield - # cleanup slot_b after test - shutil.rmtree(self.slot_b, ignore_errors=True) - - @pytest.fixture(autouse=True) - def mock_setup(self, mocker: MockerFixture, setup_test): - # ------ mock boot_controller ------ # - self._boot_control = typing.cast( - BootControllerProtocol, mocker.MagicMock(spec=BootControllerProtocol) - ) - self._boot_control.get_standby_boot_dir.return_value = self.slot_b_boot_dir - - # ------ mock otaclient cfg ------ # - mocker.patch(f"{test_cfg.OTACLIENT_MODULE_PATH}.cfg", self.otaclient_cfg) - mocker.patch( - f"{test_cfg.CREATE_STANDBY_MODULE_PATH}.rebuild_mode.cfg", - self.otaclient_cfg, - ) - mocker.patch(f"{test_cfg.OTAMETA_MODULE_PATH}.cfg", self.otaclient_cfg) - - def test_update_with_create_standby_RebuildMode(self, mocker: MockerFixture): - from otaclient.app.ota_client import _OTAUpdater, OTAClientControlFlags - from otaclient.app.create_standby.rebuild_mode import RebuildMode - - # TODO: not test process_persistent currently, - # as we currently directly compare the standby slot - # with the OTA image. - RebuildMode._process_persistents = mocker.MagicMock() - - # ------ execution ------ # - otaclient_control_flags = typing.cast( - OTAClientControlFlags, mocker.MagicMock(spec=OTAClientControlFlags) - ) - _updater = _OTAUpdater( - boot_controller=self._boot_control, - create_standby_cls=RebuildMode, - proxy=None, - control_flags=otaclient_control_flags, - ) - # NOTE: mock the shutdown method as we need to assert before the - # updater is closed. - _updater_shutdown = _updater.shutdown - _updater.shutdown = mocker.MagicMock() - - _updater.execute( - version=test_cfg.UPDATE_VERSION, - raw_url_base=test_cfg.OTA_IMAGE_URL, - cookies_json=r'{"test": "my-cookie"}', - ) - time.sleep(2) # wait for downloader to record stats - - # ------ assertions ------ # - # --- assert update finished - _updater.shutdown.assert_called_once() - otaclient_control_flags.wait_can_reboot_flag.assert_called_once() - # --- ensure the update stats are collected - _snapshot = _updater._update_stats_collector.get_snapshot() - assert _snapshot.processed_files_num - assert _snapshot.processed_files_size - assert _snapshot.downloaded_files_num - assert _snapshot.downloaded_files_size - # assert _snapshot.downloaded_bytes - # assert _snapshot.downloading_elapsed_time.export_pb().ToNanoseconds() - assert _snapshot.update_applying_elapsed_time.export_pb().ToNanoseconds() - - # --- check slot creating result, ensure slot_a and slot_b is the same --- # - # NOTE: merge contents from slot_b_boot_dir to slot_b - shutil.copytree(self.slot_b_boot_dir, self.slot_b / "boot", dirs_exist_ok=True) - # NOTE: for some reason tmp dir is created under OTA_IMAGE_DIR/data, but not listed - # in the regulars.txt, so we create one here to make the test passed - (self.slot_b / "tmp").mkdir(exist_ok=True) - - # NOTE: remove the ota-meta dir and ota-tmp dir to resolve the difference with OTA image - shutil.rmtree(self.ota_metafiles_tmp_dir, ignore_errors=True) - shutil.rmtree(self.ota_tmp_dir, ignore_errors=True) - shutil.rmtree(self.slot_b / "opt/ota", ignore_errors=True) - - # --- check standby slot, ensure it is correctly populated - compare_dir(Path(test_cfg.OTA_IMAGE_DIR) / "data", self.slot_b) - - # ------ finally close the updater ------ # - _updater_shutdown() From 807f8b5d87f44b8cb8d9830a2632827a0f03d5a8 Mon Sep 17 00:00:00 2001 From: Bodong YANG Date: Tue, 26 Dec 2023 04:13:19 +0000 Subject: [PATCH 11/17] fix forgetting rm_target when processing files/dirs under src_dir --- otaclient/app/create_standby/common.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/otaclient/app/create_standby/common.py b/otaclient/app/create_standby/common.py index f81fd13a6..981f3190c 100644 --- a/otaclient/app/create_standby/common.py +++ b/otaclient/app/create_standby/common.py @@ -635,6 +635,7 @@ def preserve_persist_entry(self, _persist_entry: StrOrPath): # ------ prepare entries in current dir ------ # for _fname in fnames: _src_fpath, _dst_fpath = src_cur_dpath / _fname, dst_cur_dpath / _fname + self._rm_target(_dst_fpath) if _src_fpath.is_symlink(): self._prepare_symlink(_src_fpath, _dst_fpath) continue @@ -642,6 +643,7 @@ def preserve_persist_entry(self, _persist_entry: StrOrPath): # symlinks to dirs also included in dnames, we must handle it for _dname in dnames: - _src_dpath = src_cur_dpath / _dname + _src_dpath, _dst_dpath = src_cur_dpath / _dname, dst_cur_dpath / _dname if _src_dpath.is_symlink(): - self._prepare_symlink(_src_dpath, dst_cur_dpath / _dname) + self._rm_target(_dst_dpath) + self._prepare_symlink(_src_dpath, _dst_dpath) From 15d7f1440240c0e9669b59f13debe6472bb70206 Mon Sep 17 00:00:00 2001 From: Bodong YANG Date: Tue, 26 Dec 2023 04:21:25 +0000 Subject: [PATCH 12/17] PersistFilesHandler: add src_missing_ok=True --- otaclient/app/create_standby/common.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/otaclient/app/create_standby/common.py b/otaclient/app/create_standby/common.py index 981f3190c..25a9677de 100644 --- a/otaclient/app/create_standby/common.py +++ b/otaclient/app/create_standby/common.py @@ -591,7 +591,9 @@ def _prepare_parent(self, _src_path: Path, _dst_path: Path) -> None: # API - def preserve_persist_entry(self, _persist_entry: StrOrPath): + def preserve_persist_entry( + self, _persist_entry: StrOrPath, *, src_missing_ok: bool = True + ): logger.info( f"preserving {_persist_entry} from {self._src_root} to {self._dst_root}" ) @@ -617,10 +619,13 @@ def preserve_persist_entry(self, _persist_entry: StrOrPath): return # we only process normal file/symlink/dir - if not src_path.is_dir(): - raise ValueError( - f"{src_path=} must be presented and either a file/symlink/dir" - ) + if src_path.exists() and not src_path.is_dir(): + raise ValueError(f"{src_path=} must be either a file/symlink/dir") + if not src_path.exists(): + _err_msg = f"{src_path=} not found" + logger.warning(_err_msg) + if not src_missing_ok: + raise ValueError(_err_msg) # for src as dir, cleanup dst_dirc, # dive into src_dir and preserve everything under the src dir From 9a9d8aac102aa327a499f2b99c82e33fa462ad08 Mon Sep 17 00:00:00 2001 From: Bodong YANG Date: Tue, 26 Dec 2023 04:43:58 +0000 Subject: [PATCH 13/17] do not prepare_parents for non-exists src --- otaclient/app/create_standby/common.py | 39 +++++++++++++------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/otaclient/app/create_standby/common.py b/otaclient/app/create_standby/common.py index 25a9677de..5c3e8edce 100644 --- a/otaclient/app/create_standby/common.py +++ b/otaclient/app/create_standby/common.py @@ -576,45 +576,45 @@ def _prepare_file(self, _src_path: Path, _dst_path: Path) -> None: os.chmod(_dst_path, _src_stat.st_mode) self._chown_with_mapping(_src_stat, _dst_path) - def _prepare_parent(self, _src_path: Path, _dst_path: Path) -> None: - if _dst_path.is_dir(): # keep the origin parent on dst as it - return - if _dst_path.is_symlink() or _dst_path.is_file(): - _dst_path.unlink(missing_ok=True) - self._prepare_dir(_src_path, _dst_path) - return - if _dst_path.exists(): - raise ValueError( - f"{_dst_path=} is not a normal file/symlink/dir, cannot cleanup" + def _prepare_parent(self, _origin_entry: Path) -> None: + for _parent in reversed(_origin_entry.parents): + _src_parent, _dst_parent = ( + self._src_root / _parent, + self._dst_root / _parent, ) - self._prepare_dir(_src_path, _dst_path) + if _dst_parent.is_dir(): # keep the origin parent on dst as it + continue + if _dst_parent.is_symlink() or _dst_parent.is_file(): + _dst_parent.unlink(missing_ok=True) + self._prepare_dir(_src_parent, _dst_parent) + continue + if _dst_parent.exists(): + raise ValueError( + f"{_dst_parent=} is not a normal file/symlink/dir, cannot cleanup" + ) + self._prepare_dir(_src_parent, _dst_parent) # API def preserve_persist_entry( self, _persist_entry: StrOrPath, *, src_missing_ok: bool = True ): - logger.info( - f"preserving {_persist_entry} from {self._src_root} to {self._dst_root}" - ) + logger.info(f"preserving {_persist_entry}") # persist_entry in persists.txt must be rooted at / origin_entry = Path(_persist_entry).relative_to(cfg.DEFAULT_ACTIVE_ROOTFS) src_path = self._src_root / origin_entry dst_path = self._dst_root / origin_entry - # ------ prepare parents ------ # - for _parent in reversed(origin_entry.parents): - self._prepare_parent(self._src_root / _parent, self._dst_root / _parent) - - # ------ prepare entry itself ------ # # NOTE: always check if symlink first! if src_path.is_symlink(): self._rm_target(dst_path) + self._prepare_parent(origin_entry) self._prepare_symlink(src_path, dst_path) return if src_path.is_file(): self._rm_target(dst_path) + self._prepare_parent(origin_entry) self._prepare_file(src_path, dst_path) return @@ -629,6 +629,7 @@ def preserve_persist_entry( # for src as dir, cleanup dst_dirc, # dive into src_dir and preserve everything under the src dir + self._prepare_parent(origin_entry) for src_curdir, dnames, fnames in os.walk(src_path, followlinks=False): src_cur_dpath = Path(src_curdir) dst_cur_dpath = self._dst_root / src_cur_dpath.relative_to(self._src_root) From d66b68ea0ff712fe5d44d2a1ee29d4439b258377 Mon Sep 17 00:00:00 2001 From: Bodong YANG Date: Tue, 26 Dec 2023 04:45:03 +0000 Subject: [PATCH 14/17] add missing return when src is not existed --- otaclient/app/create_standby/common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/otaclient/app/create_standby/common.py b/otaclient/app/create_standby/common.py index 5c3e8edce..a9cd2b803 100644 --- a/otaclient/app/create_standby/common.py +++ b/otaclient/app/create_standby/common.py @@ -626,6 +626,7 @@ def preserve_persist_entry( logger.warning(_err_msg) if not src_missing_ok: raise ValueError(_err_msg) + return # for src as dir, cleanup dst_dirc, # dive into src_dir and preserve everything under the src dir From 083da4bf91b8610299bfc246e945ee052b9f6874 Mon Sep 17 00:00:00 2001 From: Bodong YANG Date: Tue, 26 Dec 2023 04:52:53 +0000 Subject: [PATCH 15/17] minor comments --- otaclient/app/create_standby/common.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/otaclient/app/create_standby/common.py b/otaclient/app/create_standby/common.py index a9cd2b803..cf1b6b0e5 100644 --- a/otaclient/app/create_standby/common.py +++ b/otaclient/app/create_standby/common.py @@ -605,22 +605,27 @@ def preserve_persist_entry( src_path = self._src_root / origin_entry dst_path = self._dst_root / origin_entry - # NOTE: always check if symlink first! + # ------ src is symlink ------ # + # NOTE: always check if symlink first as is_file/is_dir/exists all follow_symlinks if src_path.is_symlink(): self._rm_target(dst_path) self._prepare_parent(origin_entry) self._prepare_symlink(src_path, dst_path) return + # ------ src is file ------ # if src_path.is_file(): self._rm_target(dst_path) self._prepare_parent(origin_entry) self._prepare_file(src_path, dst_path) return + # ------ src is not regular file/symlink/dir ------ # # we only process normal file/symlink/dir if src_path.exists() and not src_path.is_dir(): raise ValueError(f"{src_path=} must be either a file/symlink/dir") + + # ------ src doesn't exist ------ # if not src_path.exists(): _err_msg = f"{src_path=} not found" logger.warning(_err_msg) @@ -628,7 +633,7 @@ def preserve_persist_entry( raise ValueError(_err_msg) return - # for src as dir, cleanup dst_dirc, + # ------ src is dir ------ # # dive into src_dir and preserve everything under the src dir self._prepare_parent(origin_entry) for src_curdir, dnames, fnames in os.walk(src_path, followlinks=False): From 6ebe9bcefa11efafd1a1255377654e6fa1870164 Mon Sep 17 00:00:00 2001 From: Bodong YANG Date: Tue, 26 Dec 2023 05:45:18 +0000 Subject: [PATCH 16/17] add logging for uid/gid mapping result --- otaclient/app/create_standby/common.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/otaclient/app/create_standby/common.py b/otaclient/app/create_standby/common.py index cf1b6b0e5..41f05a752 100644 --- a/otaclient/app/create_standby/common.py +++ b/otaclient/app/create_standby/common.py @@ -46,8 +46,8 @@ from otaclient._utils.unix import ( ParsedPasswd, ParsedGroup, - map_gid_by_grpnam, - map_uid_by_pwnam, + map_gid_by_grpnam as _utils_map_gid_by_grpnam, + map_uid_by_pwnam as _utils_map_uid_by_pwnam, ) from ..common import create_tmp_fname from ..configs import config as cfg @@ -513,14 +513,14 @@ def __init__( ): self._uid_mapper = functools.lru_cache()( functools.partial( - map_uid_by_pwnam, + self.map_uid_by_pwnam, src_db=ParsedPasswd(src_passwd_file), dst_db=ParsedPasswd(dst_passwd_file), ) ) self._gid_mapper = functools.lru_cache()( functools.partial( - map_gid_by_grpnam, + self.map_gid_by_grpnam, src_db=ParsedGroup(src_group_file), dst_db=ParsedGroup(dst_group_file), ) @@ -528,6 +528,24 @@ def __init__( self._src_root = Path(src_root) self._dst_root = Path(dst_root) + @staticmethod + def map_uid_by_pwnam( + *, src_db: ParsedPasswd, dst_db: ParsedPasswd, uid: int + ) -> int: + _mapped_uid = _utils_map_uid_by_pwnam(src_db=src_db, dst_db=dst_db, uid=uid) + _usern = src_db._by_uid[uid] + + logger.info(f"{_usern=}: mapping src_{uid=} to {_mapped_uid}") + return _mapped_uid + + @staticmethod + def map_gid_by_grpnam(*, src_db: ParsedGroup, dst_db: ParsedGroup, gid: int) -> int: + _mapped_gid = _utils_map_gid_by_grpnam(src_db=src_db, dst_db=dst_db, gid=gid) + _groupn = src_db._by_gid[gid] + + logger.info(f"{_groupn=}: mapping src_{gid=} to {_mapped_gid=}") + return _mapped_gid + def _chown_with_mapping( self, _src_stat: os.stat_result, _dst_path: StrOrPath ) -> None: From cf40cd77ab7a4a9e9709239a71731741cbf74206 Mon Sep 17 00:00:00 2001 From: Bodong YANG Date: Tue, 26 Dec 2023 05:46:58 +0000 Subject: [PATCH 17/17] minor change --- otaclient/app/create_standby/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/otaclient/app/create_standby/common.py b/otaclient/app/create_standby/common.py index 41f05a752..09faab545 100644 --- a/otaclient/app/create_standby/common.py +++ b/otaclient/app/create_standby/common.py @@ -535,7 +535,7 @@ def map_uid_by_pwnam( _mapped_uid = _utils_map_uid_by_pwnam(src_db=src_db, dst_db=dst_db, uid=uid) _usern = src_db._by_uid[uid] - logger.info(f"{_usern=}: mapping src_{uid=} to {_mapped_uid}") + logger.info(f"{_usern=}: mapping src_{uid=} to {_mapped_uid=}") return _mapped_uid @staticmethod