diff --git a/otaclient/_utils/unix.py b/otaclient/_utils/unix.py new file mode 100644 index 000000000..7ac1adc39 --- /dev/null +++ b/otaclient/_utils/unix.py @@ -0,0 +1,79 @@ +from __future__ import annotations +from .typing import StrOrPath + +_SPLITTER = ":" + + +class ParsedPasswd: + """Parse passwd and store name/uid mapping. + + Example passwd entry line: + nobody:x:65534:65534:nobody:/nonexistent:/usr/sbin/nologin + + Attrs: + _by_name (dict[str, int]): name:uid mapping. + _by_uid (dict[int, str]): uid:name mapping. + """ + + __slots__ = ["_by_name", "_by_uid"] + + def __init__(self, passwd_fpath: StrOrPath) -> None: + self._by_name: dict[str, int] = {} + try: + with open(passwd_fpath, "r") as f: + for line in f: + _raw_list = line.strip().split(_SPLITTER) + _name, _uid = _raw_list[0], int(_raw_list[2]) + self._by_name[_name] = _uid + self._by_uid = {v: k for k, v in self._by_name.items()} + except Exception as e: + raise ValueError(f"invalid or missing {passwd_fpath=}: {e!r}") + + +class ParsedGroup: + """Parse group and store name/gid mapping. + + Example group entry line: + nogroup:x:65534: + + Attrs: + _by_name (dict[str, int]): name:gid mapping. + _by_gid (dict[int, str]): gid:name mapping. + """ + + __slots__ = ["_by_name", "_by_gid"] + + def __init__(self, group_fpath: StrOrPath) -> None: + self._by_name: dict[str, int] = {} + try: + with open(group_fpath, "r") as f: + for line in f: + _raw_list = line.strip().split(_SPLITTER) + self._by_name[_raw_list[0]] = int(_raw_list[2]) + self._by_gid = {v: k for k, v in self._by_name.items()} + except Exception as e: + raise ValueError(f"invalid or missing {group_fpath=}: {e!r}") + + +def map_uid_by_pwnam(*, src_db: ParsedPasswd, dst_db: ParsedPasswd, uid: int) -> int: + """Perform src_uid -> src_name -> dst_name -> dst_uid mapping. + + Raises: + ValueError on failed mapping. + """ + try: + return dst_db._by_name[src_db._by_uid[uid]] + except KeyError: + raise ValueError(f"failed to find mapping for {uid}") + + +def map_gid_by_grpnam(*, src_db: ParsedGroup, dst_db: ParsedGroup, gid: int) -> int: + """Perform src_gid -> src_name -> dst_name -> dst_gid mapping. + + Raises: + ValueError on failed mapping. + """ + try: + return dst_db._by_name[src_db._by_gid[gid]] + except KeyError: + raise ValueError(f"failed to find mapping for {gid}") diff --git a/otaclient/app/common.py b/otaclient/app/common.py index 15c6808fb..270f4e46b 100644 --- a/otaclient/app/common.py +++ b/otaclient/app/common.py @@ -14,6 +14,7 @@ r"""Utils that shared between modules are listed here.""" +from __future__ import annotations import itertools import os import shlex diff --git a/otaclient/app/copy_tree.py b/otaclient/app/copy_tree.py deleted file mode 100644 index 8e32beb17..000000000 --- a/otaclient/app/copy_tree.py +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright 2022 TIER IV, INC. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import os -import stat -import shutil -from pathlib import Path - -from . import log_setting - -logger = log_setting.get_logger(__name__) - - -class CopyTree: - def __init__( - self, - src_passwd_file: Path, - src_group_file: Path, - dst_passwd_file: Path, - dst_group_file: Path, - ): - self._src_passwd = self._id_to_name_dict(src_passwd_file) - self._src_group = self._id_to_name_dict(src_group_file) - self._dst_passwd = self._name_to_id_dict(dst_passwd_file) - self._dst_group = self._name_to_id_dict(dst_group_file) - - def copy_with_parents(self, src: Path, dst_dir: Path): - dst_path = dst_dir - if not dst_path.is_dir() or dst_path.is_symlink(): - raise ValueError(f"{dst_path} should be plain directory") - - for parent in reversed(list(src.parents)): - self._copy_preserve(parent, dst_path) - dst_path = dst_path / parent.name - self._copy_recursive(src, dst_path) - - """ private functions from here """ - - def _id_to_name_dict(self, passwd_or_group_file: Path): - lines = open(passwd_or_group_file).readlines() - entries = {} - for line in lines: - e = line.split(":") - entries[e[2]] = e[0] # {uid: name} - return entries - - def _name_to_id_dict(self, passwd_or_group_file: Path): - lines = open(passwd_or_group_file).readlines() - entries = {} - for line in lines: - e = line.split(":") - entries[e[0]] = e[2] # {name: uid} - return entries - - # src_uid -> src_name -> dst_name -> dst_uid - # src_gid -> src_name -> dst_name -> dst_gid - def _convert_src_id_to_dst_id(self, src_uid, src_gid): - logger.info(f"src_uid: {src_uid}, src_gid: {src_gid}") - user = self._src_passwd[str(src_uid)] - uid = self._dst_passwd[user] - group = self._src_group[str(src_gid)] - gid = self._dst_group[group] - logger.info(f"dst_uid: {uid}, dst_gid: {gid}") - return int(uid), int(gid) - - def _copy_stat(self, src, dst): - st = os.stat(src, follow_symlinks=False) - try: - dst_uid, dst_gid = self._convert_src_id_to_dst_id( - st[stat.ST_UID], st[stat.ST_GID] - ) - except KeyError: # In case src UID/GID not found, keep UID/GID as is. - logger.warning(f"uid: {st[stat.ST_UID]}, gid: {st[stat.ST_GID]} not found") - dst_uid, dst_gid = st[stat.ST_UID], st[stat.ST_GID] - os.chown(dst, dst_uid, dst_gid, follow_symlinks=False) - if not dst.is_symlink(): # symlink always 777 - os.chmod(dst, st[stat.ST_MODE]) - - def _copy_preserve(self, src: Path, dst_dir: Path): - if not dst_dir.is_dir() or dst_dir.is_symlink(): - raise ValueError(f"{dst_dir} should be plain directory") - - dst_path = dst_dir / src.name - # src is plain directory? - if src.is_dir() and not src.is_symlink(): - # if plain file or symlink (which links to file or directory) - if dst_path.is_file() or dst_path.is_symlink(): - logger.info(f"{src}: {dst_path} exists but is not a directory") - dst_path.unlink() - if dst_path.is_dir(): # dst_path exists as a directory - return # keep it untouched - logger.info(f"creating directory {dst_path}") - dst_path.mkdir() - self._copy_stat(src, dst_path) - # src is plain file or symlink - elif src.is_file() or src.is_symlink(): # includes broken symlink - if dst_path.is_symlink() or dst_path.is_file(): - # When source is symlink, shutil.copy2 fails to overwrite destination file. - # When destination is symlink, shutil.copy2 copy src file under destination. - # To avoid these cases, remove destination file beforehand. - dst_path.unlink() - if dst_path.is_dir(): - logger.info(f"{src}: {dst_path} exists as a directory") - shutil.rmtree(dst_path, ignore_errors=True) - - logger.info(f"copying file {dst_dir / src.name}") - shutil.copy2(src, dst_path, follow_symlinks=False) - self._copy_stat(src, dst_path) - else: - raise ValueError(f"{src} unintended file type") - - def _copy_recursive(self, src: Path, dst_dir: Path): - self._copy_preserve(src, dst_dir) - if src.is_dir() and not src.is_symlink(): - for src_child in src.glob("*"): - self._copy_recursive(src_child, dst_dir / src.name) diff --git a/otaclient/app/create_standby/common.py b/otaclient/app/create_standby/common.py index b7adc89c5..09faab545 100644 --- a/otaclient/app/create_standby/common.py +++ b/otaclient/app/create_standby/common.py @@ -15,9 +15,11 @@ from __future__ import annotations +import functools import os import random import time +import shutil from concurrent.futures import ( Future, ThreadPoolExecutor, @@ -40,6 +42,13 @@ ) from weakref import WeakKeyDictionary, WeakValueDictionary +from otaclient._utils.typing import StrOrPath +from otaclient._utils.unix import ( + ParsedPasswd, + ParsedGroup, + map_gid_by_grpnam as _utils_map_gid_by_grpnam, + map_uid_by_pwnam as _utils_map_uid_by_pwnam, +) from ..common import create_tmp_fname from ..configs import config as cfg from ..ota_metadata import OTAMetadata, MetafilesV1 @@ -480,3 +489,191 @@ def calculate_and_process_delta(self) -> DeltaBundle: total_regular_num=self.total_regulars_num, total_download_files_size=self.total_download_files_size, ) + + +class PersistFilesHandler: + """Preserving files in persist list from to . + + Files being copied will have mode bits preserved, + and uid/gid preserved with mapping as follow: + + src_uid -> src_name -> dst_name -> dst_uid + src_gid -> src_name -> dst_name -> dst_gid + """ + + def __init__( + self, + src_passwd_file: StrOrPath, + src_group_file: StrOrPath, + dst_passwd_file: StrOrPath, + dst_group_file: StrOrPath, + *, + src_root: StrOrPath, + dst_root: StrOrPath, + ): + self._uid_mapper = functools.lru_cache()( + functools.partial( + self.map_uid_by_pwnam, + src_db=ParsedPasswd(src_passwd_file), + dst_db=ParsedPasswd(dst_passwd_file), + ) + ) + self._gid_mapper = functools.lru_cache()( + functools.partial( + self.map_gid_by_grpnam, + src_db=ParsedGroup(src_group_file), + dst_db=ParsedGroup(dst_group_file), + ) + ) + self._src_root = Path(src_root) + self._dst_root = Path(dst_root) + + @staticmethod + def map_uid_by_pwnam( + *, src_db: ParsedPasswd, dst_db: ParsedPasswd, uid: int + ) -> int: + _mapped_uid = _utils_map_uid_by_pwnam(src_db=src_db, dst_db=dst_db, uid=uid) + _usern = src_db._by_uid[uid] + + logger.info(f"{_usern=}: mapping src_{uid=} to {_mapped_uid=}") + return _mapped_uid + + @staticmethod + def map_gid_by_grpnam(*, src_db: ParsedGroup, dst_db: ParsedGroup, gid: int) -> int: + _mapped_gid = _utils_map_gid_by_grpnam(src_db=src_db, dst_db=dst_db, gid=gid) + _groupn = src_db._by_gid[gid] + + logger.info(f"{_groupn=}: mapping src_{gid=} to {_mapped_gid=}") + return _mapped_gid + + def _chown_with_mapping( + self, _src_stat: os.stat_result, _dst_path: StrOrPath + ) -> None: + _src_uid, _src_gid = _src_stat.st_uid, _src_stat.st_gid + try: + _dst_uid = self._uid_mapper(uid=_src_uid) + except ValueError: + logger.warning(f"failed to find mapping for {_src_uid=}, keep unchanged") + _dst_uid = _src_uid + + try: + _dst_gid = self._gid_mapper(gid=_src_gid) + except ValueError: + logger.warning(f"failed to find mapping for {_src_gid=}, keep unchanged") + _dst_gid = _src_gid + os.chown(_dst_path, uid=_dst_uid, gid=_dst_gid, follow_symlinks=False) + + @staticmethod + def _rm_target(_target: Path) -> None: + """Remove target with proper methods.""" + if _target.is_symlink() or _target.is_file(): + return _target.unlink(missing_ok=True) + elif _target.is_dir(): + return shutil.rmtree(_target, ignore_errors=True) + elif _target.exists(): + raise ValueError( + f"{_target} is not normal file/symlink/dir, failed to remove" + ) + + def _prepare_symlink(self, _src_path: Path, _dst_path: Path) -> None: + _dst_path.symlink_to(os.readlink(_src_path)) + # NOTE: to get stat from symlink, using os.stat with follow_symlinks=False + self._chown_with_mapping(os.stat(_src_path, follow_symlinks=False), _dst_path) + + def _prepare_dir(self, _src_path: Path, _dst_path: Path) -> None: + _dst_path.mkdir(exist_ok=True) + + _src_stat = os.stat(_src_path, follow_symlinks=False) + os.chmod(_dst_path, _src_stat.st_mode) + self._chown_with_mapping(_src_stat, _dst_path) + + def _prepare_file(self, _src_path: Path, _dst_path: Path) -> None: + shutil.copy(_src_path, _dst_path, follow_symlinks=False) + + _src_stat = os.stat(_src_path, follow_symlinks=False) + os.chmod(_dst_path, _src_stat.st_mode) + self._chown_with_mapping(_src_stat, _dst_path) + + def _prepare_parent(self, _origin_entry: Path) -> None: + for _parent in reversed(_origin_entry.parents): + _src_parent, _dst_parent = ( + self._src_root / _parent, + self._dst_root / _parent, + ) + if _dst_parent.is_dir(): # keep the origin parent on dst as it + continue + if _dst_parent.is_symlink() or _dst_parent.is_file(): + _dst_parent.unlink(missing_ok=True) + self._prepare_dir(_src_parent, _dst_parent) + continue + if _dst_parent.exists(): + raise ValueError( + f"{_dst_parent=} is not a normal file/symlink/dir, cannot cleanup" + ) + self._prepare_dir(_src_parent, _dst_parent) + + # API + + def preserve_persist_entry( + self, _persist_entry: StrOrPath, *, src_missing_ok: bool = True + ): + logger.info(f"preserving {_persist_entry}") + # persist_entry in persists.txt must be rooted at / + origin_entry = Path(_persist_entry).relative_to(cfg.DEFAULT_ACTIVE_ROOTFS) + src_path = self._src_root / origin_entry + dst_path = self._dst_root / origin_entry + + # ------ src is symlink ------ # + # NOTE: always check if symlink first as is_file/is_dir/exists all follow_symlinks + if src_path.is_symlink(): + self._rm_target(dst_path) + self._prepare_parent(origin_entry) + self._prepare_symlink(src_path, dst_path) + return + + # ------ src is file ------ # + if src_path.is_file(): + self._rm_target(dst_path) + self._prepare_parent(origin_entry) + self._prepare_file(src_path, dst_path) + return + + # ------ src is not regular file/symlink/dir ------ # + # we only process normal file/symlink/dir + if src_path.exists() and not src_path.is_dir(): + raise ValueError(f"{src_path=} must be either a file/symlink/dir") + + # ------ src doesn't exist ------ # + if not src_path.exists(): + _err_msg = f"{src_path=} not found" + logger.warning(_err_msg) + if not src_missing_ok: + raise ValueError(_err_msg) + return + + # ------ src is dir ------ # + # dive into src_dir and preserve everything under the src dir + self._prepare_parent(origin_entry) + for src_curdir, dnames, fnames in os.walk(src_path, followlinks=False): + src_cur_dpath = Path(src_curdir) + dst_cur_dpath = self._dst_root / src_cur_dpath.relative_to(self._src_root) + + # ------ prepare current dir itself ------ # + self._rm_target(dst_cur_dpath) + self._prepare_dir(src_cur_dpath, dst_cur_dpath) + + # ------ prepare entries in current dir ------ # + for _fname in fnames: + _src_fpath, _dst_fpath = src_cur_dpath / _fname, dst_cur_dpath / _fname + self._rm_target(_dst_fpath) + if _src_fpath.is_symlink(): + self._prepare_symlink(_src_fpath, _dst_fpath) + continue + self._prepare_file(_src_fpath, _dst_fpath) + + # symlinks to dirs also included in dnames, we must handle it + for _dname in dnames: + _src_dpath, _dst_dpath = src_cur_dpath / _dname, dst_cur_dpath / _dname + if _src_dpath.is_symlink(): + self._rm_target(_dst_dpath) + self._prepare_symlink(_src_dpath, _dst_dpath) diff --git a/otaclient/app/create_standby/rebuild_mode.py b/otaclient/app/create_standby/rebuild_mode.py index 1decb47c1..0985e546f 100644 --- a/otaclient/app/create_standby/rebuild_mode.py +++ b/otaclient/app/create_standby/rebuild_mode.py @@ -32,7 +32,7 @@ from ..proto.wrapper import RegularInf from .. import log_setting -from .common import HardlinkRegister, DeltaGenerator, DeltaBundle +from .common import HardlinkRegister, DeltaGenerator, DeltaBundle, PersistFilesHandler from .interface import StandbySlotCreatorProtocol logger = log_setting.get_logger(__name__) @@ -78,10 +78,7 @@ def _process_dirs(self): entry.mkdir_relative_to_mount_point(self.standby_slot_mp) def _process_persistents(self): - """NOTE: just copy from legacy mode""" - from ..copy_tree import CopyTree - - _copy_tree = CopyTree( + _handler = PersistFilesHandler( src_passwd_file=Path(cfg.PASSWD_FPATH), src_group_file=Path(cfg.GROUP_FPATH), dst_passwd_file=Path( @@ -90,16 +87,12 @@ def _process_persistents(self): dst_group_file=Path( replace_root(cfg.GROUP_FPATH, cfg.ACTIVE_ROOTFS, cfg.STANDBY_SLOT_MP) ), + src_root=self.active_slot_mp, + dst_root=self.standby_slot_mp, ) for _perinf in self._ota_metadata.iter_metafile(MetafilesV1.PERSISTENT_FNAME): - _perinf_path = Path(_perinf.path) - if ( - _perinf_path.is_file() - or _perinf_path.is_dir() - or _perinf_path.is_symlink() - ): # NOTE: not equivalent to perinf.path.exists() - _copy_tree.copy_with_parents(_perinf_path, self.standby_slot_mp) + _handler.preserve_persist_entry(_perinf.path) def _process_symlinks(self): for _symlink in self._ota_metadata.iter_metafile( diff --git a/tests/test_create_standby/__init__.py b/tests/test_create_standby/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_copy_tree.py b/tests/test_create_standby/test_persist_files_handling.py similarity index 55% rename from tests/test_copy_tree.py rename to tests/test_create_standby/test_persist_files_handling.py index 2a115ff46..6f6d16508 100644 --- a/tests/test_copy_tree.py +++ b/tests/test_create_standby/test_persist_files_handling.py @@ -13,13 +13,20 @@ # limitations under the License. +from __future__ import annotations import os import stat -import pytest from pathlib import Path +from otaclient._utils.path import replace_root +from otaclient.app.create_standby.common import PersistFilesHandler + def create_files(tmp_path: Path): + """ + 20231222: this function create and folders as preserved src rootfs and save destination rootfs. + """ + dst = tmp_path / "dst" dst.mkdir() @@ -291,8 +298,6 @@ def create_passwd_group_files(tmp_path): def test_copy_tree_src_dir(mocker, tmp_path): - from otaclient.app.copy_tree import CopyTree - ( dst, src, @@ -317,65 +322,70 @@ def test_copy_tree_src_dir(mocker, tmp_path): dst_group_file, ) = create_passwd_group_files(tmp_path) - CopyTree( - src_passwd_file, src_group_file, dst_passwd_file, dst_group_file - ).copy_with_parents(B, dst) + # NOTE: persist entry must be canonical and starts with / + persist_entry = replace_root(B, src, "/") + PersistFilesHandler( + src_passwd_file=src_passwd_file, + src_group_file=src_group_file, + dst_passwd_file=dst_passwd_file, + dst_group_file=dst_group_file, + src_root=src, + dst_root=dst, + ).preserve_persist_entry(persist_entry) # src/A - assert (dst / A.relative_to("/")).is_dir() - assert not (dst / A.relative_to("/")).is_symlink() - assert_uid_gid_mode(dst / A.relative_to("/"), 18, 29, 0o115) + assert (dst / A.relative_to(src)).is_dir() + assert not (dst / A.relative_to(src)).is_symlink() + assert_uid_gid_mode(dst / A.relative_to(src), 18, 29, 0o115) # src/A/B/ - assert (dst / B.relative_to("/")).is_dir() - assert not (dst / B.relative_to("/")).is_symlink() - assert_uid_gid_mode(dst / B.relative_to("/"), 141, 265534, 0o122) + assert (dst / B.relative_to(src)).is_dir() + assert not (dst / B.relative_to(src)).is_symlink() + assert_uid_gid_mode(dst / B.relative_to(src), 141, 265534, 0o122) # src/A/B/c - assert (dst / c.relative_to("/")).is_file() - assert not (dst / c.relative_to("/")).is_symlink() - assert_uid_gid_mode(dst / c.relative_to("/"), 1100, 2102, 0o123) + assert (dst / c.relative_to(src)).is_file() + assert not (dst / c.relative_to(src)).is_symlink() + assert_uid_gid_mode(dst / c.relative_to(src), 1100, 2102, 0o123) # src/A/B/to_c - assert (dst / to_c.relative_to("/")).is_file() - assert (dst / to_c.relative_to("/")).is_symlink() + assert (dst / to_c.relative_to(src)).is_file() + assert (dst / to_c.relative_to(src)).is_symlink() # uid, gid can't be converted so original uid, gid is used. - assert_uid_gid_mode(dst / to_c.relative_to("/"), 12345678, 87654321, 0o777) + assert_uid_gid_mode(dst / to_c.relative_to(src), 12345678, 87654321, 0o777) # src/A/B/to_broken_c - assert not (dst / to_broken_c.relative_to("/")).is_file() - assert (dst / to_broken_c.relative_to("/")).is_symlink() - assert_uid_gid_mode(dst / to_broken_c.relative_to("/"), 1104, 2104, 0o777) + assert not (dst / to_broken_c.relative_to(src)).is_file() + assert (dst / to_broken_c.relative_to(src)).is_symlink() + assert_uid_gid_mode(dst / to_broken_c.relative_to(src), 1104, 2104, 0o777) # src/A/B/C/ - assert (dst / C.relative_to("/")).is_dir() - assert not (dst / C.relative_to("/")).is_symlink() - assert_uid_gid_mode(dst / C.relative_to("/"), 1105, 2105, 0o126) + assert (dst / C.relative_to(src)).is_dir() + assert not (dst / C.relative_to(src)).is_symlink() + assert_uid_gid_mode(dst / C.relative_to(src), 1105, 2105, 0o126) # followings should not exist # src/a - assert not (dst / a.relative_to("/")).exists() - assert not (dst / a.relative_to("/")).is_symlink() + assert not (dst / a.relative_to(src)).exists() + assert not (dst / a.relative_to(src)).is_symlink() # src/to_a - assert not (dst / to_a.relative_to("/")).exists() - assert not (dst / to_a.relative_to("/")).is_symlink() + assert not (dst / to_a.relative_to(src)).exists() + assert not (dst / to_a.relative_to(src)).is_symlink() # src/to_broken_a - assert not (dst / to_broken_a.relative_to("/")).exists() - assert not (dst / to_broken_a.relative_to("/")).is_symlink() + assert not (dst / to_broken_a.relative_to(src)).exists() + assert not (dst / to_broken_a.relative_to(src)).is_symlink() # src/A/b - assert not (dst / b.relative_to("/")).exists() - assert not (dst / b.relative_to("/")).is_symlink() + assert not (dst / b.relative_to(src)).exists() + assert not (dst / b.relative_to(src)).is_symlink() # src/A/to_b - assert not (dst / to_b.relative_to("/")).exists() - assert not (dst / to_b.relative_to("/")).is_symlink() + assert not (dst / to_b.relative_to(src)).exists() + assert not (dst / to_b.relative_to(src)).is_symlink() # src/A/to_broken_b - assert not (dst / to_broken_b.relative_to("/")).exists() - assert not (dst / to_broken_b.relative_to("/")).is_symlink() + assert not (dst / to_broken_b.relative_to(src)).exists() + assert not (dst / to_broken_b.relative_to(src)).is_symlink() def test_copy_tree_src_file(mocker, tmp_path): - from otaclient.app.copy_tree import CopyTree - ( dst, src, @@ -400,56 +410,59 @@ def test_copy_tree_src_file(mocker, tmp_path): dst_group_file, ) = create_passwd_group_files(tmp_path) - CopyTree( - src_passwd_file, src_group_file, dst_passwd_file, dst_group_file - ).copy_with_parents(to_b, dst) + PersistFilesHandler( + src_passwd_file=src_passwd_file, + src_group_file=src_group_file, + dst_passwd_file=dst_passwd_file, + dst_group_file=dst_group_file, + src_root=src, + dst_root=dst, + ).preserve_persist_entry(replace_root(to_b, src, "/")) # src/A - assert (dst / A.relative_to("/")).is_dir() - assert not (dst / A.relative_to("/")).is_symlink() - assert_uid_gid_mode(dst / A.relative_to("/"), 18, 29, 0o115) + assert (dst / A.relative_to(src)).is_dir() + assert not (dst / A.relative_to(src)).is_symlink() + assert_uid_gid_mode(dst / A.relative_to(src), 18, 29, 0o115) # src/A/to_b (src/A/b is not copied, so to_b.is_file() is False) - assert not (dst / to_b.relative_to("/")).is_file() - assert (dst / to_b.relative_to("/")).is_symlink() - assert_uid_gid_mode(dst / to_b.relative_to("/"), 133, 234, 0o777) + assert not (dst / to_b.relative_to(src)).is_file() + assert (dst / to_b.relative_to(src)).is_symlink() + assert_uid_gid_mode(dst / to_b.relative_to(src), 133, 234, 0o777) # followings should not exist # src/a - assert not (dst / a.relative_to("/")).exists() - assert not (dst / a.relative_to("/")).is_symlink() + assert not (dst / a.relative_to(src)).exists() + assert not (dst / a.relative_to(src)).is_symlink() # src/to_a - assert not (dst / to_a.relative_to("/")).exists() - assert not (dst / to_a.relative_to("/")).is_symlink() + assert not (dst / to_a.relative_to(src)).exists() + assert not (dst / to_a.relative_to(src)).is_symlink() # src/to_broken_a - assert not (dst / to_broken_a.relative_to("/")).exists() - assert not (dst / to_broken_a.relative_to("/")).is_symlink() + assert not (dst / to_broken_a.relative_to(src)).exists() + assert not (dst / to_broken_a.relative_to(src)).is_symlink() # src/A/b - assert not (dst / b.relative_to("/")).exists() - assert not (dst / b.relative_to("/")).is_symlink() + assert not (dst / b.relative_to(src)).exists() + assert not (dst / b.relative_to(src)).is_symlink() # src/A/to_broken_b - assert not (dst / to_broken_b.relative_to("/")).exists() - assert not (dst / to_broken_b.relative_to("/")).is_symlink() + assert not (dst / to_broken_b.relative_to(src)).exists() + assert not (dst / to_broken_b.relative_to(src)).is_symlink() # src/A/B - assert not (dst / B.relative_to("/")).exists() - assert not (dst / B.relative_to("/")).is_symlink() + assert not (dst / B.relative_to(src)).exists() + assert not (dst / B.relative_to(src)).is_symlink() # src/A/B/c - assert not (dst / c.relative_to("/")).exists() - assert not (dst / c.relative_to("/")).is_symlink() + assert not (dst / c.relative_to(src)).exists() + assert not (dst / c.relative_to(src)).is_symlink() # src/A/B/to_c - assert not (dst / to_c.relative_to("/")).exists() - assert not (dst / to_c.relative_to("/")).is_symlink() + assert not (dst / to_c.relative_to(src)).exists() + assert not (dst / to_c.relative_to(src)).is_symlink() # src/A/B/to_broken_c - assert not (dst / to_broken_c.relative_to("/")).exists() - assert not (dst / to_broken_c.relative_to("/")).is_symlink() + assert not (dst / to_broken_c.relative_to(src)).exists() + assert not (dst / to_broken_c.relative_to(src)).is_symlink() # src/A/B/C/ - assert not (dst / C.relative_to("/")).exists() - assert not (dst / C.relative_to("/")).is_symlink() + assert not (dst / C.relative_to(src)).exists() + assert not (dst / C.relative_to(src)).is_symlink() def test_copy_tree_B_exists(mocker, tmp_path): - from otaclient.app.copy_tree import CopyTree - ( dst, src, @@ -467,7 +480,7 @@ def test_copy_tree_B_exists(mocker, tmp_path): C, ) = create_files(tmp_path) - dst_A = dst / tmp_path.relative_to("/") / "src" / "A" + dst_A = dst / "A" dst_A.mkdir(parents=True) print(f"dst_A {dst_A}") dst_B = dst_A / "B" @@ -486,60 +499,63 @@ def test_copy_tree_B_exists(mocker, tmp_path): dst_group_file, ) = create_passwd_group_files(tmp_path) - CopyTree( - src_passwd_file, src_group_file, dst_passwd_file, dst_group_file - ).copy_with_parents(C, dst) + PersistFilesHandler( + src_passwd_file=src_passwd_file, + src_group_file=src_group_file, + dst_passwd_file=dst_passwd_file, + dst_group_file=dst_group_file, + src_root=src, + dst_root=dst, + ).preserve_persist_entry(replace_root(C, src, "/")) # src/A - assert (dst / A.relative_to("/")).is_dir() - assert not (dst / A.relative_to("/")).is_symlink() + assert (dst / A.relative_to(src)).is_dir() + assert not (dst / A.relative_to(src)).is_symlink() # 'A' is created by this function before hand - assert_uid_gid_mode(dst / A.relative_to("/"), 0, 1, 0o765) + assert_uid_gid_mode(dst / A.relative_to(src), 0, 1, 0o765) # src/A/B - assert (dst / B.relative_to("/")).is_dir() - assert not (dst / B.relative_to("/")).is_symlink() + assert (dst / B.relative_to(src)).is_dir() + assert not (dst / B.relative_to(src)).is_symlink() # 'B' is created by this function before hand - assert_uid_gid_mode(dst / B.relative_to("/"), 1, 2, 0o654) + assert_uid_gid_mode(dst / B.relative_to(src), 1, 2, 0o654) # src/A/B/C/ - assert (dst / C.relative_to("/")).is_dir() - assert not (dst / C.relative_to("/")).is_symlink() - assert_uid_gid_mode(dst / C.relative_to("/"), 1105, 2105, 0o126) + assert (dst / C.relative_to(src)).is_dir() + assert not (dst / C.relative_to(src)).is_symlink() + assert_uid_gid_mode(dst / C.relative_to(src), 1105, 2105, 0o126) # followings should not exist # src/a - assert not (dst / a.relative_to("/")).exists() - assert not (dst / a.relative_to("/")).is_symlink() + assert not (dst / a.relative_to(src)).exists() + assert not (dst / a.relative_to(src)).is_symlink() # src/to_a - assert not (dst / to_a.relative_to("/")).exists() - assert not (dst / to_a.relative_to("/")).is_symlink() + assert not (dst / to_a.relative_to(src)).exists() + assert not (dst / to_a.relative_to(src)).is_symlink() # src/to_broken_a - assert not (dst / to_broken_a.relative_to("/")).exists() - assert not (dst / to_broken_a.relative_to("/")).is_symlink() + assert not (dst / to_broken_a.relative_to(src)).exists() + assert not (dst / to_broken_a.relative_to(src)).is_symlink() # src/A/b - assert not (dst / b.relative_to("/")).exists() - assert not (dst / b.relative_to("/")).is_symlink() + assert not (dst / b.relative_to(src)).exists() + assert not (dst / b.relative_to(src)).is_symlink() # src/A/to_b (src/A/b is not copied, so to_b.is_file() is False) - assert not (dst / to_b.relative_to("/")).exists() - assert not (dst / to_b.relative_to("/")).is_symlink() + assert not (dst / to_b.relative_to(src)).exists() + assert not (dst / to_b.relative_to(src)).is_symlink() # src/A/to_broken_b - assert not (dst / to_broken_b.relative_to("/")).exists() - assert not (dst / to_broken_b.relative_to("/")).is_symlink() + assert not (dst / to_broken_b.relative_to(src)).exists() + assert not (dst / to_broken_b.relative_to(src)).is_symlink() # src/A/B/c - assert not (dst / c.relative_to("/")).exists() - assert not (dst / c.relative_to("/")).is_symlink() + assert not (dst / c.relative_to(src)).exists() + assert not (dst / c.relative_to(src)).is_symlink() # src/A/B/to_c - assert not (dst / to_c.relative_to("/")).exists() - assert not (dst / to_c.relative_to("/")).is_symlink() + assert not (dst / to_c.relative_to(src)).exists() + assert not (dst / to_c.relative_to(src)).is_symlink() # src/A/B/to_broken_c - assert not (dst / to_broken_c.relative_to("/")).exists() - assert not (dst / to_broken_c.relative_to("/")).is_symlink() + assert not (dst / to_broken_c.relative_to(src)).exists() + assert not (dst / to_broken_c.relative_to(src)).is_symlink() def test_copy_tree_with_symlink_overwrite(mocker, tmp_path): - from otaclient.app.copy_tree import CopyTree - ( dst, src, @@ -564,31 +580,36 @@ def test_copy_tree_with_symlink_overwrite(mocker, tmp_path): dst_group_file, ) = create_passwd_group_files(tmp_path) - ct = CopyTree(src_passwd_file, src_group_file, dst_passwd_file, dst_group_file) + ct = PersistFilesHandler( + src_passwd_file=src_passwd_file, + src_group_file=src_group_file, + dst_passwd_file=dst_passwd_file, + dst_group_file=dst_group_file, + src_root=src, + dst_root=dst, + ) - ct.copy_with_parents(to_a, dst) - ct.copy_with_parents(to_broken_a, dst) + ct.preserve_persist_entry(replace_root(to_a, src, "/")) + ct.preserve_persist_entry(replace_root(to_broken_a, src, "/")) # followings should exist # src/to_a - assert (dst / to_a.relative_to("/")).is_symlink() + assert (dst / to_a.relative_to(src)).is_symlink() # src/to_broken_a - assert (dst / to_broken_a.relative_to("/")).is_symlink() + assert (dst / to_broken_a.relative_to(src)).is_symlink() # overwrite symlinks - ct.copy_with_parents(to_a, dst) - ct.copy_with_parents(to_broken_a, dst) + ct.preserve_persist_entry(replace_root(to_a, src, "/")) + ct.preserve_persist_entry(replace_root(to_broken_a, src, "/")) # followings should exist # src/to_a - assert (dst / to_a.relative_to("/")).is_symlink() + assert (dst / to_a.relative_to(src)).is_symlink() # src/to_broken_a - assert (dst / to_broken_a.relative_to("/")).is_symlink() + assert (dst / to_broken_a.relative_to(src)).is_symlink() def test_copy_tree_src_dir_dst_file(mocker, tmp_path): - from otaclient.app.copy_tree import CopyTree - ( dst, src, @@ -613,72 +634,77 @@ def test_copy_tree_src_dir_dst_file(mocker, tmp_path): dst_group_file, ) = create_passwd_group_files(tmp_path) - ct = CopyTree(src_passwd_file, src_group_file, dst_passwd_file, dst_group_file) + ct = PersistFilesHandler( + src_passwd_file=src_passwd_file, + src_group_file=src_group_file, + dst_passwd_file=dst_passwd_file, + dst_group_file=dst_group_file, + src_root=src, + dst_root=dst, + ) (dst / src.relative_to("/") / "A").mkdir(parents=True) # NOTE: create {dst}/{src}/A/B as *file* before hand (dst / src.relative_to("/") / "A" / "B").write_text("B") - ct.copy_with_parents(B, dst) + ct.preserve_persist_entry(replace_root(B, src, "/")) # src/A - assert (dst / A.relative_to("/")).is_dir() - assert not (dst / A.relative_to("/")).is_symlink() + assert (dst / A.relative_to(src)).is_dir() + assert not (dst / A.relative_to(src)).is_symlink() # NOTE: {dst}/{src}/A exists before copy so uid, gid and mode are unchanged. assert_uid_gid_mode( - dst / A.relative_to("/"), *uid_gid_mode(dst / src.relative_to("/") / "A") + dst / A.relative_to(src), *uid_gid_mode(dst / src.relative_to(src) / "A") ) # src/A/B/ - assert (dst / B.relative_to("/")).is_dir() - assert not (dst / B.relative_to("/")).is_symlink() - assert_uid_gid_mode(dst / B.relative_to("/"), 141, 265534, 0o122) + assert (dst / B.relative_to(src)).is_dir() + assert not (dst / B.relative_to(src)).is_symlink() + assert_uid_gid_mode(dst / B.relative_to(src), 141, 265534, 0o122) # src/A/B/c - assert (dst / c.relative_to("/")).is_file() - assert not (dst / c.relative_to("/")).is_symlink() - assert_uid_gid_mode(dst / c.relative_to("/"), 1100, 2102, 0o123) + assert (dst / c.relative_to(src)).is_file() + assert not (dst / c.relative_to(src)).is_symlink() + assert_uid_gid_mode(dst / c.relative_to(src), 1100, 2102, 0o123) # src/A/B/to_c - assert (dst / to_c.relative_to("/")).is_file() - assert (dst / to_c.relative_to("/")).is_symlink() + assert (dst / to_c.relative_to(src)).is_file() + assert (dst / to_c.relative_to(src)).is_symlink() # uid, gid can't be converted so original uid, gid is used. - assert_uid_gid_mode(dst / to_c.relative_to("/"), 12345678, 87654321, 0o777) + assert_uid_gid_mode(dst / to_c.relative_to(src), 12345678, 87654321, 0o777) # src/A/B/to_broken_c - assert not (dst / to_broken_c.relative_to("/")).is_file() - assert (dst / to_broken_c.relative_to("/")).is_symlink() - assert_uid_gid_mode(dst / to_broken_c.relative_to("/"), 1104, 2104, 0o777) + assert not (dst / to_broken_c.relative_to(src)).is_file() + assert (dst / to_broken_c.relative_to(src)).is_symlink() + assert_uid_gid_mode(dst / to_broken_c.relative_to(src), 1104, 2104, 0o777) # src/A/B/C/ - assert (dst / C.relative_to("/")).is_dir() - assert not (dst / C.relative_to("/")).is_symlink() - assert_uid_gid_mode(dst / C.relative_to("/"), 1105, 2105, 0o126) + assert (dst / C.relative_to(src)).is_dir() + assert not (dst / C.relative_to(src)).is_symlink() + assert_uid_gid_mode(dst / C.relative_to(src), 1105, 2105, 0o126) # followings should not exist # src/a - assert not (dst / a.relative_to("/")).exists() - assert not (dst / a.relative_to("/")).is_symlink() + assert not (dst / a.relative_to(src)).exists() + assert not (dst / a.relative_to(src)).is_symlink() # src/to_a - assert not (dst / to_a.relative_to("/")).exists() - assert not (dst / to_a.relative_to("/")).is_symlink() + assert not (dst / to_a.relative_to(src)).exists() + assert not (dst / to_a.relative_to(src)).is_symlink() # src/to_broken_a - assert not (dst / to_broken_a.relative_to("/")).exists() - assert not (dst / to_broken_a.relative_to("/")).is_symlink() + assert not (dst / to_broken_a.relative_to(src)).exists() + assert not (dst / to_broken_a.relative_to(src)).is_symlink() # src/A/b - assert not (dst / b.relative_to("/")).exists() - assert not (dst / b.relative_to("/")).is_symlink() + assert not (dst / b.relative_to(src)).exists() + assert not (dst / b.relative_to(src)).is_symlink() # src/A/to_b - assert not (dst / to_b.relative_to("/")).exists() - assert not (dst / to_b.relative_to("/")).is_symlink() + assert not (dst / to_b.relative_to(src)).exists() + assert not (dst / to_b.relative_to(src)).is_symlink() # src/A/to_broken_b - assert not (dst / to_broken_b.relative_to("/")).exists() - assert not (dst / to_broken_b.relative_to("/")).is_symlink() + assert not (dst / to_broken_b.relative_to(src)).exists() + assert not (dst / to_broken_b.relative_to(src)).is_symlink() def test_copy_tree_src_file_dst_dir(mocker, tmp_path): - from otaclient.app.copy_tree import CopyTree - ( dst, src, @@ -703,66 +729,73 @@ def test_copy_tree_src_file_dst_dir(mocker, tmp_path): dst_group_file, ) = create_passwd_group_files(tmp_path) - ct = CopyTree(src_passwd_file, src_group_file, dst_passwd_file, dst_group_file) + ct = PersistFilesHandler( + src_passwd_file=src_passwd_file, + src_group_file=src_group_file, + dst_passwd_file=dst_passwd_file, + dst_group_file=dst_group_file, + src_root=src, + dst_root=dst, + ) # NOTE: create {dst}/{src}/A/B/c as *dir* before hand (dst / src.relative_to("/") / "A" / "B" / "c").mkdir(parents=True) - ct.copy_with_parents(B, dst) + ct.preserve_persist_entry(replace_root(B, src, "/")) # src/A - assert (dst / A.relative_to("/")).is_dir() - assert not (dst / A.relative_to("/")).is_symlink() + assert (dst / A.relative_to(src)).is_dir() + assert not (dst / A.relative_to(src)).is_symlink() # NOTE: {dst}/{src}/A exists before copy so uid, gid and mode are unchanged. assert_uid_gid_mode( - dst / A.relative_to("/"), *uid_gid_mode(dst / src.relative_to("/") / "A") + dst / A.relative_to(src), *uid_gid_mode(dst / src.relative_to(src) / "A") ) # src/A/B/ - assert (dst / B.relative_to("/")).is_dir() - assert not (dst / B.relative_to("/")).is_symlink() + assert (dst / B.relative_to(src)).is_dir() + assert not (dst / B.relative_to(src)).is_symlink() # NOTE: {dst}/{src}/A/B exists before copy so uid, gid and mode are unchanged. assert_uid_gid_mode( - dst / B.relative_to("/"), *uid_gid_mode(dst / src.relative_to("/") / "A" / "B") + dst / B.relative_to(src), *uid_gid_mode(dst / src.relative_to(src) / "A" / "B") ) # src/A/B/c - assert (dst / c.relative_to("/")).is_file() - assert not (dst / c.relative_to("/")).is_symlink() - assert_uid_gid_mode(dst / c.relative_to("/"), 1100, 2102, 0o123) + assert (dst / c.relative_to(src)).is_file() + assert not (dst / c.relative_to(src)).is_symlink() + assert_uid_gid_mode(dst / c.relative_to(src), 1100, 2102, 0o123) # src/A/B/to_c - assert (dst / to_c.relative_to("/")).is_file() - assert (dst / to_c.relative_to("/")).is_symlink() + assert (dst / to_c.relative_to(src)).is_file() + assert (dst / to_c.relative_to(src)).is_symlink() # uid, gid can't be converted so original uid, gid is used. - assert_uid_gid_mode(dst / to_c.relative_to("/"), 12345678, 87654321, 0o777) + assert_uid_gid_mode(dst / to_c.relative_to(src), 12345678, 87654321, 0o777) # src/A/B/to_broken_c - assert not (dst / to_broken_c.relative_to("/")).is_file() - assert (dst / to_broken_c.relative_to("/")).is_symlink() - assert_uid_gid_mode(dst / to_broken_c.relative_to("/"), 1104, 2104, 0o777) + assert not (dst / to_broken_c.relative_to(src)).is_file() + assert (dst / to_broken_c.relative_to(src)).is_symlink() + assert_uid_gid_mode(dst / to_broken_c.relative_to(src), 1104, 2104, 0o777) # src/A/B/C/ - assert (dst / C.relative_to("/")).is_dir() - assert not (dst / C.relative_to("/")).is_symlink() - assert_uid_gid_mode(dst / C.relative_to("/"), 1105, 2105, 0o126) + assert (dst / C.relative_to(src)).is_dir() + assert not (dst / C.relative_to(src)).is_symlink() + assert_uid_gid_mode(dst / C.relative_to(src), 1105, 2105, 0o126) # followings should not exist # src/a - assert not (dst / a.relative_to("/")).exists() - assert not (dst / a.relative_to("/")).is_symlink() + assert not (dst / a.relative_to(src)).exists() + assert not (dst / a.relative_to(src)).is_symlink() # src/to_a - assert not (dst / to_a.relative_to("/")).exists() - assert not (dst / to_a.relative_to("/")).is_symlink() + assert not (dst / to_a.relative_to(src)).exists() + assert not (dst / to_a.relative_to(src)).is_symlink() # src/to_broken_a - assert not (dst / to_broken_a.relative_to("/")).exists() - assert not (dst / to_broken_a.relative_to("/")).is_symlink() + assert not (dst / to_broken_a.relative_to(src)).exists() + assert not (dst / to_broken_a.relative_to(src)).is_symlink() # src/A/b - assert not (dst / b.relative_to("/")).exists() - assert not (dst / b.relative_to("/")).is_symlink() + assert not (dst / b.relative_to(src)).exists() + assert not (dst / b.relative_to(src)).is_symlink() # src/A/to_b - assert not (dst / to_b.relative_to("/")).exists() - assert not (dst / to_b.relative_to("/")).is_symlink() + assert not (dst / to_b.relative_to(src)).exists() + assert not (dst / to_b.relative_to(src)).is_symlink() # src/A/to_broken_b - assert not (dst / to_broken_b.relative_to("/")).exists() - assert not (dst / to_broken_b.relative_to("/")).is_symlink() + assert not (dst / to_broken_b.relative_to(src)).exists() + assert not (dst / to_broken_b.relative_to(src)).is_symlink() diff --git a/tests/test_create_standby.py b/tests/test_create_standby/test_rebuild_mode.py similarity index 100% rename from tests/test_create_standby.py rename to tests/test_create_standby/test_rebuild_mode.py