feat: use file_table and resource_table as sqlite3 db (#460)

This PR introduces file_table as sqlite3 db with three tables, naming as follow:
1. `ft_regular`: table contains all regular file entries.
2. `ft_non_regular`: table contains all non-regular file entries, including symlinks and chardev(with 0,0 devnum).
3. `ft_dir`: table contains all directory entries.

Also a new resource_table sqlite3 db with one table is introduced:
1. `rs_table`: table contains all the unique file entries(by sha256digest) and the necessary metadata of the corresponding resource.

The [simple-sqlite3-orm](https://github.com/pga2rn/simple-sqlite3-orm) python package(which is developed and maintained by me also) is used to implement the table definitions and ORMs.

For each tables definition(in `file_table._table` module), helper methods to prepare the entries onto the target(standby slot mount point during OTA) are implemented. 
Comparing to previous implementation, xattrs and chardev files(required by supporting storing docker images on system image) are supported now.

pyproject.toml

@@ -29,12 +29,13 @@ dependencies = [
   "cryptography>=43.0.1,<45",
   "grpcio>=1.53.2,<1.69",
   "multidict<7.0,>=4.5",
+  "msgpack>=1,<1.2",
   "protobuf>=4.21.12,<5.29",
   "pydantic<3,>=2.10",
   "pydantic-settings<3,>=2.3",
   "pyyaml<7,>=6.0.1",
   "requests<2.33,>=2.32",
-  "simple-sqlite3-orm<0.6,>=0.5",
+  "simple-sqlite3-orm<0.7,>=0.6",
   "typing-extensions>=4.6.3",
   "urllib3<2.3,>=2.2.2",
   "uvicorn[standard]>=0.30,<0.35",

requirements.txt

@@ -5,12 +5,13 @@ aiohttp>=3.10.11,<3.12
 cryptography>=43.0.1,<45
 grpcio>=1.53.2,<1.69
 multidict<7.0,>=4.5
+msgpack>=1,<1.2
 protobuf>=4.21.12,<5.29
 pydantic<3,>=2.10
 pydantic-settings<3,>=2.3
 pyyaml<7,>=6.0.1
 requests<2.33,>=2.32
-simple-sqlite3-orm<0.6,>=0.5
+simple-sqlite3-orm<0.7,>=0.6
 typing-extensions>=4.6.3
 urllib3<2.3,>=2.2.2
 uvicorn[standard]>=0.30,<0.35

src/ota_metadata/file_table/__init__.py

@@ -0,0 +1,38 @@
+# Copyright 2022 TIER IV, INC. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from ._orm import (
+    FileTableDirORM,
+    FileTableNonRegularORM,
+    FileTableRegularORM,
+    FileTableRegularORMPool,
+)
+from ._table import (
+    FileTableDirectories,
+    FileTableNonRegularFiles,
+    FileTableRegularFiles,
+)
+from ._types import FileEntryAttrs
+
+__all__ = [
+    "FileTableNonRegularORM",
+    "FileTableRegularORM",
+    "FileTableDirORM",
+    "FileTableRegularORMPool",
+    "FileTableNonRegularFiles",
+    "FileTableRegularFiles",
+    "FileTableDirectories",
+    "FileEntryAttrs",
+]

src/ota_metadata/file_table/_orm.py

@@ -0,0 +1,53 @@
+# Copyright 2022 TIER IV, INC. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from __future__ import annotations
+
+from simple_sqlite3_orm import ORMBase, ORMThreadPoolBase
+
+from ._table import (
+    FileTableDirectories,
+    FileTableNonRegularFiles,
+    FileTableRegularFiles,
+)
+
+FT_REGULAR_TABLE_NAME = "ft_regular"
+FT_NON_REGULAR_TABLE_NAME = "ft_non_regular"
+FT_DIR_TABLE_NAME = "ft_dir"
+
+
+class FileTableRegularORM(ORMBase[FileTableRegularFiles]):
+
+    _orm_table_name = FT_REGULAR_TABLE_NAME
+
+
+class FileTableRegularORMPool(ORMThreadPoolBase[FileTableRegularFiles]):
+
+    _orm_table_name = FT_REGULAR_TABLE_NAME
+
+
+class FileTableNonRegularORM(ORMBase[FileTableNonRegularFiles]):
+
+    _orm_table_name = FT_NON_REGULAR_TABLE_NAME
+
+
+class FileTableDirORM(ORMBase[FileTableDirectories]):
+
+    _orm_table_name = FT_DIR_TABLE_NAME
+
+
+class FileTableDirORMPool(ORMThreadPoolBase[FileTableDirectories]):
+
+    _orm_table_name = FT_DIR_TABLE_NAME

src/ota_metadata/file_table/_table.py

@@ -0,0 +1,205 @@
+# Copyright 2022 TIER IV, INC. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from __future__ import annotations
+
+import os
+import shutil
+import stat
+from abc import abstractmethod
+from pathlib import Path
+from typing import Any, ClassVar, Literal, Optional
+
+from pydantic import BaseModel, SkipValidation
+from simple_sqlite3_orm import ConstrainRepr, TableSpec, TypeAffinityRepr
+from typing_extensions import Annotated
+
+from ota_metadata.file_table._types import EntryAttrsType
+from otaclient_common.typing import StrOrPath
+
+CANONICAL_ROOT = "/"
+
+
+class FileTableBase(BaseModel):
+    schema_ver: ClassVar[Literal[1]] = 1
+
+    path: Annotated[
+        str,
+        TypeAffinityRepr(str),
+        ConstrainRepr("PRIMARY KEY"),
+        SkipValidation,
+    ]
+
+    entry_attrs: Annotated[
+        EntryAttrsType,
+        TypeAffinityRepr(bytes),
+        ConstrainRepr("NOT NULL"),
+    ]
+    """msgpacked basic attrs for this file entry.
+
+    Ref: https://www.kernel.org/doc/html/latest/filesystems/ext4/inodes.html
+
+    including the following fields from inode table:
+    1. mode bits
+    2. uid
+    3. gid
+    4. inode(when the file is hardlinked)
+    5. xattrs
+
+    See file_table._types for more details.
+    """
+
+    def set_xattr(self, _target: StrOrPath) -> None:
+        """Set the xattr of self onto the <_target>.
+
+        NOTE: this method always don't follow symlink.
+        """
+        if xattrs := self.entry_attrs.xattrs:
+            for k, v in xattrs.items():
+                os.setxattr(
+                    path=_target,
+                    attribute=k,
+                    value=v.encode(),
+                    follow_symlinks=False,
+                )
+
+    def set_perm(self, _target: StrOrPath) -> None:
+        """Set the mode,uid,gid of self onto the <_target>."""
+        entry_attrs = self.entry_attrs
+        # NOTE(20241213): chown will reset the sticky bit of the file!!!
+        #   Remember to always put chown before chmod !!!
+        os.chown(_target, uid=entry_attrs.uid, gid=entry_attrs.gid)
+        os.chmod(_target, mode=entry_attrs.mode)
+
+    def fpath_on_target(self, target_mnt: StrOrPath) -> Path:
+        """Return the fpath of self joined to <target_mnt>."""
+        _canonical_path = Path(self.path)
+        _target_on_mnt = Path(target_mnt) / _canonical_path.relative_to(CANONICAL_ROOT)
+        return _target_on_mnt
+
+    @abstractmethod
+    def prepare_target(self, *args: Any, target_mnt: StrOrPath, **kwargs) -> None:
+        raise NotImplementedError
+
+
+class FileTableRegularFiles(TableSpec, FileTableBase):
+    """DB table for regular file entries."""
+
+    digest: Annotated[
+        bytes,
+        TypeAffinityRepr(bytes),
+        SkipValidation,
+    ]
+
+    def prepare_target(
+        self,
+        _rs: StrOrPath,
+        *,
+        target_mnt: StrOrPath,
+        prepare_method: Literal["move", "hardlink", "copy"],
+    ) -> None:
+        _target_on_mnt = self.fpath_on_target(target_mnt=target_mnt)
+
+        if prepare_method == "copy":
+            shutil.copy(_rs, _target_on_mnt)
+            self.set_perm(_target_on_mnt)
+            self.set_xattr(_target_on_mnt)
+            return
+
+        if prepare_method == "hardlink":
+            # NOTE: os.link will make dst a hardlink to src.
+            os.link(_rs, _target_on_mnt)
+            # NOTE: although we actually don't need to set_perm and set_xattr everytime
+            #   to file paths point to the same inode, for simplicity here we just
+            #   do it everytime.
+            self.set_perm(_target_on_mnt)
+            self.set_xattr(_target_on_mnt)
+            return
+
+        if prepare_method == "move":
+            shutil.move(str(_rs), _target_on_mnt)
+            self.set_perm(_target_on_mnt)
+            self.set_xattr(_target_on_mnt)
+
+
+class FileTableNonRegularFiles(TableSpec, FileTableBase):
+    """DB table for non-regular file entries.
+
+    This includes:
+    1. symlink.
+    2. chardev file.
+
+    NOTE that support for chardev file is only for overlayfs' whiteout file,
+        so only device num as 0,0 will be allowed.
+    """
+
+    contents: Annotated[
+        Optional[bytes],
+        TypeAffinityRepr(bytes),
+        SkipValidation,
+    ] = None
+    """The contents of the file. Currently only used by symlink."""
+
+    def set_perm(self, _target: StrOrPath) -> None:
+        """Set the mode,uid,gid of self onto the <_target>.
+
+        NOTE: this method always don't follow symlink.
+        """
+        entry_attrs = self.entry_attrs
+
+        # NOTE(20241213): chown will reset the sticky bit of the file!!!
+        #   Remember to always put chown before chmod !!!
+        os.chown(
+            _target, uid=entry_attrs.uid, gid=entry_attrs.gid, follow_symlinks=False
+        )
+        # NOTE: changing mode of symlink is not needed and uneffective, and on some platform
+        #   changing mode of symlink will even result in exception raised.
+        if not stat.S_ISLNK(entry_attrs.mode):
+            os.chmod(_target, mode=entry_attrs.mode)
+
+    def prepare_target(self, *, target_mnt: StrOrPath) -> None:
+        _target_on_mnt = self.fpath_on_target(target_mnt=target_mnt)
+
+        entry_attrs = self.entry_attrs
+        _mode = entry_attrs.mode
+        if stat.S_ISLNK(_mode):
+            assert (
+                _symlink_target_raw := self.contents
+            ), f"invalid entry {self}, entry is a symlink but no link target is defined"
+
+            _symlink_target = _symlink_target_raw.decode()
+            _target_on_mnt.symlink_to(_symlink_target)
+            self.set_perm(_target_on_mnt)
+            self.set_xattr(_target_on_mnt)
+            return
+
+        if stat.S_ISCHR(_mode):
+            _device_num = os.makedev(0, 0)
+            os.mknod(_target_on_mnt, mode=_mode, device=_device_num)
+            self.set_perm(_target_on_mnt)
+            self.set_xattr(_target_on_mnt)
+            return
+
+        raise ValueError(f"invalid entry {self}")
+
+
+class FileTableDirectories(TableSpec, FileTableBase):
+    """DB table for directory entries."""
+
+    def prepare_target(self, *, target_mnt: StrOrPath) -> None:
+        _target_on_mnt = self.fpath_on_target(target_mnt=target_mnt)
+        _target_on_mnt.mkdir(exist_ok=True, parents=True)
+        self.set_perm(_target_on_mnt)
+        self.set_xattr(_target_on_mnt)

src/ota_metadata/file_table/_types.py

@@ -0,0 +1,63 @@
+# Copyright 2022 TIER IV, INC. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from __future__ import annotations
+
+from typing import Dict, NamedTuple, Optional
+
+from msgpack import Unpacker, packb
+from pydantic import PlainSerializer, PlainValidator
+from typing_extensions import Annotated
+
+FILE_ENTRY_MAX_SIZE = 1024**2  # 1MiB
+
+
+class FileEntryAttrs(NamedTuple):
+    mode: int
+    uid: int
+    gid: int
+    size: Optional[int] = None
+    inode: Optional[int] = None
+    xattrs: Optional[Dict[str, str]] = None
+
+    @classmethod
+    def _validator(cls, _in: bytes | FileEntryAttrs) -> FileEntryAttrs:
+        if isinstance(_in, FileEntryAttrs):
+            return _in
+
+        _unpacker = Unpacker(max_buffer_size=FILE_ENTRY_MAX_SIZE)
+        _unpacker.feed(_in)  # feed all the data into the internal buffer
+
+        # get exactly one list from buffer.
+        # NOTE that msgpack only has two container types when unpacking: list and dict.
+        _obj = _unpacker.unpack()
+        if not isinstance(_obj, list):
+            raise TypeError(f"expect unpack to a list, get {type(_obj)=}")
+        return cls(*_obj)
+
+    def _serializer(self: FileEntryAttrs) -> bytes:
+        try:
+            if _res := packb(self, buf_size=FILE_ENTRY_MAX_SIZE):
+                return _res
+            raise ValueError("nothing is packed")
+        except Exception as e:
+            raise ValueError(f"failed to pack {self}: {e!r}") from e
+
+
+EntryAttrsType = Annotated[
+    FileEntryAttrs,
+    PlainValidator(FileEntryAttrs._validator),
+    PlainSerializer(FileEntryAttrs._serializer),
+]