fix(csaf): finalize db schema

.github/workflows/tests.yml

@@ -76,4 +76,4 @@ jobs:
           flake8 vmaas/
       - name: Run mypy
         run: |
-          mypy .
+          mypy

database/upgrade_scripts/020-csaf_tables2.sql

@@ -0,0 +1,68 @@
+DROP TABLE csaf_cves;
+DROP TABLE csaf_products;
+UPDATE csaf_file SET updated = NULL;
+
+-- -----------------------------------------------------
+-- Table vmaas.csaf_product
+-- -----------------------------------------------------
+CREATE TABLE IF NOT EXISTS csaf_product (
+  id                SERIAL,
+  cpe_id            INT NOT NULL,
+  package_name_id   INT NULL,
+  package_id        INT NULL,
+  module_stream     TEXT NULL CHECK (NOT empty(module_stream)),
+  PRIMARY KEY (id),
+  CONSTRAINT cpe_id
+    FOREIGN KEY (cpe_id)
+    REFERENCES cpe (id),
+  CONSTRAINT package_name_id
+    FOREIGN KEY (package_name_id)
+    REFERENCES package_name (id),
+  CONSTRAINT package_id
+    FOREIGN KEY (package_id)
+    REFERENCES package (id),
+  CONSTRAINT pkg_id CHECK(
+    (package_name_id IS NOT NULL AND package_id IS NULL) OR
+    (package_name_id IS NULL AND package_id IS NOT NULL))
+)TABLESPACE pg_default;
+
+CREATE UNIQUE INDEX ON csaf_product(cpe_id, package_name_id) WHERE package_name_id IS NOT NULL AND package_id IS NULL AND module_stream IS NULL;
+CREATE UNIQUE INDEX ON csaf_product(cpe_id, package_id) WHERE package_id IS NOT NULL AND package_name_id IS NULL AND module_stream IS NULL;
+CREATE UNIQUE INDEX ON csaf_product(cpe_id, package_name_id, module_stream) WHERE package_name_id IS NOT NULL AND package_id IS NULL AND module_stream IS NOT NULL;
+CREATE UNIQUE INDEX ON csaf_product(cpe_id, package_id, module_stream) WHERE package_id IS NOT NULL AND package_name_id IS NULL AND module_stream IS NOT NULL;
+
+-- -----------------------------------------------------
+-- Table vmaas.csaf_cve_product
+-- -----------------------------------------------------
+CREATE TABLE IF NOT EXISTS csaf_cve_product (
+  id                     SERIAL,
+  cve_id                 INT NOT NULL,
+  csaf_product_id        INT NOT NULL,
+  csaf_product_status_id INT NOT NULL,
+  csaf_file_id           INT NOT NULL,
+  PRIMARY KEY (id),
+  CONSTRAINT cve_id
+    FOREIGN KEY (cve_id)
+    REFERENCES cve (id),
+  CONSTRAINT csaf_product_id
+    FOREIGN KEY (csaf_product_id)
+    REFERENCES csaf_product (id),
+  CONSTRAINT csaf_product_status_id
+    FOREIGN KEY (csaf_product_status_id)
+    REFERENCES csaf_product_status (id),
+  CONSTRAINT csaf_file_id
+    FOREIGN KEY (csaf_file_id)
+    REFERENCES csaf_file (id),
+  CONSTRAINT csaf_cve_product_ids_uq
+    UNIQUE (cve_id, csaf_product_id)
+)TABLESPACE pg_default;
+
+
+-- -----------------------------------------------------
+-- vmaas users permission setup:
+-- vmaas_writer - has rights to INSERT/UPDATE/DELETE; used by reposcan
+-- vmaas_reader - has SELECT only; used by webapp
+-- -----------------------------------------------------
+GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO vmaas_writer;
+GRANT USAGE, SELECT, UPDATE ON ALL SEQUENCES IN SCHEMA public TO vmaas_writer;
+GRANT SELECT ON ALL TABLES IN SCHEMA public TO vmaas_reader;

database/vmaas_db_postgresql.sql

@@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS db_version (
 )TABLESPACE pg_default;
 
 -- Increment this when editing this file
-INSERT INTO db_version (name, version) VALUES ('schema_version', 19);
+INSERT INTO db_version (name, version) VALUES ('schema_version', 20);
 
 -- -----------------------------------------------------
 -- evr type
@@ -1307,34 +1307,58 @@ ON CONFLICT DO NOTHING;
 
 
 -- -----------------------------------------------------
--- Table vmaas.csaf_products
+-- Table vmaas.csaf_product
 -- -----------------------------------------------------
-CREATE TABLE IF NOT EXISTS csaf_products (
-  id             SERIAL,
-  cpe            TEXT NOT NULL, CHECK (NOT empty(cpe)),
-  package        TEXT NULL, CHECK (NOT empty(package)),
-  module         TEXT NULL, CHECK (NOT empty(module)),
-  CONSTRAINT unique_csaf_product UNIQUE (cpe, package, module),
-  PRIMARY KEY (id)
+CREATE TABLE IF NOT EXISTS csaf_product (
+  id                SERIAL,
+  cpe_id            INT NOT NULL,
+  package_name_id   INT NULL,
+  package_id        INT NULL,
+  module_stream     TEXT NULL CHECK (NOT empty(module_stream)),
+  PRIMARY KEY (id),
+  CONSTRAINT cpe_id
+    FOREIGN KEY (cpe_id)
+    REFERENCES cpe (id),
+  CONSTRAINT package_name_id
+    FOREIGN KEY (package_name_id)
+    REFERENCES package_name (id),
+  CONSTRAINT package_id
+    FOREIGN KEY (package_id)
+    REFERENCES package (id),
+  CONSTRAINT pkg_id CHECK(
+    (package_name_id IS NOT NULL AND package_id IS NULL) OR
+    (package_name_id IS NULL AND package_id IS NOT NULL))
 )TABLESPACE pg_default;
 
+CREATE UNIQUE INDEX ON csaf_product(cpe_id, package_name_id) WHERE package_name_id IS NOT NULL AND package_id IS NULL AND module_stream IS NULL;
+CREATE UNIQUE INDEX ON csaf_product(cpe_id, package_id) WHERE package_id IS NOT NULL AND package_name_id IS NULL AND module_stream IS NULL;
+CREATE UNIQUE INDEX ON csaf_product(cpe_id, package_name_id, module_stream) WHERE package_name_id IS NOT NULL AND package_id IS NULL AND module_stream IS NOT NULL;
+CREATE UNIQUE INDEX ON csaf_product(cpe_id, package_id, module_stream) WHERE package_id IS NOT NULL AND package_name_id IS NULL AND module_stream IS NOT NULL;
 
 -- -----------------------------------------------------
--- Table vmaas.csaf_cves
+-- Table vmaas.csaf_cve_product
 -- -----------------------------------------------------
-CREATE TABLE IF NOT EXISTS csaf_cves (
-  id                SERIAL,
-  cve               TEXT NOT NULL, CHECK (NOT empty(cve)),
-  csaf_product_id   INT NOT NULL,
-  product_status_id INT NOT NULL,
-  CONSTRAINT unique_csaf_cve_product UNIQUE (cve, csaf_product_id),
+CREATE TABLE IF NOT EXISTS csaf_cve_product (
+  id                     SERIAL,
+  cve_id                 INT NOT NULL,
+  csaf_product_id        INT NOT NULL,
+  csaf_product_status_id INT NOT NULL,
+  csaf_file_id           INT NOT NULL,
   PRIMARY KEY (id),
+  CONSTRAINT cve_id
+    FOREIGN KEY (cve_id)
+    REFERENCES cve (id),
   CONSTRAINT csaf_product_id
     FOREIGN KEY (csaf_product_id)
-    REFERENCES csaf_products (id),
+    REFERENCES csaf_product (id),
   CONSTRAINT csaf_product_status_id
-    FOREIGN KEY (product_status_id)
-    REFERENCES csaf_product_status (id)
+    FOREIGN KEY (csaf_product_status_id)
+    REFERENCES csaf_product_status (id),
+  CONSTRAINT csaf_file_id
+    FOREIGN KEY (csaf_file_id)
+    REFERENCES csaf_file (id),
+  CONSTRAINT csaf_cve_product_ids_uq
+    UNIQUE (cve_id, csaf_product_id)
 )TABLESPACE pg_default;
 
 

pyproject.toml

@@ -54,7 +54,14 @@ requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.mypy]
-ignore_missing_imports = true
-exclude = '''(?x)(
-    ^vmaas/webapp        # exclude vmaas/webapp dir
-)'''
+disallow_any_generics = true
+disallow_untyped_defs = true
+disallow_untyped_calls = true
+warn_return_any = true
+warn_unreachable = true
+follow_imports = "silent"
+packages = """
+    vmaas.reposcan.database.csaf_store,
+    vmaas.reposcan.database.test.test_csaf,
+    vmaas.reposcan.redhatcsaf
+"""

run_tests.sh

@@ -24,7 +24,7 @@ for test_dir in $test_dirs; do
 done
 
 # Find and run tests
-pytest -vvv --cov-report=xml --cov=. --color=yes --durations=1
+pytest -vvv --cov-report=xml --cov=. --color=yes --durations=1 -rP
 
 rc=$(($rc+$?))
 

vmaas/common/batch_list.py

@@ -2,6 +2,7 @@
 Module containing class for list of batches.
 """
 import os
+import typing as t
 
 BATCH_MAX_SIZE = int(os.getenv('BATCH_MAX_SIZE', "500"))
 BATCH_MAX_FILESIZE = int(os.getenv('BATCH_MAX_FILESIZE', "14_000_000_000"))
@@ -10,18 +11,18 @@
 class BatchList:
     """List of lists with defined maximum size of inner lists or by arbitrary file_size of each item"""
 
-    def __init__(self):
+    def __init__(self) -> None:
         self.batches = []
         self.last_batch_filesize = 0
 
-    def __iter__(self):
+    def __iter__(self) -> t.Iterator[list]:
         return iter(self.batches)
 
-    def clear(self):
+    def clear(self) -> None:
         """Clear all previously added items."""
         self.batches = []
 
-    def add_item(self, item, file_size=0):
+    def add_item(self, item: t.Any, file_size: int = 0) -> None:
         """Add item into the last batch. Create new batch if there is no batch or last batch is full."""
         if self.batches:
             last_batch = self.batches[-1]
@@ -41,9 +42,9 @@ def add_item(self, item, file_size=0):
         last_batch.append(item)
         self.last_batch_filesize += file_size
 
-    def __len__(self):
+    def __len__(self) -> int:
         return len(self.batches)
 
-    def get_total_items(self):
+    def get_total_items(self) -> int:
         """Return total item count in all batches."""
         return sum(len(batch) for batch in self.batches)

vmaas/common/config.py

@@ -23,7 +23,7 @@ def __call__(cls, *args, **kwargs):
 class BaseConfig:
     """Base configuration, same for clowder and non-clowder."""
 
-    def __init__(self):
+    def __init__(self) -> None:
         self.postgresql_writer_password = os.getenv(
             "POSTGRESQL_WRITER_PASSWORD", "vmaas_writer_pwd"
         )
@@ -49,15 +49,15 @@ def __init__(self):
 class Config(BaseConfig, metaclass=Singleton):
     """VMaaS configuration singleton."""
 
-    def __init__(self):
+    def __init__(self) -> None:
         if not app_common_python.isClowderEnabled():
             raise EnvironmentError("Missing Clowder config.")
 
         super().__init__()
         self._cfg = app_common_python.LoadedConfig
         self.clowder()
 
-    def clowder(self):
+    def clowder(self) -> None:
         """Configuration from Clowder."""
         self.db_name = getattr(self._cfg.database, "name", "")
         self.db_user = getattr(self._cfg.database, "username", "")
@@ -86,7 +86,7 @@ def clowder(self):
         self.private_port = self._cfg.privatePort
         self.metrics_port = self._cfg.metricsPort
 
-    def _build_url(self, endpoint, scheme="http"):
+    def _build_url(self, endpoint: object, scheme="http") -> str:
         scheme = f"{scheme}s" if self.tls_ca_path else scheme
         port = endpoint.tlsPort if self.tls_ca_path else endpoint.port
         return f"{scheme}://{endpoint.hostname}:{port}"

vmaas/common/date_utils.py

@@ -7,7 +7,7 @@
 from dateutil import tz as dateutil_tz
 
 
-def parse_datetime(date):
+def parse_datetime(date: str) -> datetime:
     """Parse date from string in ISO format."""
     if date is None:
         return None

vmaas/common/logging_utils.py

@@ -133,7 +133,7 @@ def init_logging(num_servers=1):
         setup_cw_logging(logger)
 
 
-def get_logger(name):
+def get_logger(name: str) -> logging.Logger:
     """
     Set logging level and return logger.
     Don't set custom logging level in root handler to not display debug messages from underlying libraries.

vmaas/common/rpm_utils.py

@@ -21,7 +21,8 @@ class RPMParseException(Exception):
     r'((?P<e1>[0-9]+):)?(?P<pn>[^:]+)(?(e1)-|-((?P<e2>[0-9]+):)?)(?P<ver>[^-:]+)-(?P<rel>[^-:]+)\.(?P<arch>[a-z0-9_]+)')
 
 
-def parse_rpm_name(rpm_name, default_epoch=None, raise_exception=False):
+def parse_rpm_name(rpm_name: str, default_epoch: str | None = None,
+                   raise_exception: bool = False) -> tuple[str, str, str, str, str]:
     """
     Extract components from rpm name.
     """

vmaas/reposcan/conftest.py

@@ -8,11 +8,42 @@
 
 from vmaas.common.paths import USER_CREATE_SQL_PATH, DB_CREATE_SQL_PATH
 from vmaas.reposcan.database.database_handler import init_db
+from vmaas.reposcan.redhatcsaf import modeling as csaf_model
 
 VMAAS_DIR = Path(__file__).resolve().parent.parent.parent
 VMAAS_DB_DATA = VMAAS_DIR.joinpath("vmaas", "reposcan", "test_data", "database", "test_data.sql")
 VMAAS_PG_OLD = VMAAS_DIR.joinpath("vmaas", "reposcan", "test_data", "database", "vmaas_db_postgresql_old.sql")
 
+EXPECTED_CSAF = (
+    ("cve-2023-0030.json", csaf_model.CsafCves({"CVE-2023-0030": csaf_model.CsafProducts()})),
+    (
+        "cve-2023-0049.json",
+        csaf_model.CsafCves(
+            {
+                "CVE-2023-0049": csaf_model.CsafProducts([
+                    csaf_model.CsafProduct("cpe:/o:redhat:enterprise_linux:6", "vim", 4),
+                    csaf_model.CsafProduct("cpe:/o:redhat:enterprise_linux:7", "vim", 4),
+                    csaf_model.CsafProduct("cpe:/o:redhat:enterprise_linux:8", "vim", 4),
+                    csaf_model.CsafProduct("cpe:/o:redhat:enterprise_linux:9", "vim", 4),
+                ])
+            }
+        ),
+    ),
+    (
+        "cve-2023-1017.json",
+        csaf_model.CsafCves(
+            {
+                "CVE-2023-1017": csaf_model.CsafProducts([
+                    csaf_model.CsafProduct("cpe:/o:redhat:enterprise_linux:8", "libtpms", 4, "virt:rhel"),
+                    csaf_model.CsafProduct("cpe:/a:redhat:advanced_virtualization:8::el8", "libtpms", 4, "virt:8.2"),
+                    csaf_model.CsafProduct("cpe:/a:redhat:advanced_virtualization:8::el8", "libtpms", 4, "virt:8.3"),
+                    csaf_model.CsafProduct("cpe:/a:redhat:advanced_virtualization:8::el8", "libtpms", 4, "virt:av"),
+                ])
+            }
+        ),
+    ),
+)
+
 
 @pytest.fixture(scope="session")
 def db_conn():
@@ -55,7 +86,7 @@ def reset_db(conn, old_schema: bool = False):
     conn.commit()
 
 
-def write_testing_data(conn):
+def write_testing_data(conn: psycopg2.extensions.connection) -> None:
     """Write testing data to the database."""
     with conn.cursor() as cursor:
         cursor.execute(VMAAS_DB_DATA.read_text(encoding="utf-8"))

vmaas/reposcan/database/cpe_store.py

@@ -42,7 +42,7 @@ def _save_lastmodified(self, lastmodified, key):
         cur.close()
         self.conn.commit()
 
-    def populate_cpes(self, cpes):  # pylint: disable=too-many-branches
+    def populate_cpes(self, cpes: dict[str, str | None]) -> None:  # pylint: disable=too-many-branches
         """
         Insert or update CPE items.
         """