From 6a684127015eaf6f8cbb8f9fa554f1b9884393bc Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Tue, 21 Jan 2025 10:59:54 -0500 Subject: [PATCH] fix verbose config processing Signed-off-by: Alex Goodman --- .github/workflows/daily-db-publisher-r2.yaml | 4 ++-- .github/workflows/daily-db-publisher.yaml | 4 ++-- .github/workflows/staging-db-publisher.yaml | 4 ++-- manager/src/grype_db_manager/cli/cli.py | 10 +++++----- manager/src/grype_db_manager/cli/config.py | 7 +++++-- manager/src/grype_db_manager/cli/db.py | 11 +++-------- manager/tests/cli/test_legacy_workflows.py | 6 +++--- manager/tests/cli/test_workflows.py | 6 +++--- manager/tests/unit/cli/fixtures/config/full.yaml | 1 + manager/tests/unit/cli/test_config.py | 2 ++ test/db/acceptance.sh | 2 +- 11 files changed, 29 insertions(+), 28 deletions(-) diff --git a/.github/workflows/daily-db-publisher-r2.yaml b/.github/workflows/daily-db-publisher-r2.yaml index 7be4b3bf..1402b7e9 100644 --- a/.github/workflows/daily-db-publisher-r2.yaml +++ b/.github/workflows/daily-db-publisher-r2.yaml @@ -85,10 +85,10 @@ jobs: run: | poetry run \ grype-db-manager \ + -vv \ -c ./config/grype-db-manager/publish-production-r2.yaml \ db build-and-upload \ - --schema-version ${{ matrix.schema-version }} \ - -vvv + --schema-version ${{ matrix.schema-version }} env: AWS_ACCESS_KEY_ID: ${{ secrets.TOOLBOX_CLOUDFLARE_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLBOX_CLOUDFLARE_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/daily-db-publisher.yaml b/.github/workflows/daily-db-publisher.yaml index 9fc35797..f3d621e7 100644 --- a/.github/workflows/daily-db-publisher.yaml +++ b/.github/workflows/daily-db-publisher.yaml @@ -85,10 +85,10 @@ jobs: run: | poetry run \ grype-db-manager \ + -vv \ -c ./config/grype-db-manager/publish-production.yaml \ db build-and-upload \ - --schema-version ${{ matrix.schema-version }} \ - -vvv + --schema-version ${{ matrix.schema-version }} env: AWS_ACCESS_KEY_ID: ${{ secrets.TOOLBOX_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLBOX_AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/staging-db-publisher.yaml b/.github/workflows/staging-db-publisher.yaml index 006515e9..7d292a55 100644 --- a/.github/workflows/staging-db-publisher.yaml +++ b/.github/workflows/staging-db-publisher.yaml @@ -91,10 +91,10 @@ jobs: run: | poetry run \ grype-db-manager \ + -vv \ -c ./config/grype-db-manager/publish-staging.yaml \ db build-and-upload \ - --schema-version ${{ github.event.inputs.schema-version }} \ - -vvv + --schema-version ${{ github.event.inputs.schema-version }} env: AWS_ACCESS_KEY_ID: ${{ secrets.TOOLBOX_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLBOX_AWS_SECRET_ACCESS_KEY }} diff --git a/manager/src/grype_db_manager/cli/cli.py b/manager/src/grype_db_manager/cli/cli.py index d74cd165..8a0cf839 100644 --- a/manager/src/grype_db_manager/cli/cli.py +++ b/manager/src/grype_db_manager/cli/cli.py @@ -11,17 +11,17 @@ from grype_db_manager.db.format import Format -@click.option("--verbose", "-v", default=False, help="show more verbose logging", count=True) +@click.option("--verbose", "-v", "verbosity", count=True, help="show details of all comparisons") @click.option("--config", "-c", "config_path", default=None, help="override config path") @click.group(help="A tool for publishing validated grype databases to S3 for distribution.") @click.version_option(package_name=package_name, message="%(prog)s %(version)s") @click.pass_context -def cli(ctx: click.core.Context, verbose: bool, config_path: str | None) -> None: +def cli(ctx: click.core.Context, verbosity: int, config_path: str | None) -> None: import logging.config import colorlog - ctx.obj = config.load(path=config_path) + ctx.obj = config.load(path=config_path, verbosity=verbosity) class DeltaTimeFormatter(colorlog.ColoredFormatter): def __init__(self, *args: Any, **kwargs: Any): @@ -35,9 +35,9 @@ def format(self, record: logging.LogRecord) -> str: # noqa: A003 return super().format(record) log_level = ctx.obj.log.level - if verbose == 1: + if verbosity == 1: log_level = "DEBUG" - elif verbose >= 2: + elif verbosity >= 2: log_level = "TRACE" logging.config.dictConfig( diff --git a/manager/src/grype_db_manager/cli/config.py b/manager/src/grype_db_manager/cli/config.py index 75724dde..1afb228e 100644 --- a/manager/src/grype_db_manager/cli/config.py +++ b/manager/src/grype_db_manager/cli/config.py @@ -127,6 +127,7 @@ class Data: @dataclass class Application: + verbosity: int = 0 data: Data = field(default_factory=Data) log: Log = field(default_factory=Log) schema_mapping_file: str = "" # default is to use built-in schema mapping @@ -192,9 +193,10 @@ def convert_value(obj: Any) -> Any: def load( path: None | str | list[str] | tuple[str] = DEFAULT_CONFIGS, wire_values: bool = True, + verbosity: int = 0, env: Mapping | None = None, ) -> Application: - cfg = _load_paths(path, wire_values=wire_values, env=env) + cfg = _load_paths(path, wire_values=wire_values, env=env, verbosity=verbosity) if not cfg: msg = "no config found" @@ -207,6 +209,7 @@ def _load_paths( path: None | str | list[str] | tuple[str], wire_values: bool = True, env: Mapping | None = None, + verbosity: int = 0, ) -> Application | None: if not path: path = DEFAULT_CONFIGS @@ -225,7 +228,7 @@ def _load_paths( return _load(p, wire_values=wire_values, env=env) # use the default application config - return Application() + return Application(verbosity=verbosity) msg = f"invalid path type {type(path)}" raise ValueError(msg) diff --git a/manager/src/grype_db_manager/cli/db.py b/manager/src/grype_db_manager/cli/db.py index 4b3ebfc7..5772fbfa 100644 --- a/manager/src/grype_db_manager/cli/db.py +++ b/manager/src/grype_db_manager/cli/db.py @@ -97,7 +97,6 @@ def show_db(cfg: config.Application, db_uuid: str) -> None: multiple=True, help="the image (or images) to validate with (default is to use all configured images)", ) -@click.option("--verbose", "-v", "verbosity", count=True, help="show details of all comparisons") @click.option("--recapture", "-r", is_flag=True, help="recapture grype results (even if not stale)") @click.option( "--skip-namespace-check", @@ -114,7 +113,6 @@ def validate_db( cfg: config.Application, db_uuid: str, images: list[str], - verbosity: int, recapture: bool, skip_namespace_check: bool, force: bool, @@ -136,7 +134,7 @@ def validate_db( # ensure the minimum number of namespaces are present db_manager.validate_providers(db_uuid=db_uuid, expected=cfg.validate.expected_providers) - _validate_db(ctx, cfg, db_info, images, db_uuid, verbosity, recapture, force=force) + _validate_db(ctx, cfg, db_info, images, db_uuid, recapture, force=force) if db_info.schema_version >= 6: logging.info(f"validating latest.json {db_uuid}") @@ -151,7 +149,6 @@ def _validate_db( db_info: grypedb.DBInfo, images: list[str], db_uuid: str, - verbosity: int, recapture: bool, force: bool = False, ) -> None: @@ -238,7 +235,7 @@ def _validate_db( yardstick_validate, always_run_label_comparison=False, breakdown_by_ecosystem=False, - verbosity=verbosity, + verbosity=cfg.verbosity, result_sets=[], all_result_sets=True, ) @@ -330,7 +327,6 @@ def upload_db(cfg: config.Application, db_uuid: str, ttl_seconds: int) -> None: is_flag=True, help="do not ensure the minimum expected namespaces are present (for v6+ this is a providers-based check)", ) -@click.option("--verbose", "-v", "verbosity", count=True, help="show details of all comparisons") @click.pass_obj @click.pass_context @error.handle_exception(handle=(ValueError, s3utils.CredentialsError)) @@ -341,7 +337,6 @@ def build_and_upload_db( skip_validate: bool, skip_namespace_check: bool, dry_run: bool, - verbosity: bool, ) -> None: if dry_run: click.echo(f"{Format.ITALIC}Dry run! Will skip uploading the listing file to S3{Format.RESET}") @@ -355,7 +350,7 @@ def build_and_upload_db( click.echo(f"{Format.ITALIC}Skipping validation of DB {db_uuid!r}{Format.RESET}") else: click.echo(f"{Format.BOLD}Validating DB {db_uuid!r}{Format.RESET}") - ctx.invoke(validate_db, db_uuid=db_uuid, verbosity=verbosity, skip_namespace_check=skip_namespace_check) + ctx.invoke(validate_db, db_uuid=db_uuid, skip_namespace_check=skip_namespace_check) if not dry_run: click.echo(f"{Format.BOLD}Uploading DB {db_uuid!r}{Format.RESET}") diff --git a/manager/tests/cli/test_legacy_workflows.py b/manager/tests/cli/test_legacy_workflows.py index d73d87f4..f8c80d89 100644 --- a/manager/tests/cli/test_legacy_workflows.py +++ b/manager/tests/cli/test_legacy_workflows.py @@ -51,7 +51,7 @@ def test_workflow_2(cli_env, command, logger): cli_env["GOWORK"] = "off" stdout, _ = command.run( - f"grype-db-manager db validate {db_id} -vvv --skip-namespace-check --recapture", + f"grype-db-manager -vv db validate {db_id} --skip-namespace-check --recapture", env=cli_env, expect_fail=True, ) @@ -65,7 +65,7 @@ def test_workflow_2(cli_env, command, logger): command.run("make install-oracle-labels", env=cli_env) _, stderr = command.run( - f"grype-db-manager db validate {db_id} -vvv", + f"grype-db-manager -vv db validate {db_id}", env=cli_env, expect_fail=True, ) @@ -79,7 +79,7 @@ def test_workflow_2(cli_env, command, logger): command.run("make install-oracle-labels", env=cli_env) stdout, _ = command.run( - f"grype-db-manager db validate {db_id} -vvv --skip-namespace-check", + f"grype-db-manager -vv db validate {db_id} --skip-namespace-check", env=cli_env, ) assert "Quality gate passed!" in stdout diff --git a/manager/tests/cli/test_workflows.py b/manager/tests/cli/test_workflows.py index 149944f3..8500cd34 100644 --- a/manager/tests/cli/test_workflows.py +++ b/manager/tests/cli/test_workflows.py @@ -101,7 +101,7 @@ def test_workflow_2(cli_env, command, logger): # note: we add --force to ensure we're checking validations (even if it's disabled for the schema) stdout, _ = command.run( - f"grype-db-manager db validate {db_id} --skip-namespace-check --force -vvv --recapture", + f"grype-db-manager -vv db validate {db_id} --skip-namespace-check --force --recapture", env=cli_env, expect_fail=True, ) @@ -115,7 +115,7 @@ def test_workflow_2(cli_env, command, logger): command.run("make install-oracle-labels", env=cli_env) _, stderr = command.run( - f"grype-db-manager db validate {db_id} --force -vvv", + f"grype-db-manager -vv db validate {db_id} --force", env=cli_env, expect_fail=True, ) @@ -129,7 +129,7 @@ def test_workflow_2(cli_env, command, logger): command.run("make install-oracle-labels", env=cli_env) stdout, _ = command.run( - f"grype-db-manager db validate {db_id} --skip-namespace-check --force -vvv", + f"grype-db-manager -vv db validate {db_id} --skip-namespace-check --force", env=cli_env, ) assert "Quality gate passed!" in stdout diff --git a/manager/tests/unit/cli/fixtures/config/full.yaml b/manager/tests/unit/cli/fixtures/config/full.yaml index 6226a3ec..799e3be0 100644 --- a/manager/tests/unit/cli/fixtures/config/full.yaml +++ b/manager/tests/unit/cli/fixtures/config/full.yaml @@ -47,3 +47,4 @@ validate: allow_empty_results_for_schemas: [1,2,3] images: - docker.io/cloudbees/cloudbees-core-agent:2.289.2.2@sha256:d48f0546b4cf5ef4626136242ce302f94a42751156b7be42f4b1b75a66608880 +verbosity: 2 \ No newline at end of file diff --git a/manager/tests/unit/cli/test_config.py b/manager/tests/unit/cli/test_config.py index da774599..e77a1e1a 100644 --- a/manager/tests/unit/cli/test_config.py +++ b/manager/tests/unit/cli/test_config.py @@ -82,6 +82,7 @@ def test_load_default(): minimumVulnerabilities: null overrideDbSchemaVersion: null overrideGrypeVersion: null +verbosity: 0 """ assert actual == expected @@ -148,6 +149,7 @@ def test_load(test_dir_path): minimumVulnerabilities: 400 overrideDbSchemaVersion: null overrideGrypeVersion: null +verbosity: 2 """ assert actual == expected diff --git a/test/db/acceptance.sh b/test/db/acceptance.sh index 7925b2c7..5f197bbc 100755 --- a/test/db/acceptance.sh +++ b/test/db/acceptance.sh @@ -26,4 +26,4 @@ fi title "Validating DB" -grype-db-manager db validate $DB_ID -vvv +grype-db-manager -vv db validate $DB_ID