diff --git a/requirements.txt b/requirements.txt index 465132d3..398c2989 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,4 +13,4 @@ pubtools-iib kerberos marshmallow urllib3<2 -pubtools-sign>0.0.6 +pubtools-sign>0.0.7 diff --git a/src/pubtools/_quay/clear_repo.py b/src/pubtools/_quay/clear_repo.py index 1dc636b2..ac46b449 100644 --- a/src/pubtools/_quay/clear_repo.py +++ b/src/pubtools/_quay/clear_repo.py @@ -120,7 +120,7 @@ def clear_repositories(repositories: str, settings: Dict[str, Any]) -> None: LOG.info("Clearing repositories '{0}'".format(repositories)) quay_client = QuayClient(settings["quay_user"], settings["quay_password"]) item_processor = item_processor_for_internal_data( - quay_client, "quay.io", 5, settings["quay_org"] + quay_client, "quay.io", [""], 5, settings["quay_org"] ) item_processor.extractor.full_extract = True diff --git a/src/pubtools/_quay/iib_operations.py b/src/pubtools/_quay/iib_operations.py index 6b7c82c4..d93a7141 100644 --- a/src/pubtools/_quay/iib_operations.py +++ b/src/pubtools/_quay/iib_operations.py @@ -96,7 +96,9 @@ def _index_image_to_sign_entries( reference """ iib_repo = target_settings["quay_operator_repository"] + pub_iib_repo = target_settings["quay_operator_repository"] dest_registries = target_settings["docker_settings"]["docker_reference_registry"] + pub_registry = dest_registries[0] dest_registries = dest_registries if isinstance(dest_registries, list) else [dest_registries] if internal: dest_registries = ["quay.io"] @@ -130,6 +132,7 @@ def _index_image_to_sign_entries( to_sign_entries.append( SignEntry( reference=f"{registry}/{iib_repo}:{_dest_tag}", + pub_reference=f"{pub_registry}/{pub_iib_repo}", repo=iib_repo, digest=headers["docker-content-digest"], arch="amd64", @@ -145,6 +148,7 @@ def _index_image_to_sign_entries( to_sign_entries.append( SignEntry( reference=reference, + pub_reference=f"{pub_registry}/{pub_iib_repo}", repo=iib_repo, digest=digest, arch="amd64", @@ -267,6 +271,7 @@ def task_iib_add_bundles( item_processor = item_processor_for_internal_data( quay_client, target_settings["quay_host"].rstrip("/"), + target_settings["docker_settings"]["docker_reference_registry"], target_settings.get("retry_sleep_time", 5), quay_operator_namespace, ) @@ -401,6 +406,7 @@ def task_iib_remove_operators( item_processor = item_processor_for_internal_data( quay_client, target_settings["quay_host"].rstrip("/"), + target_settings["docker_settings"]["docker_reference_registry"], target_settings.get("retry_sleep_time", 5), quay_operator_namespace, ) @@ -530,6 +536,7 @@ def task_iib_build_from_scratch( item_processor = item_processor_for_internal_data( quay_client, target_settings["quay_host"].rstrip("/"), + target_settings["docker_settings"]["docker_reference_registry"], target_settings.get("retry_sleep_time", 5), quay_operator_namespace, ) diff --git a/src/pubtools/_quay/item_processor.py b/src/pubtools/_quay/item_processor.py index 12e193d8..147d8da0 100644 --- a/src/pubtools/_quay/item_processor.py +++ b/src/pubtools/_quay/item_processor.py @@ -38,12 +38,14 @@ class SignEntry: signing_key (str): Signing key. repo (str): Repo reference in format / reference (str): Reference in format /: + pub_reference (str): Public repo reference in format /repo digest (str): Digest of the manifest. arch (str): Architecture of the manifest. """ repo: str reference: str + pub_reference: str digest: str signing_key: str arch: str @@ -393,6 +395,7 @@ class ItemProcesor: reference_processor: Union[ReferenceProcessorExternal, ReferenceProcessorInternal] reference_registries: List[str] source_registry: Optional[str] + public_registry: Optional[str] INTERNAL_DELIMITER = "----" @@ -505,6 +508,7 @@ def generate_to_sign(self, item: Any, include_manifest_lists: bool = False) -> L SignEntry( repo=repo, reference=reference, + pub_reference=cast(str, self.public_registry) + "/" + repo, digest=mad.digest, arch=mad.arch, signing_key=item.claims_signing_key, @@ -700,17 +704,23 @@ def item_processor_for_external_data( reference_processor=ReferenceProcessorExternal(), reference_registries=external_registries, source_registry=None, + public_registry=external_registries[0], ) def item_processor_for_internal_data( - quay_client: QuayClient, internal_registry: str, retry_sleep_time: int, internal_namespace: str + quay_client: QuayClient, + internal_registry: str, + external_registries: List[str], + retry_sleep_time: int, + internal_namespace: str, ) -> ItemProcesor: """Get instance of item processor configured to produce internal data. Args: quay_client (QuayClient): Quay client. internal_registry (str): Docker registry where containers are stored + external registries (str): List of external registries used for container identity. retry_sleep_time (int): sleep time bewteen retries for fetching data from registry. internal_namespace (str): Namespace of internal organization in the registry. Returns: @@ -723,4 +733,5 @@ def item_processor_for_internal_data( reference_processor=reference_processor, reference_registries=["quay.io"], source_registry=internal_registry, + public_registry=external_registries[0], ) diff --git a/src/pubtools/_quay/push_docker.py b/src/pubtools/_quay/push_docker.py index 651023f9..74545892 100644 --- a/src/pubtools/_quay/push_docker.py +++ b/src/pubtools/_quay/push_docker.py @@ -348,6 +348,7 @@ def generate_backup_mapping( internal_item_processor = item_processor_for_internal_data( self.dest_quay_client, self.target_settings["quay_host"].rstrip("/"), + self.dest_registries, self.target_settings.get("retry_sleep_time", 5), self.target_settings["quay_namespace"], ) @@ -503,6 +504,7 @@ def fetch_missing_push_items_digests( item_processor = item_processor_for_internal_data( self.dest_quay_client, self.target_settings["quay_host"].rstrip("/"), + self.dest_registries, self.target_settings.get("retry_sleep_time", 5), self.target_settings["quay_namespace"], ) @@ -557,10 +559,13 @@ def sign_new_manifests(self, docker_push_items: List[Any]) -> List[Tuple[str, st + ":" + tag ) + registry = self.dest_registries[0] + pub_reference = f"{registry}/{repo}" # add entries in internal format for cosign to_sign_entries_internal.append( SignEntry( reference=reference, + pub_reference=pub_reference, repo=repo, digest=digest, signing_key=key, @@ -572,6 +577,7 @@ def sign_new_manifests(self, docker_push_items: List[Any]) -> List[Tuple[str, st to_sign_entries.append( SignEntry( reference=reference, + pub_reference="", repo=repo, digest=digest, signing_key=key, @@ -666,6 +672,7 @@ def run(self) -> None: to_sign_entries = [] item_processor = item_processor_for_internal_data( self.src_quay_client, + self.target_settings["quay_host"].rstrip("/"), self.dest_registries, self.target_settings.get("retry_sleep_time", 5), self.target_settings["quay_namespace"], diff --git a/src/pubtools/_quay/remove_repo.py b/src/pubtools/_quay/remove_repo.py index b79455d9..d5311618 100644 --- a/src/pubtools/_quay/remove_repo.py +++ b/src/pubtools/_quay/remove_repo.py @@ -119,7 +119,7 @@ def remove_repositories(repositories: str, settings: Dict[str, Any]) -> None: quay_api_client = QuayApiClient(settings["quay_api_token"]) quay_client = QuayClient(settings["quay_user"], settings["quay_password"], "quay.io") item_processor = item_processor_for_internal_data( - quay_client, "quay.io", 5, settings["quay_org"] + quay_client, "quay.io", [""], 5, settings["quay_org"] ) item_processor.extractor.full_extract = True # Remove repository doesn't work with push item by default, therefore we create VirtualPushItem diff --git a/src/pubtools/_quay/signer_wrapper.py b/src/pubtools/_quay/signer_wrapper.py index b4517237..89741352 100644 --- a/src/pubtools/_quay/signer_wrapper.py +++ b/src/pubtools/_quay/signer_wrapper.py @@ -528,6 +528,20 @@ class CosignSignerWrapper(SignerWrapper): # LOG.warning("Fetch existing signatures error:" + esig[1]) # return rets + def sign_container_opt_args( + self, sign_entry: SignEntry, task_id: Optional[str] = None + ) -> Dict[str, Any]: + """Return optional arguments for signing a container. + + Args: + sign_entry (SignEntry): SignEntry to sign. + task_id (str): Task ID to identify the signing task if needed. + + Returns: + dict: Optional arguments for signing a container. + """ + return {"identity": sign_entry.pub_reference} + def _filter_to_remove( self, signatures: List[Tuple[str, str, str]], diff --git a/src/pubtools/_quay/tag_docker.py b/src/pubtools/_quay/tag_docker.py index 28bb8228..c3ca330f 100644 --- a/src/pubtools/_quay/tag_docker.py +++ b/src/pubtools/_quay/tag_docker.py @@ -548,6 +548,7 @@ def copy_tag_sign_images(self, push_item: Any, tag: str, executor: Executor) -> to_sign_entries_internal.append( SignEntry( repo=repo, + pub_reference=registries[0] + "/" + list(push_item.repos.keys())[0], reference="quay.io/" + self.target_settings["quay_namespace"] + "/" @@ -566,6 +567,7 @@ def copy_tag_sign_images(self, push_item: Any, tag: str, executor: Executor) -> to_sign_entries.append( SignEntry( repo=repo, + pub_reference="", reference=reference, digest=details.digest, signing_key=push_item.claims_signing_key, @@ -577,6 +579,7 @@ def copy_tag_sign_images(self, push_item: Any, tag: str, executor: Executor) -> item_processor = item_processor_for_internal_data( self.quay_client, self.target_settings["quay_host"].rstrip("/"), + self.dest_registries[0], self.target_settings.get("retry_sleep_time", 5), self.target_settings["quay_namespace"], ) @@ -666,9 +669,11 @@ def merge_manifest_lists_sign_images( to_sign_entries_internal = [] for manifest in new_manifest_list["manifests"]: + registry = dest_registries[0] to_sign_entries_internal.append( SignEntry( repo=list(push_item.repos.keys())[0], + pub_reference=registry + "/" + list(push_item.repos.keys())[0], reference="quay.io/" + self.target_settings["quay_namespace"] + "/" @@ -687,6 +692,7 @@ def merge_manifest_lists_sign_images( to_sign_entries.append( SignEntry( repo=list(push_item.repos.keys())[0], + pub_reference="", reference=reference, digest=manifest["digest"], arch=manifest["platform"]["architecture"], @@ -701,6 +707,7 @@ def merge_manifest_lists_sign_images( item_processor = item_processor_for_internal_data( self.quay_client, self.target_settings["quay_host"].rstrip("/"), + self.dest_registries[0], self.target_settings.get("retry_sleep_time", 5), self.target_settings["quay_namespace"], ) @@ -751,9 +758,11 @@ def merge_manifest_lists_sign_images( if push_item.claims_signing_key: # for cosign sign also manifest list + pub_reference = "https://" + registry + "/" + list(push_item.repos.keys())[0] to_sign_entries_internal.append( SignEntry( repo=list(push_item.repos.keys())[0], + pub_reference=pub_reference, reference="quay.io/" + self.target_settings["quay_namespace"] + "/" @@ -818,6 +827,7 @@ def untag_image(self, push_item: Any, tag: str) -> None: item_processor = item_processor_for_internal_data( self.quay_client, self.target_settings["quay_host"].rstrip("/"), + self.dest_registries[0], self.target_settings.get("retry_sleep_time", 5), self.target_settings["quay_namespace"], ) diff --git a/tests/test_iib_operations.py b/tests/test_iib_operations.py index 657aa63e..e38c02cb 100644 --- a/tests/test_iib_operations.py +++ b/tests/test_iib_operations.py @@ -359,6 +359,7 @@ def test_task_iib_add_bundles( "sha256:bd6eba96070efe86b64b9a212680ca6d46a2e30f0a7d8e539f657eabc45c35a6", "sha256:bd6eba96070efe86b64b9a212680ca6d46a2e30f0a7d8e539f657eabc45c35a6", ], + identity="some-registry1.com/operators/index-image", ), ] ) @@ -473,6 +474,7 @@ def test_task_iib_add_bundles_missing_manifest_list( "sha256:bd6eba96070efe86b64b9a212680ca6d46a2e30f0a7d8e539f657eabc45c35a6", "sha256:bd6eba96070efe86b64b9a212680ca6d46a2e30f0a7d8e539f657eabc45c35a6", ], + identity="some-registry1.com/operators/index-image", ), ] ) @@ -637,6 +639,7 @@ def test_task_iib_add_bundles_operator_ns( "sha256:bd6eba96070efe86b64b9a212680ca6d46a2e30f0a7d8e539f657eabc45c35a6", "sha256:bd6eba96070efe86b64b9a212680ca6d46a2e30f0a7d8e539f657eabc45c35a6", ], + identity="some-registry1.com/operators/index-image", ), ] ) @@ -1154,6 +1157,7 @@ def test_task_iib_build_from_scratch( "sha256:bd6eba96070efe86b64b9a212680ca6d46a2e30f0a7d8e539f657eabc45c35a6", "sha256:bd6eba96070efe86b64b9a212680ca6d46a2e30f0a7d8e539f657eabc45c35a6", ], + identity="some-registry1.com/operators/index-image", ), ] ) @@ -1407,6 +1411,7 @@ def test_task_iib_build_from_scratch_missing_manifest_list( "sha256:bd6eba96070efe86b64b9a212680ca6d46a2e30f0a7d8e539f657eabc45c35a6", "sha256:bd6eba96070efe86b64b9a212680ca6d46a2e30f0a7d8e539f657eabc45c35a6", ], + identity="some-registry1.com/operators/index-image", ), ] ) @@ -1544,6 +1549,7 @@ def test_task_iib_build_from_scratch_operator_ns( ], task_id="1", ), + # cosign mock.call( config_file="test-config.yml", signing_key="some-key", @@ -1559,6 +1565,7 @@ def test_task_iib_build_from_scratch_operator_ns( "sha256:bd6eba96070efe86b64b9a212680ca6d46a2e30f0a7d8e539f657eabc45c35a6", "sha256:bd6eba96070efe86b64b9a212680ca6d46a2e30f0a7d8e539f657eabc45c35a6", ], + identity="some-registry1.com/operators/index-image", ), ] ) diff --git a/tests/test_integration.py b/tests/test_integration.py index 87d5a4ca..ca0991d2 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -330,6 +330,7 @@ def test_push_docker_multiarch_merge_ml_operator( "sha256:3333333333", "sha256:5555555555", ], + identity="some-registry1.com/target/repo", ), mock.call( config_file="test-config.yml", @@ -380,6 +381,7 @@ def test_push_docker_multiarch_merge_ml_operator( "sha256:5555555555", "sha256:5555555555", ], + identity="some-registry1.com/operators/index-image", ), mock.call( config_file="test-config.yml", @@ -396,6 +398,7 @@ def test_push_docker_multiarch_merge_ml_operator( "sha256:5555555555", "sha256:5555555555", ], + identity="some-registry1.com/operators/index-image", ), ] ) @@ -512,6 +515,7 @@ def test_push_docker_multiarch_simple_workflow( ], task_id="1", ), + # cosign mock.call( config_file="test-config.yml", signing_key="some-key", @@ -529,6 +533,7 @@ def test_push_docker_multiarch_simple_workflow( "sha256:3333333333", "sha256:5555555555", ], + identity="some-registry1.com/target/repo", ), ] ) @@ -662,6 +667,7 @@ def mock_fetch_missing_push_items_digests_sf(push_items): signing_key="fake-sign-key", reference=["quay.io/some-namespace/target----repo:latest-test-tag"], digest=["fake-digest-0"], + identity="some-registry1.com/target/repo", ), ] ) @@ -841,6 +847,7 @@ def test_tag_docker_multiarch_merge_ml( ], task_id="1", ), + # cosign mock.call( config_file="test-config.yml", signing_key="some-key", @@ -854,6 +861,7 @@ def test_tag_docker_multiarch_merge_ml( "sha256:5555555555", "sha256:71e75d5344d529631eaf40a8f9522edb7a66620d73eda6aff667572d511c6519", ], + identity="some-registry1.com/namespace/test_repo", ), ] ) @@ -1051,6 +1059,7 @@ def test_tag_docker_source_copy_untag( ], task_id="1", ), + # cosign mock.call( config_file="test-config.yml", signing_key="some-key", @@ -1060,6 +1069,7 @@ def test_tag_docker_source_copy_untag( digest=[ "sha256:6ef06d8c90c863ba4eb4297f1073ba8cb28c1f6570e2206cdaad2084e2a4715d", ], + identity="some-registry1.com/namespace/test_repo", ), ] ) @@ -1350,6 +1360,7 @@ def test_task_iib_add_bundles( "sha256:5555555555", "sha256:5555555555", ], + identity="some-registry1.com/operators/index-image", ), ] ) @@ -1466,6 +1477,7 @@ def test_task_iib_remove_operators( "quay.io/some-namespace/operators----index-image:8-timestamp", ], digest=["sha256:a1a1a1", "sha256:a1a1a1", "sha256:5555555555", "sha256:5555555555"], + identity="some-registry1.com/operators/index-image", ), ] ) @@ -1877,6 +1889,7 @@ def test_push_docker_operator_verify_bundle_fail( "sha256:3333333333", "sha256:5555555555", ], + identity="some-registry1.com/target/repo", ), ] ) diff --git a/tests/test_item_processor.py b/tests/test_item_processor.py index 70c66930..542e887a 100644 --- a/tests/test_item_processor.py +++ b/tests/test_item_processor.py @@ -21,6 +21,7 @@ def test_generate_existing_tags_tolerate_missing(operator_signing_push_item): ip = ItemProcesor( source_registry="test-registry.io", reference_registries=["dest-registry.io"], + public_registry="", reference_processor=rp, extractor=ContentExtractor(quay_client=mock_client), ) @@ -37,6 +38,7 @@ def test_generate_existing_tags_no_tolerate_missing(operator_signing_push_item): ip = ItemProcesor( source_registry="test-registry.io", reference_registries=["dest-registry.io"], + public_registry="", reference_processor=rp, extractor=ContentExtractor(quay_client=mock_client), ) @@ -53,6 +55,7 @@ def test_generate_existing_tags_server_error(operator_signing_push_item): ip = ItemProcesor( source_registry="test-registry.io", reference_registries=["dest-registry.io"], + public_registry="", reference_processor=rp, extractor=ContentExtractor(quay_client=mock_client), ) @@ -69,6 +72,7 @@ def test_generate_existing_tags_server_error_tolerate_missing(operator_signing_p ip = ItemProcesor( source_registry="test-registry.io", reference_registries=["dest-registry.io"], + public_registry="", reference_processor=rp, extractor=ContentExtractor(quay_client=mock_client), ) @@ -164,6 +168,7 @@ def test_generate_existing_manifest_map_tolerate_429(operator_signing_push_item) ip = ItemProcesor( source_registry="test-registry.io", reference_registries=["dest-registry.io"], + public_registry="", reference_processor=rp, extractor=ContentExtractor(quay_client=mock_client, sleep_time=0), ) diff --git a/tests/test_signer.py b/tests/test_signer.py index 749e7a78..7036876e 100644 --- a/tests/test_signer.py +++ b/tests/test_signer.py @@ -33,6 +33,7 @@ def test_sign_containers_failed(): sw._sign_containers( [ SignEntry( + pub_reference="", reference="fake-reference", digest="fake-digest", signing_key="fake-signing-key",