diff --git a/tests/test_ubipop.py b/tests/test_ubipop.py index 42bfe70..65bdf48 100644 --- a/tests/test_ubipop.py +++ b/tests/test_ubipop.py @@ -687,7 +687,7 @@ def test_create_output_file_all_repos( "foo.pulp.com", ("foo", "foo"), False, - output_repos=out_file_path, + output_repos_file=out_file_path, ubi_manifest_url="https://ubi-manifest.com", ) ubipop.populate_ubi_repos() @@ -814,9 +814,7 @@ def test_get_pulp_actions(mock_ubipop_runner, mock_current_content): unassociations, mdd_association, mdd_unassociation, - ) = mock_ubipop_runner._get_pulp_actions( - mock_current_content, modular_binary=[], modular_debug=[], modular_source=[] - ) + ) = mock_ubipop_runner._get_pulp_actions(mock_current_content) # firstly, check correct associations, there should 1 unit of each type associated modules, rpms, srpms, debug_rpms = associations @@ -929,9 +927,7 @@ def test_get_pulp_actions_no_actions(mock_ubipop_runner, mock_current_content): unassociations, mdd_association, mdd_unassociation, - ) = mock_ubipop_runner._get_pulp_actions( - mock_current_content, modular_binary=[], modular_debug=[], modular_source=[] - ) + ) = mock_ubipop_runner._get_pulp_actions(mock_current_content) # firstly, check correct associations, there should 0 units associated modules, rpms, srpms, debug_rpms = associations @@ -1022,100 +1018,6 @@ def test_log_pulp_action_no_actions(capsys, set_logging, mock_ubipop_runner): assert unassoc_line.strip() == "No unassociation expected for modules from test_dst" -def test_get_pulp_no_duplicates(mock_ubipop_runner, mock_current_content): - binary_rpm = get_rpm_unit( - name="rpm_current", - version="1", - release="2", - arch="x86_64", - filename="rpm_current.rpm", - src_repo_id="foo-rpms", - ) - - debug_rpm = get_rpm_unit( - name="debug_rpm_current", - version="1", - release="2", - arch="x86_64", - filename="debug_rpm_current.rpm", - src_repo_id="foo-debug", - ) - - source_rpms = [ - get_srpm_unit( - name="test_srpm", - version="1.0", - release="1", - filename="test_srpm-1.0-1.src.rpm", - src_repo_id="foo-source", - ), - get_srpm_unit( - name="test_srpm", - version="1.0", - release="2", - filename="test_srpm-1.0-2.src.rpm", - src_repo_id="foo-source", - ), - get_srpm_unit( - name="test_srpm", - version="1.0", - release="2", - filename="test_srpm-1.1-1.src.rpm", - src_repo_id="foo-source", - ), - get_srpm_unit( - name="test_pkg", - version="1", - release="2", - filename="srpm_new.src.rpm", - src_repo_id="foo-source", - ), - get_srpm_unit( - name="foo_pkg", - version="1", - release="2", - filename="srpm_new.src.rpm", - src_repo_id="foo-source", - ), - get_srpm_unit( - name="bar_pkg", - version="1", - release="2", - filename="srpm_new_next.src.rpm", - src_repo_id="foo-source", - ), - ] - modulemd = get_modulemd_unit( - name="md_current", - stream="fake-stream", - version=1, - context="fake-context", - arch="x86_64", - src_repo_id="foo-rpms", - ) - modulemd_defaults = get_modulemd_defaults_unit( - name="mdd_current", - stream="rhel", - profiles={"2.5": ["common"]}, - repo_id="foo-rpms", - src_repo_id="foo-rpms", - ) - mock_ubipop_runner.repos.packages = f_proxy(f_return([binary_rpm])) - mock_ubipop_runner.repos.debug_rpms = f_proxy(f_return([debug_rpm])) - mock_ubipop_runner.repos.source_rpms = f_proxy(f_return(source_rpms)) - mock_ubipop_runner.repos.modules = f_proxy(f_return([modulemd])) - mock_ubipop_runner.repos.module_defaults = f_proxy(f_return([modulemd_defaults])) - - # pylint: disable=protected-access - associations, _, _, _ = mock_ubipop_runner._get_pulp_actions( - mock_current_content, modular_binary=[], modular_debug=[], modular_source=[] - ) - - _, _, srpms, _ = associations - # only 5 srpm associations, no duplicates - assert len(srpms.units) == 5 - - def test_associate_units(mock_ubipop_runner): src_repo = get_test_repo(id="test_src") dst_repo = get_test_repo(id="test_dst") @@ -1304,8 +1206,8 @@ def test_populate_ubi_repos( get_debug_repository, get_source_repository, cache_purge_env, requests_mock ): # pylint: disable=unused-argument - """Test run of populate_ubi_repos that check correct number of repo publication. It's simplified to - contain only actions on RPM packages.""" + """Test run of populate_ubi_repos that checks correct number of repo publication. + It's simplified to contain only actions on RPM packages.""" dt = datetime(2019, 9, 12, 0, 0, 0) d1 = Distributor( @@ -1449,6 +1351,7 @@ def test_populate_ubi_repos( _create_ubi_manifest_mocks(requests_mock) _create_fastpurge_mocks(requests_mock) _create_cdn_mocks(requests_mock) + # let's run actual population ubi_populate.populate_ubi_repos() history = fake_pulp.publish_history @@ -1489,6 +1392,187 @@ def test_populate_ubi_repos( assert sorted(request.json()["objects"]) == sorted(expected_req) +@patch("ubipop._cdn.Publisher") +@patch("ubipop._pulp_client.Pulp.wait_for_tasks") +@patch("pubtools.pulplib.YumRepository.get_debug_repository") +@patch("pubtools.pulplib.YumRepository.get_source_repository") +def test_populate_ubi_repos_no_publish( + get_debug_repository, + get_source_repository, + wait_for_tasks, + publisher, + requests_mock, + monkeypatch, +): + """Test run of populate_ubi_repos which checks that correct asssociations and + unassociations were made in Pulp but no publish was performed. It's simplified to contain + only actions on RPM packages.""" + monkeypatch.setenv("UBIPOP_SKIP_PUBLISH", "true") + + dt = datetime(2019, 9, 12, 0, 0, 0) + + d1 = Distributor( + id="yum_distributor", + type_id="yum_distributor", + repo_id="ubi_binary", + last_publish=dt, + relative_url="content/unit/2/client", + ) + + d2 = Distributor( + id="yum_distributor", + type_id="yum_distributor", + repo_id="ubi_source", + last_publish=dt, + relative_url="content/unit/3/client", + ) + + d3 = Distributor( + id="yum_distributor", + type_id="yum_distributor", + repo_id="ubi_debug", + last_publish=dt, + relative_url="content/unit/4/client", + ) + + output_binary_repo = YumRepository( + id="ubi_binary", + content_set="ubi-8-for-x86_64-appstream-rpms", + population_sources=["input_binary"], + ubi_population=True, + ubi_config_version="8", + eng_product_id=102, + distributors=[d1], + relative_url="content/unit/1/client", + ) + input_binary_repo = YumRepository(id="input_binary") + input_source_repo = YumRepository(id="input_source") + input_debug_repo = YumRepository(id="input_debug") + + output_source_repo = YumRepository( + id="ubi_source", + population_sources=["input_source"], + eng_product_id=102, + distributors=[d2], + relative_url="content/unit/2/client", + ) + output_debug_repo = YumRepository( + id="ubi_debug", + population_sources=["input_debug"], + eng_product_id=102, + distributors=[d3], + relative_url="content/unit/3/client", + ) + + ubi_populate = FakeUbiPopulate( + "foo.pulp.com", + ("foo", "foo"), + False, + ubiconfig_dir_or_url=TEST_DATA_DIR, + ubi_manifest_url="https://ubi-manifest.com", + ) + + fake_pulp = ubi_populate.pulp_client_controller + fake_pulp.insert_repository(input_binary_repo) + fake_pulp.insert_repository(input_source_repo) + fake_pulp.insert_repository(input_debug_repo) + + fake_pulp.insert_repository(output_binary_repo) + fake_pulp.insert_repository(output_source_repo) + fake_pulp.insert_repository(output_debug_repo) + + get_debug_repository.return_value = fake_pulp.client.get_repository("ubi_debug") + get_source_repository.return_value = fake_pulp.client.get_repository("ubi_source") + + old_rpm = RpmUnit( + name="golang", + version="1", + release="a", + arch="x86_64", + filename="golang-1.a.x86_64.rpm", + sourcerpm="golang-1.a.x86_64.src.rpm", + ) + + new_rpm = RpmUnit( + name="golang", + version="2", + release="a", + arch="x86_64", + filename="golang-2.a.x86_64.rpm", + sourcerpm="golang-2.a.x86_64.src.rpm", + ) + + old_modulemd = ModulemdUnit( + name="test_md", stream="s", version="100", context="c", arch="x86_64" + ) + new_modulemd = ModulemdUnit( + name="test_md", stream="s", version="200", context="c", arch="x86_64" + ) + + old_modulemd_defaults = ModulemdDefaultsUnit( + name="test_md_defaults", + stream="stream", + profiles={"minimal": ["name_1"]}, + repo_id="ubi_binary", + ) + new_modulemd_defaults = ModulemdDefaultsUnit( + name="test_md_defaults", + stream="stream", + profiles={"minimal": ["name_1", "name_2"]}, + repo_id="input_binary", + ) + + fake_pulp.insert_units( + output_binary_repo, [old_rpm, old_modulemd, old_modulemd_defaults] + ) + fake_pulp.insert_units( + input_binary_repo, [new_rpm, new_modulemd, new_modulemd_defaults] + ) + + url = "/pulp/api/v2/repositories/{dst_repo}/actions/associate/".format( + dst_repo="ubi_binary" + ) + requests_mock.register_uri( + "POST", url, json={"spawned_tasks": [{"task_id": "association_task_id"}]} + ) + + url = "/pulp/api/v2/repositories/{dst_repo}/actions/unassociate/".format( + dst_repo="ubi_binary" + ) + requests_mock.register_uri( + "POST", url, json={"spawned_tasks": [{"task_id": "unassociation_task_id"}]} + ) + + url = "/pulp/api/v2/tasks/{task_id}/".format(task_id="association_task_id") + requests_mock.register_uri( + "GET", url, json={"state": "finished", "task_id": "association_task_id"} + ) + url = "/pulp/api/v2/tasks/{task_id}/".format(task_id="unassociation_task_id") + requests_mock.register_uri( + "GET", url, json={"state": "finished", "task_id": "unassociation_task_id"} + ) + + # mock calls to ubi-manifest service + _create_ubi_manifest_mocks(requests_mock) + + # let's run actual population + ubi_populate.populate_ubi_repos() + + publisher.assert_not_called() + # there should be 3 associations and 3 unassociations + wait_for_tasks.assert_has_calls( + [ + call(["association_task_id"]), + call(["association_task_id"]), + call(["association_task_id"]), + call(["unassociation_task_id"]), + call(["unassociation_task_id"]), + call(["unassociation_task_id"]), + ], + any_order=True, + ) + + def _create_fastpurge_mocks(requests_mock): url = "https://some-host/ccu/v3/delete/url/production" seconds = 0.1 diff --git a/ubipop/__init__.py b/ubipop/__init__.py index fd6e5cf..16edc52 100644 --- a/ubipop/__init__.py +++ b/ubipop/__init__.py @@ -115,7 +115,7 @@ def __init__( ubiconfig_dir_or_url=None, verify=True, workers_count=4, - output_repos=None, + output_repos_file=None, **kwargs ): # legacy client implemeted in this repo, it's expected to be replaced by pubtools.pulplib.Client @@ -125,7 +125,7 @@ def __init__( self._verify = verify self._pulp_client = None self.dry_run = dry_run - self.output_repos = output_repos + self.output_repos_file = output_repos_file self._executor = Executors.thread_pool(max_workers=workers_count).with_retry() self._ubiconfig_list = None self._ubiconfig_filename_list = ubiconfig_filename_list @@ -152,6 +152,7 @@ def __init__( ), "verify": os.getenv("UBIPOP_CDN_CA_CERT", ""), } + self._skip_publish = os.getenv("UBIPOP_SKIP_PUBLISH", "false") @property def pulp_client(self): @@ -240,7 +241,6 @@ def _filter_ubi_conf_list(self, config_list): conf.content_sets.debuginfo.input, conf.content_sets.debuginfo.output, ]: - # matching by specific content sets takes precedence # or if empty content_set, content_set_regex - take the config immediately # we don't have any other filter to use @@ -249,7 +249,6 @@ def _filter_ubi_conf_list(self, config_list): or label in content_sets or (content_set_regex and re.search(content_set_regex, label)) ): - filtered_conf_list.append(conf) break @@ -305,13 +304,14 @@ def _get_config(self, repo, config): return right_config def populate_ubi_repos(self): - out_repos = set() - used_content_sets = set() + repo_pairs_list = [] + out_repos = set() # list of all affected repositories + ubi_binary_repos = [] # binary repos used for generating manifest + # since repos are searched by content sets, same repo could be searched and populated # multiple times, to avoid that, cache the content sets already used and skip the config # whose content sets are all in the cache - repo_pairs_list = [] - ubi_binary_repos = [] + used_content_sets = set() for config in sorted(self.ubiconfig_list, key=str): content_sets = [ config.content_sets.rpm.output, @@ -329,37 +329,43 @@ def populate_ubi_repos(self): ) continue try: - repo_pairs = self._get_ubi_repo_sets(config.content_sets.rpm.output) + repo_sets = self._get_ubi_repo_sets(config.content_sets.rpm.output) except RepoMissing: _LOG.warning("Skipping current content triplet, some repos are missing") continue - repo_pairs_list.append((repo_pairs, config)) + repo_pairs_list.append((repo_sets, config)) ubi_binary_repos.extend( - [repo_set.out_repos.rpm.id for repo_set in repo_pairs] + [repo_set.out_repos.rpm.id for repo_set in repo_sets] ) with UbimClient(self._ubi_manifest_url) as ubim_client: tasks = ubim_client.generate_manifest(ubi_binary_repos) tasks.result() - with Publisher(**self._publisher_args) as publisher: + if self._skip_publish != "false": + _LOG.warning("Repository publishing via ubipop will be skipped") self._run_ubi_population( - repo_pairs_list, out_repos, ubim_client, publisher + repo_pairs_list, out_repos, ubim_client, publisher=None ) - publisher.wait_publish_and_purge_cache() + else: + with Publisher(**self._publisher_args) as publisher: + self._run_ubi_population( + repo_pairs_list, out_repos, ubim_client, publisher + ) + publisher.wait_publish_and_purge_cache() - if self.output_repos: - with open(self.output_repos, "w") as f: + if self.output_repos_file: + with open(self.output_repos_file, "w") as f: for repo in out_repos: f.write(repo.id.strip() + "\n") def _run_ubi_population( self, repo_pairs_list, out_repos, ubim_client=None, publisher=None ): - for repo_pairs, config in repo_pairs_list: - for repo_set in repo_pairs: + for repo_sets, config in repo_pairs_list: + for repo_set in repo_sets: right_config = self._get_config(repo_set.out_repos.rpm, config) UbiPopulateRunner( @@ -448,10 +454,8 @@ def __init__( self._executor = executor self._publisher = publisher - def _determine_pulp_actions(self, units, current, diff_f, extra_units=None): + def _determine_pulp_actions(self, units, current, diff_f): expected = list(units) - if extra_units: - expected += list(extra_units) to_associate = diff_f(expected, current) to_unassociate = diff_f(current, expected) return to_associate, to_unassociate @@ -468,37 +472,12 @@ def _get_pulp_actions_md_defaults(self, module_defaults, current): self._diff_md_defaults_by_profiles, ) - def _get_pulp_actions_pkgs(self, pkgs, current, modular_pkgs): - return self._determine_pulp_actions( - pkgs, current, self._diff_packages_by_filename, modular_pkgs - ) - - def _get_pulp_actions_src_pkgs(self, pkgs, current, modular): - """ - Get required pulp actions to make sure existing and desired source packages are in - match. - """ - uniq_srpms = {} - - all_pkgs = list(pkgs) + list(modular) - - # filter out packages that share same source rpm - for pkg in all_pkgs: - fn = pkg.filename or pkg.sourcerpm - uniq_srpms[fn] = pkg - - src_pkgs = list(uniq_srpms.values()) + def _get_pulp_actions_pkgs(self, pkgs, current): return self._determine_pulp_actions( - src_pkgs, current, self._diff_packages_by_filename + pkgs, current, self._diff_packages_by_filename ) - def _get_pulp_actions( - self, - current_content, - modular_binary, - modular_debug, - modular_source, - ): + def _get_pulp_actions(self, current_content): """ Determines expected pulp actions by comparing current content of output repos and expected content. @@ -516,17 +495,17 @@ def _get_pulp_actions( ) rpms_assoc, rpms_unassoc = self._get_pulp_actions_pkgs( - self.repos.packages, current_content.binary_rpms, modular_binary + self.repos.packages, current_content.binary_rpms ) - srpms_assoc, srpms_unassoc = self._get_pulp_actions_src_pkgs( - self.repos.source_rpms, current_content.source_rpms, modular_source + srpms_assoc, srpms_unassoc = self._get_pulp_actions_pkgs( + self.repos.source_rpms, current_content.source_rpms ) debug_assoc = None debug_unassoc = None if current_content.debug_rpms: debug_assoc, debug_unassoc = self._get_pulp_actions_pkgs( - self.repos.debug_rpms, current_content.debug_rpms, modular_debug + self.repos.debug_rpms, current_content.debug_rpms ) associations = ( @@ -596,10 +575,6 @@ def _search_expected_modulemd_defaults(self, modulemd_defaults): def run_ubi_population(self): current_content = self._get_current_content() - modular_rpms = [] - modular_debug_rpms = [] - modular_source_rpms = [] - # start async querying for modulemds and modular and non-modular packages binary_manifest = self.ubim_client.get_manifest(self.repos.out_repos.rpm.id) debug_manifest = self.ubim_client.get_manifest(self.repos.out_repos.debug.id) @@ -617,12 +592,7 @@ def run_ubi_population(self): unassociations, mdd_association, mdd_unassociation, - ) = self._get_pulp_actions( - current_content, - modular_rpms, - modular_debug_rpms, - modular_source_rpms, - ) + ) = self._get_pulp_actions(current_content) if self.dry_run: self.log_curent_content(current_content) @@ -639,8 +609,8 @@ def run_ubi_population(self): self._associate_unassociate_md_defaults( (mdd_association,), (mdd_unassociation,) ) - - self._publish_out_repos() + if self._publisher: + self._publish_out_repos() def _associate_unassociate_units(self, action_list): fts = []