From d6b2b93fdddda9196f25c348f51789d8ab421276 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 13 Jun 2024 12:45:05 +0200 Subject: [PATCH 1/8] Enable master (2.11) tests --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7f9810da..ad09ab64 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,7 +17,7 @@ jobs: needs: lint strategy: matrix: - ckan-version: ["2.10", "2.9", "2.9-py2"] + ckan-version: ["master", "2.10", "2.9", "2.9-py2"] fail-fast: false name: CKAN ${{ matrix.ckan-version }} From e835d525d6a39987de09949a6a806a596bc80239 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 13 Jun 2024 12:46:32 +0200 Subject: [PATCH 2/8] Upgrade action versions --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ad09ab64..28231fc7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,8 +4,8 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: '3.7' - name: Install requirements @@ -44,7 +44,7 @@ jobs: CKAN_REDIS_URL: redis://redis:6379/1 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Install requirements run: | pip install -e . From 318f033d605eec4189cee5f4cf9d230f12e92be7 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 13 Jun 2024 12:51:01 +0200 Subject: [PATCH 3/8] Use the ckan images, not the okfn ones --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 28231fc7..9db29c13 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,7 +23,7 @@ jobs: name: CKAN ${{ matrix.ckan-version }} runs-on: ubuntu-latest container: - image: openknowledge/ckan-dev:${{ matrix.ckan-version }} + image: ckan/ckan-dev:${{ matrix.ckan-version }} services: solr: image: ckan/ckan-solr:${{ matrix.ckan-version }} From e9fb899e2674b44f54901af9e60b3be4de4660a6 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 13 Jun 2024 12:59:33 +0200 Subject: [PATCH 4/8] Drop 2.9-py2 images as there are no longer build --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9db29c13..31d10f04 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,7 +17,7 @@ jobs: needs: lint strategy: matrix: - ckan-version: ["master", "2.10", "2.9", "2.9-py2"] + ckan-version: ["master", "2.10", "2.9"] fail-fast: false name: CKAN ${{ matrix.ckan-version }} From 604fb672c3deae0ad2522dd31d9b904967b0a5fc Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 13 Jun 2024 15:21:09 +0200 Subject: [PATCH 5/8] Adapt functional tests to change in auth handling Starting from CKAN 2.11 test client calls are authenticated via `headers={"Authorization": token}` rather than `environ_overrides={"Authorization": token}`. Created some helper functions to handle different CKAN versions from a single place. Also there is a new form in the page so index-based selectors needed to be updated. --- ckanext/scheming/tests/test_form.py | 143 +++++++++++++++++----------- 1 file changed, 85 insertions(+), 58 deletions(-) diff --git a/ckanext/scheming/tests/test_form.py b/ckanext/scheming/tests/test_form.py index b441ba8d..b4ae4502 100644 --- a/ckanext/scheming/tests/test_form.py +++ b/ckanext/scheming/tests/test_form.py @@ -22,34 +22,77 @@ def sysadmin_env(): def _get_package_new_page(app, env, type_='test-schema'): - return app.get(url="/{0}/new".format(type_), extra_environ=env) + if ckantoolkit.check_ckan_version(min_version="2.10.0"): + return app.get(url="/{0}/new".format(type_), headers=env) + else: + return app.get(url="/{0}/new".format(type_), extra_environ=env) + def _get_package_update_page(app, id, env): - return app.get( - url="/test-schema/edit/{}".format(id), extra_environ=env - ) + + if ckantoolkit.check_ckan_version(min_version="2.10.0"): + return app.get(url="/test-schema/edit/{}".format(id), headers=env) + else: + return app.get(url="/test-schema/edit/{}".format(id), extra_environ=env) def _get_resource_new_page(app, id, env): url = '/dataset/{}/resource/new'.format(id) - return app.get( - url, extra_environ=env - ) + + if ckantoolkit.check_ckan_version(min_version="2.10.0"): + return app.get(url, headers=env) + else: + return app.get(url, extra_environ=env) def _get_resource_update_page(app, id, resource_id, env): url = '/dataset/{}/resource/{}/edit'.format(id, resource_id) - return app.get( - url, extra_environ=env, - ) + + if ckantoolkit.check_ckan_version(min_version="2.10.0"): + return app.get(url, headers=env) + else: + return app.get(url, extra_environ=env) def _get_organization_new_page(app, env, type_="organization"): - return app.get(url="/{0}/new".format(type_), extra_environ=env) + + if ckantoolkit.check_ckan_version(min_version="2.10.0"): + return app.get(url="/{0}/new".format(type_), headers=env) + else: + return app.get(url="/{0}/new".format(type_), extra_environ=env) def _get_group_new_page(app, env, type_="group"): - return app.get(url="/{0}/new".format(type_), extra_environ=env) + + if ckantoolkit.check_ckan_version(min_version="2.10.0"): + return app.get(url="/{0}/new".format(type_), headers=env) + else: + return app.get(url="/{0}/new".format(type_), extra_environ=env) + + +def _get_organization_form(html): + # FIXME: add an id to this form + if ckantoolkit.check_ckan_version(min_version="2.11.0a0"): + form = BeautifulSoup(html).select("form")[2] + else: + form = BeautifulSoup(html).select("form")[1] + return form + + +def _get_group_form(html): + return _get_organization_form(html) + + +def _post_data(app, url, data, env): + try: + if ckantoolkit.check_ckan_version(min_version="2.11.0a0"): + app.post(url, headers=env, data=data, follow_redirects=False) + else: + app.post( + url, environ_overrides=env, data=data, follow_redirects=False + ) + except TypeError: + app.post(url.encode('ascii'), params=data, extra_environ=sysadmin_env) @pytest.mark.usefixtures("clean_db") @@ -69,12 +112,8 @@ def test_dataset_form_slug_says_dataset(self, app, sysadmin_env): def test_resource_form_includes_custom_fields(self, app, sysadmin_env): dataset = Dataset(type="test-schema", name="resource-includes-custom") - url = '/dataset/{}/resource/new'.format(dataset["id"]) + response = _get_resource_new_page(app, dataset["id"], sysadmin_env) - response = app.get( - url, - extra_environ=sysadmin_env, - ) form = BeautifulSoup(response.body).select_one("#resource-edit") assert form.select("input[name=camels_in_photo]") @@ -84,18 +123,19 @@ def test_dataset_form_includes_licenses(self, app, sysadmin_env): `DefaultDatasetForm::setup_template_variables` in order to change it. """ - response = app.get(url="/dataset/new", extra_environ=sysadmin_env) + response = _get_package_new_page(app, sysadmin_env, type_="dataset") page = BeautifulSoup(response.body) licenses = page.select('#field-license_id option') assert licenses + @pytest.mark.usefixtures("clean_db") class TestOrganizationFormNew(object): def test_organization_form_includes_custom_field(self, app, sysadmin_env): response = _get_organization_new_page(app, sysadmin_env) - # FIXME: add an id to this form - form = BeautifulSoup(response.body).select("form")[1] + + form = _get_organization_form(response.body) # FIXME: generate the form for orgs (this is currently missing) assert form.select("input[name=department_id]") @@ -114,8 +154,7 @@ class TestGroupFormNew(object): def test_group_form_includes_custom_field(self, app, sysadmin_env): response = _get_group_new_page(app, sysadmin_env) - # FIXME: add an id to this form - form = BeautifulSoup(response.body).select("form")[1] + form = _get_organization_form(response.body) assert form.select("input[name=bookface]") @@ -132,7 +171,9 @@ def test_group_form_slug_says_group(self, app, sysadmin_env): class TestCustomGroupFormNew(object): def test_group_form_includes_custom_field(self, app, sysadmin_env): response = _get_group_new_page(app, sysadmin_env, "theme") - form = BeautifulSoup(response.body).select("form")[1] + + form = _get_group_form(response.body) + assert form.select("input[name=status]") def test_group_form_slug_uses_custom_type(self, app, sysadmin_env): @@ -148,7 +189,8 @@ def test_org_form_includes_custom_field(self, app, sysadmin_env): response = _get_organization_new_page( app, sysadmin_env, "publisher" ) - form = BeautifulSoup(response.body).select("form")[1] + + form = _get_organization_form(response.body) assert form.select("input[name=address]") def test_org_form_slug_uses_custom_type(self, app, sysadmin_env): @@ -163,7 +205,7 @@ def test_org_form_slug_uses_custom_type(self, app, sysadmin_env): class TestJSONDatasetForm(object): def test_dataset_form_includes_json_fields(self, app, sysadmin_env): response = _get_package_new_page(app, sysadmin_env) - form = BeautifulSoup(response.body).select("form")[1] + form = BeautifulSoup(response.body).select("#dataset-edit")[0] assert form.select("textarea[name=a_json_field]") def test_dataset_form_create(self, app, sysadmin_env): @@ -175,10 +217,8 @@ def test_dataset_form_create(self, app, sysadmin_env): data["a_json_field"] = json_value url = '/test-schema/new' - try: - app.post(url, environ_overrides=sysadmin_env, data=data, follow_redirects=False) - except TypeError: - app.post(url.encode('ascii'), params=data, extra_environ=sysadmin_env) + + _post_data(app, url, data, sysadmin_env) dataset = call_action("package_show", id="json_dataset_1") assert dataset["a_json_field"] == value @@ -205,10 +245,8 @@ def test_dataset_form_update(self, app, sysadmin_env): } url = '/dataset/edit/' + dataset["id"] - try: - app.post(url, environ_overrides=sysadmin_env, data=data, follow_redirects=False) - except TypeError: - app.post(url.encode('ascii'), params=data, extra_environ=sysadmin_env) + + _post_data(app, url, data, sysadmin_env) dataset = call_action("package_show", id=dataset["id"]) @@ -245,10 +283,9 @@ def test_resource_form_create(self, app, sysadmin_env): "a_resource_json_field": json_value, "name": dataset["name"], } - try: - app.post(url, environ_overrides=sysadmin_env, data=data, follow_redirects=False) - except TypeError: - app.post(url.encode('ascii'), params=data, extra_environ=sysadmin_env) + + _post_data(app, url, data, sysadmin_env) + dataset = call_action("package_show", id=dataset["id"]) assert dataset["resources"][0]["a_resource_json_field"] == value @@ -293,10 +330,8 @@ def test_resource_form_update(self, app, sysadmin_env): "a_resource_json_field": json_value, "name": dataset["name"], } - try: - app.post(url, environ_overrides=sysadmin_env, data=data, follow_redirects=False) - except TypeError: - app.post(url.encode('ascii'), params=data, extra_environ=sysadmin_env) + + _post_data(app, url, data, sysadmin_env) dataset = call_action("package_show", id=dataset["id"]) @@ -307,7 +342,7 @@ def test_resource_form_update(self, app, sysadmin_env): class TestSubfieldDatasetForm(object): def test_dataset_form_includes_subfields(self, app, sysadmin_env): response = _get_package_new_page(app, sysadmin_env, 'test-subfields') - form = BeautifulSoup(response.body).select("form")[1] + form = BeautifulSoup(response.body).select("#dataset-edit")[0] assert form.select("fieldset[name=scheming-repeating-subfields]") def test_dataset_form_create(self, app, sysadmin_env): @@ -318,10 +353,8 @@ def test_dataset_form_create(self, app, sysadmin_env): data["contact_address-0-address"] = 'anyplace' url = '/test-subfields/new' - try: - app.post(url, environ_overrides=sysadmin_env, data=data, follow_redirects=False) - except TypeError: - app.post(url.encode('ascii'), params=data, extra_environ=sysadmin_env) + + _post_data(app, url, data, sysadmin_env) dataset = call_action("package_show", id="subfield_dataset_1") assert dataset["citation"] == [{'originator': ['mei', 'ahmed']}] @@ -348,10 +381,8 @@ def test_dataset_form_update(self, app, sysadmin_env): data["name"] = dataset["name"] url = '/test-subfields/edit/' + dataset["id"] - try: - app.post(url, environ_overrides=sysadmin_env, data=data, follow_redirects=False) - except TypeError: - app.post(url.encode('ascii'), params=data, extra_environ=sysadmin_env) + + _post_data(app, url, data, sysadmin_env) dataset = call_action("package_show", id=dataset["id"]) @@ -382,10 +413,9 @@ def test_resource_form_create(self, app, sysadmin_env): data = {"id": "", "save": ""} data["schedule-0-impact"] = "P" - try: - app.post(url, environ_overrides=sysadmin_env, data=data, follow_redirects=False) - except TypeError: - app.post(url.encode('ascii'), params=data, extra_environ=sysadmin_env) + + _post_data(app, url, data, sysadmin_env) + dataset = call_action("package_show", id=dataset["id"]) assert dataset["resources"][0]["schedule"] == [{"impact": "P"}] @@ -431,10 +461,7 @@ def test_resource_form_update(self, app, sysadmin_env): data["schedule-1-frequency"] = '1m' data["schedule-1-impact"] = 'P' - try: - app.post(url, environ_overrides=sysadmin_env, data=data, follow_redirects=False) - except TypeError: - app.post(url.encode('ascii'), params=data, extra_environ=sysadmin_env) + _post_data(app, url, data, sysadmin_env) dataset = call_action("package_show", id=dataset["id"]) From 65ae24d0ca9d7168b0b885246e1b0f25a24474c5 Mon Sep 17 00:00:00 2001 From: amercader Date: Fri, 14 Jun 2024 14:34:55 +0200 Subject: [PATCH 6/8] Load plugins in validation/subfields tests --- ckanext/scheming/tests/test_validation.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/ckanext/scheming/tests/test_validation.py b/ckanext/scheming/tests/test_validation.py index 590c43d3..fcb5a9da 100644 --- a/ckanext/scheming/tests/test_validation.py +++ b/ckanext/scheming/tests/test_validation.py @@ -21,6 +21,22 @@ not_empty = get_validator("not_empty") +pytestmark = [ + pytest.mark.usefixtures("with_plugins"), + pytest.mark.ckan_config( + "ckan.plugins", + " ".join([ + "scheming_datasets", + "scheming_groups", + "scheming_organizations", + "scheming_test_plugin", + "scheming_subfields_index", + "scheming_test_validation", + ]) + ) +] + + class TestGetValidatorOrConverter(object): def test_missing(self): with pytest.raises(SchemingException): @@ -941,8 +957,6 @@ def test_invalid_choice(self): raise AssertionError("ValidationError not raised") -@pytest.mark.ckan_config("ckan.plugins", "scheming_test_validation") -@pytest.mark.usefixtures("with_plugins") class TestValidatorsFromString: def test_empty(self): assert validators_from_string("", {}, {}) == [] From 0983735ca163f906693c46f5315ee5abf68d2742 Mon Sep 17 00:00:00 2001 From: amercader Date: Fri, 14 Jun 2024 14:58:32 +0200 Subject: [PATCH 7/8] Replace wrong plugin --- ckanext/scheming/tests/test_validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/scheming/tests/test_validation.py b/ckanext/scheming/tests/test_validation.py index fcb5a9da..8e0678c7 100644 --- a/ckanext/scheming/tests/test_validation.py +++ b/ckanext/scheming/tests/test_validation.py @@ -30,7 +30,7 @@ "scheming_groups", "scheming_organizations", "scheming_test_plugin", - "scheming_subfields_index", + "scheming_nerf_index", "scheming_test_validation", ]) ) From a64ec05411e0ed275cd4336786380021c763f33d Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 27 Jun 2024 16:07:31 +0200 Subject: [PATCH 8/8] Update image tags --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 31d10f04..1f87559f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,7 +17,7 @@ jobs: needs: lint strategy: matrix: - ckan-version: ["master", "2.10", "2.9"] + ckan-version: ["2.11", "2.10", "2.9"] fail-fast: false name: CKAN ${{ matrix.ckan-version }} @@ -26,7 +26,7 @@ jobs: image: ckan/ckan-dev:${{ matrix.ckan-version }} services: solr: - image: ckan/ckan-solr:${{ matrix.ckan-version }} + image: ckan/ckan-solr:${{ matrix.ckan-version }}-solr9 postgres: image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }} env: