Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support for CKAN 2.11 #413

Merged
merged 8 commits into from
Jun 27, 2024
Merged

Support for CKAN 2.11 #413

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
d6b2b93
Enable master (2.11) tests
amercader Jun 13, 2024
e835d52
Upgrade action versions
amercader Jun 13, 2024
318f033
Use the ckan images, not the okfn ones
amercader Jun 13, 2024
e9fb899
Drop 2.9-py2 images as there are no longer build
amercader Jun 13, 2024
604fb67
Adapt functional tests to change in auth handling
amercader Jun 13, 2024
65ae24d
Load plugins in validation/subfields tests
amercader Jun 14, 2024
0983735
Replace wrong plugin
amercader Jun 14, 2024
a64ec05
Update image tags
amercader Jun 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions .github/workflows/test.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.7'
- name: Install requirements
Expand All @@ -17,16 +17,16 @@ jobs:
needs: lint
strategy:
matrix:
ckan-version: ["2.10", "2.9", "2.9-py2"]
ckan-version: ["2.11", "2.10", "2.9"]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

don't want to keep testing against master?

fail-fast: false

name: CKAN ${{ matrix.ckan-version }}
runs-on: ubuntu-latest
container:
image: openknowledge/ckan-dev:${{ matrix.ckan-version }}
image: ckan/ckan-dev:${{ matrix.ckan-version }}
services:
solr:
image: ckan/ckan-solr:${{ matrix.ckan-version }}
image: ckan/ckan-solr:${{ matrix.ckan-version }}-solr9
postgres:
image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }}
env:
Expand All @@ -44,7 +44,7 @@ jobs:
CKAN_REDIS_URL: redis://redis:6379/1

steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v4
- name: Install requirements
run: |
pip install -e .
Expand Down
143 changes: 85 additions & 58 deletions ckanext/scheming/tests/test_form.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,34 +22,77 @@ def sysadmin_env():


def _get_package_new_page(app, env, type_='test-schema'):
return app.get(url="/{0}/new".format(type_), extra_environ=env)
if ckantoolkit.check_ckan_version(min_version="2.10.0"):
return app.get(url="/{0}/new".format(type_), headers=env)
else:
return app.get(url="/{0}/new".format(type_), extra_environ=env)


def _get_package_update_page(app, id, env):
return app.get(
url="/test-schema/edit/{}".format(id), extra_environ=env
)

if ckantoolkit.check_ckan_version(min_version="2.10.0"):
return app.get(url="/test-schema/edit/{}".format(id), headers=env)
else:
return app.get(url="/test-schema/edit/{}".format(id), extra_environ=env)


def _get_resource_new_page(app, id, env):
url = '/dataset/{}/resource/new'.format(id)
return app.get(
url, extra_environ=env
)

if ckantoolkit.check_ckan_version(min_version="2.10.0"):
return app.get(url, headers=env)
else:
return app.get(url, extra_environ=env)


def _get_resource_update_page(app, id, resource_id, env):
url = '/dataset/{}/resource/{}/edit'.format(id, resource_id)
return app.get(
url, extra_environ=env,
)

if ckantoolkit.check_ckan_version(min_version="2.10.0"):
return app.get(url, headers=env)
else:
return app.get(url, extra_environ=env)


def _get_organization_new_page(app, env, type_="organization"):
return app.get(url="/{0}/new".format(type_), extra_environ=env)

if ckantoolkit.check_ckan_version(min_version="2.10.0"):
return app.get(url="/{0}/new".format(type_), headers=env)
else:
return app.get(url="/{0}/new".format(type_), extra_environ=env)


def _get_group_new_page(app, env, type_="group"):
return app.get(url="/{0}/new".format(type_), extra_environ=env)

if ckantoolkit.check_ckan_version(min_version="2.10.0"):
return app.get(url="/{0}/new".format(type_), headers=env)
else:
return app.get(url="/{0}/new".format(type_), extra_environ=env)


def _get_organization_form(html):
# FIXME: add an id to this form
if ckantoolkit.check_ckan_version(min_version="2.11.0a0"):
form = BeautifulSoup(html).select("form")[2]
else:
form = BeautifulSoup(html).select("form")[1]
return form


def _get_group_form(html):
return _get_organization_form(html)


def _post_data(app, url, data, env):
try:
if ckantoolkit.check_ckan_version(min_version="2.11.0a0"):
app.post(url, headers=env, data=data, follow_redirects=False)
else:
app.post(
url, environ_overrides=env, data=data, follow_redirects=False
)
except TypeError:
app.post(url.encode('ascii'), params=data, extra_environ=sysadmin_env)


@pytest.mark.usefixtures("clean_db")
Expand All @@ -69,12 +112,8 @@ def test_dataset_form_slug_says_dataset(self, app, sysadmin_env):
def test_resource_form_includes_custom_fields(self, app, sysadmin_env):
dataset = Dataset(type="test-schema", name="resource-includes-custom")

url = '/dataset/{}/resource/new'.format(dataset["id"])
response = _get_resource_new_page(app, dataset["id"], sysadmin_env)

response = app.get(
url,
extra_environ=sysadmin_env,
)
form = BeautifulSoup(response.body).select_one("#resource-edit")
assert form.select("input[name=camels_in_photo]")

Expand All @@ -84,18 +123,19 @@ def test_dataset_form_includes_licenses(self, app, sysadmin_env):
`DefaultDatasetForm::setup_template_variables` in order to change
it.
"""
response = app.get(url="/dataset/new", extra_environ=sysadmin_env)
response = _get_package_new_page(app, sysadmin_env, type_="dataset")
page = BeautifulSoup(response.body)
licenses = page.select('#field-license_id option')
assert licenses


@pytest.mark.usefixtures("clean_db")
class TestOrganizationFormNew(object):
def test_organization_form_includes_custom_field(self, app, sysadmin_env):

response = _get_organization_new_page(app, sysadmin_env)
# FIXME: add an id to this form
form = BeautifulSoup(response.body).select("form")[1]

form = _get_organization_form(response.body)

# FIXME: generate the form for orgs (this is currently missing)
assert form.select("input[name=department_id]")
Expand All @@ -114,8 +154,7 @@ class TestGroupFormNew(object):
def test_group_form_includes_custom_field(self, app, sysadmin_env):

response = _get_group_new_page(app, sysadmin_env)
# FIXME: add an id to this form
form = BeautifulSoup(response.body).select("form")[1]
form = _get_organization_form(response.body)

assert form.select("input[name=bookface]")

Expand All @@ -132,7 +171,9 @@ def test_group_form_slug_says_group(self, app, sysadmin_env):
class TestCustomGroupFormNew(object):
def test_group_form_includes_custom_field(self, app, sysadmin_env):
response = _get_group_new_page(app, sysadmin_env, "theme")
form = BeautifulSoup(response.body).select("form")[1]

form = _get_group_form(response.body)

assert form.select("input[name=status]")

def test_group_form_slug_uses_custom_type(self, app, sysadmin_env):
Expand All @@ -148,7 +189,8 @@ def test_org_form_includes_custom_field(self, app, sysadmin_env):
response = _get_organization_new_page(
app, sysadmin_env, "publisher"
)
form = BeautifulSoup(response.body).select("form")[1]

form = _get_organization_form(response.body)
assert form.select("input[name=address]")

def test_org_form_slug_uses_custom_type(self, app, sysadmin_env):
Expand All @@ -163,7 +205,7 @@ def test_org_form_slug_uses_custom_type(self, app, sysadmin_env):
class TestJSONDatasetForm(object):
def test_dataset_form_includes_json_fields(self, app, sysadmin_env):
response = _get_package_new_page(app, sysadmin_env)
form = BeautifulSoup(response.body).select("form")[1]
form = BeautifulSoup(response.body).select("#dataset-edit")[0]
assert form.select("textarea[name=a_json_field]")

def test_dataset_form_create(self, app, sysadmin_env):
Expand All @@ -175,10 +217,8 @@ def test_dataset_form_create(self, app, sysadmin_env):
data["a_json_field"] = json_value

url = '/test-schema/new'
try:
app.post(url, environ_overrides=sysadmin_env, data=data, follow_redirects=False)
except TypeError:
app.post(url.encode('ascii'), params=data, extra_environ=sysadmin_env)

_post_data(app, url, data, sysadmin_env)

dataset = call_action("package_show", id="json_dataset_1")
assert dataset["a_json_field"] == value
Expand All @@ -205,10 +245,8 @@ def test_dataset_form_update(self, app, sysadmin_env):
}

url = '/dataset/edit/' + dataset["id"]
try:
app.post(url, environ_overrides=sysadmin_env, data=data, follow_redirects=False)
except TypeError:
app.post(url.encode('ascii'), params=data, extra_environ=sysadmin_env)

_post_data(app, url, data, sysadmin_env)

dataset = call_action("package_show", id=dataset["id"])

Expand Down Expand Up @@ -245,10 +283,9 @@ def test_resource_form_create(self, app, sysadmin_env):
"a_resource_json_field": json_value,
"name": dataset["name"],
}
try:
app.post(url, environ_overrides=sysadmin_env, data=data, follow_redirects=False)
except TypeError:
app.post(url.encode('ascii'), params=data, extra_environ=sysadmin_env)

_post_data(app, url, data, sysadmin_env)

dataset = call_action("package_show", id=dataset["id"])

assert dataset["resources"][0]["a_resource_json_field"] == value
Expand Down Expand Up @@ -293,10 +330,8 @@ def test_resource_form_update(self, app, sysadmin_env):
"a_resource_json_field": json_value,
"name": dataset["name"],
}
try:
app.post(url, environ_overrides=sysadmin_env, data=data, follow_redirects=False)
except TypeError:
app.post(url.encode('ascii'), params=data, extra_environ=sysadmin_env)

_post_data(app, url, data, sysadmin_env)

dataset = call_action("package_show", id=dataset["id"])

Expand All @@ -307,7 +342,7 @@ def test_resource_form_update(self, app, sysadmin_env):
class TestSubfieldDatasetForm(object):
def test_dataset_form_includes_subfields(self, app, sysadmin_env):
response = _get_package_new_page(app, sysadmin_env, 'test-subfields')
form = BeautifulSoup(response.body).select("form")[1]
form = BeautifulSoup(response.body).select("#dataset-edit")[0]
assert form.select("fieldset[name=scheming-repeating-subfields]")

def test_dataset_form_create(self, app, sysadmin_env):
Expand All @@ -318,10 +353,8 @@ def test_dataset_form_create(self, app, sysadmin_env):
data["contact_address-0-address"] = 'anyplace'

url = '/test-subfields/new'
try:
app.post(url, environ_overrides=sysadmin_env, data=data, follow_redirects=False)
except TypeError:
app.post(url.encode('ascii'), params=data, extra_environ=sysadmin_env)

_post_data(app, url, data, sysadmin_env)

dataset = call_action("package_show", id="subfield_dataset_1")
assert dataset["citation"] == [{'originator': ['mei', 'ahmed']}]
Expand All @@ -348,10 +381,8 @@ def test_dataset_form_update(self, app, sysadmin_env):
data["name"] = dataset["name"]

url = '/test-subfields/edit/' + dataset["id"]
try:
app.post(url, environ_overrides=sysadmin_env, data=data, follow_redirects=False)
except TypeError:
app.post(url.encode('ascii'), params=data, extra_environ=sysadmin_env)

_post_data(app, url, data, sysadmin_env)

dataset = call_action("package_show", id=dataset["id"])

Expand Down Expand Up @@ -382,10 +413,9 @@ def test_resource_form_create(self, app, sysadmin_env):

data = {"id": "", "save": ""}
data["schedule-0-impact"] = "P"
try:
app.post(url, environ_overrides=sysadmin_env, data=data, follow_redirects=False)
except TypeError:
app.post(url.encode('ascii'), params=data, extra_environ=sysadmin_env)

_post_data(app, url, data, sysadmin_env)

dataset = call_action("package_show", id=dataset["id"])

assert dataset["resources"][0]["schedule"] == [{"impact": "P"}]
Expand Down Expand Up @@ -431,10 +461,7 @@ def test_resource_form_update(self, app, sysadmin_env):
data["schedule-1-frequency"] = '1m'
data["schedule-1-impact"] = 'P'

try:
app.post(url, environ_overrides=sysadmin_env, data=data, follow_redirects=False)
except TypeError:
app.post(url.encode('ascii'), params=data, extra_environ=sysadmin_env)
_post_data(app, url, data, sysadmin_env)

dataset = call_action("package_show", id=dataset["id"])

Expand Down
18 changes: 16 additions & 2 deletions ckanext/scheming/tests/test_validation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,22 @@
not_empty = get_validator("not_empty")


pytestmark = [
pytest.mark.usefixtures("with_plugins"),
pytest.mark.ckan_config(
"ckan.plugins",
" ".join([
"scheming_datasets",
"scheming_groups",
"scheming_organizations",
"scheming_test_plugin",
"scheming_nerf_index",
"scheming_test_validation",
])
)
]


class TestGetValidatorOrConverter(object):
def test_missing(self):
with pytest.raises(SchemingException):
Expand Down Expand Up @@ -941,8 +957,6 @@ def test_invalid_choice(self):
raise AssertionError("ValidationError not raised")


@pytest.mark.ckan_config("ckan.plugins", "scheming_test_validation")
@pytest.mark.usefixtures("with_plugins")
class TestValidatorsFromString:
def test_empty(self):
assert validators_from_string("", {}, {}) == []
Expand Down