From cf588d5734eb2ee4e9bef2011cfe58b17191166d Mon Sep 17 00:00:00 2001 From: kyokukou Date: Mon, 4 Nov 2024 13:32:09 -0800 Subject: [PATCH 01/29] set up tweaks --- README.md | 1 + oaipmh/config.py | 2 -- tests/conftest.py | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 3e05612..70a1fd7 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ poetry install run: python main.py +Then check the app is running with http://localhost:8080/oai tests: diff --git a/oaipmh/config.py b/oaipmh/config.py index a937a70..6ec1a28 100644 --- a/oaipmh/config.py +++ b/oaipmh/config.py @@ -14,8 +14,6 @@ class Settings(arxiv_base.Settings): SQLALCHEMY_MAX_OVERFLOW: Optional[int] = 0 SQLALCHEMY_POOL_SIZE: Optional[int] = 10 - APPLICATION_ROOT: Optional[str] = None - def check(self) -> None: """A check and fix up of a settings object.""" if 'sqlite' in self.CLASSIC_DB_URI: diff --git a/tests/conftest.py b/tests/conftest.py index 1fd4402..5db8b2b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,7 +2,6 @@ from oaipmh.factory import create_web_app TESTING_CONFIG = { - "APPLICATION_ROOT": "", "TESTING": True } From 1a9b1c27a91be2b3c353ff4b711cdbd9451d2855 Mon Sep 17 00:00:00 2001 From: kyokukou Date: Mon, 4 Nov 2024 14:20:42 -0800 Subject: [PATCH 02/29] created layout with empty files --- oaipmh/data/oai_config.py | 0 oaipmh/data/oai_errors.py | 0 oaipmh/factory.py | 2 +- oaipmh/processors/db.py | 0 oaipmh/processors/queries.py | 0 oaipmh/processors/resume.py | 0 oaipmh/requests/data_queries.py | 0 oaipmh/requests/info_queries.py | 0 oaipmh/requests/resumption.py | 0 oaipmh/{ => requests}/routes.py | 2 +- oaipmh/requests/verb_sorter.py | 0 oaipmh/serializers/config_srl.py | 0 oaipmh/serializers/database_srl.py | 0 oaipmh/serializers/output_formats.py | 0 oaipmh/templates/base.html | 1 + oaipmh/templates/error.html | 0 oaipmh/templates/headers.html | 0 oaipmh/templates/records.html | 0 oaipmh/templates/token.html | 0 19 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 oaipmh/data/oai_config.py create mode 100644 oaipmh/data/oai_errors.py create mode 100644 oaipmh/processors/db.py create mode 100644 oaipmh/processors/queries.py create mode 100644 oaipmh/processors/resume.py create mode 100644 oaipmh/requests/data_queries.py create mode 100644 oaipmh/requests/info_queries.py create mode 100644 oaipmh/requests/resumption.py rename oaipmh/{ => requests}/routes.py (69%) create mode 100644 oaipmh/requests/verb_sorter.py create mode 100644 oaipmh/serializers/config_srl.py create mode 100644 oaipmh/serializers/database_srl.py create mode 100644 oaipmh/serializers/output_formats.py create mode 100644 oaipmh/templates/base.html create mode 100644 oaipmh/templates/error.html create mode 100644 oaipmh/templates/headers.html create mode 100644 oaipmh/templates/records.html create mode 100644 oaipmh/templates/token.html diff --git a/oaipmh/data/oai_config.py b/oaipmh/data/oai_config.py new file mode 100644 index 0000000..e69de29 diff --git a/oaipmh/data/oai_errors.py b/oaipmh/data/oai_errors.py new file mode 100644 index 0000000..e69de29 diff --git a/oaipmh/factory.py b/oaipmh/factory.py index 6663194..5c27bf1 100644 --- a/oaipmh/factory.py +++ b/oaipmh/factory.py @@ -8,7 +8,7 @@ from arxiv.db import config_query_timing, configure_db from oaipmh.config import Settings -from oaipmh import routes +from oaipmh.requests import routes s3 = FlaskS3() diff --git a/oaipmh/processors/db.py b/oaipmh/processors/db.py new file mode 100644 index 0000000..e69de29 diff --git a/oaipmh/processors/queries.py b/oaipmh/processors/queries.py new file mode 100644 index 0000000..e69de29 diff --git a/oaipmh/processors/resume.py b/oaipmh/processors/resume.py new file mode 100644 index 0000000..e69de29 diff --git a/oaipmh/requests/data_queries.py b/oaipmh/requests/data_queries.py new file mode 100644 index 0000000..e69de29 diff --git a/oaipmh/requests/info_queries.py b/oaipmh/requests/info_queries.py new file mode 100644 index 0000000..e69de29 diff --git a/oaipmh/requests/resumption.py b/oaipmh/requests/resumption.py new file mode 100644 index 0000000..e69de29 diff --git a/oaipmh/routes.py b/oaipmh/requests/routes.py similarity index 69% rename from oaipmh/routes.py rename to oaipmh/requests/routes.py index 6268f9b..9ae8143 100644 --- a/oaipmh/routes.py +++ b/oaipmh/requests/routes.py @@ -3,6 +3,6 @@ blueprint = Blueprint('general', __name__) -@blueprint.route("/oai", methods=['GET', 'HEAD']) +@blueprint.route("/oai", methods=['GET', 'HEAD', 'POST']) def oai(): # type: ignore return "working!", 200, {} \ No newline at end of file diff --git a/oaipmh/requests/verb_sorter.py b/oaipmh/requests/verb_sorter.py new file mode 100644 index 0000000..e69de29 diff --git a/oaipmh/serializers/config_srl.py b/oaipmh/serializers/config_srl.py new file mode 100644 index 0000000..e69de29 diff --git a/oaipmh/serializers/database_srl.py b/oaipmh/serializers/database_srl.py new file mode 100644 index 0000000..e69de29 diff --git a/oaipmh/serializers/output_formats.py b/oaipmh/serializers/output_formats.py new file mode 100644 index 0000000..e69de29 diff --git a/oaipmh/templates/base.html b/oaipmh/templates/base.html new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/oaipmh/templates/base.html @@ -0,0 +1 @@ + diff --git a/oaipmh/templates/error.html b/oaipmh/templates/error.html new file mode 100644 index 0000000..e69de29 diff --git a/oaipmh/templates/headers.html b/oaipmh/templates/headers.html new file mode 100644 index 0000000..e69de29 diff --git a/oaipmh/templates/records.html b/oaipmh/templates/records.html new file mode 100644 index 0000000..e69de29 diff --git a/oaipmh/templates/token.html b/oaipmh/templates/token.html new file mode 100644 index 0000000..e69de29 From f5c8b482e17fd06fd2500d2de5fa5247d7404409 Mon Sep 17 00:00:00 2001 From: kyokukou Date: Tue, 5 Nov 2024 10:30:32 -0800 Subject: [PATCH 03/29] sort request into handlers based on verb --- oaipmh/requests/data_queries.py | 12 +++++ oaipmh/requests/info_queries.py | 11 ++++ oaipmh/requests/routes.py | 16 ++++-- oaipmh/requests/verb_sorter.py | 30 +++++++++++ oaipmh/serializers/output_formats.py | 3 ++ tests/test_verb_sorting.py | 81 ++++++++++++++++++++++++++++ 6 files changed, 149 insertions(+), 4 deletions(-) create mode 100644 tests/test_verb_sorting.py diff --git a/oaipmh/requests/data_queries.py b/oaipmh/requests/data_queries.py index e69de29..0d7071c 100644 --- a/oaipmh/requests/data_queries.py +++ b/oaipmh/requests/data_queries.py @@ -0,0 +1,12 @@ +from typing import Dict + +from oaipmh.serializers.output_formats import InteriorData + +def get_record(params: Dict[str, str]) -> InteriorData: + return + +def list_records(params: Dict[str, str]) -> InteriorData: + return + +def list_identifiers(params: Dict[str, str]) -> InteriorData: + return diff --git a/oaipmh/requests/info_queries.py b/oaipmh/requests/info_queries.py index e69de29..c6e8a99 100644 --- a/oaipmh/requests/info_queries.py +++ b/oaipmh/requests/info_queries.py @@ -0,0 +1,11 @@ +from typing import Dict +from oaipmh.serializers.output_formats import InteriorData + +def identify(params: Dict[str, str]) -> InteriorData: + return + +def list_metadata_formats(params: Dict[str, str]) -> InteriorData: + return + +def list_sets(params: Dict[str, str]) -> InteriorData: + return diff --git a/oaipmh/requests/routes.py b/oaipmh/requests/routes.py index 9ae8143..d3ff678 100644 --- a/oaipmh/requests/routes.py +++ b/oaipmh/requests/routes.py @@ -1,8 +1,16 @@ -from flask import Blueprint +from typing import Dict +from flask import Blueprint, request, Response + +from oaipmh.requests.verb_sorter import verb_sorter blueprint = Blueprint('general', __name__) -@blueprint.route("/oai", methods=['GET', 'HEAD', 'POST']) -def oai(): # type: ignore - return "working!", 200, {} \ No newline at end of file +@blueprint.route("/oai", methods=['GET', 'POST']) +def oai() -> Response: + + params: Dict[str, str] = request.args.to_dict() if request.method == 'GET' else request.form.to_dict() + result=verb_sorter(params) + #TODO package interior data in page + + return "working", 200, {} \ No newline at end of file diff --git a/oaipmh/requests/verb_sorter.py b/oaipmh/requests/verb_sorter.py index e69de29..60f9e09 100644 --- a/oaipmh/requests/verb_sorter.py +++ b/oaipmh/requests/verb_sorter.py @@ -0,0 +1,30 @@ +from typing import Dict + +from oaipmh.requests.info_queries import identify, list_metadata_formats, list_sets +from oaipmh.requests.data_queries import get_record, list_identifiers, list_records +from oaipmh.serializers.output_formats import InteriorData + +def verb_sorter(params: Dict[str, str]) -> InteriorData: + """ + sorts OAI queries to the appropriate handler based on their verb statement + this defines what the client is asking for as per the OAI standard + further verification of parameters is done with the handlers for individual verbs + returns the interior xml for the response + """ + verb = params.get("verb", "") + match verb: + case "GetRecord": + return get_record(params) + case "ListRecords": + return list_records(params) + case "ListIdentifiers": + return list_identifiers(params) + case "Identify": + return identify(params) + case "ListMetadataFormats": + return list_metadata_formats(params) + case "ListSets": + return list_sets(params) + case _: + #TODO bad/no verb case error + return InteriorData() diff --git a/oaipmh/serializers/output_formats.py b/oaipmh/serializers/output_formats.py index e69de29..c73df8a 100644 --- a/oaipmh/serializers/output_formats.py +++ b/oaipmh/serializers/output_formats.py @@ -0,0 +1,3 @@ + +class InteriorData: #TODO placeholder return type + value="" \ No newline at end of file diff --git a/tests/test_verb_sorting.py b/tests/test_verb_sorting.py new file mode 100644 index 0000000..e2eed25 --- /dev/null +++ b/tests/test_verb_sorting.py @@ -0,0 +1,81 @@ +from unittest.mock import patch + +def test_get_record(test_client): + params = {"verb": "GetRecord", "identifier": "oai:example.org:record123"} + + with patch('oaipmh.requests.verb_sorter.get_record') as mock_get_record: + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + mock_get_record.assert_called_once_with(params) + + with patch('oaipmh.requests.verb_sorter.get_record') as mock_get_record: + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + mock_get_record.assert_called_once_with(params) + +def test_list_records(test_client): + params = {"verb": "ListRecords", "metadataPrefix": "oai_dc"} + + with patch('oaipmh.requests.verb_sorter.list_records') as mock_list_records: + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + mock_list_records.assert_called_once_with(params) + + with patch('oaipmh.requests.verb_sorter.list_records') as mock_list_records: + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + mock_list_records.assert_called_once_with(params) + +def test_list_identifiers(test_client): + params = {"verb": "ListIdentifiers", "metadataPrefix": "oai_dc"} + + with patch('oaipmh.requests.verb_sorter.list_identifiers') as mock_list_identifiers: + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + mock_list_identifiers.assert_called_once_with(params) + + with patch('oaipmh.requests.verb_sorter.list_identifiers') as mock_list_identifiers: + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + mock_list_identifiers.assert_called_once_with(params) + +def test_identify(test_client): + params = {"verb": "Identify"} + + with patch('oaipmh.requests.verb_sorter.identify') as mock_identify: + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + mock_identify.assert_called_once_with(params) + + with patch('oaipmh.requests.verb_sorter.identify') as mock_identify: + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + mock_identify.assert_called_once_with(params) + +def test_list_metadata_formats(test_client): + params = {"verb": "ListMetadataFormats"} + + with patch('oaipmh.requests.verb_sorter.list_metadata_formats') as mock_list_metadata_formats: + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + mock_list_metadata_formats.assert_called_once_with(params) + + with patch('oaipmh.requests.verb_sorter.list_metadata_formats') as mock_list_metadata_formats: + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + mock_list_metadata_formats.assert_called_once_with(params) + +def test_list_sets(test_client): + params = {"verb": "ListSets"} + + with patch('oaipmh.requests.verb_sorter.list_sets') as mock_list_sets: + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + mock_list_sets.assert_called_once_with(params) + + with patch('oaipmh.requests.verb_sorter.list_sets') as mock_list_sets: + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + mock_list_sets.assert_called_once_with(params) + +#TODO test no/bad verb \ No newline at end of file From 7b68c943e9b0a4ef690a10b2fe0445cb6942c0b4 Mon Sep 17 00:00:00 2001 From: kyokukou Date: Tue, 5 Nov 2024 12:52:37 -0800 Subject: [PATCH 04/29] render a base template --- oaipmh/requests/routes.py | 15 ++++++++++++--- oaipmh/templates/base.html | 1 - oaipmh/templates/base.xml | 8 ++++++++ 3 files changed, 20 insertions(+), 4 deletions(-) delete mode 100644 oaipmh/templates/base.html create mode 100644 oaipmh/templates/base.xml diff --git a/oaipmh/requests/routes.py b/oaipmh/requests/routes.py index d3ff678..c63906e 100644 --- a/oaipmh/requests/routes.py +++ b/oaipmh/requests/routes.py @@ -1,5 +1,6 @@ from typing import Dict -from flask import Blueprint, request, Response +from datetime import datetime, timezone +from flask import Blueprint, request, Response, render_template from oaipmh.requests.verb_sorter import verb_sorter @@ -11,6 +12,14 @@ def oai() -> Response: params: Dict[str, str] = request.args.to_dict() if request.method == 'GET' else request.form.to_dict() result=verb_sorter(params) - #TODO package interior data in page + + + response_xml=render_template("base.xml", + response_date=datetime.now(timezone.utc), + request_info="request info", #TODO + interior_xml="interior data" #TODO + ) + + headers={"Content-Type":"application/xml"} - return "working", 200, {} \ No newline at end of file + return response_xml, 200, headers \ No newline at end of file diff --git a/oaipmh/templates/base.html b/oaipmh/templates/base.html deleted file mode 100644 index 8b13789..0000000 --- a/oaipmh/templates/base.html +++ /dev/null @@ -1 +0,0 @@ - diff --git a/oaipmh/templates/base.xml b/oaipmh/templates/base.xml new file mode 100644 index 0000000..822b43c --- /dev/null +++ b/oaipmh/templates/base.xml @@ -0,0 +1,8 @@ + + + {{ response_date.strftime("%Y-%m-%dT%H:%M:%SZ") }} + {{ request_info }} + {{interior_xml}} + \ No newline at end of file From a66d6ad8cf6cbd161f69a99fe2331736224b5e46 Mon Sep 17 00:00:00 2001 From: kyokukou Date: Tue, 5 Nov 2024 13:31:22 -0800 Subject: [PATCH 05/29] create basic request element --- oaipmh/config.py | 2 ++ oaipmh/requests/routes.py | 8 ++++++-- oaipmh/templates/base.xml | 3 ++- oaipmh/templates/macros.html | 3 +++ 4 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 oaipmh/templates/macros.html diff --git a/oaipmh/config.py b/oaipmh/config.py index 6ec1a28..5e445cd 100644 --- a/oaipmh/config.py +++ b/oaipmh/config.py @@ -14,6 +14,8 @@ class Settings(arxiv_base.Settings): SQLALCHEMY_MAX_OVERFLOW: Optional[int] = 0 SQLALCHEMY_POOL_SIZE: Optional[int] = 10 + FLASKS3_BUCKET_NAME: str = "some_bucket" #TODO needed to use url for for some reason? + def check(self) -> None: """A check and fix up of a settings object.""" if 'sqlite' in self.CLASSIC_DB_URI: diff --git a/oaipmh/requests/routes.py b/oaipmh/requests/routes.py index c63906e..115b913 100644 --- a/oaipmh/requests/routes.py +++ b/oaipmh/requests/routes.py @@ -19,7 +19,11 @@ def oai() -> Response: request_info="request info", #TODO interior_xml="interior data" #TODO ) - headers={"Content-Type":"application/xml"} - return response_xml, 200, headers \ No newline at end of file + return response_xml, 200, headers + +@blueprint.route('/favicon.ico') +def favicon(): + #TODO + return '', 204 \ No newline at end of file diff --git a/oaipmh/templates/base.xml b/oaipmh/templates/base.xml index 822b43c..546ed87 100644 --- a/oaipmh/templates/base.xml +++ b/oaipmh/templates/base.xml @@ -1,8 +1,9 @@ +{% import 'macros.html' as macros %} {{ response_date.strftime("%Y-%m-%dT%H:%M:%SZ") }} - {{ request_info }} + {{macros.request_element()}} {{interior_xml}} \ No newline at end of file diff --git a/oaipmh/templates/macros.html b/oaipmh/templates/macros.html new file mode 100644 index 0000000..b647bca --- /dev/null +++ b/oaipmh/templates/macros.html @@ -0,0 +1,3 @@ +{% macro request_element( attributes={}) %} + {{ url_for("general.oai", _external=True) }} +{% endmacro %} \ No newline at end of file From abd284a06e567b3b1724ee7b4d1ee17175bfd80c Mon Sep 17 00:00:00 2001 From: kyokukou Date: Wed, 6 Nov 2024 10:09:08 -0800 Subject: [PATCH 06/29] added OAI errors, handler --- oaipmh/data/oai_errors.py | 46 ++++++++++++++++++++++++++++++++++ oaipmh/factory.py | 19 +++++++++++++- oaipmh/requests/routes.py | 4 +-- oaipmh/requests/verb_sorter.py | 4 +-- oaipmh/templates/base.xml | 7 ++++-- oaipmh/templates/error.html | 0 oaipmh/templates/errors.xml | 9 +++++++ tests/test_basic.py | 6 ++++- tests/test_verb_sorting.py | 22 +++++++++++++++- 9 files changed, 108 insertions(+), 9 deletions(-) delete mode 100644 oaipmh/templates/error.html create mode 100644 oaipmh/templates/errors.xml diff --git a/oaipmh/data/oai_errors.py b/oaipmh/data/oai_errors.py index e69de29..291e957 100644 --- a/oaipmh/data/oai_errors.py +++ b/oaipmh/data/oai_errors.py @@ -0,0 +1,46 @@ + +class OAIException(Exception): + """General class for all OAI defined errors""" + code: str + description: str + pass + +class OAIBadArgument(OAIException): + #dont include attributes + code="badArgument" + description="The request includes illegal arguments, is missing required arguments, includes a repeated argument, or values for arguments have an illegal syntax." + +class OAIBadResumptionToken(OAIException): + #TODO consider including params + code="badResumptionToken" + description="The value of the resumptionToken argument is invalid or expired." + +class OAIBadVerb(OAIException): + #dont include attributes + code="badVerb" + description="Value of the verb argument is not a legal OAI-PMH verb, the verb argument is missing, or the verb argument is repeated." + +class OAIBadFormat(OAIException): + #TODO consider including params + code="cannotDisseminateFormat" + description="The metadata format identified by the value given for the metadataPrefix argument is not supported by the item or by the repository." + +class OAINonexistentID(OAIException): + #TODO consider including params + code="idDoesNotExist" + description="The value of the identifier argument is unknown or illegal in this repository." + +class OAINoRecordsMatch(OAIException): + #TODO params + code="noRecordsMatch" + description="The combination of the values of the from, until, set and metadataPrefix arguments results in an empty list." + +class OAINoMetadataFormats(OAIException): + #TODO consider including params + code="noMetadataFormats" + description="There are no metadata formats available for the specified item." + +class OAINoSetHierarchy(OAIException): + #should not be triggered for arXiv implementation + code="noSetHierarchy" + description="The repository does not support sets. This exception should not be true for the arXiv implementation." \ No newline at end of file diff --git a/oaipmh/factory.py b/oaipmh/factory.py index 5c27bf1..e3f1400 100644 --- a/oaipmh/factory.py +++ b/oaipmh/factory.py @@ -1,12 +1,15 @@ import logging +from datetime import datetime, timezone -from flask import Flask +from flask import Flask, render_template from flask_s3 import FlaskS3 from flask.logging import default_handler +from werkzeug.exceptions import HTTPException from arxiv.base import Base from arxiv.db import config_query_timing, configure_db +from oaipmh.data.oai_errors import OAIException from oaipmh.config import Settings from oaipmh.requests import routes @@ -33,6 +36,20 @@ def create_web_app(**kwargs) -> Flask: # type: ignore app.register_blueprint(routes.blueprint) s3.init_app(app) + @app.errorhandler(OAIException) + def handle_oai_error(e): + response=render_template("errors.xml", + response_date=datetime.now(timezone.utc), + error=e) + headers={"Content-Type":"application/xml"} + return response, 200, headers + + #TODO make this actually trigger + @app.errorhandler(HTTPException) + def handle_http_error(e): + print("main error handler ran!") + return e.description, e.code, {} + app.jinja_env.trim_blocks = True app.jinja_env.lstrip_blocks = True if not app.jinja_env.globals: diff --git a/oaipmh/requests/routes.py b/oaipmh/requests/routes.py index 115b913..ac75de7 100644 --- a/oaipmh/requests/routes.py +++ b/oaipmh/requests/routes.py @@ -10,10 +10,10 @@ @blueprint.route("/oai", methods=['GET', 'POST']) def oai() -> Response: + #TODO what happens if duplicate params params: Dict[str, str] = request.args.to_dict() if request.method == 'GET' else request.form.to_dict() result=verb_sorter(params) - response_xml=render_template("base.xml", response_date=datetime.now(timezone.utc), request_info="request info", #TODO @@ -26,4 +26,4 @@ def oai() -> Response: @blueprint.route('/favicon.ico') def favicon(): #TODO - return '', 204 \ No newline at end of file + return '', 204 diff --git a/oaipmh/requests/verb_sorter.py b/oaipmh/requests/verb_sorter.py index 60f9e09..d49e7cb 100644 --- a/oaipmh/requests/verb_sorter.py +++ b/oaipmh/requests/verb_sorter.py @@ -3,6 +3,7 @@ from oaipmh.requests.info_queries import identify, list_metadata_formats, list_sets from oaipmh.requests.data_queries import get_record, list_identifiers, list_records from oaipmh.serializers.output_formats import InteriorData +from oaipmh.data.oai_errors import OAIBadVerb def verb_sorter(params: Dict[str, str]) -> InteriorData: """ @@ -26,5 +27,4 @@ def verb_sorter(params: Dict[str, str]) -> InteriorData: case "ListSets": return list_sets(params) case _: - #TODO bad/no verb case error - return InteriorData() + raise OAIBadVerb #dont keep invalid verb diff --git a/oaipmh/templates/base.xml b/oaipmh/templates/base.xml index 546ed87..cf966db 100644 --- a/oaipmh/templates/base.xml +++ b/oaipmh/templates/base.xml @@ -4,6 +4,9 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd"> {{ response_date.strftime("%Y-%m-%dT%H:%M:%SZ") }} - {{macros.request_element()}} - {{interior_xml}} + {% block request_element %} + {% endblock %} + + {% block interior_xml %} + {% endblock %} \ No newline at end of file diff --git a/oaipmh/templates/error.html b/oaipmh/templates/error.html deleted file mode 100644 index e69de29..0000000 diff --git a/oaipmh/templates/errors.xml b/oaipmh/templates/errors.xml new file mode 100644 index 0000000..2822712 --- /dev/null +++ b/oaipmh/templates/errors.xml @@ -0,0 +1,9 @@ +{% extends "base.xml" %} + +{% block request_element %} + {{ macros.request_element() }} +{% endblock %} + +{% block interior_xml %} + {{error.description}} +{% endblock %} \ No newline at end of file diff --git a/tests/test_basic.py b/tests/test_basic.py index 939ef34..f74f6e8 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -2,4 +2,8 @@ def test_basic(test_client): response = test_client.get("/oai") - assert response.status_code == 200 \ No newline at end of file + text=response.text + assert response.status_code == 200 + assert 'xmlns="http://www.openarchives.org/OAI/2.0/"' in text + assert '' in text + assert '/oai' in text \ No newline at end of file diff --git a/tests/test_verb_sorting.py b/tests/test_verb_sorting.py index e2eed25..ca713b4 100644 --- a/tests/test_verb_sorting.py +++ b/tests/test_verb_sorting.py @@ -78,4 +78,24 @@ def test_list_sets(test_client): assert response.status_code == 200 mock_list_sets.assert_called_once_with(params) -#TODO test no/bad verb \ No newline at end of file +def test_no_verb(test_client): + params = {"not_verb": "ListSets"} + + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + assert "" in response.text + + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + assert "" in response.text + +def test_bad_verb(test_client): + params = {"verb": "chaos!"} + + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + assert "" in response.text + + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + assert "" in response.text \ No newline at end of file From 7261955a2a7a04da1309aa2a4ae4ca86c9a9474a Mon Sep 17 00:00:00 2001 From: kyokukou Date: Wed, 6 Nov 2024 11:16:55 -0800 Subject: [PATCH 07/29] refactor using custom response object --- oaipmh/requests/data_queries.py | 14 +++++++------- oaipmh/requests/info_queries.py | 14 +++++++------- oaipmh/requests/routes.py | 16 ++++++---------- oaipmh/requests/verb_sorter.py | 5 +++-- oaipmh/serializers/output_formats.py | 4 ++-- tests/test_verb_sorting.py | 24 ++++++++++++------------ 6 files changed, 37 insertions(+), 40 deletions(-) diff --git a/oaipmh/requests/data_queries.py b/oaipmh/requests/data_queries.py index 0d7071c..c553a2f 100644 --- a/oaipmh/requests/data_queries.py +++ b/oaipmh/requests/data_queries.py @@ -1,12 +1,12 @@ from typing import Dict +from oaipmh.serializers.output_formats import Response -from oaipmh.serializers.output_formats import InteriorData -def get_record(params: Dict[str, str]) -> InteriorData: - return +def get_record(params: Dict[str, str]) -> Response: + return "b", 200, {} -def list_records(params: Dict[str, str]) -> InteriorData: - return +def list_records(params: Dict[str, str]) -> Response: + return "b", 200, {} -def list_identifiers(params: Dict[str, str]) -> InteriorData: - return +def list_identifiers(params: Dict[str, str]) -> Response: + return "b", 200, {} diff --git a/oaipmh/requests/info_queries.py b/oaipmh/requests/info_queries.py index c6e8a99..9dece80 100644 --- a/oaipmh/requests/info_queries.py +++ b/oaipmh/requests/info_queries.py @@ -1,11 +1,11 @@ from typing import Dict -from oaipmh.serializers.output_formats import InteriorData +from oaipmh.serializers.output_formats import Response -def identify(params: Dict[str, str]) -> InteriorData: - return +def identify(params: Dict[str, str]) -> Response: + return "b", 200, {} -def list_metadata_formats(params: Dict[str, str]) -> InteriorData: - return +def list_metadata_formats(params: Dict[str, str]) -> Response: + return "b", 200, {} -def list_sets(params: Dict[str, str]) -> InteriorData: - return +def list_sets(params: Dict[str, str]) -> Response: + return "b", 200, {} diff --git a/oaipmh/requests/routes.py b/oaipmh/requests/routes.py index ac75de7..6dfef39 100644 --- a/oaipmh/requests/routes.py +++ b/oaipmh/requests/routes.py @@ -1,8 +1,9 @@ from typing import Dict from datetime import datetime, timezone -from flask import Blueprint, request, Response, render_template +from flask import Blueprint, request, render_template from oaipmh.requests.verb_sorter import verb_sorter +from oaipmh.serializers.output_formats import Response blueprint = Blueprint('general', __name__) @@ -10,18 +11,13 @@ @blueprint.route("/oai", methods=['GET', 'POST']) def oai() -> Response: - #TODO what happens if duplicate params + #TODO duplicate params dont create errors, technically not to spec params: Dict[str, str] = request.args.to_dict() if request.method == 'GET' else request.form.to_dict() - result=verb_sorter(params) - response_xml=render_template("base.xml", - response_date=datetime.now(timezone.utc), - request_info="request info", #TODO - interior_xml="interior data" #TODO - ) - headers={"Content-Type":"application/xml"} + response, code, headers=verb_sorter(params) + headers["Content-Type"]="application/xml" - return response_xml, 200, headers + return response, code, headers @blueprint.route('/favicon.ico') def favicon(): diff --git a/oaipmh/requests/verb_sorter.py b/oaipmh/requests/verb_sorter.py index d49e7cb..41eb9d5 100644 --- a/oaipmh/requests/verb_sorter.py +++ b/oaipmh/requests/verb_sorter.py @@ -1,11 +1,12 @@ from typing import Dict + from oaipmh.requests.info_queries import identify, list_metadata_formats, list_sets from oaipmh.requests.data_queries import get_record, list_identifiers, list_records -from oaipmh.serializers.output_formats import InteriorData +from oaipmh.serializers.output_formats import Response from oaipmh.data.oai_errors import OAIBadVerb -def verb_sorter(params: Dict[str, str]) -> InteriorData: +def verb_sorter(params: Dict[str, str]) -> Response: """ sorts OAI queries to the appropriate handler based on their verb statement this defines what the client is asking for as per the OAI standard diff --git a/oaipmh/serializers/output_formats.py b/oaipmh/serializers/output_formats.py index c73df8a..7a37bf2 100644 --- a/oaipmh/serializers/output_formats.py +++ b/oaipmh/serializers/output_formats.py @@ -1,3 +1,3 @@ +from typing import Tuple, Dict, Any -class InteriorData: #TODO placeholder return type - value="" \ No newline at end of file +Response = Tuple[Dict[str, Any], int, Dict[str, Any]] \ No newline at end of file diff --git a/tests/test_verb_sorting.py b/tests/test_verb_sorting.py index ca713b4..856670c 100644 --- a/tests/test_verb_sorting.py +++ b/tests/test_verb_sorting.py @@ -3,12 +3,12 @@ def test_get_record(test_client): params = {"verb": "GetRecord", "identifier": "oai:example.org:record123"} - with patch('oaipmh.requests.verb_sorter.get_record') as mock_get_record: + with patch('oaipmh.requests.verb_sorter.get_record', return_value=("working", 200, {})) as mock_get_record: response = test_client.get("/oai", query_string=params) assert response.status_code == 200 mock_get_record.assert_called_once_with(params) - with patch('oaipmh.requests.verb_sorter.get_record') as mock_get_record: + with patch('oaipmh.requests.verb_sorter.get_record', return_value=("working", 200, {})) as mock_get_record: response = test_client.post("/oai", data=params) assert response.status_code == 200 mock_get_record.assert_called_once_with(params) @@ -16,12 +16,12 @@ def test_get_record(test_client): def test_list_records(test_client): params = {"verb": "ListRecords", "metadataPrefix": "oai_dc"} - with patch('oaipmh.requests.verb_sorter.list_records') as mock_list_records: + with patch('oaipmh.requests.verb_sorter.list_records', return_value=("working", 200, {})) as mock_list_records: response = test_client.get("/oai", query_string=params) assert response.status_code == 200 mock_list_records.assert_called_once_with(params) - with patch('oaipmh.requests.verb_sorter.list_records') as mock_list_records: + with patch('oaipmh.requests.verb_sorter.list_records', return_value=("working", 200, {})) as mock_list_records: response = test_client.post("/oai", data=params) assert response.status_code == 200 mock_list_records.assert_called_once_with(params) @@ -29,12 +29,12 @@ def test_list_records(test_client): def test_list_identifiers(test_client): params = {"verb": "ListIdentifiers", "metadataPrefix": "oai_dc"} - with patch('oaipmh.requests.verb_sorter.list_identifiers') as mock_list_identifiers: + with patch('oaipmh.requests.verb_sorter.list_identifiers', return_value=("working", 200, {})) as mock_list_identifiers: response = test_client.get("/oai", query_string=params) assert response.status_code == 200 mock_list_identifiers.assert_called_once_with(params) - with patch('oaipmh.requests.verb_sorter.list_identifiers') as mock_list_identifiers: + with patch('oaipmh.requests.verb_sorter.list_identifiers', return_value=("working", 200, {})) as mock_list_identifiers: response = test_client.post("/oai", data=params) assert response.status_code == 200 mock_list_identifiers.assert_called_once_with(params) @@ -42,12 +42,12 @@ def test_list_identifiers(test_client): def test_identify(test_client): params = {"verb": "Identify"} - with patch('oaipmh.requests.verb_sorter.identify') as mock_identify: + with patch('oaipmh.requests.verb_sorter.identify', return_value=("working", 200, {})) as mock_identify: response = test_client.get("/oai", query_string=params) assert response.status_code == 200 mock_identify.assert_called_once_with(params) - with patch('oaipmh.requests.verb_sorter.identify') as mock_identify: + with patch('oaipmh.requests.verb_sorter.identify', return_value=("working", 200, {})) as mock_identify: response = test_client.post("/oai", data=params) assert response.status_code == 200 mock_identify.assert_called_once_with(params) @@ -55,12 +55,12 @@ def test_identify(test_client): def test_list_metadata_formats(test_client): params = {"verb": "ListMetadataFormats"} - with patch('oaipmh.requests.verb_sorter.list_metadata_formats') as mock_list_metadata_formats: + with patch('oaipmh.requests.verb_sorter.list_metadata_formats', return_value=("working", 200, {})) as mock_list_metadata_formats: response = test_client.get("/oai", query_string=params) assert response.status_code == 200 mock_list_metadata_formats.assert_called_once_with(params) - with patch('oaipmh.requests.verb_sorter.list_metadata_formats') as mock_list_metadata_formats: + with patch('oaipmh.requests.verb_sorter.list_metadata_formats', return_value=("working", 200, {})) as mock_list_metadata_formats: response = test_client.post("/oai", data=params) assert response.status_code == 200 mock_list_metadata_formats.assert_called_once_with(params) @@ -68,12 +68,12 @@ def test_list_metadata_formats(test_client): def test_list_sets(test_client): params = {"verb": "ListSets"} - with patch('oaipmh.requests.verb_sorter.list_sets') as mock_list_sets: + with patch('oaipmh.requests.verb_sorter.list_sets', return_value=("working", 200, {})) as mock_list_sets: response = test_client.get("/oai", query_string=params) assert response.status_code == 200 mock_list_sets.assert_called_once_with(params) - with patch('oaipmh.requests.verb_sorter.list_sets') as mock_list_sets: + with patch('oaipmh.requests.verb_sorter.list_sets', return_value=("working", 200, {})) as mock_list_sets: response = test_client.post("/oai", data=params) assert response.status_code == 200 mock_list_sets.assert_called_once_with(params) From 5a7266b897e75cf6d425dbea129b5eeef979248f Mon Sep 17 00:00:00 2001 From: kyokukou Date: Wed, 6 Nov 2024 13:15:12 -0800 Subject: [PATCH 08/29] parameter check for get record --- oaipmh/requests/data_queries.py | 9 +++++ tests/inputs/test_get_record.py | 51 +++++++++++++++++++++++++ tests/{ => inputs}/test_verb_sorting.py | 0 3 files changed, 60 insertions(+) create mode 100644 tests/inputs/test_get_record.py rename tests/{ => inputs}/test_verb_sorting.py (100%) diff --git a/oaipmh/requests/data_queries.py b/oaipmh/requests/data_queries.py index c553a2f..746cb5e 100644 --- a/oaipmh/requests/data_queries.py +++ b/oaipmh/requests/data_queries.py @@ -1,8 +1,17 @@ from typing import Dict + +from oaipmh.data.oai_errors import OAIBadArgument from oaipmh.serializers.output_formats import Response def get_record(params: Dict[str, str]) -> Response: + # get parameters + expected_params={"identifier", "metadata_prefix", "verb"} + if set(params.keys()) != expected_params: + raise OAIBadArgument + identifier=params["identifier"] + meta_type=params["metadata_prefix"] + return "b", 200, {} def list_records(params: Dict[str, str]) -> Response: diff --git a/tests/inputs/test_get_record.py b/tests/inputs/test_get_record.py new file mode 100644 index 0000000..97a74e8 --- /dev/null +++ b/tests/inputs/test_get_record.py @@ -0,0 +1,51 @@ + +def test_good_params(test_client): + + params = {"verb": "GetRecord", "identifier": "oai:example.org:record123", "metadata_prefix": "oai_dc"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + +def test_extra_params(test_client): + params = {"verb": "GetRecord", "identifier": "oai:example.org:record123", "metadata_prefix": "oai_dc", "cookie":"chocolate"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text + + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text + +def test_missing_params(test_client): + + # missing metadata_prefix + params = {"verb": "GetRecord", "identifier": "oai:example.org:record123"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text + + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text + + # missing metadata_prefix + params = {"verb": "GetRecord", "metadata_prefix": "oai_dc"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text + + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text \ No newline at end of file diff --git a/tests/test_verb_sorting.py b/tests/inputs/test_verb_sorting.py similarity index 100% rename from tests/test_verb_sorting.py rename to tests/inputs/test_verb_sorting.py From 64a1cdba04cccd1defdf9134fa3dff07687cd3ce Mon Sep 17 00:00:00 2001 From: kyokukou Date: Wed, 6 Nov 2024 13:17:57 -0800 Subject: [PATCH 09/29] small reorganize --- tests/{inputs => }/test_get_record.py | 0 tests/{inputs => }/test_verb_sorting.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/{inputs => }/test_get_record.py (100%) rename tests/{inputs => }/test_verb_sorting.py (100%) diff --git a/tests/inputs/test_get_record.py b/tests/test_get_record.py similarity index 100% rename from tests/inputs/test_get_record.py rename to tests/test_get_record.py diff --git a/tests/inputs/test_verb_sorting.py b/tests/test_verb_sorting.py similarity index 100% rename from tests/inputs/test_verb_sorting.py rename to tests/test_verb_sorting.py From 3fab0a661893043e6fa67ef389a26aedadcc1fda Mon Sep 17 00:00:00 2001 From: kyokukou Date: Wed, 6 Nov 2024 13:23:30 -0800 Subject: [PATCH 10/29] check correct params for identify --- oaipmh/requests/data_queries.py | 2 ++ oaipmh/requests/info_queries.py | 6 ++++++ tests/test_identify.py | 25 +++++++++++++++++++++++++ 3 files changed, 33 insertions(+) create mode 100644 tests/test_identify.py diff --git a/oaipmh/requests/data_queries.py b/oaipmh/requests/data_queries.py index 746cb5e..54abca7 100644 --- a/oaipmh/requests/data_queries.py +++ b/oaipmh/requests/data_queries.py @@ -5,6 +5,8 @@ def get_record(params: Dict[str, str]) -> Response: + """used to get data on a particular record in a particular metadata format""" + # get parameters expected_params={"identifier", "metadata_prefix", "verb"} if set(params.keys()) != expected_params: diff --git a/oaipmh/requests/info_queries.py b/oaipmh/requests/info_queries.py index 9dece80..9a3919e 100644 --- a/oaipmh/requests/info_queries.py +++ b/oaipmh/requests/info_queries.py @@ -1,7 +1,13 @@ from typing import Dict + +from oaipmh.data.oai_errors import OAIBadArgument from oaipmh.serializers.output_formats import Response def identify(params: Dict[str, str]) -> Response: + """used to retrieve information about the repository""" + if set(params.keys()) != {"verb"}: + raise OAIBadArgument + return "b", 200, {} def list_metadata_formats(params: Dict[str, str]) -> Response: diff --git a/tests/test_identify.py b/tests/test_identify.py new file mode 100644 index 0000000..b84fc96 --- /dev/null +++ b/tests/test_identify.py @@ -0,0 +1,25 @@ + + +def test_good_params(test_client): + params = {"verb": "Identify"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + +def test_extra_params(test_client): + params = {"verb": "Identify", "identifier": "oai:example.org:record123"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text + + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text \ No newline at end of file From e70fb2618a159e3198f7c01f074be3e1aea8bd2a Mon Sep 17 00:00:00 2001 From: kyokukou Date: Thu, 7 Nov 2024 08:06:08 -0800 Subject: [PATCH 11/29] check parameters for list_identifiers --- oaipmh/requests/data_queries.py | 32 +++++++++++++- tests/test_get_record.py | 8 ++-- tests/test_list_indentifiers.py | 75 +++++++++++++++++++++++++++++++++ 3 files changed, 109 insertions(+), 6 deletions(-) create mode 100644 tests/test_list_indentifiers.py diff --git a/oaipmh/requests/data_queries.py b/oaipmh/requests/data_queries.py index 54abca7..e194e96 100644 --- a/oaipmh/requests/data_queries.py +++ b/oaipmh/requests/data_queries.py @@ -8,16 +8,44 @@ def get_record(params: Dict[str, str]) -> Response: """used to get data on a particular record in a particular metadata format""" # get parameters - expected_params={"identifier", "metadata_prefix", "verb"} + expected_params={"identifier", "metadataPrefix", "verb"} if set(params.keys()) != expected_params: raise OAIBadArgument identifier=params["identifier"] - meta_type=params["metadata_prefix"] + meta_type=params["metadataPrefix"] + + #TODO rest of function return "b", 200, {} def list_records(params: Dict[str, str]) -> Response: + + #TODO rest of function + return "b", 200, {} def list_identifiers(params: Dict[str, str]) -> Response: + """retrieves headers of all records matching certain parameters""" + token=None + + #get parameters + given_params=set(params.keys()) + if "resumptionToken" in given_params: + if given_params != {"resumptionToken", "verb"}: #resumption token is exclusive + raise OAIBadArgument + token=params["resumptionToken"] + else: + if "metadataPrefix" not in given_params: + raise OAIBadArgument + allowed_params={"verb","metadataPrefix", "from", "until", "set" } + if given_params-allowed_params: #no extra keys allowed + raise OAIBadArgument + + meta_type=params["metadataPrefix"] + from_str=params.get("from") + until_str=params.get("until") + set_str=params.get("set") + + #TODO rest of function + return "b", 200, {} diff --git a/tests/test_get_record.py b/tests/test_get_record.py index 97a74e8..8fe0567 100644 --- a/tests/test_get_record.py +++ b/tests/test_get_record.py @@ -1,7 +1,7 @@ def test_good_params(test_client): - params = {"verb": "GetRecord", "identifier": "oai:example.org:record123", "metadata_prefix": "oai_dc"} + params = {"verb": "GetRecord", "identifier": "oai:example.org:record123", "metadataPrefix": "oai_dc"} response = test_client.get("/oai", query_string=params) assert response.status_code == 200 text=response.get_data(as_text=True) @@ -13,7 +13,7 @@ def test_good_params(test_client): assert "" not in text def test_extra_params(test_client): - params = {"verb": "GetRecord", "identifier": "oai:example.org:record123", "metadata_prefix": "oai_dc", "cookie":"chocolate"} + params = {"verb": "GetRecord", "identifier": "oai:example.org:record123", "metadataPrefix": "oai_dc", "cookie":"chocolate"} response = test_client.get("/oai", query_string=params) assert response.status_code == 200 text=response.get_data(as_text=True) @@ -38,8 +38,8 @@ def test_missing_params(test_client): text=response.get_data(as_text=True) assert "" in text - # missing metadata_prefix - params = {"verb": "GetRecord", "metadata_prefix": "oai_dc"} + # missing identifier + params = {"verb": "GetRecord", "metadataPrefix": "oai_dc"} response = test_client.get("/oai", query_string=params) assert response.status_code == 200 text=response.get_data(as_text=True) diff --git a/tests/test_list_indentifiers.py b/tests/test_list_indentifiers.py new file mode 100644 index 0000000..d5d53d2 --- /dev/null +++ b/tests/test_list_indentifiers.py @@ -0,0 +1,75 @@ + +def test_good_params(test_client): + #good minimal params + params = {"verb": "ListIdentifiers", "metadataPrefix": "oai_dc"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + + #good maximal params + params = {"verb": "ListIdentifiers", "metadataPrefix": "oai_dc", "from": "now", "until":"later", "set": "math"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + + #good partial params + params = {"verb": "ListIdentifiers", "metadataPrefix": "oai_dc", "until":"later", "set": "math"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + +def test_extra_params(test_client): + #good minimal params + params = {"verb": "ListIdentifiers", "metadataPrefix": "oai_dc", "color":"green"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text + + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text + +def test_token_params(test_client): + #correct params + params = {"verb": "ListIdentifiers", "resumptionToken": "rainbow"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + + #cant have other valid params along with token + params = {"verb": "ListIdentifiers", "resumptionToken": "rainbow", "metadataPrefix": "oai_dc"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text + + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text From 266a75fe1320584569d68279507e91a3b79cf006 Mon Sep 17 00:00:00 2001 From: kyokukou Date: Thu, 7 Nov 2024 08:28:06 -0800 Subject: [PATCH 12/29] turn verbs and params into classes no more typos! --- oaipmh/data/oai_properties.py | 16 ++++++++++++++++ oaipmh/requests/data_queries.py | 25 +++++++++++++------------ oaipmh/requests/info_queries.py | 3 ++- oaipmh/requests/verb_sorter.py | 13 +++++++------ tests/test_get_record.py | 9 +++++---- tests/test_identify.py | 6 +++--- tests/test_list_indentifiers.py | 13 +++++++------ tests/test_verb_sorting.py | 18 ++++++++++-------- 8 files changed, 63 insertions(+), 40 deletions(-) create mode 100644 oaipmh/data/oai_properties.py diff --git a/oaipmh/data/oai_properties.py b/oaipmh/data/oai_properties.py new file mode 100644 index 0000000..46ac3fa --- /dev/null +++ b/oaipmh/data/oai_properties.py @@ -0,0 +1,16 @@ +class OAIParams: + VERB = "verb" + ID = "identifier" + META_PREFIX = "metadataPrefix" + SET = "set" + FROM = "from" + UNTIL = "until" + RES_TOKEN = "resumptionToken" + +class OAIVerbs: + GET_RECORD = "GetRecord" + LIST_RECORDS = "ListRecords" + LIST_IDS = "ListIdentifiers" + IDENTIFY = "Identify" + LIST_META_FORMATS = "ListMetadataFormats" + LIST_SETS = "ListSets" \ No newline at end of file diff --git a/oaipmh/requests/data_queries.py b/oaipmh/requests/data_queries.py index e194e96..5a5705d 100644 --- a/oaipmh/requests/data_queries.py +++ b/oaipmh/requests/data_queries.py @@ -1,6 +1,7 @@ from typing import Dict from oaipmh.data.oai_errors import OAIBadArgument +from oaipmh.data.oai_properties import OAIParams from oaipmh.serializers.output_formats import Response @@ -8,11 +9,11 @@ def get_record(params: Dict[str, str]) -> Response: """used to get data on a particular record in a particular metadata format""" # get parameters - expected_params={"identifier", "metadataPrefix", "verb"} + expected_params={OAIParams.ID, OAIParams.META_PREFIX, OAIParams.VERB} if set(params.keys()) != expected_params: raise OAIBadArgument - identifier=params["identifier"] - meta_type=params["metadataPrefix"] + identifier=params[OAIParams.ID] + meta_type=params[OAIParams.META_PREFIX] #TODO rest of function @@ -30,21 +31,21 @@ def list_identifiers(params: Dict[str, str]) -> Response: #get parameters given_params=set(params.keys()) - if "resumptionToken" in given_params: - if given_params != {"resumptionToken", "verb"}: #resumption token is exclusive + if OAIParams.RES_TOKEN in given_params: + if given_params != {OAIParams.RES_TOKEN, OAIParams.VERB}: #resumption token is exclusive raise OAIBadArgument - token=params["resumptionToken"] + token=params[OAIParams.RES_TOKEN] else: - if "metadataPrefix" not in given_params: + if OAIParams.META_PREFIX not in given_params: raise OAIBadArgument - allowed_params={"verb","metadataPrefix", "from", "until", "set" } + allowed_params={OAIParams.VERB,OAIParams.META_PREFIX, OAIParams.FROM, OAIParams.UNTIL, OAIParams.SET } if given_params-allowed_params: #no extra keys allowed raise OAIBadArgument - meta_type=params["metadataPrefix"] - from_str=params.get("from") - until_str=params.get("until") - set_str=params.get("set") + meta_type=params[OAIParams.META_PREFIX] + from_str=params.get(OAIParams.FROM) + until_str=params.get(OAIParams.UNTIL) + set_str=params.get(OAIParams.SET) #TODO rest of function diff --git a/oaipmh/requests/info_queries.py b/oaipmh/requests/info_queries.py index 9a3919e..0f37d6a 100644 --- a/oaipmh/requests/info_queries.py +++ b/oaipmh/requests/info_queries.py @@ -1,11 +1,12 @@ from typing import Dict from oaipmh.data.oai_errors import OAIBadArgument +from oaipmh.data.oai_properties import OAIParams from oaipmh.serializers.output_formats import Response def identify(params: Dict[str, str]) -> Response: """used to retrieve information about the repository""" - if set(params.keys()) != {"verb"}: + if set(params.keys()) != {OAIParams.VERB}: raise OAIBadArgument return "b", 200, {} diff --git a/oaipmh/requests/verb_sorter.py b/oaipmh/requests/verb_sorter.py index 41eb9d5..d478d7d 100644 --- a/oaipmh/requests/verb_sorter.py +++ b/oaipmh/requests/verb_sorter.py @@ -5,6 +5,7 @@ from oaipmh.requests.data_queries import get_record, list_identifiers, list_records from oaipmh.serializers.output_formats import Response from oaipmh.data.oai_errors import OAIBadVerb +from oaipmh.data.oai_properties import OAIVerbs def verb_sorter(params: Dict[str, str]) -> Response: """ @@ -15,17 +16,17 @@ def verb_sorter(params: Dict[str, str]) -> Response: """ verb = params.get("verb", "") match verb: - case "GetRecord": + case OAIVerbs.GET_RECORD: return get_record(params) - case "ListRecords": + case OAIVerbs.LIST_RECORDS: return list_records(params) - case "ListIdentifiers": + case OAIVerbs.LIST_IDS: return list_identifiers(params) - case "Identify": + case OAIVerbs.IDENTIFY: return identify(params) - case "ListMetadataFormats": + case OAIVerbs.LIST_META_FORMATS: return list_metadata_formats(params) - case "ListSets": + case OAIVerbs.LIST_SETS: return list_sets(params) case _: raise OAIBadVerb #dont keep invalid verb diff --git a/tests/test_get_record.py b/tests/test_get_record.py index 8fe0567..620899b 100644 --- a/tests/test_get_record.py +++ b/tests/test_get_record.py @@ -1,7 +1,8 @@ +from oaipmh.data.oai_properties import OAIParams, OAIVerbs def test_good_params(test_client): - params = {"verb": "GetRecord", "identifier": "oai:example.org:record123", "metadataPrefix": "oai_dc"} + params = {OAIParams.VERB: OAIVerbs.GET_RECORD, OAIParams.ID: "oai:example.org:record123", OAIParams.META_PREFIX: "oai_dc"} response = test_client.get("/oai", query_string=params) assert response.status_code == 200 text=response.get_data(as_text=True) @@ -13,7 +14,7 @@ def test_good_params(test_client): assert "" not in text def test_extra_params(test_client): - params = {"verb": "GetRecord", "identifier": "oai:example.org:record123", "metadataPrefix": "oai_dc", "cookie":"chocolate"} + params = {OAIParams.VERB: OAIVerbs.GET_RECORD, OAIParams.ID: "oai:example.org:record123", OAIParams.META_PREFIX: "oai_dc", "cookie":"chocolate"} response = test_client.get("/oai", query_string=params) assert response.status_code == 200 text=response.get_data(as_text=True) @@ -27,7 +28,7 @@ def test_extra_params(test_client): def test_missing_params(test_client): # missing metadata_prefix - params = {"verb": "GetRecord", "identifier": "oai:example.org:record123"} + params = {OAIParams.VERB: OAIVerbs.GET_RECORD, OAIParams.ID: "oai:example.org:record123"} response = test_client.get("/oai", query_string=params) assert response.status_code == 200 text=response.get_data(as_text=True) @@ -39,7 +40,7 @@ def test_missing_params(test_client): assert "" in text # missing identifier - params = {"verb": "GetRecord", "metadataPrefix": "oai_dc"} + params = {OAIParams.VERB: OAIVerbs.GET_RECORD, OAIParams.META_PREFIX: "oai_dc"} response = test_client.get("/oai", query_string=params) assert response.status_code == 200 text=response.get_data(as_text=True) diff --git a/tests/test_identify.py b/tests/test_identify.py index b84fc96..c229dcc 100644 --- a/tests/test_identify.py +++ b/tests/test_identify.py @@ -1,7 +1,7 @@ - +from oaipmh.data.oai_properties import OAIParams, OAIVerbs def test_good_params(test_client): - params = {"verb": "Identify"} + params = {OAIParams.VERB: OAIVerbs.IDENTIFY} response = test_client.get("/oai", query_string=params) assert response.status_code == 200 text=response.get_data(as_text=True) @@ -13,7 +13,7 @@ def test_good_params(test_client): assert "" not in text def test_extra_params(test_client): - params = {"verb": "Identify", "identifier": "oai:example.org:record123"} + params = {OAIParams.VERB: OAIVerbs.IDENTIFY, OAIParams.ID: "oai:example.org:record123"} response = test_client.get("/oai", query_string=params) assert response.status_code == 200 text=response.get_data(as_text=True) diff --git a/tests/test_list_indentifiers.py b/tests/test_list_indentifiers.py index d5d53d2..5b0a408 100644 --- a/tests/test_list_indentifiers.py +++ b/tests/test_list_indentifiers.py @@ -1,7 +1,8 @@ +from oaipmh.data.oai_properties import OAIParams, OAIVerbs def test_good_params(test_client): #good minimal params - params = {"verb": "ListIdentifiers", "metadataPrefix": "oai_dc"} + params = {OAIParams.VERB: OAIVerbs.LIST_IDS, OAIParams.META_PREFIX: "oai_dc"} response = test_client.get("/oai", query_string=params) assert response.status_code == 200 text=response.get_data(as_text=True) @@ -13,7 +14,7 @@ def test_good_params(test_client): assert "" not in text #good maximal params - params = {"verb": "ListIdentifiers", "metadataPrefix": "oai_dc", "from": "now", "until":"later", "set": "math"} + params = {OAIParams.VERB: OAIVerbs.LIST_IDS, OAIParams.META_PREFIX: "oai_dc", OAIParams.FROM: "now", OAIParams.UNTIL:"later", OAIParams.SET: "math"} response = test_client.get("/oai", query_string=params) assert response.status_code == 200 text=response.get_data(as_text=True) @@ -25,7 +26,7 @@ def test_good_params(test_client): assert "" not in text #good partial params - params = {"verb": "ListIdentifiers", "metadataPrefix": "oai_dc", "until":"later", "set": "math"} + params = {OAIParams.VERB: OAIVerbs.LIST_IDS, OAIParams.META_PREFIX: "oai_dc", OAIParams.UNTIL:"later", OAIParams.SET: "math"} response = test_client.get("/oai", query_string=params) assert response.status_code == 200 text=response.get_data(as_text=True) @@ -38,7 +39,7 @@ def test_good_params(test_client): def test_extra_params(test_client): #good minimal params - params = {"verb": "ListIdentifiers", "metadataPrefix": "oai_dc", "color":"green"} + params = {OAIParams.VERB: OAIVerbs.LIST_IDS, OAIParams.META_PREFIX: "oai_dc", "color":"green"} response = test_client.get("/oai", query_string=params) assert response.status_code == 200 text=response.get_data(as_text=True) @@ -51,7 +52,7 @@ def test_extra_params(test_client): def test_token_params(test_client): #correct params - params = {"verb": "ListIdentifiers", "resumptionToken": "rainbow"} + params = {OAIParams.VERB: OAIVerbs.LIST_IDS, OAIParams.RES_TOKEN: "rainbow"} response = test_client.get("/oai", query_string=params) assert response.status_code == 200 text=response.get_data(as_text=True) @@ -63,7 +64,7 @@ def test_token_params(test_client): assert "" not in text #cant have other valid params along with token - params = {"verb": "ListIdentifiers", "resumptionToken": "rainbow", "metadataPrefix": "oai_dc"} + params = {OAIParams.VERB: OAIVerbs.LIST_IDS, OAIParams.RES_TOKEN: "rainbow", OAIParams.META_PREFIX: "oai_dc"} response = test_client.get("/oai", query_string=params) assert response.status_code == 200 text=response.get_data(as_text=True) diff --git a/tests/test_verb_sorting.py b/tests/test_verb_sorting.py index 856670c..34f92fb 100644 --- a/tests/test_verb_sorting.py +++ b/tests/test_verb_sorting.py @@ -1,7 +1,9 @@ from unittest.mock import patch +from oaipmh.data.oai_properties import OAIVerbs, OAIParams + def test_get_record(test_client): - params = {"verb": "GetRecord", "identifier": "oai:example.org:record123"} + params = {OAIParams.VERB: OAIVerbs.GET_RECORD, OAIParams.ID: "oai:example.org:record123"} with patch('oaipmh.requests.verb_sorter.get_record', return_value=("working", 200, {})) as mock_get_record: response = test_client.get("/oai", query_string=params) @@ -14,7 +16,7 @@ def test_get_record(test_client): mock_get_record.assert_called_once_with(params) def test_list_records(test_client): - params = {"verb": "ListRecords", "metadataPrefix": "oai_dc"} + params = {OAIParams.VERB: OAIVerbs.LIST_RECORDS, OAIParams.META_PREFIX: "oai_dc"} with patch('oaipmh.requests.verb_sorter.list_records', return_value=("working", 200, {})) as mock_list_records: response = test_client.get("/oai", query_string=params) @@ -27,7 +29,7 @@ def test_list_records(test_client): mock_list_records.assert_called_once_with(params) def test_list_identifiers(test_client): - params = {"verb": "ListIdentifiers", "metadataPrefix": "oai_dc"} + params = {OAIParams.VERB: OAIVerbs.LIST_IDS, OAIParams.META_PREFIX: "oai_dc"} with patch('oaipmh.requests.verb_sorter.list_identifiers', return_value=("working", 200, {})) as mock_list_identifiers: response = test_client.get("/oai", query_string=params) @@ -40,7 +42,7 @@ def test_list_identifiers(test_client): mock_list_identifiers.assert_called_once_with(params) def test_identify(test_client): - params = {"verb": "Identify"} + params = {OAIParams.VERB: OAIVerbs.IDENTIFY} with patch('oaipmh.requests.verb_sorter.identify', return_value=("working", 200, {})) as mock_identify: response = test_client.get("/oai", query_string=params) @@ -53,7 +55,7 @@ def test_identify(test_client): mock_identify.assert_called_once_with(params) def test_list_metadata_formats(test_client): - params = {"verb": "ListMetadataFormats"} + params = {OAIParams.VERB: OAIVerbs.LIST_META_FORMATS} with patch('oaipmh.requests.verb_sorter.list_metadata_formats', return_value=("working", 200, {})) as mock_list_metadata_formats: response = test_client.get("/oai", query_string=params) @@ -66,7 +68,7 @@ def test_list_metadata_formats(test_client): mock_list_metadata_formats.assert_called_once_with(params) def test_list_sets(test_client): - params = {"verb": "ListSets"} + params = {OAIParams.VERB: OAIVerbs.LIST_SETS} with patch('oaipmh.requests.verb_sorter.list_sets', return_value=("working", 200, {})) as mock_list_sets: response = test_client.get("/oai", query_string=params) @@ -79,7 +81,7 @@ def test_list_sets(test_client): mock_list_sets.assert_called_once_with(params) def test_no_verb(test_client): - params = {"not_verb": "ListSets"} + params = {"not_verb": OAIVerbs.LIST_SETS} response = test_client.get("/oai", query_string=params) assert response.status_code == 200 @@ -90,7 +92,7 @@ def test_no_verb(test_client): assert "" in response.text def test_bad_verb(test_client): - params = {"verb": "chaos!"} + params = {OAIParams.VERB: "chaos!"} response = test_client.get("/oai", query_string=params) assert response.status_code == 200 From b470b9e40dbd2d73922d32ca5fb626d570e5bd11 Mon Sep 17 00:00:00 2001 From: kyokukou Date: Thu, 7 Nov 2024 08:46:21 -0800 Subject: [PATCH 13/29] check parameters given for list_metadata_formats --- oaipmh/requests/info_queries.py | 17 +++++++++++++- tests/test_list_meta_formats.py | 39 +++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 tests/test_list_meta_formats.py diff --git a/oaipmh/requests/info_queries.py b/oaipmh/requests/info_queries.py index 0f37d6a..afd4638 100644 --- a/oaipmh/requests/info_queries.py +++ b/oaipmh/requests/info_queries.py @@ -12,7 +12,22 @@ def identify(params: Dict[str, str]) -> Response: return "b", 200, {} def list_metadata_formats(params: Dict[str, str]) -> Response: - return "b", 200, {} + """used to retrieve the metadata formats available from a repository. + An optional argument restricts the request to the formats available for a specific item. + """ + + given_params=set(params.keys()) + if OAIParams.ID in given_params: #give formats for one item + if given_params != {OAIParams.VERB, OAIParams.ID}: + raise OAIBadArgument + #TODO get formats for an item + return "b", 200, {} + + else: #give formats repository supports + if given_params != {OAIParams.VERB}: + raise OAIBadArgument + #TODO get formats for repository + return "b", 200, {} def list_sets(params: Dict[str, str]) -> Response: return "b", 200, {} diff --git a/tests/test_list_meta_formats.py b/tests/test_list_meta_formats.py new file mode 100644 index 0000000..ee33187 --- /dev/null +++ b/tests/test_list_meta_formats.py @@ -0,0 +1,39 @@ +from oaipmh.data.oai_properties import OAIParams, OAIVerbs + +def test_good_params(test_client): + #general case + params = {OAIParams.VERB: OAIVerbs.LIST_META_FORMATS} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + + #for an item + params = {OAIParams.VERB: OAIVerbs.LIST_META_FORMATS, OAIParams.ID : "item"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + +def test_extra_params(test_client): + #general case + params = {OAIParams.VERB: OAIVerbs.LIST_META_FORMATS, OAIParams.FROM:"now"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text + + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text \ No newline at end of file From 2a5635a14e3068cff324d9a1f016bfc0e9013216 Mon Sep 17 00:00:00 2001 From: kyokukou Date: Thu, 7 Nov 2024 09:25:03 -0800 Subject: [PATCH 14/29] checking params for list records --- oaipmh/requests/data_queries.py | 19 ++++++++ tests/test_list_records.py | 77 +++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 tests/test_list_records.py diff --git a/oaipmh/requests/data_queries.py b/oaipmh/requests/data_queries.py index 5a5705d..6f5a7d8 100644 --- a/oaipmh/requests/data_queries.py +++ b/oaipmh/requests/data_queries.py @@ -20,7 +20,26 @@ def get_record(params: Dict[str, str]) -> Response: return "b", 200, {} def list_records(params: Dict[str, str]) -> Response: + """used to harvest records from a repository with support for selective harvesting""" + token=None + #get parameters + given_params=set(params.keys()) + if OAIParams.RES_TOKEN in given_params: + if given_params != {OAIParams.RES_TOKEN, OAIParams.VERB}: #resumption token is exclusive + raise OAIBadArgument + token=params[OAIParams.RES_TOKEN] + else: + if OAIParams.META_PREFIX not in given_params: + raise OAIBadArgument + allowed_params={OAIParams.VERB,OAIParams.META_PREFIX, OAIParams.FROM, OAIParams.UNTIL, OAIParams.SET } + if given_params-allowed_params: #no extra keys allowed + raise OAIBadArgument + + meta_type=params[OAIParams.META_PREFIX] + from_str=params.get(OAIParams.FROM) + until_str=params.get(OAIParams.UNTIL) + set_str=params.get(OAIParams.SET) #TODO rest of function return "b", 200, {} diff --git a/tests/test_list_records.py b/tests/test_list_records.py new file mode 100644 index 0000000..0d5339d --- /dev/null +++ b/tests/test_list_records.py @@ -0,0 +1,77 @@ + +from oaipmh.data.oai_properties import OAIParams, OAIVerbs + +def test_good_params(test_client): + #good minimal params + params = {OAIParams.VERB: OAIVerbs.LIST_RECORDS, OAIParams.META_PREFIX: "oai_dc"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + + #good maximal params + params = {OAIParams.VERB: OAIVerbs.LIST_RECORDS, OAIParams.META_PREFIX: "oai_dc", OAIParams.FROM: "now", OAIParams.UNTIL:"later", OAIParams.SET: "math"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + + #good partial params + params = {OAIParams.VERB: OAIVerbs.LIST_RECORDS, OAIParams.META_PREFIX: "oai_dc", OAIParams.UNTIL:"later", OAIParams.SET: "math"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + +def test_extra_params(test_client): + #good minimal params + params = {OAIParams.VERB: OAIVerbs.LIST_RECORDS, OAIParams.META_PREFIX: "oai_dc", "color":"green"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text + + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text + +def test_token_params(test_client): + #correct params + params = {OAIParams.VERB: OAIVerbs.LIST_RECORDS, OAIParams.RES_TOKEN: "rainbow"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + + #cant have other valid params along with token + params = {OAIParams.VERB: OAIVerbs.LIST_RECORDS, OAIParams.RES_TOKEN: "rainbow", OAIParams.META_PREFIX: "oai_dc"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text + + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text From 58ad3160fd1cca1b194ee79ea2b48613d5714f3f Mon Sep 17 00:00:00 2001 From: kyokukou Date: Thu, 7 Nov 2024 09:43:33 -0800 Subject: [PATCH 15/29] check corrent params present for list sets --- oaipmh/requests/info_queries.py | 16 +++++++++++++ tests/test_list_sets.py | 40 +++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 tests/test_list_sets.py diff --git a/oaipmh/requests/info_queries.py b/oaipmh/requests/info_queries.py index afd4638..a179332 100644 --- a/oaipmh/requests/info_queries.py +++ b/oaipmh/requests/info_queries.py @@ -30,4 +30,20 @@ def list_metadata_formats(params: Dict[str, str]) -> Response: return "b", 200, {} def list_sets(params: Dict[str, str]) -> Response: + """used to retrieve the set structure of a repository""" + + token=None + #get parameters + given_params=set(params.keys()) + if OAIParams.RES_TOKEN in given_params: + if given_params != {OAIParams.RES_TOKEN, OAIParams.VERB}: #resumption token is exclusive + raise OAIBadArgument + token=params[OAIParams.RES_TOKEN] + #TODO will we ever hit this, or will we always return our set structure in full? + else: + if given_params != {OAIParams.VERB}: + raise OAIBadArgument + + #TODO rest of function + return "b", 200, {} diff --git a/tests/test_list_sets.py b/tests/test_list_sets.py new file mode 100644 index 0000000..fe581dd --- /dev/null +++ b/tests/test_list_sets.py @@ -0,0 +1,40 @@ + +from oaipmh.data.oai_properties import OAIParams, OAIVerbs + +def test_good_params(test_client): + #general case + params = {OAIParams.VERB: OAIVerbs.LIST_SETS} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + + #with token + params = {OAIParams.VERB: OAIVerbs.LIST_SETS, OAIParams.RES_TOKEN : "item"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + +def test_extra_params(test_client): + #general case + params = {OAIParams.VERB: OAIVerbs.LIST_SETS, OAIParams.FROM:"now"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text + + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text \ No newline at end of file From babdcf7aba74d65454e031ca91e599807caa3980 Mon Sep 17 00:00:00 2001 From: kyokukou Date: Thu, 7 Nov 2024 11:24:25 -0800 Subject: [PATCH 16/29] draft of creating set list method to display query data in request xml --- oaipmh/processors/create_set_list.py | 19 +++++++++++++++++++ oaipmh/requests/info_queries.py | 17 +++++++++-------- oaipmh/templates/setSpec.xml | 20 ++++++++++++++++++++ 3 files changed, 48 insertions(+), 8 deletions(-) create mode 100644 oaipmh/processors/create_set_list.py create mode 100644 oaipmh/templates/setSpec.xml diff --git a/oaipmh/processors/create_set_list.py b/oaipmh/processors/create_set_list.py new file mode 100644 index 0000000..4f71030 --- /dev/null +++ b/oaipmh/processors/create_set_list.py @@ -0,0 +1,19 @@ +from datetime import datetime, timezone +from typing import Dict, Any + +from flask import render_template + +from arxiv.taxonomy.definitions import ARCHIVES_ACTIVE + +from oaipmh.data.oai_properties import OAIParams +from oaipmh.serializers.output_formats import Response + +def produce_set_list(query_data: Dict[OAIParams, Any]) -> Response: + """create the set structure of a repository""" + #TODO display in desired form/ level of depth once decided + #TODO filter out hidden entries + response=render_template("setSpec.xml", + response_date=datetime.now(timezone.utc), + query_data=query_data, + archives=ARCHIVES_ACTIVE) + return response, 200, {} \ No newline at end of file diff --git a/oaipmh/requests/info_queries.py b/oaipmh/requests/info_queries.py index a179332..aadebbc 100644 --- a/oaipmh/requests/info_queries.py +++ b/oaipmh/requests/info_queries.py @@ -1,8 +1,9 @@ -from typing import Dict +from typing import Dict, Any from oaipmh.data.oai_errors import OAIBadArgument -from oaipmh.data.oai_properties import OAIParams +from oaipmh.data.oai_properties import OAIParams, OAIVerbs from oaipmh.serializers.output_formats import Response +from oaipmh.processors.create_set_list import produce_set_list def identify(params: Dict[str, str]) -> Response: """used to retrieve information about the repository""" @@ -31,19 +32,19 @@ def list_metadata_formats(params: Dict[str, str]) -> Response: def list_sets(params: Dict[str, str]) -> Response: """used to retrieve the set structure of a repository""" - - token=None - #get parameters + + query_data: Dict[OAIParams, Any]={OAIParams.VERB:OAIVerbs.LIST_SETS} given_params=set(params.keys()) if OAIParams.RES_TOKEN in given_params: if given_params != {OAIParams.RES_TOKEN, OAIParams.VERB}: #resumption token is exclusive raise OAIBadArgument - token=params[OAIParams.RES_TOKEN] + token_str=params[OAIParams.RES_TOKEN] + #TODO token validation/processing + query_data[OAIParams.RES_TOKEN]=token_str #TODO will we ever hit this, or will we always return our set structure in full? else: if given_params != {OAIParams.VERB}: raise OAIBadArgument - #TODO rest of function + return produce_set_list(query_data) - return "b", 200, {} diff --git a/oaipmh/templates/setSpec.xml b/oaipmh/templates/setSpec.xml new file mode 100644 index 0000000..84370ed --- /dev/null +++ b/oaipmh/templates/setSpec.xml @@ -0,0 +1,20 @@ +{% extends "base.xml" %} + +{% block request_element %} + {{ macros.request_element(query_data) }} +{% endblock %} + +{% block interior_xml %} + + {% for item in archives.values() %} + {{ set_item(item) }} + {% endfor %} + +{% endblock %} + +{% macro set_item(item) %} + + {{item.id}} + {{item.full_name}} + +{% endmacro %} \ No newline at end of file From 1745fa499793b84b156ab88d1cc8bc2751fd9492 Mon Sep 17 00:00:00 2001 From: kyokukou Date: Thu, 7 Nov 2024 12:42:30 -0800 Subject: [PATCH 17/29] validating identifiers --- oaipmh/requests/data_queries.py | 10 ++++++++-- oaipmh/requests/info_queries.py | 6 +++--- oaipmh/requests/param_processing.py | 23 +++++++++++++++++++++++ tests/test_get_record.py | 6 +++--- tests/test_param_process.py | 29 +++++++++++++++++++++++++++++ 5 files changed, 66 insertions(+), 8 deletions(-) create mode 100644 oaipmh/requests/param_processing.py create mode 100644 tests/test_param_process.py diff --git a/oaipmh/requests/data_queries.py b/oaipmh/requests/data_queries.py index 6f5a7d8..71db552 100644 --- a/oaipmh/requests/data_queries.py +++ b/oaipmh/requests/data_queries.py @@ -1,18 +1,24 @@ from typing import Dict from oaipmh.data.oai_errors import OAIBadArgument -from oaipmh.data.oai_properties import OAIParams +from oaipmh.data.oai_properties import OAIParams, OAIVerbs from oaipmh.serializers.output_formats import Response +from oaipmh.requests.param_processing import process_identifier def get_record(params: Dict[str, str]) -> Response: """used to get data on a particular record in a particular metadata format""" + query_data: Dict[OAIParams, str]={OAIParams.VERB:OAIVerbs.GET_RECORD} # get parameters expected_params={OAIParams.ID, OAIParams.META_PREFIX, OAIParams.VERB} if set(params.keys()) != expected_params: raise OAIBadArgument - identifier=params[OAIParams.ID] + + identifier_str=params[OAIParams.ID] + arxiv_id=process_identifier(identifier_str) + query_data[OAIParams.ID]=identifier_str + meta_type=params[OAIParams.META_PREFIX] #TODO rest of function diff --git a/oaipmh/requests/info_queries.py b/oaipmh/requests/info_queries.py index aadebbc..1b3e84a 100644 --- a/oaipmh/requests/info_queries.py +++ b/oaipmh/requests/info_queries.py @@ -1,4 +1,4 @@ -from typing import Dict, Any +from typing import Dict from oaipmh.data.oai_errors import OAIBadArgument from oaipmh.data.oai_properties import OAIParams, OAIVerbs @@ -32,8 +32,8 @@ def list_metadata_formats(params: Dict[str, str]) -> Response: def list_sets(params: Dict[str, str]) -> Response: """used to retrieve the set structure of a repository""" - - query_data: Dict[OAIParams, Any]={OAIParams.VERB:OAIVerbs.LIST_SETS} + + query_data: Dict[OAIParams, str]={OAIParams.VERB:OAIVerbs.LIST_SETS} given_params=set(params.keys()) if OAIParams.RES_TOKEN in given_params: if given_params != {OAIParams.RES_TOKEN, OAIParams.VERB}: #resumption token is exclusive diff --git a/oaipmh/requests/param_processing.py b/oaipmh/requests/param_processing.py new file mode 100644 index 0000000..1702e78 --- /dev/null +++ b/oaipmh/requests/param_processing.py @@ -0,0 +1,23 @@ + + +from arxiv.identifier import Identifier, IdentifierException + +from oaipmh.data.oai_errors import OAIBadArgument + +ID_PREFIX="oai:arXiv.org:" +def process_identifier(id_str:str) -> Identifier: + """transform an OAI identifier into an arxiv ID""" + if not id_str.startswith(ID_PREFIX): + raise OAIBadArgument + + short_id = id_str[len(ID_PREFIX):] + try: + arxiv_id=Identifier(short_id) + except IdentifierException: + raise OAIBadArgument + + return arxiv_id + +def create_oai_id(arxiv_id: Identifier) -> str: + #turns arxiv style id into oai style id + return f"{ID_PREFIX}{arxiv_id.id}" \ No newline at end of file diff --git a/tests/test_get_record.py b/tests/test_get_record.py index 620899b..08039ce 100644 --- a/tests/test_get_record.py +++ b/tests/test_get_record.py @@ -2,7 +2,7 @@ def test_good_params(test_client): - params = {OAIParams.VERB: OAIVerbs.GET_RECORD, OAIParams.ID: "oai:example.org:record123", OAIParams.META_PREFIX: "oai_dc"} + params = {OAIParams.VERB: OAIVerbs.GET_RECORD, OAIParams.ID: "oai:arXiv.org:2307.10651", OAIParams.META_PREFIX: "oai_dc"} response = test_client.get("/oai", query_string=params) assert response.status_code == 200 text=response.get_data(as_text=True) @@ -14,7 +14,7 @@ def test_good_params(test_client): assert "" not in text def test_extra_params(test_client): - params = {OAIParams.VERB: OAIVerbs.GET_RECORD, OAIParams.ID: "oai:example.org:record123", OAIParams.META_PREFIX: "oai_dc", "cookie":"chocolate"} + params = {OAIParams.VERB: OAIVerbs.GET_RECORD, OAIParams.ID: "oai:arXiv.org:2307.10651", OAIParams.META_PREFIX: "oai_dc", "cookie":"chocolate"} response = test_client.get("/oai", query_string=params) assert response.status_code == 200 text=response.get_data(as_text=True) @@ -28,7 +28,7 @@ def test_extra_params(test_client): def test_missing_params(test_client): # missing metadata_prefix - params = {OAIParams.VERB: OAIVerbs.GET_RECORD, OAIParams.ID: "oai:example.org:record123"} + params = {OAIParams.VERB: OAIVerbs.GET_RECORD, OAIParams.ID: "oai:arXiv.org:2307.10651"} response = test_client.get("/oai", query_string=params) assert response.status_code == 200 text=response.get_data(as_text=True) diff --git a/tests/test_param_process.py b/tests/test_param_process.py new file mode 100644 index 0000000..fa13b4a --- /dev/null +++ b/tests/test_param_process.py @@ -0,0 +1,29 @@ +import pytest + +from arxiv.identifier import Identifier + +from oaipmh.data.oai_errors import OAIBadArgument +from oaipmh.requests.param_processing import process_identifier, create_oai_id + +def test_process_old_id(): + expected= Identifier("cs/0007002") + result=process_identifier("oai:arXiv.org:cs/0007002") + assert result==expected + +def test_process_new_id(): + expected= Identifier("2307.10651") + result=process_identifier("oai:arXiv.org:2307.10651") + assert result==expected + +def test_process_bad_id(): + with pytest.raises(OAIBadArgument): + process_identifier("cs/0007002") + + with pytest.raises(OAIBadArgument): + process_identifier("oai:arXiv.org:cs/0007.002") + + with pytest.raises(OAIBadArgument): + process_identifier("oai:arXiv.org:99.9999") + + with pytest.raises(OAIBadArgument): + process_identifier("oai:arXiv.org:totally_an_id//sdfkj34o") \ No newline at end of file From 321fe235311953955c51ab33b9d1b6b2a0ce50de Mon Sep 17 00:00:00 2001 From: kyokukou Date: Thu, 7 Nov 2024 13:13:34 -0800 Subject: [PATCH 18/29] use error for nonexistant id --- oaipmh/requests/param_processing.py | 6 +++--- tests/test_get_record.py | 8 ++++++++ tests/test_param_process.py | 10 +++++----- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/oaipmh/requests/param_processing.py b/oaipmh/requests/param_processing.py index 1702e78..6808fa9 100644 --- a/oaipmh/requests/param_processing.py +++ b/oaipmh/requests/param_processing.py @@ -2,19 +2,19 @@ from arxiv.identifier import Identifier, IdentifierException -from oaipmh.data.oai_errors import OAIBadArgument +from oaipmh.data.oai_errors import OAINonexistentID ID_PREFIX="oai:arXiv.org:" def process_identifier(id_str:str) -> Identifier: """transform an OAI identifier into an arxiv ID""" if not id_str.startswith(ID_PREFIX): - raise OAIBadArgument + raise OAINonexistentID short_id = id_str[len(ID_PREFIX):] try: arxiv_id=Identifier(short_id) except IdentifierException: - raise OAIBadArgument + raise OAINonexistentID return arxiv_id diff --git a/tests/test_get_record.py b/tests/test_get_record.py index 08039ce..7ce5ffc 100644 --- a/tests/test_get_record.py +++ b/tests/test_get_record.py @@ -13,6 +13,14 @@ def test_good_params(test_client): text=response.get_data(as_text=True) assert "" not in text +def test_bad_id(test_client): + + params = {OAIParams.VERB: OAIVerbs.GET_RECORD, OAIParams.ID: "arXiv.org:2307.10651", OAIParams.META_PREFIX: "oai_dc"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text + def test_extra_params(test_client): params = {OAIParams.VERB: OAIVerbs.GET_RECORD, OAIParams.ID: "oai:arXiv.org:2307.10651", OAIParams.META_PREFIX: "oai_dc", "cookie":"chocolate"} response = test_client.get("/oai", query_string=params) diff --git a/tests/test_param_process.py b/tests/test_param_process.py index fa13b4a..48bce28 100644 --- a/tests/test_param_process.py +++ b/tests/test_param_process.py @@ -2,7 +2,7 @@ from arxiv.identifier import Identifier -from oaipmh.data.oai_errors import OAIBadArgument +from oaipmh.data.oai_errors import OAINonexistentID from oaipmh.requests.param_processing import process_identifier, create_oai_id def test_process_old_id(): @@ -16,14 +16,14 @@ def test_process_new_id(): assert result==expected def test_process_bad_id(): - with pytest.raises(OAIBadArgument): + with pytest.raises(OAINonexistentID): process_identifier("cs/0007002") - with pytest.raises(OAIBadArgument): + with pytest.raises(OAINonexistentID): process_identifier("oai:arXiv.org:cs/0007.002") - with pytest.raises(OAIBadArgument): + with pytest.raises(OAINonexistentID): process_identifier("oai:arXiv.org:99.9999") - with pytest.raises(OAIBadArgument): + with pytest.raises(OAINonexistentID): process_identifier("oai:arXiv.org:totally_an_id//sdfkj34o") \ No newline at end of file From 1194c286b6e05d0fbfc229dfb7d35be0d063d7c3 Mon Sep 17 00:00:00 2001 From: kyokukou Date: Thu, 7 Nov 2024 13:53:52 -0800 Subject: [PATCH 19/29] collect query data for verbs --- oaipmh/requests/data_queries.py | 11 ++++++++--- oaipmh/requests/info_queries.py | 9 +++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/oaipmh/requests/data_queries.py b/oaipmh/requests/data_queries.py index 71db552..075626d 100644 --- a/oaipmh/requests/data_queries.py +++ b/oaipmh/requests/data_queries.py @@ -27,14 +27,15 @@ def get_record(params: Dict[str, str]) -> Response: def list_records(params: Dict[str, str]) -> Response: """used to harvest records from a repository with support for selective harvesting""" - - token=None + query_data: Dict[OAIParams, str]={OAIParams.VERB:OAIVerbs.LIST_RECORDS} + #get parameters given_params=set(params.keys()) if OAIParams.RES_TOKEN in given_params: if given_params != {OAIParams.RES_TOKEN, OAIParams.VERB}: #resumption token is exclusive raise OAIBadArgument token=params[OAIParams.RES_TOKEN] + #TODO token validation else: if OAIParams.META_PREFIX not in given_params: raise OAIBadArgument @@ -46,13 +47,15 @@ def list_records(params: Dict[str, str]) -> Response: from_str=params.get(OAIParams.FROM) until_str=params.get(OAIParams.UNTIL) set_str=params.get(OAIParams.SET) + #TODO parameter validation + #TODO rest of function return "b", 200, {} def list_identifiers(params: Dict[str, str]) -> Response: """retrieves headers of all records matching certain parameters""" - token=None + query_data: Dict[OAIParams, str]={OAIParams.VERB:OAIVerbs.LIST_IDS} #get parameters given_params=set(params.keys()) @@ -60,6 +63,7 @@ def list_identifiers(params: Dict[str, str]) -> Response: if given_params != {OAIParams.RES_TOKEN, OAIParams.VERB}: #resumption token is exclusive raise OAIBadArgument token=params[OAIParams.RES_TOKEN] + #TODO token processing and validation else: if OAIParams.META_PREFIX not in given_params: raise OAIBadArgument @@ -71,6 +75,7 @@ def list_identifiers(params: Dict[str, str]) -> Response: from_str=params.get(OAIParams.FROM) until_str=params.get(OAIParams.UNTIL) set_str=params.get(OAIParams.SET) + #TODO paramter processing #TODO rest of function diff --git a/oaipmh/requests/info_queries.py b/oaipmh/requests/info_queries.py index 1b3e84a..7f0251a 100644 --- a/oaipmh/requests/info_queries.py +++ b/oaipmh/requests/info_queries.py @@ -4,23 +4,32 @@ from oaipmh.data.oai_properties import OAIParams, OAIVerbs from oaipmh.serializers.output_formats import Response from oaipmh.processors.create_set_list import produce_set_list +from oaipmh.requests.param_processing import process_identifier def identify(params: Dict[str, str]) -> Response: """used to retrieve information about the repository""" + query_data: Dict[OAIParams, str]={OAIParams.VERB : OAIVerbs.IDENTIFY} if set(params.keys()) != {OAIParams.VERB}: raise OAIBadArgument + #TODO return "b", 200, {} def list_metadata_formats(params: Dict[str, str]) -> Response: """used to retrieve the metadata formats available from a repository. An optional argument restricts the request to the formats available for a specific item. """ + query_data: Dict[OAIParams, str]={OAIParams.VERB : OAIVerbs.LIST_META_FORMATS} given_params=set(params.keys()) if OAIParams.ID in given_params: #give formats for one item if given_params != {OAIParams.VERB, OAIParams.ID}: raise OAIBadArgument + + identifier_str=params[OAIParams.ID] + arxiv_id=process_identifier(identifier_str) + query_data[OAIParams.ID]=identifier_str + #TODO get formats for an item return "b", 200, {} From e7dad0a14ac68cd451f43d5f084dbabde713f99e Mon Sep 17 00:00:00 2001 From: kyokukou Date: Fri, 8 Nov 2024 09:08:16 -0800 Subject: [PATCH 20/29] errors contain valid query parameters and more info on error causes that gets displayed data and validation for metadata formats --- oaipmh/data/oai_config.py | 25 ++++++++++++++++++ oaipmh/data/oai_errors.py | 41 ++++++++++++++++++++++------- oaipmh/data/oai_properties.py | 8 +++++- oaipmh/requests/data_queries.py | 22 +++++++++------- oaipmh/requests/info_queries.py | 13 ++++----- oaipmh/requests/param_processing.py | 4 +-- oaipmh/requests/verb_sorter.py | 2 +- oaipmh/templates/errors.xml | 8 ++++-- tests/test_get_record.py | 9 ++++++- tests/test_identify.py | 4 ++- tests/test_list_indentifiers.py | 4 +++ tests/test_list_meta_formats.py | 18 ++++++++++++- tests/test_list_records.py | 5 +++- tests/test_list_sets.py | 4 ++- tests/test_verb_sorting.py | 6 ++++- 15 files changed, 136 insertions(+), 37 deletions(-) diff --git a/oaipmh/data/oai_config.py b/oaipmh/data/oai_config.py index e69de29..3ea5239 100644 --- a/oaipmh/data/oai_config.py +++ b/oaipmh/data/oai_config.py @@ -0,0 +1,25 @@ + +from oaipmh.data.oai_properties import MetadataFormat + +SUPPORTED_METADATA_FORMATS={ + "oai_dc":MetadataFormat( + prefix="oai_dc", + schema="http://www.openarchives.org/OAI/2.0/oai_dc.xsd", + namespace="http://www.openarchives.org/OAI/2.0/oai_dc/" + ), + "arXiv":MetadataFormat( + prefix="arXiv", + schema="http://arxiv.org/OAI/arXiv.xsd", + namespace="http://arxiv.org/OAI/arXiv/" + ), + "arXivOld":MetadataFormat( + prefix="arXiv", + schema="http://arxiv.org/OAI/arXivOld.xsd", + namespace="http://arxiv.org/OAI/arXivOld/" + ), + "arXiv":MetadataFormat( + prefix="arXivRaw", + schema="http://arxiv.org/OAI/arXivRaw.xsd", + namespace="http://arxiv.org/OAI/arXivRaw/" + ), +} \ No newline at end of file diff --git a/oaipmh/data/oai_errors.py b/oaipmh/data/oai_errors.py index 291e957..c8b3fa5 100644 --- a/oaipmh/data/oai_errors.py +++ b/oaipmh/data/oai_errors.py @@ -1,46 +1,67 @@ +from typing import Dict, Optional + +from oaipmh.data.oai_properties import OAIParams class OAIException(Exception): """General class for all OAI defined errors""" code: str description: str - pass + query_params: Optional[Dict[OAIParams, str]] + reason: Optional[str] class OAIBadArgument(OAIException): - #dont include attributes code="badArgument" description="The request includes illegal arguments, is missing required arguments, includes a repeated argument, or values for arguments have an illegal syntax." - + query_params=None #dont include attributes + def __init__(self, reason:str= None): + self.reason=reason + class OAIBadResumptionToken(OAIException): - #TODO consider including params code="badResumptionToken" description="The value of the resumptionToken argument is invalid or expired." + def __init__(self, reason:str, query_params: Dict[OAIParams, str] = None): + self.query_params=query_params + self.reason=reason class OAIBadVerb(OAIException): - #dont include attributes code="badVerb" description="Value of the verb argument is not a legal OAI-PMH verb, the verb argument is missing, or the verb argument is repeated." + query_params=None #dont include attributes + def __init__(self, reason:str= None): + self.reason=reason class OAIBadFormat(OAIException): - #TODO consider including params code="cannotDisseminateFormat" description="The metadata format identified by the value given for the metadataPrefix argument is not supported by the item or by the repository." + def __init__(self, reason:str= None, query_params: Dict[OAIParams, str] = None): + self.query_params=query_params + self.reason=reason class OAINonexistentID(OAIException): - #TODO consider including params code="idDoesNotExist" description="The value of the identifier argument is unknown or illegal in this repository." + def __init__(self, reason:str= None, query_params: Dict[OAIParams, str] = None): + self.query_params=query_params + self.reason=reason class OAINoRecordsMatch(OAIException): - #TODO params code="noRecordsMatch" description="The combination of the values of the from, until, set and metadataPrefix arguments results in an empty list." + def __init__(self, reason:str= None, query_params: Dict[OAIParams, str] = None): + self.query_params=query_params + self.reason=reason class OAINoMetadataFormats(OAIException): - #TODO consider including params code="noMetadataFormats" description="There are no metadata formats available for the specified item." + def __init__(self, reason:str= None, query_params: Dict[OAIParams, str] = None): + self.query_params=query_params + self.reason=reason class OAINoSetHierarchy(OAIException): #should not be triggered for arXiv implementation code="noSetHierarchy" - description="The repository does not support sets. This exception should not be true for the arXiv implementation." \ No newline at end of file + description="The repository does not support sets. This exception should not be true for the arXiv implementation." + def __init__(self, reason:str= None, query_params: Dict[OAIParams, str] = None): + self.query_params=query_params + self.reason=reason \ No newline at end of file diff --git a/oaipmh/data/oai_properties.py b/oaipmh/data/oai_properties.py index 46ac3fa..a28e42c 100644 --- a/oaipmh/data/oai_properties.py +++ b/oaipmh/data/oai_properties.py @@ -13,4 +13,10 @@ class OAIVerbs: LIST_IDS = "ListIdentifiers" IDENTIFY = "Identify" LIST_META_FORMATS = "ListMetadataFormats" - LIST_SETS = "ListSets" \ No newline at end of file + LIST_SETS = "ListSets" + +class MetadataFormat: + def __init__(self, prefix: str, schema: str, namespace: str): + self.prefix = prefix + self.schema = schema + self.namespace = namespace diff --git a/oaipmh/requests/data_queries.py b/oaipmh/requests/data_queries.py index 075626d..d9dec79 100644 --- a/oaipmh/requests/data_queries.py +++ b/oaipmh/requests/data_queries.py @@ -1,6 +1,7 @@ from typing import Dict -from oaipmh.data.oai_errors import OAIBadArgument +from oaipmh.data.oai_config import SUPPORTED_METADATA_FORMATS +from oaipmh.data.oai_errors import OAIBadArgument, OAIBadFormat from oaipmh.data.oai_properties import OAIParams, OAIVerbs from oaipmh.serializers.output_formats import Response from oaipmh.requests.param_processing import process_identifier @@ -13,13 +14,16 @@ def get_record(params: Dict[str, str]) -> Response: # get parameters expected_params={OAIParams.ID, OAIParams.META_PREFIX, OAIParams.VERB} if set(params.keys()) != expected_params: - raise OAIBadArgument + raise OAIBadArgument(f"Parameters provided did not match expected. Expected: {', '.join(str(param) for param in expected_params)}") identifier_str=params[OAIParams.ID] arxiv_id=process_identifier(identifier_str) query_data[OAIParams.ID]=identifier_str - meta_type=params[OAIParams.META_PREFIX] + meta_type_str=params[OAIParams.META_PREFIX] + if meta_type_str not in SUPPORTED_METADATA_FORMATS: + raise OAIBadFormat(reason="Did not recognize requested format", query_params=query_data) + meta_type=SUPPORTED_METADATA_FORMATS[meta_type_str] #TODO rest of function @@ -33,15 +37,15 @@ def list_records(params: Dict[str, str]) -> Response: given_params=set(params.keys()) if OAIParams.RES_TOKEN in given_params: if given_params != {OAIParams.RES_TOKEN, OAIParams.VERB}: #resumption token is exclusive - raise OAIBadArgument + raise OAIBadArgument(f"No other paramters allowed with {OAIParams.RES_TOKEN}") token=params[OAIParams.RES_TOKEN] #TODO token validation else: if OAIParams.META_PREFIX not in given_params: - raise OAIBadArgument + raise OAIBadArgument(f"{OAIParams.META_PREFIX} required.") allowed_params={OAIParams.VERB,OAIParams.META_PREFIX, OAIParams.FROM, OAIParams.UNTIL, OAIParams.SET } if given_params-allowed_params: #no extra keys allowed - raise OAIBadArgument + raise OAIBadArgument(f"Unallowed parameter. Allowed parameters: {', '.join(str(param) for param in allowed_params)}") meta_type=params[OAIParams.META_PREFIX] from_str=params.get(OAIParams.FROM) @@ -61,15 +65,15 @@ def list_identifiers(params: Dict[str, str]) -> Response: given_params=set(params.keys()) if OAIParams.RES_TOKEN in given_params: if given_params != {OAIParams.RES_TOKEN, OAIParams.VERB}: #resumption token is exclusive - raise OAIBadArgument + raise OAIBadArgument(f"No other paramters allowed with {OAIParams.RES_TOKEN}") token=params[OAIParams.RES_TOKEN] #TODO token processing and validation else: if OAIParams.META_PREFIX not in given_params: - raise OAIBadArgument + raise OAIBadArgument(f"{OAIParams.META_PREFIX} required.") allowed_params={OAIParams.VERB,OAIParams.META_PREFIX, OAIParams.FROM, OAIParams.UNTIL, OAIParams.SET } if given_params-allowed_params: #no extra keys allowed - raise OAIBadArgument + raise OAIBadArgument(f"Unallowed parameter. Allowed parameters: {', '.join(str(param) for param in allowed_params)}") meta_type=params[OAIParams.META_PREFIX] from_str=params.get(OAIParams.FROM) diff --git a/oaipmh/requests/info_queries.py b/oaipmh/requests/info_queries.py index 7f0251a..fe97e7c 100644 --- a/oaipmh/requests/info_queries.py +++ b/oaipmh/requests/info_queries.py @@ -10,7 +10,7 @@ def identify(params: Dict[str, str]) -> Response: """used to retrieve information about the repository""" query_data: Dict[OAIParams, str]={OAIParams.VERB : OAIVerbs.IDENTIFY} if set(params.keys()) != {OAIParams.VERB}: - raise OAIBadArgument + raise OAIBadArgument(f"No other parameters allowed for {OAIVerbs.IDENTIFY}") #TODO return "b", 200, {} @@ -22,9 +22,10 @@ def list_metadata_formats(params: Dict[str, str]) -> Response: query_data: Dict[OAIParams, str]={OAIParams.VERB : OAIVerbs.LIST_META_FORMATS} given_params=set(params.keys()) + expected_params={OAIParams.VERB, OAIParams.ID} if OAIParams.ID in given_params: #give formats for one item - if given_params != {OAIParams.VERB, OAIParams.ID}: - raise OAIBadArgument + if given_params != expected_params: + raise OAIBadArgument(f"Only {OAIParams.ID} parameter allowed") identifier_str=params[OAIParams.ID] arxiv_id=process_identifier(identifier_str) @@ -35,7 +36,7 @@ def list_metadata_formats(params: Dict[str, str]) -> Response: else: #give formats repository supports if given_params != {OAIParams.VERB}: - raise OAIBadArgument + raise OAIBadArgument(f"Only allowed parameters are {', '.join(str(param) for param in expected_params)}") #TODO get formats for repository return "b", 200, {} @@ -46,14 +47,14 @@ def list_sets(params: Dict[str, str]) -> Response: given_params=set(params.keys()) if OAIParams.RES_TOKEN in given_params: if given_params != {OAIParams.RES_TOKEN, OAIParams.VERB}: #resumption token is exclusive - raise OAIBadArgument + raise OAIBadArgument(f"No other paramters allowed with {OAIParams.RES_TOKEN}") token_str=params[OAIParams.RES_TOKEN] #TODO token validation/processing query_data[OAIParams.RES_TOKEN]=token_str #TODO will we ever hit this, or will we always return our set structure in full? else: if given_params != {OAIParams.VERB}: - raise OAIBadArgument + raise OAIBadArgument(f"No other parameters allowed") return produce_set_list(query_data) diff --git a/oaipmh/requests/param_processing.py b/oaipmh/requests/param_processing.py index 6808fa9..952ed92 100644 --- a/oaipmh/requests/param_processing.py +++ b/oaipmh/requests/param_processing.py @@ -8,13 +8,13 @@ def process_identifier(id_str:str) -> Identifier: """transform an OAI identifier into an arxiv ID""" if not id_str.startswith(ID_PREFIX): - raise OAINonexistentID + raise OAINonexistentID(reason=f"All identifiers start with: {ID_PREFIX}") short_id = id_str[len(ID_PREFIX):] try: arxiv_id=Identifier(short_id) except IdentifierException: - raise OAINonexistentID + raise OAINonexistentID(reason="Invalid arXiv_id structure") return arxiv_id diff --git a/oaipmh/requests/verb_sorter.py b/oaipmh/requests/verb_sorter.py index d478d7d..11e9753 100644 --- a/oaipmh/requests/verb_sorter.py +++ b/oaipmh/requests/verb_sorter.py @@ -29,4 +29,4 @@ def verb_sorter(params: Dict[str, str]) -> Response: case OAIVerbs.LIST_SETS: return list_sets(params) case _: - raise OAIBadVerb #dont keep invalid verb + raise OAIBadVerb(f"Invalid verb provided") #dont keep invalid verb diff --git a/oaipmh/templates/errors.xml b/oaipmh/templates/errors.xml index 2822712..5641a4e 100644 --- a/oaipmh/templates/errors.xml +++ b/oaipmh/templates/errors.xml @@ -1,9 +1,13 @@ {% extends "base.xml" %} {% block request_element %} - {{ macros.request_element() }} + {% if error.query_params %} + {{ macros.request_element(error.query_params) }} + {% else %} + {{ macros.request_element() }} + {% endif %} {% endblock %} {% block interior_xml %} - {{error.description}} + {{error.description}}{% if error.reason %} Reason: {{ error.reason }}{% endif %} {% endblock %} \ No newline at end of file diff --git a/tests/test_get_record.py b/tests/test_get_record.py index 7ce5ffc..dc33b52 100644 --- a/tests/test_get_record.py +++ b/tests/test_get_record.py @@ -20,6 +20,7 @@ def test_bad_id(test_client): assert response.status_code == 200 text=response.get_data(as_text=True) assert "" in text + assert "All identifiers start with:" in text def test_extra_params(test_client): params = {OAIParams.VERB: OAIVerbs.GET_RECORD, OAIParams.ID: "oai:arXiv.org:2307.10651", OAIParams.META_PREFIX: "oai_dc", "cookie":"chocolate"} @@ -27,11 +28,13 @@ def test_extra_params(test_client): assert response.status_code == 200 text=response.get_data(as_text=True) assert "" in text + assert "Parameters provided did not match expected." in text response = test_client.post("/oai", data=params) assert response.status_code == 200 text=response.get_data(as_text=True) assert "" in text + assert "Parameters provided did not match expected." in text def test_missing_params(test_client): @@ -41,11 +44,13 @@ def test_missing_params(test_client): assert response.status_code == 200 text=response.get_data(as_text=True) assert "" in text + assert "Parameters provided did not match expected." in text response = test_client.post("/oai", data=params) assert response.status_code == 200 text=response.get_data(as_text=True) assert "" in text + assert "Parameters provided did not match expected." in text # missing identifier params = {OAIParams.VERB: OAIVerbs.GET_RECORD, OAIParams.META_PREFIX: "oai_dc"} @@ -53,8 +58,10 @@ def test_missing_params(test_client): assert response.status_code == 200 text=response.get_data(as_text=True) assert "" in text + assert "Parameters provided did not match expected." in text response = test_client.post("/oai", data=params) assert response.status_code == 200 text=response.get_data(as_text=True) - assert "" in text \ No newline at end of file + assert "" in text + assert "Parameters provided did not match expected." in text \ No newline at end of file diff --git a/tests/test_identify.py b/tests/test_identify.py index c229dcc..1cce421 100644 --- a/tests/test_identify.py +++ b/tests/test_identify.py @@ -18,8 +18,10 @@ def test_extra_params(test_client): assert response.status_code == 200 text=response.get_data(as_text=True) assert "" in text + assert "No other parameters allowed" in text response = test_client.post("/oai", data=params) assert response.status_code == 200 text=response.get_data(as_text=True) - assert "" in text \ No newline at end of file + assert "" in text + assert "No other parameters allowed" in text \ No newline at end of file diff --git a/tests/test_list_indentifiers.py b/tests/test_list_indentifiers.py index 5b0a408..a9c7a46 100644 --- a/tests/test_list_indentifiers.py +++ b/tests/test_list_indentifiers.py @@ -44,11 +44,13 @@ def test_extra_params(test_client): assert response.status_code == 200 text=response.get_data(as_text=True) assert "" in text + assert "Unallowed parameter." in text response = test_client.post("/oai", data=params) assert response.status_code == 200 text=response.get_data(as_text=True) assert "" in text + assert "Unallowed parameter." in text def test_token_params(test_client): #correct params @@ -69,8 +71,10 @@ def test_token_params(test_client): assert response.status_code == 200 text=response.get_data(as_text=True) assert "" in text + assert "No other paramters allowed with" in text response = test_client.post("/oai", data=params) assert response.status_code == 200 text=response.get_data(as_text=True) assert "" in text + assert "No other paramters allowed with" in text diff --git a/tests/test_list_meta_formats.py b/tests/test_list_meta_formats.py index ee33187..41b4c4d 100644 --- a/tests/test_list_meta_formats.py +++ b/tests/test_list_meta_formats.py @@ -32,8 +32,24 @@ def test_extra_params(test_client): assert response.status_code == 200 text=response.get_data(as_text=True) assert "" in text + assert "Only allowed parameters are" in text response = test_client.post("/oai", data=params) assert response.status_code == 200 text=response.get_data(as_text=True) - assert "" in text \ No newline at end of file + assert "" in text + assert "Only allowed parameters are" in text + + #with an id + params = {OAIParams.VERB: OAIVerbs.LIST_META_FORMATS, OAIParams.ID:"something", OAIParams.FROM:"now"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text + assert f"Only {OAIParams.ID} parameter allowed" in text + + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text + assert f"Only {OAIParams.ID} parameter allowed" in text \ No newline at end of file diff --git a/tests/test_list_records.py b/tests/test_list_records.py index 0d5339d..1104da6 100644 --- a/tests/test_list_records.py +++ b/tests/test_list_records.py @@ -39,17 +39,18 @@ def test_good_params(test_client): assert "" not in text def test_extra_params(test_client): - #good minimal params params = {OAIParams.VERB: OAIVerbs.LIST_RECORDS, OAIParams.META_PREFIX: "oai_dc", "color":"green"} response = test_client.get("/oai", query_string=params) assert response.status_code == 200 text=response.get_data(as_text=True) assert "" in text + assert "Unallowed parameter." in text response = test_client.post("/oai", data=params) assert response.status_code == 200 text=response.get_data(as_text=True) assert "" in text + assert "Unallowed parameter." in text def test_token_params(test_client): #correct params @@ -70,8 +71,10 @@ def test_token_params(test_client): assert response.status_code == 200 text=response.get_data(as_text=True) assert "" in text + assert "No other paramters allowed with" in text response = test_client.post("/oai", data=params) assert response.status_code == 200 text=response.get_data(as_text=True) assert "" in text + assert "No other paramters allowed with" in text diff --git a/tests/test_list_sets.py b/tests/test_list_sets.py index fe581dd..ee1f13b 100644 --- a/tests/test_list_sets.py +++ b/tests/test_list_sets.py @@ -33,8 +33,10 @@ def test_extra_params(test_client): assert response.status_code == 200 text=response.get_data(as_text=True) assert "" in text + assert "No other parameters allowed" in text response = test_client.post("/oai", data=params) assert response.status_code == 200 text=response.get_data(as_text=True) - assert "" in text \ No newline at end of file + assert "" in text + assert "No other parameters allowed" in text \ No newline at end of file diff --git a/tests/test_verb_sorting.py b/tests/test_verb_sorting.py index 34f92fb..6a1affa 100644 --- a/tests/test_verb_sorting.py +++ b/tests/test_verb_sorting.py @@ -86,10 +86,12 @@ def test_no_verb(test_client): response = test_client.get("/oai", query_string=params) assert response.status_code == 200 assert "" in response.text + assert "Invalid verb provided" in response.text response = test_client.post("/oai", data=params) assert response.status_code == 200 assert "" in response.text + assert "Invalid verb provided" in response.text def test_bad_verb(test_client): params = {OAIParams.VERB: "chaos!"} @@ -97,7 +99,9 @@ def test_bad_verb(test_client): response = test_client.get("/oai", query_string=params) assert response.status_code == 200 assert "" in response.text + assert "Invalid verb provided" in response.text response = test_client.post("/oai", data=params) assert response.status_code == 200 - assert "" in response.text \ No newline at end of file + assert "" in response.text + assert "Invalid verb provided" in response.text \ No newline at end of file From 0098d64be61169b4102b18ff6dac0b6dbf5d4753 Mon Sep 17 00:00:00 2001 From: kyokukou Date: Fri, 8 Nov 2024 11:05:56 -0800 Subject: [PATCH 21/29] refactor shared get_list code --- oaipmh/requests/data_queries.py | 42 +++++---------- tests/test_get_record.py | 8 +++ tests/test_list_data.py | 90 +++++++++++++++++++++++++++++++++ tests/test_list_indentifiers.py | 80 ----------------------------- tests/test_list_records.py | 80 ----------------------------- 5 files changed, 111 insertions(+), 189 deletions(-) create mode 100644 tests/test_list_data.py diff --git a/oaipmh/requests/data_queries.py b/oaipmh/requests/data_queries.py index d9dec79..9524a64 100644 --- a/oaipmh/requests/data_queries.py +++ b/oaipmh/requests/data_queries.py @@ -1,4 +1,5 @@ from typing import Dict +from datetime import datetime from oaipmh.data.oai_config import SUPPORTED_METADATA_FORMATS from oaipmh.data.oai_errors import OAIBadArgument, OAIBadFormat @@ -31,34 +32,15 @@ def get_record(params: Dict[str, str]) -> Response: def list_records(params: Dict[str, str]) -> Response: """used to harvest records from a repository with support for selective harvesting""" - query_data: Dict[OAIParams, str]={OAIParams.VERB:OAIVerbs.LIST_RECORDS} - - #get parameters - given_params=set(params.keys()) - if OAIParams.RES_TOKEN in given_params: - if given_params != {OAIParams.RES_TOKEN, OAIParams.VERB}: #resumption token is exclusive - raise OAIBadArgument(f"No other paramters allowed with {OAIParams.RES_TOKEN}") - token=params[OAIParams.RES_TOKEN] - #TODO token validation - else: - if OAIParams.META_PREFIX not in given_params: - raise OAIBadArgument(f"{OAIParams.META_PREFIX} required.") - allowed_params={OAIParams.VERB,OAIParams.META_PREFIX, OAIParams.FROM, OAIParams.UNTIL, OAIParams.SET } - if given_params-allowed_params: #no extra keys allowed - raise OAIBadArgument(f"Unallowed parameter. Allowed parameters: {', '.join(str(param) for param in allowed_params)}") - - meta_type=params[OAIParams.META_PREFIX] - from_str=params.get(OAIParams.FROM) - until_str=params.get(OAIParams.UNTIL) - set_str=params.get(OAIParams.SET) - #TODO parameter validation - - #TODO rest of function - - return "b", 200, {} + return _list_data(params, False) def list_identifiers(params: Dict[str, str]) -> Response: """retrieves headers of all records matching certain parameters""" + return _list_data(params, True) + + +def _list_data(params: Dict[str, str], just_ids: bool)-> Response: + """runs both list queries. just_ids true for list identifiers, false for list records""" query_data: Dict[OAIParams, str]={OAIParams.VERB:OAIVerbs.LIST_IDS} #get parameters @@ -75,12 +57,14 @@ def list_identifiers(params: Dict[str, str]) -> Response: if given_params-allowed_params: #no extra keys allowed raise OAIBadArgument(f"Unallowed parameter. Allowed parameters: {', '.join(str(param) for param in allowed_params)}") - meta_type=params[OAIParams.META_PREFIX] + meta_type_str=params[OAIParams.META_PREFIX] + if meta_type_str not in SUPPORTED_METADATA_FORMATS: + raise OAIBadFormat(reason="Did not recognize requested format", query_params=query_data) + meta_type=SUPPORTED_METADATA_FORMATS[meta_type_str] + from_str=params.get(OAIParams.FROM) until_str=params.get(OAIParams.UNTIL) set_str=params.get(OAIParams.SET) #TODO paramter processing - #TODO rest of function - - return "b", 200, {} + return "b", 200, {} \ No newline at end of file diff --git a/tests/test_get_record.py b/tests/test_get_record.py index dc33b52..0181805 100644 --- a/tests/test_get_record.py +++ b/tests/test_get_record.py @@ -13,6 +13,14 @@ def test_good_params(test_client): text=response.get_data(as_text=True) assert "" not in text +def test_bad_meta_format(test_client): + params = {OAIParams.VERB: OAIVerbs.GET_RECORD, OAIParams.ID: "oai:arXiv.org:2307.10651", OAIParams.META_PREFIX: "pictures"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text + assert "Did not recognize requested format" in text + def test_bad_id(test_client): params = {OAIParams.VERB: OAIVerbs.GET_RECORD, OAIParams.ID: "arXiv.org:2307.10651", OAIParams.META_PREFIX: "oai_dc"} diff --git a/tests/test_list_data.py b/tests/test_list_data.py new file mode 100644 index 0000000..931fd90 --- /dev/null +++ b/tests/test_list_data.py @@ -0,0 +1,90 @@ +#runs tests for the code list_records and list_indetifiers share + + +from oaipmh.data.oai_properties import OAIParams, OAIVerbs + +def test_good_params(test_client): + #good minimal params + params = {OAIParams.VERB: OAIVerbs.LIST_RECORDS, OAIParams.META_PREFIX: "oai_dc"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + + #good maximal params + params = {OAIParams.VERB: OAIVerbs.LIST_IDS, OAIParams.META_PREFIX: "oai_dc", OAIParams.FROM: "now", OAIParams.UNTIL:"later", OAIParams.SET: "math"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + + #good partial params + params = {OAIParams.VERB: OAIVerbs.LIST_RECORDS, OAIParams.META_PREFIX: "oai_dc", OAIParams.UNTIL:"later", OAIParams.SET: "math"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + +def test_extra_params(test_client): + params = {OAIParams.VERB: OAIVerbs.LIST_IDS, OAIParams.META_PREFIX: "oai_dc", "color":"green"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text + assert "Unallowed parameter." in text + + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text + assert "Unallowed parameter." in text + +def test_bad_meta_format(test_client): + params = {OAIParams.VERB: OAIVerbs.LIST_RECORDS, OAIParams.META_PREFIX: "pictures"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text + assert "Did not recognize requested format" in text + +def test_token_params(test_client): + #correct params + params = {OAIParams.VERB: OAIVerbs.LIST_RECORDS, OAIParams.RES_TOKEN: "rainbow"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" not in text + + #cant have other valid params along with token + params = {OAIParams.VERB: OAIVerbs.LIST_RECORDS, OAIParams.RES_TOKEN: "rainbow", OAIParams.META_PREFIX: "oai_dc"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text + assert "No other paramters allowed with" in text + + response = test_client.post("/oai", data=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text + assert "No other paramters allowed with" in text diff --git a/tests/test_list_indentifiers.py b/tests/test_list_indentifiers.py index a9c7a46..e69de29 100644 --- a/tests/test_list_indentifiers.py +++ b/tests/test_list_indentifiers.py @@ -1,80 +0,0 @@ -from oaipmh.data.oai_properties import OAIParams, OAIVerbs - -def test_good_params(test_client): - #good minimal params - params = {OAIParams.VERB: OAIVerbs.LIST_IDS, OAIParams.META_PREFIX: "oai_dc"} - response = test_client.get("/oai", query_string=params) - assert response.status_code == 200 - text=response.get_data(as_text=True) - assert "" not in text - - response = test_client.post("/oai", data=params) - assert response.status_code == 200 - text=response.get_data(as_text=True) - assert "" not in text - - #good maximal params - params = {OAIParams.VERB: OAIVerbs.LIST_IDS, OAIParams.META_PREFIX: "oai_dc", OAIParams.FROM: "now", OAIParams.UNTIL:"later", OAIParams.SET: "math"} - response = test_client.get("/oai", query_string=params) - assert response.status_code == 200 - text=response.get_data(as_text=True) - assert "" not in text - - response = test_client.post("/oai", data=params) - assert response.status_code == 200 - text=response.get_data(as_text=True) - assert "" not in text - - #good partial params - params = {OAIParams.VERB: OAIVerbs.LIST_IDS, OAIParams.META_PREFIX: "oai_dc", OAIParams.UNTIL:"later", OAIParams.SET: "math"} - response = test_client.get("/oai", query_string=params) - assert response.status_code == 200 - text=response.get_data(as_text=True) - assert "" not in text - - response = test_client.post("/oai", data=params) - assert response.status_code == 200 - text=response.get_data(as_text=True) - assert "" not in text - -def test_extra_params(test_client): - #good minimal params - params = {OAIParams.VERB: OAIVerbs.LIST_IDS, OAIParams.META_PREFIX: "oai_dc", "color":"green"} - response = test_client.get("/oai", query_string=params) - assert response.status_code == 200 - text=response.get_data(as_text=True) - assert "" in text - assert "Unallowed parameter." in text - - response = test_client.post("/oai", data=params) - assert response.status_code == 200 - text=response.get_data(as_text=True) - assert "" in text - assert "Unallowed parameter." in text - -def test_token_params(test_client): - #correct params - params = {OAIParams.VERB: OAIVerbs.LIST_IDS, OAIParams.RES_TOKEN: "rainbow"} - response = test_client.get("/oai", query_string=params) - assert response.status_code == 200 - text=response.get_data(as_text=True) - assert "" not in text - - response = test_client.post("/oai", data=params) - assert response.status_code == 200 - text=response.get_data(as_text=True) - assert "" not in text - - #cant have other valid params along with token - params = {OAIParams.VERB: OAIVerbs.LIST_IDS, OAIParams.RES_TOKEN: "rainbow", OAIParams.META_PREFIX: "oai_dc"} - response = test_client.get("/oai", query_string=params) - assert response.status_code == 200 - text=response.get_data(as_text=True) - assert "" in text - assert "No other paramters allowed with" in text - - response = test_client.post("/oai", data=params) - assert response.status_code == 200 - text=response.get_data(as_text=True) - assert "" in text - assert "No other paramters allowed with" in text diff --git a/tests/test_list_records.py b/tests/test_list_records.py index 1104da6..e69de29 100644 --- a/tests/test_list_records.py +++ b/tests/test_list_records.py @@ -1,80 +0,0 @@ - -from oaipmh.data.oai_properties import OAIParams, OAIVerbs - -def test_good_params(test_client): - #good minimal params - params = {OAIParams.VERB: OAIVerbs.LIST_RECORDS, OAIParams.META_PREFIX: "oai_dc"} - response = test_client.get("/oai", query_string=params) - assert response.status_code == 200 - text=response.get_data(as_text=True) - assert "" not in text - - response = test_client.post("/oai", data=params) - assert response.status_code == 200 - text=response.get_data(as_text=True) - assert "" not in text - - #good maximal params - params = {OAIParams.VERB: OAIVerbs.LIST_RECORDS, OAIParams.META_PREFIX: "oai_dc", OAIParams.FROM: "now", OAIParams.UNTIL:"later", OAIParams.SET: "math"} - response = test_client.get("/oai", query_string=params) - assert response.status_code == 200 - text=response.get_data(as_text=True) - assert "" not in text - - response = test_client.post("/oai", data=params) - assert response.status_code == 200 - text=response.get_data(as_text=True) - assert "" not in text - - #good partial params - params = {OAIParams.VERB: OAIVerbs.LIST_RECORDS, OAIParams.META_PREFIX: "oai_dc", OAIParams.UNTIL:"later", OAIParams.SET: "math"} - response = test_client.get("/oai", query_string=params) - assert response.status_code == 200 - text=response.get_data(as_text=True) - assert "" not in text - - response = test_client.post("/oai", data=params) - assert response.status_code == 200 - text=response.get_data(as_text=True) - assert "" not in text - -def test_extra_params(test_client): - params = {OAIParams.VERB: OAIVerbs.LIST_RECORDS, OAIParams.META_PREFIX: "oai_dc", "color":"green"} - response = test_client.get("/oai", query_string=params) - assert response.status_code == 200 - text=response.get_data(as_text=True) - assert "" in text - assert "Unallowed parameter." in text - - response = test_client.post("/oai", data=params) - assert response.status_code == 200 - text=response.get_data(as_text=True) - assert "" in text - assert "Unallowed parameter." in text - -def test_token_params(test_client): - #correct params - params = {OAIParams.VERB: OAIVerbs.LIST_RECORDS, OAIParams.RES_TOKEN: "rainbow"} - response = test_client.get("/oai", query_string=params) - assert response.status_code == 200 - text=response.get_data(as_text=True) - assert "" not in text - - response = test_client.post("/oai", data=params) - assert response.status_code == 200 - text=response.get_data(as_text=True) - assert "" not in text - - #cant have other valid params along with token - params = {OAIParams.VERB: OAIVerbs.LIST_RECORDS, OAIParams.RES_TOKEN: "rainbow", OAIParams.META_PREFIX: "oai_dc"} - response = test_client.get("/oai", query_string=params) - assert response.status_code == 200 - text=response.get_data(as_text=True) - assert "" in text - assert "No other paramters allowed with" in text - - response = test_client.post("/oai", data=params) - assert response.status_code == 200 - text=response.get_data(as_text=True) - assert "" in text - assert "No other paramters allowed with" in text From 48a5d8b2ba96ade3b2335434fd35abe8ee5f1831 Mon Sep 17 00:00:00 2001 From: kyokukou Date: Mon, 11 Nov 2024 10:01:14 -0800 Subject: [PATCH 22/29] display metadata formats page --- oaipmh/requests/info_queries.py | 12 +++++++++++- oaipmh/templates/metaformats.xml | 25 +++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 oaipmh/templates/metaformats.xml diff --git a/oaipmh/requests/info_queries.py b/oaipmh/requests/info_queries.py index fe97e7c..292eff9 100644 --- a/oaipmh/requests/info_queries.py +++ b/oaipmh/requests/info_queries.py @@ -1,5 +1,9 @@ from typing import Dict +from datetime import timezone, datetime +from flask import render_template + +from oaipmh.data.oai_config import SUPPORTED_METADATA_FORMATS from oaipmh.data.oai_errors import OAIBadArgument from oaipmh.data.oai_properties import OAIParams, OAIVerbs from oaipmh.serializers.output_formats import Response @@ -38,7 +42,13 @@ def list_metadata_formats(params: Dict[str, str]) -> Response: if given_params != {OAIParams.VERB}: raise OAIBadArgument(f"Only allowed parameters are {', '.join(str(param) for param in expected_params)}") #TODO get formats for repository - return "b", 200, {} + response=render_template("metaformats.xml", + response_date=datetime.now(timezone.utc), + query_params=query_data, + formats=SUPPORTED_METADATA_FORMATS + ) + headers={"Content-Type":"application/xml"} + return response, 200, headers def list_sets(params: Dict[str, str]) -> Response: """used to retrieve the set structure of a repository""" diff --git a/oaipmh/templates/metaformats.xml b/oaipmh/templates/metaformats.xml new file mode 100644 index 0000000..74fd08d --- /dev/null +++ b/oaipmh/templates/metaformats.xml @@ -0,0 +1,25 @@ +{% extends "base.xml" %} + +{% macro metadata_format(format) %} + + {{ format.prefix }} + {{ format.schema }} + {{ format.namespace }} + +{% endmacro %} + +{% block request_element %} + {{ macros.request_element(query_params) }} +{% endblock %} + +{% block interior_xml %} + + {% for item in formats.values() %} + {{ metadata_format(item) }} + {% endfor %} + +{% endblock %} + + + + From 456c2f532465881cd52ebb5eafdc54500e4d7343 Mon Sep 17 00:00:00 2001 From: kyokukou Date: Mon, 11 Nov 2024 10:06:04 -0800 Subject: [PATCH 23/29] datetime checking --- oaipmh/data/oai_config.py | 7 +++- oaipmh/requests/data_queries.py | 47 +++++++++++++++++++++--- tests/test_list_data.py | 65 ++++++++++++++++++++++++++++++++- 3 files changed, 109 insertions(+), 10 deletions(-) diff --git a/oaipmh/data/oai_config.py b/oaipmh/data/oai_config.py index 3ea5239..341f94f 100644 --- a/oaipmh/data/oai_config.py +++ b/oaipmh/data/oai_config.py @@ -1,6 +1,9 @@ - +from datetime import datetime, timezone from oaipmh.data.oai_properties import MetadataFormat +#TODO do we want to change this +EARLIEST_DATE=datetime(2007, 5, 23, 0, 0, tzinfo=timezone.utc) + SUPPORTED_METADATA_FORMATS={ "oai_dc":MetadataFormat( prefix="oai_dc", @@ -17,7 +20,7 @@ schema="http://arxiv.org/OAI/arXivOld.xsd", namespace="http://arxiv.org/OAI/arXivOld/" ), - "arXiv":MetadataFormat( + "arXivRaw":MetadataFormat( prefix="arXivRaw", schema="http://arxiv.org/OAI/arXivRaw.xsd", namespace="http://arxiv.org/OAI/arXivRaw/" diff --git a/oaipmh/requests/data_queries.py b/oaipmh/requests/data_queries.py index 9524a64..5c469aa 100644 --- a/oaipmh/requests/data_queries.py +++ b/oaipmh/requests/data_queries.py @@ -1,7 +1,8 @@ from typing import Dict -from datetime import datetime +import re +from datetime import datetime, timezone -from oaipmh.data.oai_config import SUPPORTED_METADATA_FORMATS +from oaipmh.data.oai_config import SUPPORTED_METADATA_FORMATS, EARLIEST_DATE from oaipmh.data.oai_errors import OAIBadArgument, OAIBadFormat from oaipmh.data.oai_properties import OAIParams, OAIVerbs from oaipmh.serializers.output_formats import Response @@ -43,28 +44,62 @@ def _list_data(params: Dict[str, str], just_ids: bool)-> Response: """runs both list queries. just_ids true for list identifiers, false for list records""" query_data: Dict[OAIParams, str]={OAIParams.VERB:OAIVerbs.LIST_IDS} - #get parameters + #parameter processing given_params=set(params.keys()) - if OAIParams.RES_TOKEN in given_params: + if OAIParams.RES_TOKEN in given_params: #using resumption token if given_params != {OAIParams.RES_TOKEN, OAIParams.VERB}: #resumption token is exclusive raise OAIBadArgument(f"No other paramters allowed with {OAIParams.RES_TOKEN}") token=params[OAIParams.RES_TOKEN] #TODO token processing and validation - else: + + else: #using request parameters + #correct parameters present if OAIParams.META_PREFIX not in given_params: raise OAIBadArgument(f"{OAIParams.META_PREFIX} required.") allowed_params={OAIParams.VERB,OAIParams.META_PREFIX, OAIParams.FROM, OAIParams.UNTIL, OAIParams.SET } if given_params-allowed_params: #no extra keys allowed raise OAIBadArgument(f"Unallowed parameter. Allowed parameters: {', '.join(str(param) for param in allowed_params)}") + #metadata meta_type_str=params[OAIParams.META_PREFIX] if meta_type_str not in SUPPORTED_METADATA_FORMATS: raise OAIBadFormat(reason="Did not recognize requested format", query_params=query_data) meta_type=SUPPORTED_METADATA_FORMATS[meta_type_str] - + query_data[OAIParams.META_PREFIX]=meta_type_str + + #dates from_str=params.get(OAIParams.FROM) + if from_str: + try: + if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", from_str): + raise ValueError + start_date=datetime.strptime(from_str, "%Y-%m-%d") + start_date = start_date.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=timezone.utc) + query_data[OAIParams.FROM]=from_str + except Exception: + raise OAIBadArgument("from date format must be YYYY-MM-DD") + else: + start_date=EARLIEST_DATE + until_str=params.get(OAIParams.UNTIL) + if until_str: + try: + if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", until_str): + raise ValueError + end_date=datetime.strptime(until_str, "%Y-%m-%d") + end_date = end_date.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=timezone.utc) + query_data[OAIParams.UNTIL]=until_str + except Exception: + raise OAIBadArgument("until date format must be YYYY-MM-DD") + else: + end_date=datetime.now(timezone.utc) + + #sets set_str=params.get(OAIParams.SET) #TODO paramter processing + #TODO check that combined parameters are valid (dates are okay) + + #TODO rest of function + return "b", 200, {} \ No newline at end of file diff --git a/tests/test_list_data.py b/tests/test_list_data.py index 931fd90..a6179b5 100644 --- a/tests/test_list_data.py +++ b/tests/test_list_data.py @@ -17,7 +17,7 @@ def test_good_params(test_client): assert "" not in text #good maximal params - params = {OAIParams.VERB: OAIVerbs.LIST_IDS, OAIParams.META_PREFIX: "oai_dc", OAIParams.FROM: "now", OAIParams.UNTIL:"later", OAIParams.SET: "math"} + params = {OAIParams.VERB: OAIVerbs.LIST_IDS, OAIParams.META_PREFIX: "oai_dc", OAIParams.FROM: "2020-01-05", OAIParams.UNTIL:"2020-02-05", OAIParams.SET: "math"} response = test_client.get("/oai", query_string=params) assert response.status_code == 200 text=response.get_data(as_text=True) @@ -29,7 +29,7 @@ def test_good_params(test_client): assert "" not in text #good partial params - params = {OAIParams.VERB: OAIVerbs.LIST_RECORDS, OAIParams.META_PREFIX: "oai_dc", OAIParams.UNTIL:"later", OAIParams.SET: "math"} + params = {OAIParams.VERB: OAIVerbs.LIST_RECORDS, OAIParams.META_PREFIX: "oai_dc", OAIParams.UNTIL:"2020-02-05", OAIParams.SET: "math"} response = test_client.get("/oai", query_string=params) assert response.status_code == 200 text=response.get_data(as_text=True) @@ -62,6 +62,67 @@ def test_bad_meta_format(test_client): assert "" in text assert "Did not recognize requested format" in text +def test_bad_date_params(test_client): + + #invalid from types + params = {OAIParams.VERB: OAIVerbs.LIST_IDS, OAIParams.META_PREFIX: "oai_dc", OAIParams.FROM: "a/37", OAIParams.UNTIL:"2020-02-05", OAIParams.SET: "math"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text + assert "from date format must be YYYY-MM-DD" in text + + params = {OAIParams.VERB: OAIVerbs.LIST_IDS, OAIParams.META_PREFIX: "oai_dc", OAIParams.FROM: "2020-5-6", OAIParams.UNTIL:"2020-02-05", OAIParams.SET: "math"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text + assert "from date format must be YYYY-MM-DD" in text + + params = {OAIParams.VERB: OAIVerbs.LIST_IDS, OAIParams.META_PREFIX: "oai_dc", OAIParams.FROM: "2024-11-08T19:49:17Z", OAIParams.UNTIL:"2020-02-05", OAIParams.SET: "math"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text + assert "from date format must be YYYY-MM-DD" in text + + params = {OAIParams.VERB: OAIVerbs.LIST_IDS, OAIParams.META_PREFIX: "oai_dc", OAIParams.FROM: "2024-32-08", OAIParams.UNTIL:"2020-02-05", OAIParams.SET: "math"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text + assert "from date format must be YYYY-MM-DD" in text + + #invalid until types + params = {OAIParams.VERB: OAIVerbs.LIST_IDS, OAIParams.META_PREFIX: "oai_dc", OAIParams.UNTIL: "a/37", OAIParams.FROM:"2020-02-05", OAIParams.SET: "math"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text + assert "until date format must be YYYY-MM-DD" in text + + params = {OAIParams.VERB: OAIVerbs.LIST_IDS, OAIParams.META_PREFIX: "oai_dc", OAIParams.UNTIL: "2020-5-6", OAIParams.FROM:"2020-02-05", OAIParams.SET: "math"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text + assert "until date format must be YYYY-MM-DD" in text + + params = {OAIParams.VERB: OAIVerbs.LIST_IDS, OAIParams.META_PREFIX: "oai_dc", OAIParams.UNTIL: "2024-11-08T19:49:17Z", OAIParams.FROM:"2020-02-05", OAIParams.SET: "math"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text + assert "until date format must be YYYY-MM-DD" in text + + params = {OAIParams.VERB: OAIVerbs.LIST_IDS, OAIParams.META_PREFIX: "oai_dc", OAIParams.UNTIL: "2024-32-08", OAIParams.FROM:"2020-02-05", OAIParams.SET: "math"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text + assert "until date format must be YYYY-MM-DD" in text + + def test_token_params(test_client): #correct params params = {OAIParams.VERB: OAIVerbs.LIST_RECORDS, OAIParams.RES_TOKEN: "rainbow"} From 9412b914a54a4726c3ce6702a269d3820396943f Mon Sep 17 00:00:00 2001 From: kyokukou Date: Tue, 12 Nov 2024 11:06:02 -0800 Subject: [PATCH 24/29] validate set parameters --- oaipmh/requests/data_queries.py | 54 ++++++++++++++++++++++++++++----- oaipmh/requests/info_queries.py | 3 +- tests/test_list_data.py | 46 ++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 10 deletions(-) diff --git a/oaipmh/requests/data_queries.py b/oaipmh/requests/data_queries.py index 5c469aa..4168d2e 100644 --- a/oaipmh/requests/data_queries.py +++ b/oaipmh/requests/data_queries.py @@ -1,13 +1,17 @@ -from typing import Dict +from typing import Dict, Union import re from datetime import datetime, timezone +from arxiv.taxonomy.definitions import GROUPS, ARCHIVES_ACTIVE, CATEGORIES_ACTIVE +from arxiv.taxonomy.category import Group, Archive, Category + from oaipmh.data.oai_config import SUPPORTED_METADATA_FORMATS, EARLIEST_DATE from oaipmh.data.oai_errors import OAIBadArgument, OAIBadFormat from oaipmh.data.oai_properties import OAIParams, OAIVerbs from oaipmh.serializers.output_formats import Response from oaipmh.requests.param_processing import process_identifier +DATE_REGEX = r"\d{4}-\d{2}-\d{2}" def get_record(params: Dict[str, str]) -> Response: """used to get data on a particular record in a particular metadata format""" @@ -27,7 +31,7 @@ def get_record(params: Dict[str, str]) -> Response: raise OAIBadFormat(reason="Did not recognize requested format", query_params=query_data) meta_type=SUPPORTED_METADATA_FORMATS[meta_type_str] - #TODO rest of function + #TODO paramters done, do rest of function return "b", 200, {} @@ -71,7 +75,7 @@ def _list_data(params: Dict[str, str], just_ids: bool)-> Response: from_str=params.get(OAIParams.FROM) if from_str: try: - if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", from_str): + if not re.fullmatch(DATE_REGEX, from_str): raise ValueError start_date=datetime.strptime(from_str, "%Y-%m-%d") start_date = start_date.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=timezone.utc) @@ -84,7 +88,7 @@ def _list_data(params: Dict[str, str], just_ids: bool)-> Response: until_str=params.get(OAIParams.UNTIL) if until_str: try: - if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", until_str): + if not re.fullmatch(DATE_REGEX, until_str): raise ValueError end_date=datetime.strptime(until_str, "%Y-%m-%d") end_date = end_date.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=timezone.utc) @@ -94,12 +98,46 @@ def _list_data(params: Dict[str, str], just_ids: bool)-> Response: else: end_date=datetime.now(timezone.utc) - #sets + #sets set_str=params.get(OAIParams.SET) - #TODO paramter processing + if set_str: + rq_set= _parse_set(set_str) + query_data[OAIParams.SET]=set_str - #TODO check that combined parameters are valid (dates are okay) + #TODO check that combined parameters are valid (dates are okay, sets are active and not test) combined with token data #TODO rest of function - return "b", 200, {} \ No newline at end of file + return "b", 200, {} + +def _parse_set(set_str:str)-> Union[Group, Archive, Category]: + """turns OAI style string into taxonomy item + validates item + """ + set_parts=set_str.split(":") + match len(set_parts): + case 1: #asking for a group + rq_set = GROUPS.get(f'grp_{set_str}') + if not rq_set: + raise OAIBadArgument("Set does not exist") + case 2: #archive (including archive as category) + grp_str, archive_str = set_parts + rq_set = ARCHIVES_ACTIVE.get(archive_str) + if not rq_set or f'grp_{grp_str}' != rq_set.in_group: + raise OAIBadArgument("Set does not exist") + case 3: #full category + grp_str, archive_str, category_suffix = set_parts + cat_str = f"{archive_str}.{category_suffix}" + if cat_str not in CATEGORIES_ACTIVE: + raise OAIBadArgument("Set does not exist") + rq_set= CATEGORIES_ACTIVE[cat_str] + archive= rq_set.get_archive() + if archive_str!= archive.id or f'grp_{grp_str}' != archive.in_group: + raise OAIBadArgument("Set does not exist") + case _: + raise OAIBadArgument("Set has too many levels") + + return rq_set + + + diff --git a/oaipmh/requests/info_queries.py b/oaipmh/requests/info_queries.py index 292eff9..cd1ae39 100644 --- a/oaipmh/requests/info_queries.py +++ b/oaipmh/requests/info_queries.py @@ -41,7 +41,6 @@ def list_metadata_formats(params: Dict[str, str]) -> Response: else: #give formats repository supports if given_params != {OAIParams.VERB}: raise OAIBadArgument(f"Only allowed parameters are {', '.join(str(param) for param in expected_params)}") - #TODO get formats for repository response=render_template("metaformats.xml", response_date=datetime.now(timezone.utc), query_params=query_data, @@ -66,5 +65,5 @@ def list_sets(params: Dict[str, str]) -> Response: if given_params != {OAIParams.VERB}: raise OAIBadArgument(f"No other parameters allowed") - return produce_set_list(query_data) + return produce_set_list(query_data) #TODO update with categories diff --git a/tests/test_list_data.py b/tests/test_list_data.py index a6179b5..7e69714 100644 --- a/tests/test_list_data.py +++ b/tests/test_list_data.py @@ -1,7 +1,11 @@ #runs tests for the code list_records and list_indetifiers share +import pytest +from arxiv.taxonomy.definitions import GROUPS, ARCHIVES, CATEGORIES from oaipmh.data.oai_properties import OAIParams, OAIVerbs +from oaipmh.data.oai_errors import OAIBadArgument +from oaipmh.requests.data_queries import _parse_set def test_good_params(test_client): #good minimal params @@ -122,6 +126,13 @@ def test_bad_date_params(test_client): assert "" in text assert "until date format must be YYYY-MM-DD" in text +def test_bad_set_params(test_client): + params = {OAIParams.VERB: OAIVerbs.LIST_IDS, OAIParams.META_PREFIX: "oai_dc", OAIParams.SET: "math:physics"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text + assert "Set does not exist" in text def test_token_params(test_client): #correct params @@ -149,3 +160,38 @@ def test_token_params(test_client): text=response.get_data(as_text=True) assert "" in text assert "No other paramters allowed with" in text + +def test_set_parser(): + #good values + assert _parse_set('physics') == GROUPS['grp_physics'] + assert _parse_set('physics:physics') == ARCHIVES['physics'] + assert _parse_set('physics:physics:flu-dyn') == CATEGORIES['physics.flu-dyn'] + + #bad values + #gibberish + with pytest.raises(OAIBadArgument, match="Set does not exist"): + _parse_set("2020-5-6") + with pytest.raises(OAIBadArgument, match="Set does not exist"): + _parse_set("maths") + with pytest.raises(OAIBadArgument, match="Set does not exist"): + _parse_set("math:nonsense") + with pytest.raises(OAIBadArgument, match="Set does not exist"): + _parse_set("math:math:nonsense") + with pytest.raises(OAIBadArgument, match="Set does not exist"): + _parse_set("nonsense:math") + with pytest.raises(OAIBadArgument, match="Set does not exist"): + _parse_set("nonsense:math:NA") + with pytest.raises(OAIBadArgument, match="Set does not exist"): + _parse_set("math:nonsense:NA") + + with pytest.raises(OAIBadArgument, match="Set has too many levels"): + _parse_set("math:math:math:NA") + + #incorrect structure + with pytest.raises(OAIBadArgument, match="Set does not exist"): + _parse_set("math:math:flu-dyn") + with pytest.raises(OAIBadArgument, match="Set does not exist"): + _parse_set("math:physics:flu-dyn") + with pytest.raises(OAIBadArgument, match="Set does not exist"): + _parse_set("math:nucl-th") + From ebbccb9d321e3d61d047c636e6553f9d85d58724 Mon Sep 17 00:00:00 2001 From: kyokukou Date: Tue, 12 Nov 2024 11:58:23 -0800 Subject: [PATCH 25/29] helper function to turn arxiv categories into set strings --- oaipmh/processors/create_set_list.py | 17 +++++++++++++++-- tests/test_list_sets.py | 15 ++++++++++++++- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/oaipmh/processors/create_set_list.py b/oaipmh/processors/create_set_list.py index 4f71030..0a8373d 100644 --- a/oaipmh/processors/create_set_list.py +++ b/oaipmh/processors/create_set_list.py @@ -1,8 +1,9 @@ from datetime import datetime, timezone -from typing import Dict, Any +from typing import Dict, Any, Union from flask import render_template +from arxiv.taxonomy.category import Group, Archive, Category from arxiv.taxonomy.definitions import ARCHIVES_ACTIVE from oaipmh.data.oai_properties import OAIParams @@ -16,4 +17,16 @@ def produce_set_list(query_data: Dict[OAIParams, Any]) -> Response: response_date=datetime.now(timezone.utc), query_data=query_data, archives=ARCHIVES_ACTIVE) - return response, 200, {} \ No newline at end of file + return response, 200, {} + +def make_set_str(item: Union[Group, Archive, Category]) -> str: + """helper function to convert arXiv category data into OAI set structure + the grp_ prefix should be removed from group ids + """ + if isinstance(item, Group): + return item.id[4:] + elif isinstance(item, Archive): + return f"{item.in_group[4:]}:{item.id}" + elif isinstance(item, Category): + archive=item.get_archive() + return f"{archive.in_group[4:]}:{item.id.replace('.',':')}" diff --git a/tests/test_list_sets.py b/tests/test_list_sets.py index ee1f13b..915e163 100644 --- a/tests/test_list_sets.py +++ b/tests/test_list_sets.py @@ -1,5 +1,8 @@ +from arxiv.taxonomy.definitions import GROUPS, ARCHIVES, CATEGORIES + from oaipmh.data.oai_properties import OAIParams, OAIVerbs +from oaipmh.processors.create_set_list import make_set_str def test_good_params(test_client): #general case @@ -39,4 +42,14 @@ def test_extra_params(test_client): assert response.status_code == 200 text=response.get_data(as_text=True) assert "" in text - assert "No other parameters allowed" in text \ No newline at end of file + assert "No other parameters allowed" in text + +def test_make_set_str(): + assert 'physics'==make_set_str(GROUPS['grp_physics']) + assert 'math'==make_set_str(GROUPS['grp_math']) + assert 'math:math' == make_set_str(ARCHIVES['math']) + assert 'physics:hep-ph' == make_set_str(ARCHIVES['hep-ph']) + assert 'physics:physics:flu-dyn' == make_set_str(CATEGORIES['physics.flu-dyn']) + assert 'cs:cs:GT' == make_set_str(CATEGORIES['cs.GT']) + assert 'physics:astro-ph'==make_set_str(CATEGORIES['astro-ph']) + \ No newline at end of file From 55591b97a9fcd6113c8266855b1a3735fa3c90fe Mon Sep 17 00:00:00 2001 From: kyokukou Date: Tue, 12 Nov 2024 13:45:43 -0800 Subject: [PATCH 26/29] use group:archive:cat_suffix structure for sets --- oaipmh/processors/create_set_list.py | 17 +++++++++++++---- oaipmh/requests/info_queries.py | 9 ++------- oaipmh/templates/setSpec.xml | 12 +++++++++--- tests/test_list_sets.py | 12 ++++++++++++ 4 files changed, 36 insertions(+), 14 deletions(-) diff --git a/oaipmh/processors/create_set_list.py b/oaipmh/processors/create_set_list.py index 0a8373d..9dcd55b 100644 --- a/oaipmh/processors/create_set_list.py +++ b/oaipmh/processors/create_set_list.py @@ -4,19 +4,28 @@ from flask import render_template from arxiv.taxonomy.category import Group, Archive, Category -from arxiv.taxonomy.definitions import ARCHIVES_ACTIVE +from arxiv.taxonomy.definitions import ARCHIVES, GROUPS, CATEGORIES_ACTIVE from oaipmh.data.oai_properties import OAIParams from oaipmh.serializers.output_formats import Response def produce_set_list(query_data: Dict[OAIParams, Any]) -> Response: """create the set structure of a repository""" - #TODO display in desired form/ level of depth once decided - #TODO filter out hidden entries + groups = {key: grp for key, + grp in GROUPS.items() + if grp.is_active and not grp.is_test} + archives= {key: arch for key, + arch in ARCHIVES.items() + if arch.is_active and not arch.in_group == "grp_test"} + response=render_template("setSpec.xml", response_date=datetime.now(timezone.utc), query_data=query_data, - archives=ARCHIVES_ACTIVE) + groups=groups, + archives=archives, + categories= CATEGORIES_ACTIVE, + to_set= make_set_str + ) return response, 200, {} def make_set_str(item: Union[Group, Archive, Category]) -> str: diff --git a/oaipmh/requests/info_queries.py b/oaipmh/requests/info_queries.py index cd1ae39..3faddb6 100644 --- a/oaipmh/requests/info_queries.py +++ b/oaipmh/requests/info_queries.py @@ -51,19 +51,14 @@ def list_metadata_formats(params: Dict[str, str]) -> Response: def list_sets(params: Dict[str, str]) -> Response: """used to retrieve the set structure of a repository""" - query_data: Dict[OAIParams, str]={OAIParams.VERB:OAIVerbs.LIST_SETS} given_params=set(params.keys()) if OAIParams.RES_TOKEN in given_params: if given_params != {OAIParams.RES_TOKEN, OAIParams.VERB}: #resumption token is exclusive raise OAIBadArgument(f"No other paramters allowed with {OAIParams.RES_TOKEN}") - token_str=params[OAIParams.RES_TOKEN] - #TODO token validation/processing - query_data[OAIParams.RES_TOKEN]=token_str - #TODO will we ever hit this, or will we always return our set structure in full? + raise OAIBadArgument(f"Invalid token") #we never give out a resumption token for sets else: if given_params != {OAIParams.VERB}: raise OAIBadArgument(f"No other parameters allowed") - - return produce_set_list(query_data) #TODO update with categories + return produce_set_list(query_data) diff --git a/oaipmh/templates/setSpec.xml b/oaipmh/templates/setSpec.xml index 84370ed..45e53fe 100644 --- a/oaipmh/templates/setSpec.xml +++ b/oaipmh/templates/setSpec.xml @@ -6,15 +6,21 @@ {% block interior_xml %} + {% for item in groups.values() %} + {{ set_item(item, to_set) }} + {% endfor %} {% for item in archives.values() %} - {{ set_item(item) }} + {{ set_item(item, to_set) }} + {% endfor %} + {% for item in categories.values() %} + {{ set_item(item, to_set) }} {% endfor %} {% endblock %} -{% macro set_item(item) %} +{% macro set_item(item, to_set) %} - {{item.id}} + {{to_set(item)}} {{item.full_name}} {% endmacro %} \ No newline at end of file diff --git a/tests/test_list_sets.py b/tests/test_list_sets.py index 915e163..5f4cb37 100644 --- a/tests/test_list_sets.py +++ b/tests/test_list_sets.py @@ -4,6 +4,18 @@ from oaipmh.data.oai_properties import OAIParams, OAIVerbs from oaipmh.processors.create_set_list import make_set_str +def test_output(test_client): + #general case + params = {OAIParams.VERB: OAIVerbs.LIST_SETS} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "test" not in text #test items + assert "adap-org" not in text #inactive + assert "math" in text + assert "math:math" in text + assert "math:math:NA" in text + def test_good_params(test_client): #general case params = {OAIParams.VERB: OAIVerbs.LIST_SETS} From a43f658b15c1ca918a54b97e3642092c301d3955 Mon Sep 17 00:00:00 2001 From: kyokukou Date: Tue, 12 Nov 2024 13:49:27 -0800 Subject: [PATCH 27/29] update tests, no resumption token for list_sets --- tests/test_list_sets.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/tests/test_list_sets.py b/tests/test_list_sets.py index 5f4cb37..c8cdc6a 100644 --- a/tests/test_list_sets.py +++ b/tests/test_list_sets.py @@ -29,18 +29,6 @@ def test_good_params(test_client): text=response.get_data(as_text=True) assert "" not in text - #with token - params = {OAIParams.VERB: OAIVerbs.LIST_SETS, OAIParams.RES_TOKEN : "item"} - response = test_client.get("/oai", query_string=params) - assert response.status_code == 200 - text=response.get_data(as_text=True) - assert "" not in text - - response = test_client.post("/oai", data=params) - assert response.status_code == 200 - text=response.get_data(as_text=True) - assert "" not in text - def test_extra_params(test_client): #general case params = {OAIParams.VERB: OAIVerbs.LIST_SETS, OAIParams.FROM:"now"} @@ -56,6 +44,14 @@ def test_extra_params(test_client): assert "" in text assert "No other parameters allowed" in text + #with token + params = {OAIParams.VERB: OAIVerbs.LIST_SETS, OAIParams.RES_TOKEN : "math"} + response = test_client.get("/oai", query_string=params) + assert response.status_code == 200 + text=response.get_data(as_text=True) + assert "" in text + assert "Invalid token" in text + def test_make_set_str(): assert 'physics'==make_set_str(GROUPS['grp_physics']) assert 'math'==make_set_str(GROUPS['grp_math']) From cb403195a7a8c3fcd14b9a1172083ce84714efde Mon Sep 17 00:00:00 2001 From: kyokukou Date: Wed, 13 Nov 2024 07:24:22 -0800 Subject: [PATCH 28/29] moved verb sorting into route --- oaipmh/requests/routes.py | 34 ++++++++++++++++++++++++++++------ oaipmh/requests/verb_sorter.py | 32 -------------------------------- tests/test_verb_sorting.py | 24 ++++++++++++------------ 3 files changed, 40 insertions(+), 50 deletions(-) delete mode 100644 oaipmh/requests/verb_sorter.py diff --git a/oaipmh/requests/routes.py b/oaipmh/requests/routes.py index 6dfef39..87ec172 100644 --- a/oaipmh/requests/routes.py +++ b/oaipmh/requests/routes.py @@ -1,8 +1,11 @@ from typing import Dict -from datetime import datetime, timezone -from flask import Blueprint, request, render_template +from flask import Blueprint, request -from oaipmh.requests.verb_sorter import verb_sorter +from oaipmh.requests.info_queries import identify, list_metadata_formats, list_sets +from oaipmh.requests.data_queries import get_record, list_identifiers, list_records +from oaipmh.serializers.output_formats import Response +from oaipmh.data.oai_errors import OAIBadVerb +from oaipmh.data.oai_properties import OAIVerbs from oaipmh.serializers.output_formats import Response blueprint = Blueprint('general', __name__) @@ -10,11 +13,30 @@ @blueprint.route("/oai", methods=['GET', 'POST']) def oai() -> Response: - + """ + sorts OAI queries to the appropriate handler based on their verb statement + this defines what the client is asking for as per the OAI standard + further verification of parameters is done with the handlers for individual verbs + """ #TODO duplicate params dont create errors, technically not to spec params: Dict[str, str] = request.args.to_dict() if request.method == 'GET' else request.form.to_dict() - - response, code, headers=verb_sorter(params) + verb = params.get("verb", "") + match verb: + case OAIVerbs.GET_RECORD: + response, code, headers= get_record(params) + case OAIVerbs.LIST_RECORDS: + response, code, headers= list_records(params) + case OAIVerbs.LIST_IDS: + response, code, headers= list_identifiers(params) + case OAIVerbs.IDENTIFY: + response, code, headers= identify(params) + case OAIVerbs.LIST_META_FORMATS: + response, code, headers= list_metadata_formats(params) + case OAIVerbs.LIST_SETS: + response, code, headers= list_sets(params) + case _: + raise OAIBadVerb(f"Invalid verb provided") #dont keep invalid verb + headers["Content-Type"]="application/xml" return response, code, headers diff --git a/oaipmh/requests/verb_sorter.py b/oaipmh/requests/verb_sorter.py deleted file mode 100644 index 11e9753..0000000 --- a/oaipmh/requests/verb_sorter.py +++ /dev/null @@ -1,32 +0,0 @@ -from typing import Dict - - -from oaipmh.requests.info_queries import identify, list_metadata_formats, list_sets -from oaipmh.requests.data_queries import get_record, list_identifiers, list_records -from oaipmh.serializers.output_formats import Response -from oaipmh.data.oai_errors import OAIBadVerb -from oaipmh.data.oai_properties import OAIVerbs - -def verb_sorter(params: Dict[str, str]) -> Response: - """ - sorts OAI queries to the appropriate handler based on their verb statement - this defines what the client is asking for as per the OAI standard - further verification of parameters is done with the handlers for individual verbs - returns the interior xml for the response - """ - verb = params.get("verb", "") - match verb: - case OAIVerbs.GET_RECORD: - return get_record(params) - case OAIVerbs.LIST_RECORDS: - return list_records(params) - case OAIVerbs.LIST_IDS: - return list_identifiers(params) - case OAIVerbs.IDENTIFY: - return identify(params) - case OAIVerbs.LIST_META_FORMATS: - return list_metadata_formats(params) - case OAIVerbs.LIST_SETS: - return list_sets(params) - case _: - raise OAIBadVerb(f"Invalid verb provided") #dont keep invalid verb diff --git a/tests/test_verb_sorting.py b/tests/test_verb_sorting.py index 6a1affa..17366cb 100644 --- a/tests/test_verb_sorting.py +++ b/tests/test_verb_sorting.py @@ -5,12 +5,12 @@ def test_get_record(test_client): params = {OAIParams.VERB: OAIVerbs.GET_RECORD, OAIParams.ID: "oai:example.org:record123"} - with patch('oaipmh.requests.verb_sorter.get_record', return_value=("working", 200, {})) as mock_get_record: + with patch('oaipmh.requests.routes.get_record', return_value=("working", 200, {})) as mock_get_record: response = test_client.get("/oai", query_string=params) assert response.status_code == 200 mock_get_record.assert_called_once_with(params) - with patch('oaipmh.requests.verb_sorter.get_record', return_value=("working", 200, {})) as mock_get_record: + with patch('oaipmh.requests.routes.get_record', return_value=("working", 200, {})) as mock_get_record: response = test_client.post("/oai", data=params) assert response.status_code == 200 mock_get_record.assert_called_once_with(params) @@ -18,12 +18,12 @@ def test_get_record(test_client): def test_list_records(test_client): params = {OAIParams.VERB: OAIVerbs.LIST_RECORDS, OAIParams.META_PREFIX: "oai_dc"} - with patch('oaipmh.requests.verb_sorter.list_records', return_value=("working", 200, {})) as mock_list_records: + with patch('oaipmh.requests.routes.list_records', return_value=("working", 200, {})) as mock_list_records: response = test_client.get("/oai", query_string=params) assert response.status_code == 200 mock_list_records.assert_called_once_with(params) - with patch('oaipmh.requests.verb_sorter.list_records', return_value=("working", 200, {})) as mock_list_records: + with patch('oaipmh.requests.routes.list_records', return_value=("working", 200, {})) as mock_list_records: response = test_client.post("/oai", data=params) assert response.status_code == 200 mock_list_records.assert_called_once_with(params) @@ -31,12 +31,12 @@ def test_list_records(test_client): def test_list_identifiers(test_client): params = {OAIParams.VERB: OAIVerbs.LIST_IDS, OAIParams.META_PREFIX: "oai_dc"} - with patch('oaipmh.requests.verb_sorter.list_identifiers', return_value=("working", 200, {})) as mock_list_identifiers: + with patch('oaipmh.requests.routes.list_identifiers', return_value=("working", 200, {})) as mock_list_identifiers: response = test_client.get("/oai", query_string=params) assert response.status_code == 200 mock_list_identifiers.assert_called_once_with(params) - with patch('oaipmh.requests.verb_sorter.list_identifiers', return_value=("working", 200, {})) as mock_list_identifiers: + with patch('oaipmh.requests.routes.list_identifiers', return_value=("working", 200, {})) as mock_list_identifiers: response = test_client.post("/oai", data=params) assert response.status_code == 200 mock_list_identifiers.assert_called_once_with(params) @@ -44,12 +44,12 @@ def test_list_identifiers(test_client): def test_identify(test_client): params = {OAIParams.VERB: OAIVerbs.IDENTIFY} - with patch('oaipmh.requests.verb_sorter.identify', return_value=("working", 200, {})) as mock_identify: + with patch('oaipmh.requests.routes.identify', return_value=("working", 200, {})) as mock_identify: response = test_client.get("/oai", query_string=params) assert response.status_code == 200 mock_identify.assert_called_once_with(params) - with patch('oaipmh.requests.verb_sorter.identify', return_value=("working", 200, {})) as mock_identify: + with patch('oaipmh.requests.routes.identify', return_value=("working", 200, {})) as mock_identify: response = test_client.post("/oai", data=params) assert response.status_code == 200 mock_identify.assert_called_once_with(params) @@ -57,12 +57,12 @@ def test_identify(test_client): def test_list_metadata_formats(test_client): params = {OAIParams.VERB: OAIVerbs.LIST_META_FORMATS} - with patch('oaipmh.requests.verb_sorter.list_metadata_formats', return_value=("working", 200, {})) as mock_list_metadata_formats: + with patch('oaipmh.requests.routes.list_metadata_formats', return_value=("working", 200, {})) as mock_list_metadata_formats: response = test_client.get("/oai", query_string=params) assert response.status_code == 200 mock_list_metadata_formats.assert_called_once_with(params) - with patch('oaipmh.requests.verb_sorter.list_metadata_formats', return_value=("working", 200, {})) as mock_list_metadata_formats: + with patch('oaipmh.requests.routes.list_metadata_formats', return_value=("working", 200, {})) as mock_list_metadata_formats: response = test_client.post("/oai", data=params) assert response.status_code == 200 mock_list_metadata_formats.assert_called_once_with(params) @@ -70,12 +70,12 @@ def test_list_metadata_formats(test_client): def test_list_sets(test_client): params = {OAIParams.VERB: OAIVerbs.LIST_SETS} - with patch('oaipmh.requests.verb_sorter.list_sets', return_value=("working", 200, {})) as mock_list_sets: + with patch('oaipmh.requests.routes.list_sets', return_value=("working", 200, {})) as mock_list_sets: response = test_client.get("/oai", query_string=params) assert response.status_code == 200 mock_list_sets.assert_called_once_with(params) - with patch('oaipmh.requests.verb_sorter.list_sets', return_value=("working", 200, {})) as mock_list_sets: + with patch('oaipmh.requests.routes.list_sets', return_value=("working", 200, {})) as mock_list_sets: response = test_client.post("/oai", data=params) assert response.status_code == 200 mock_list_sets.assert_called_once_with(params) From 798408a753622f02e02fefb456dd64c8aff31592 Mon Sep 17 00:00:00 2001 From: kyokukou Date: Wed, 13 Nov 2024 08:01:38 -0800 Subject: [PATCH 29/29] minor refactoring --- oaipmh/requests/data_queries.py | 11 +---------- oaipmh/requests/routes.py | 6 +++--- oaipmh/templates/base.xml | 2 +- oaipmh/templates/{macros.html => macros.xml} | 0 tests/test_verb_sorting.py | 16 ++++++++-------- 5 files changed, 13 insertions(+), 22 deletions(-) rename oaipmh/templates/{macros.html => macros.xml} (100%) diff --git a/oaipmh/requests/data_queries.py b/oaipmh/requests/data_queries.py index 4168d2e..38c4577 100644 --- a/oaipmh/requests/data_queries.py +++ b/oaipmh/requests/data_queries.py @@ -35,16 +35,7 @@ def get_record(params: Dict[str, str]) -> Response: return "b", 200, {} -def list_records(params: Dict[str, str]) -> Response: - """used to harvest records from a repository with support for selective harvesting""" - return _list_data(params, False) - -def list_identifiers(params: Dict[str, str]) -> Response: - """retrieves headers of all records matching certain parameters""" - return _list_data(params, True) - - -def _list_data(params: Dict[str, str], just_ids: bool)-> Response: +def list_data(params: Dict[str, str], just_ids: bool)-> Response: """runs both list queries. just_ids true for list identifiers, false for list records""" query_data: Dict[OAIParams, str]={OAIParams.VERB:OAIVerbs.LIST_IDS} diff --git a/oaipmh/requests/routes.py b/oaipmh/requests/routes.py index 87ec172..52688d0 100644 --- a/oaipmh/requests/routes.py +++ b/oaipmh/requests/routes.py @@ -2,7 +2,7 @@ from flask import Blueprint, request from oaipmh.requests.info_queries import identify, list_metadata_formats, list_sets -from oaipmh.requests.data_queries import get_record, list_identifiers, list_records +from oaipmh.requests.data_queries import get_record, list_data from oaipmh.serializers.output_formats import Response from oaipmh.data.oai_errors import OAIBadVerb from oaipmh.data.oai_properties import OAIVerbs @@ -25,9 +25,9 @@ def oai() -> Response: case OAIVerbs.GET_RECORD: response, code, headers= get_record(params) case OAIVerbs.LIST_RECORDS: - response, code, headers= list_records(params) + response, code, headers= list_data(params, False) case OAIVerbs.LIST_IDS: - response, code, headers= list_identifiers(params) + response, code, headers= list_data(params, True) case OAIVerbs.IDENTIFY: response, code, headers= identify(params) case OAIVerbs.LIST_META_FORMATS: diff --git a/oaipmh/templates/base.xml b/oaipmh/templates/base.xml index cf966db..0869915 100644 --- a/oaipmh/templates/base.xml +++ b/oaipmh/templates/base.xml @@ -1,5 +1,5 @@ -{% import 'macros.html' as macros %} +{% import 'macros.xml' as macros %} diff --git a/oaipmh/templates/macros.html b/oaipmh/templates/macros.xml similarity index 100% rename from oaipmh/templates/macros.html rename to oaipmh/templates/macros.xml diff --git a/tests/test_verb_sorting.py b/tests/test_verb_sorting.py index 17366cb..f0b62cb 100644 --- a/tests/test_verb_sorting.py +++ b/tests/test_verb_sorting.py @@ -18,28 +18,28 @@ def test_get_record(test_client): def test_list_records(test_client): params = {OAIParams.VERB: OAIVerbs.LIST_RECORDS, OAIParams.META_PREFIX: "oai_dc"} - with patch('oaipmh.requests.routes.list_records', return_value=("working", 200, {})) as mock_list_records: + with patch('oaipmh.requests.routes.list_data', return_value=("working", 200, {})) as mock_list_records: response = test_client.get("/oai", query_string=params) assert response.status_code == 200 - mock_list_records.assert_called_once_with(params) + mock_list_records.assert_called_once_with(params, False) - with patch('oaipmh.requests.routes.list_records', return_value=("working", 200, {})) as mock_list_records: + with patch('oaipmh.requests.routes.list_data', return_value=("working", 200, {})) as mock_list_records: response = test_client.post("/oai", data=params) assert response.status_code == 200 - mock_list_records.assert_called_once_with(params) + mock_list_records.assert_called_once_with(params, False) def test_list_identifiers(test_client): params = {OAIParams.VERB: OAIVerbs.LIST_IDS, OAIParams.META_PREFIX: "oai_dc"} - with patch('oaipmh.requests.routes.list_identifiers', return_value=("working", 200, {})) as mock_list_identifiers: + with patch('oaipmh.requests.routes.list_data', return_value=("working", 200, {})) as mock_list_identifiers: response = test_client.get("/oai", query_string=params) assert response.status_code == 200 - mock_list_identifiers.assert_called_once_with(params) + mock_list_identifiers.assert_called_once_with(params, True) - with patch('oaipmh.requests.routes.list_identifiers', return_value=("working", 200, {})) as mock_list_identifiers: + with patch('oaipmh.requests.routes.list_data', return_value=("working", 200, {})) as mock_list_identifiers: response = test_client.post("/oai", data=params) assert response.status_code == 200 - mock_list_identifiers.assert_called_once_with(params) + mock_list_identifiers.assert_called_once_with(params, True) def test_identify(test_client): params = {OAIParams.VERB: OAIVerbs.IDENTIFY}