diff --git a/hepdata/modules/records/api.py b/hepdata/modules/records/api.py index 2483f442..b1c6b105 100644 --- a/hepdata/modules/records/api.py +++ b/hepdata/modules/records/api.py @@ -27,7 +27,6 @@ from collections import OrderedDict from functools import wraps import mimetypes -import requests import time from celery import shared_task @@ -35,7 +34,7 @@ from flask_login import current_user from invenio_accounts.models import User from invenio_db import db -from sqlalchemy import and_ +from sqlalchemy import and_, func from sqlalchemy.orm.exc import NoResultFound from werkzeug.utils import secure_filename @@ -57,7 +56,14 @@ from hepdata.modules.records.utils.workflow import update_action_for_submission_participant from hepdata.modules.records.utils.yaml_utils import split_files from hepdata.modules.stats.views import increment, get_count -from hepdata.modules.submission.models import RecordVersionCommitMessage, DataSubmission, HEPSubmission, DataReview +from hepdata.modules.submission.models import ( + DataReview, + DataSubmission, + HEPSubmission, + RecordVersionCommitMessage, + RelatedRecid, + RelatedTable +) from hepdata.utils.file_extractor import extract from hepdata.utils.miscellaneous import sanitize_html from hepdata.utils.users import get_user_from_id @@ -262,8 +268,8 @@ def format_resource(resource, contents, content_url): ctx['file_mimetype'] = get_resource_mimetype(resource, contents) ctx['resource_filename'] = os.path.basename(resource.file_location) ctx['resource_filetype'] = f'{resource.file_type} File' - ctx['related_recids'] = [r.related_recid for r in hepsubmission.related_recids] - ctx['related_to_this_recids'] = [s.publication_recid for s in hepsubmission.get_related_hepsubmissions()] + ctx['related_recids'] = get_record_data_list(hepsubmission, "related") + ctx['related_to_this_recids'] = get_record_data_list(hepsubmission, "related_to_this") if resource.file_type in IMAGE_TYPES: ctx['display_type'] = 'image' @@ -401,8 +407,8 @@ def render_record(recid, record, version, output_format, light_mode=False): elif not hepdata_submission.overall_status.startswith('sandbox'): ctx = format_submission(recid, record, version, version_count, hepdata_submission) ctx['record_type'] = 'publication' - ctx['related_recids'] = [r.related_recid for r in hepdata_submission.related_recids] - ctx['related_to_this_recids'] = [s.publication_recid for s in hepdata_submission.get_related_hepsubmissions()] + ctx['related_recids'] = get_record_data_list(hepdata_submission, "related") + ctx['related_to_this_recids'] = get_record_data_list(hepdata_submission, "related_to_this") increment(recid) @@ -457,9 +463,8 @@ def render_record(recid, record, version, output_format, light_mode=False): ctx['record_type'] = 'table' ctx['related_publication_id'] = publication_recid ctx['table_name'] = record['title'] - ctx['related_recids'] = [r.related_recid for r in hepdata_submission.related_recids] - ctx['related_to_this_recids'] = [s.publication_recid for s in - hepdata_submission.get_related_hepsubmissions()] + ctx['related_recids'] = get_record_data_list(hepdata_submission, "related") + ctx['related_to_this_recids'] = get_record_data_list(hepdata_submission, "related_to_this") if output_format == 'html' or output_format == 'json_ld': ctx['json_ld'] = get_json_ld( @@ -1043,3 +1048,144 @@ def get_all_ids(index=None, id_field='recid', last_updated=None, latest_first=Fa else: query = query.order_by(HEPSubmission.publication_recid).distinct() return [int(x[0]) for x in query.all()] + + +def get_related_hepsubmissions(submission): + """ + Queries the database for all HEPSubmission objects contained in + this object's related record ID list. + (All submissions this one is relating to) + + :return: [list] A list of HEPSubmission objects + """ + related_submissions = [] + for related in submission.related_recids: + data_submission = get_latest_hepsubmission( + publication_recid=related.related_recid, + overall_status='finished' + ) + if data_submission: + related_submissions.append(data_submission) + return related_submissions + + +def get_related_to_this_hepsubmissions(submission): + """ + Queries the database for all records in the RelatedRecId table + that have THIS record's id as a related record. + Then returns the HEPSubmission object marked in the RelatedRecid table. + Returns only submissions marked as 'finished' + + :return: [list] List containing related records. + """ + related_sub_ids = ( + HEPSubmission.query + .join(RelatedRecid, RelatedRecid.this_recid == HEPSubmission.publication_recid) + .filter(RelatedRecid.related_recid == submission.publication_recid) + .filter(HEPSubmission.overall_status == 'finished') # Only finished submissions + .with_entities(HEPSubmission.publication_recid) + .all() + ) + + # Filter out the non-unique submission results returned by different versions + unique_recids = set(sub[0] for sub in related_sub_ids) + return [get_latest_hepsubmission(publication_recid=recid, overall_status='finished') for recid in unique_recids] + + +def get_related_datasubmissions(data_submission): + """ + Queries the database for all DataSubmission objects contained in + this object's related DOI list. + Only returns an object if associated HEPSubmission status is 'finished' + (All submissions this one is relating to) + + :param data_submission: The datasubmission object to find related data for. + :return: [list] A list of DataSubmission objects + """ + related_submissions = [] + for related in data_submission.related_tables: + submission = ( + DataSubmission.query + .filter(DataSubmission.doi == related.related_doi) + .join(HEPSubmission, HEPSubmission.publication_recid == DataSubmission.publication_recid) + .filter(HEPSubmission.overall_status == 'finished') + .first() + ) + if submission: + related_submissions.append(submission) + return related_submissions + + +def get_related_to_this_datasubmissions(data_submission): + """ + Get the DataSubmission Objects with a RelatedTable entry + where this doi is referred to in related_doi. + + :param data_submission: The datasubmission to find the related entries for. + :return: [List] List of DataSubmission objects. + """ + related_submissions = ( + DataSubmission.query + .join(RelatedTable, RelatedTable.table_doi == DataSubmission.doi) + .join(HEPSubmission,(HEPSubmission.publication_recid == DataSubmission.publication_recid)) + .group_by(DataSubmission.id) + .having(func.max(HEPSubmission.version) == DataSubmission.version) + .filter(RelatedTable.related_doi == data_submission.doi) + .filter(HEPSubmission.overall_status == 'finished') + .all() + ) + return related_submissions + + +def get_record_data_list(record, data_type): + """ + Generates a dictionary (title/recid) from a list of record IDs. + This must be done as the record contents are not stored within the hepsubmission object. + + :param record: The record used for the query. + :param data_type: Either the related, or related to this data. + :return: [list] A list of dictionary objects containing record ID and title pairs + """ + # Selects the related data based on the data_type flag + data = [] + if data_type == "related": + data = get_related_hepsubmissions(record) + elif data_type == "related_to_this": + data = get_related_to_this_hepsubmissions(record) + + record_data = [] + for datum in data: + record_data.append( + { + "recid": datum.publication_recid, + "title": get_record_contents(datum.publication_recid)["title"], + "version": datum.version + }) + return record_data + + +def get_table_data_list(table, data_type): + """ + Generates a list of general information (name, doi, desc) dictionaries of related DataSubmission objects. + Will either use the related data list (get_related_data_submissions) + OR the `related to this` list (generated by get_related_to_this_datasubmissions) + + :param table: The DataSubmission object used for querying. + :param data_type: The flag to decide which relation data to use. + :return: [list] A list of dictionaries with the name, doi and description of the object. + """ + # Selects the related data based on the data_type flag + if data_type == "related": + data = get_related_datasubmissions(table) + elif data_type == "related_to_this": + data = get_related_to_this_datasubmissions(table) + + record_data = [] + if data: + for datum in data: + record_data.append({ + "name": datum.name, + "doi": datum.doi, + "description": datum.description + }) + return record_data diff --git a/hepdata/modules/records/assets/js/hepdata_tables.js b/hepdata/modules/records/assets/js/hepdata_tables.js index ca4ffe75..5e90d494 100644 --- a/hepdata/modules/records/assets/js/hepdata_tables.js +++ b/hepdata/modules/records/assets/js/hepdata_tables.js @@ -195,6 +195,8 @@ HEPDATA.table_renderer = { The `placement` argument is used to specify between both related record types. The records `this is relating to`, and records `that relate to this` record. + + This function also toggles visibility of the related object area. */ var relatedTablesWrapper = $(placement); relatedTablesWrapper.find("ul").empty(); @@ -203,13 +205,21 @@ HEPDATA.table_renderer = { var relatedList = relatedTablesWrapper.find(".related-list") for (var i = 0; i < relatedDois.length; i++) { - var doi = relatedDois[i]; - var relatedItem = $('
  • ').append($('').text(doi).attr('href', "https://doi.org/" + doi)); + var related_object = relatedDois[i]; + // Create the the `li` tag, setting the text to the table name and DOI + // Adds a `title` tag containing the table's description + // Example: "Table 1" + var relatedItem = $('
  • ').append($('' + ).text(related_object.name + ).attr('href', 'https://doi.org/' + related_object.doi).attr('title', related_object.description)); + // Add the `li` tag to the list relatedList.append(relatedItem); } + // Ensure that the container is visible. relatedTablesWrapper.show(); } else { + // Hide the container if there is no information. relatedTablesWrapper.hide(); } }, diff --git a/hepdata/modules/records/templates/hepdata_records/components/related_recids.html b/hepdata/modules/records/templates/hepdata_records/components/related_recids.html index 649305cf..153fe4df 100644 --- a/hepdata/modules/records/templates/hepdata_records/components/related_recids.html +++ b/hepdata/modules/records/templates/hepdata_records/components/related_recids.html @@ -4,7 +4,7 @@
    This record is related to:
    @@ -17,7 +17,7 @@
    This record is referred to by:
    diff --git a/hepdata/modules/records/utils/common.py b/hepdata/modules/records/utils/common.py index 03ed0f19..d30fe59b 100644 --- a/hepdata/modules/records/utils/common.py +++ b/hepdata/modules/records/utils/common.py @@ -28,7 +28,7 @@ import os from sqlalchemy.orm.exc import NoResultFound -from hepdata.config import CFG_PUB_TYPE, HISTFACTORY_FILE_TYPE +from hepdata.config import HISTFACTORY_FILE_TYPE from hepdata.ext.opensearch.api import get_record from hepdata.modules.submission.models import HEPSubmission, License @@ -251,3 +251,5 @@ def get_record_by_id(recid): def record_exists(*args, **kwargs): count = HEPSubmission.query.filter_by(**kwargs).count() return count > 0 + + diff --git a/hepdata/modules/records/utils/submission.py b/hepdata/modules/records/utils/submission.py index efb52b9d..cc94425f 100644 --- a/hepdata/modules/records/utils/submission.py +++ b/hepdata/modules/records/utils/submission.py @@ -218,13 +218,15 @@ def cleanup_data_keywords(data_submission): def cleanup_data_related_recid(recid): """ Deletes all related record ID entries of a HEPSubmission object of a given recid + :param recid: The record ID of the HEPSubmission object to be cleaned :return: """ - hepsubmission = HEPSubmission.query.filter_by(publication_recid=recid).first() - for related in hepsubmission.related_recids: - db.session.delete(related) - db.session.commit() + hepsubmission = get_latest_hepsubmission(publication_recid=recid) + if hepsubmission: + for related in hepsubmission.related_recids: + db.session.delete(related) + db.session.commit() def process_data_file(recid, version, basepath, data_obj, datasubmission, main_file_path, tablenum, overall_status): @@ -270,6 +272,7 @@ def process_data_file(recid, version, basepath, data_obj, datasubmission, main_f datasubmission.keywords.append(keyword) if overall_status not in ("sandbox", "sandbox_processing"): + # Process the related doi data (only if not in sandbox mode) if "related_to_table_dois" in data_obj: for related_doi in data_obj["related_to_table_dois"]: this_doi = f"{HEPDATA_DOI_PREFIX}/hepdata.{recid}.v{version}/t{tablenum}" diff --git a/hepdata/modules/records/views.py b/hepdata/modules/records/views.py index b13da287..e3b38c17 100644 --- a/hepdata/modules/records/views.py +++ b/hepdata/modules/records/views.py @@ -49,7 +49,7 @@ render_record, current_user, db, jsonify, get_user_from_id, get_record_contents, extract_journal_info, \ user_allowed_to_perform_action, NoResultFound, OrderedDict, query_messages_for_data_review, returns_json, \ process_payload, has_upload_permissions, has_coordinator_permissions, create_new_version, format_resource, \ - should_send_json_ld, JSON_LD_MIMETYPES, get_resource_mimetype + should_send_json_ld, JSON_LD_MIMETYPES, get_resource_mimetype, get_table_data_list from hepdata.modules.submission.api import get_submission_participants_for_record from hepdata.modules.submission.models import HEPSubmission, DataSubmission, \ DataResource, DataReview, Message, Question @@ -325,9 +325,8 @@ def get_table_details(recid, data_recid, version): table_contents["name"] = datasub_record.name table_contents["title"] = datasub_record.description table_contents["keywords"] = datasub_record.keywords - # The functions used only return the objects themselves. - table_contents["related_tables"] = [r.related_doi for r in datasub_record.related_tables] - table_contents["related_to_this"] = [r.doi for r in datasub_record.get_related_datasubmissions()] + table_contents["related_tables"] = get_table_data_list(datasub_record, "related") + table_contents["related_to_this"] = get_table_data_list(datasub_record, "related_to_this") table_contents["doi"] = datasub_record.doi table_contents["location"] = datasub_record.location_in_publication diff --git a/hepdata/modules/submission/models.py b/hepdata/modules/submission/models.py index 986b4d00..ca5addb8 100644 --- a/hepdata/modules/submission/models.py +++ b/hepdata/modules/submission/models.py @@ -121,23 +121,6 @@ class HEPSubmission(db.Model): cascade="all,delete") - def get_related_hepsubmissions(self): - """ - Queries the database for all records in the RelatedRecId table - that have THIS record's id as a related record. - Then returns the HEPSubmission object marked in the RelatedRecid table. - Returns only submissions marked as 'finished' - :return: [list] List containing related records. - """ - related_submissions = ( - HEPSubmission.query - .join(RelatedRecid, RelatedRecid.this_recid == HEPSubmission.publication_recid) - .filter(RelatedRecid.related_recid == self.publication_recid) - .filter(HEPSubmission.overall_status == 'finished') # Only finished submissions - .all()) - return related_submissions - - # Declarations of the helper tables used to manage many-to-many relationships. datafile_identifier = db.Table( 'datafile_identifier', @@ -208,22 +191,6 @@ class DataSubmission(db.Model): # through a submissions review stages. version = db.Column(db.Integer, default=0) - def get_related_datasubmissions(self): - """ - Get the DataSubmission Objects with a RelatedTable entry - where this doi is referred to in related_doi. - - :return: [List] List of DataSubmission objects. - """ - related_submissions = ( - DataSubmission.query.join(RelatedTable, RelatedTable.table_doi == DataSubmission.doi) - .join(HEPSubmission, HEPSubmission.publication_recid == DataSubmission.publication_recid) - .filter(RelatedTable.related_doi == self.doi) - .filter(HEPSubmission.overall_status == 'finished') - .all() - ) - return related_submissions - class RelatedTable(db.Model): """ diff --git a/tests/e2e/test_records.py b/tests/e2e/test_records.py index d1c7bd70..1d1b0c99 100644 --- a/tests/e2e/test_records.py +++ b/tests/e2e/test_records.py @@ -24,6 +24,8 @@ """HEPData end to end testing of updating/reviewing records""" import os.path +import shutil +import time import flask import os @@ -35,15 +37,17 @@ from selenium.webdriver.support import expected_conditions as EC from hepdata.config import HEPDATA_DOI_PREFIX +from hepdata.modules.records.utils.data_files import get_data_path_for_record from hepdata.modules.submission.models import HEPSubmission, RelatedRecid, DataSubmission, RelatedTable from hepdata.modules.dashboard.api import create_record_for_dashboard from hepdata.modules.records.utils.common import get_record_by_id -from hepdata.modules.records.utils.submission import get_or_create_hepsubmission +from hepdata.modules.records.utils.submission import get_or_create_hepsubmission, process_submission_directory from hepdata.modules.records.utils.workflow import create_record from invenio_accounts.models import User from invenio_db import db +from hepdata.modules.submission.views import process_submission_payload from .conftest import e2e_assert_url @@ -257,15 +261,15 @@ def test_related_records(live_server, logged_in_browser): # Dictionary to store the generated data. # The two objects should have flipped recid/expected values # i.e. Related to each other + # TODO - Maybe make expected_number a list, to allow for testing of multiple expected values. test_data = [ - {"recid": None, "related_recid": None, "submission": None}, - {"recid": None, "related_recid": None, "submission": None} + {"recid": None, "submission": None, "related_recid": None, "submission_number": 1, "expected_number": 2}, + {"recid": None, "submission": None, "related_recid": None, "submission_number": 2, "expected_number": 1} ] - # Creates two records. for test in test_data: record_information = create_record( - {'journal_info': 'Journal', 'title': 'Test Paper'}) + {'journal_info': 'Journal', 'title': f"Test Record {test['submission_number']}"}) test['submission'] = get_or_create_hepsubmission(record_information['recid']) # Set overall status to finished so related data appears on dashboard test['submission'].overall_status = 'finished' @@ -277,6 +281,7 @@ def test_related_records(live_server, logged_in_browser): create_record_for_dashboard(record['recid'], test_submissions, user) # Recids for the test are set dynamically, based on what comes out of the minter + # TODO - Would need to be changed to extend for more than one-to-one cases test_data[0]['related_recid'] = test_data[1]['recid'] test_data[1]['related_recid'] = test_data[0]['recid'] @@ -291,14 +296,14 @@ def test_related_records(live_server, logged_in_browser): # Create a test DataSubmission object datasubmission = DataSubmission( - name="Test", + name=f"Test Table {test['submission_number']}", location_in_publication="Somewhere", data_file=1, publication_recid=test['recid'], associated_recid=test['recid'], doi=doi_string, version=1, - description="Test") + description=f"Test Description {test['submission_number']}") # Generate the test DOI string for the related DOI related_doi_string = f"{HEPDATA_DOI_PREFIX}/hepdata.{test['related_recid']}.v1/t1" @@ -319,39 +324,198 @@ def test_related_records(live_server, logged_in_browser): browser.get(record_url) # The page elements to test and their type (Record/Data) related_elements = [ - {'element':'related-recids', 'type':'recid'}, - {'element':'related-to-this-recids', 'type':'recid'}, - {'element':'related-tables', 'type':'doi'}, - {'element':'related-to-this-tables', 'type':'doi'} + {'id':'related-recids', 'type':'recid'}, + {'id':'related-to-this-recids', 'type':'recid'}, + {'id':'related-tables', 'type':'doi'}, + {'id':'related-to-this-tables', 'type':'doi'} ] + for element in related_elements: - html_element = browser.find_element(By.ID, element['element']) + html_element = browser.find_element(By.ID, element['id']) + # Get the related data list html based on the current element data_list = html_element.find_element(By.CLASS_NAME, 'related-list') list_items = data_list.find_elements(By.TAG_NAME, 'li') # There should be only one entry for each related test category assert len(list_items) == 1 + url_tag = list_items[0].find_element(By.TAG_NAME, 'a') # Get the URL of the found `li` tag. - url_text = list_items[0].find_element(By.TAG_NAME, 'a').get_attribute('href') - + url_loc = url_tag.get_attribute('href') # Expected ul and a tag contents differ based on which elements are tested # Records expect a link to the HEPData site. Tables link to doi.org if element['type'] == 'recid': # Check the URL against the regex - pattern = rf"http://localhost:\d+/record/{test['related_recid']}" - assert re.match(pattern, url_text) - # Check that the tag text is the expected record ID number + pattern = rf"^.+/record/{test['related_recid']}$" + assert re.match(pattern, url_loc) + # Check that the tag text is the expected string + # The result will be the string "Test Paper(X)" Where X is the ID. assert list_items[0].text == str(test['related_recid']) + # Check the expected title of the related record tag + assert url_tag.get_attribute('title') == f"Test Record {test['expected_number']}" elif element['type'] == 'doi': # Generate the test DOI string + # Currently only testing v1/t1, maybe needs to be extended. related_doi_check = f"{HEPDATA_DOI_PREFIX}/hepdata.{test['related_recid']}.v1/t1" # Generate expected DOI URL (linking to doi.org/DOI) doi_url = "https://doi.org/" + related_doi_check - assert url_text == doi_url - # Check that the tag text is the expected DOI string - assert list_items[0].text == related_doi_check + assert url_loc == doi_url + # Check the expected text of the related table DOI tag + assert url_tag.text == f"Test Table {test['expected_number']}" + # Check the expected title of the related table DOI tag + assert url_tag.get_attribute('title') == f"Test Description {test['expected_number']}" + +def test_version_related_table(live_server, logged_in_browser): + """ + Tests the related table data on the records page. + Checks that the tooltip and description are correct. + """ + browser = logged_in_browser + browser.file_detector = LocalFileDetector() + record = {'title': "Test Title", + 'reviewer': {'name': 'Testy McTester', 'email': 'test@test.com'}, + 'uploader': {'name': 'Testy McTester', 'email': 'test@test.com'}, + 'message': 'This is ready', + 'user_id': 1} + """ + Test data and the expected test outcomes + Expected data is only checked on the second version + Each version relates to both version 1 and 2 of the other submission + """ + test_data = [ + { + "title": "Submission1", + "recid" : None, + "other_recid": None, + "inspire_id": 1, + "versions": [ + { + "version": 1, + "submission" : None, + "directory": "test_data/test_version/test_1_version_1" + }, + { + "version" : 2, + "submission" : None, + "directory": "test_data/test_version/test_1_version_2", + "related_to_expected": [{ + "url_text": "TestTable2-V1", + "url_title": "TestTable2-description-V1" + }, + { + "url_text": "TestTable2-V2", + "url_title": "TestTable2-description-V2" + }], + "related_to_this_expected": "TestTable2-V2" + } + ] + }, + { + "title": "Submission2", + "recid": None, + "other_recid" : None, + "inspire_id": 25, + "versions": [ + { + "version": 1, + "submission": None, + "directory": "test_data/test_version/test_2_version_1" + }, + { + "version": 2, + "submission": None, + "directory": "test_data/test_version/test_2_version_2", + "related_to_expected": [ + { + "url_text": "TestTable1-V1", + "url_title": "TestTable1-description-V1" + }, + { + "url_text": "TestTable1-V2", + "url_title": "TestTable1-description-V2" + }], + "related_to_this_expected": "TestTable1-V2" + } + ] + } + ] + + # Insert the data + for test in test_data: + for version in test["versions"]: + # Version 1 needs a different submission setup to v2 + if version["version"] == 1: + version["submission"] = process_submission_payload(**record) + version["submission"].overall_status = "finished" + test["recid"] = version["submission"].publication_recid + else: + # Creates a new version of the submission and inserts it. + version["submission"] = HEPSubmission( + publication_recid=test["recid"], + inspire_id=test["inspire_id"], + version=version["version"], + overall_status='finished') + db.session.add(version["submission"]) + db.session.commit() + + # Fixes pathing when tests are ran together as it could not be figured out at the time + if not os.path.exists(version["directory"]): + version["directory"] = "tests/" + version["directory"] + + # Insertion of data, this is done for both versions + data_dir = get_data_path_for_record(test["recid"], str(int(round(time.time())+version["version"])) ) + shutil.copytree(os.path.abspath(version["directory"]), data_dir) + process_submission_directory( + data_dir, + os.path.join(data_dir, 'submission.yaml'), + test["recid"] + ) + + # Set the expected recids for the test data + test_data[0]["other_recid"] = test_data[1]["recid"] + test_data[1]["other_recid"] = test_data[0]["recid"] + + # Insert related data dynamically to ensure the ids are correct + # Ids differ when ran alongside other tests due to the test database having extra submissions + for test in test_data: + for v in test["versions"]: + sub = v["submission"] + related_recid = RelatedRecid(this_recid=test["recid"], related_recid=test["other_recid"]) + related_table_one = RelatedTable(table_doi=f"10.17182/hepdata.{test['recid']}.v1/t1", related_doi=f"10.17182/hepdata.{test['other_recid']}.v1/t1") + related_table_two = RelatedTable(table_doi=f"10.17182/hepdata.{test['recid']}.v2/t1", related_doi=f"10.17182/hepdata.{test['other_recid']}.v2/t1") + sub.related_recids.append(related_recid) + datasub = DataSubmission.query.filter_by(doi=f"10.17182/hepdata.{test['recid']}.v{v['version']}/t1").first() + datasub.related_tables.append(related_table_one) + datasub.related_tables.append(related_table_two) + db.session.add_all([related_recid, related_table_one, related_table_two]) + db.session.commit() + + # The checks + for test in test_data: + # We only need to use one submission version from each record for testing, so we're using the most recent + version = test["versions"][1] + record_url = flask.url_for('hepdata_records.get_metadata_by_alternative_id', + recid=version["submission"].publication_recid, _external=True) + browser.get(record_url) + related_area = browser.find_element(By.ID, "related-tables") + # Get the related data list html based on the current element + data_list = (related_area.find_element(By.CLASS_NAME, "related-list-container") + .find_element(By.CLASS_NAME, "related-list") + .find_elements(By.TAG_NAME, "li")) + for d in data_list: + tag = d.find_element(By.TAG_NAME, "a") + # Set the found attributes and check against the expected data + data = { "url_text" : tag.text, "url_title": tag.get_attribute("title")} + assert data in version["related_to_expected"] + + # Related to this + related_to_this_area = browser.find_element(By.ID, "related-to-this-tables") + related_to_this_list = (related_to_this_area.find_element(By.CLASS_NAME, "related-list-container") + .find_element(By.CLASS_NAME, "related-list") + .find_elements(By.TAG_NAME, "li")) + assert len(related_to_this_list) == 1 + assert related_to_this_list[0].text == version["related_to_this_expected"] def test_sandbox(live_server, logged_in_browser): diff --git a/tests/submission_test.py b/tests/submission_test.py index c752094d..97f4a066 100644 --- a/tests/submission_test.py +++ b/tests/submission_test.py @@ -33,7 +33,17 @@ from hepdata.ext.opensearch.api import get_records_matching_field from hepdata.modules.permissions.models import SubmissionParticipant -from hepdata.modules.records.api import format_submission, process_saved_file, create_new_version +from hepdata.modules.records.api import ( + create_new_version, + format_submission, + get_record_data_list, + get_related_datasubmissions, + get_related_hepsubmissions, + get_related_to_this_datasubmissions, + get_related_to_this_hepsubmissions, + get_table_data_list, + process_saved_file +) from hepdata.modules.records.utils.common import infer_file_type, contains_accepted_url, allowed_file, record_exists, \ get_record_contents, is_histfactory from hepdata.modules.records.utils.data_files import get_data_path_for_record @@ -327,13 +337,22 @@ def test_related_records(app, admin_idx): # First two are valid, and relate to each other # 3 has invalid record entry (a string), 4 has invalid data DOI string (doesn't match regex) test_data = [ - {"dir" : "related_submission_1", "related" : 2}, - {"dir" : "related_submission_2", "related" : 1}, - {"dir" : "related_submission_3", "related" : None}, - {"dir": "related_submission_4", "related": None} + {"dir": "related_submission_1", + "related": 2, + "record_title": "Title 1", + "expected_title": "Title 2", + "expected_version": 1}, + {"dir": "related_submission_2", + "related": 1, + "record_title": "Title 2", + "expected_title": "Title 1", + "expected_version": 1}, + {"dir": "related_submission_3", "related": None, "record_title": "Title 3"}, + {"dir": "related_submission_4", "related": None, "record_title": "Title 4"} ] # Dummy record data - record = {'title': 'HEPData Testing', + # The title will be set later based on test_data values + record = {'title': None, 'reviewer': {'name': 'Testy McTester', 'email': 'test@test.com'}, 'uploader': {'name': 'Testy McTester', 'email': 'test@test.com'}, 'message': 'This is ready', @@ -343,8 +362,9 @@ def test_related_records(app, admin_idx): # Begin submission of test submissions for data in test_data: # Set up a new test submission - test_sub = process_submission_payload(**record) - data['sub'] = test_sub + record['title'] = data['record_title'] + data['sub'] = process_submission_payload(**record) + test_sub = data['sub'] # Ensure the status is set to `finished` so the related data can be accessed. test_sub.overall_status = 'finished' test_directory = os.path.join(base_dir, test_dir, data['dir']) @@ -356,21 +376,41 @@ def test_related_records(app, admin_idx): # Checking against results in test_data for data in test_data: submission = data['sub'] - related_hepsubmissions = submission.get_related_hepsubmissions() - # Set some test criteria based on the current data. # If related_id is None, then some tests should yield empty lists. submission_count, table_count = (1, 3) if data['related'] is not None else (0, 0) + related_hepsubmissions = get_related_hepsubmissions(submission) + related_to_this_hepsubmissions = get_related_to_this_hepsubmissions(submission) + # Check that the correct amount of objects are returned from the queries. assert len(submission.related_recids) == submission_count + assert len(related_to_this_hepsubmissions) == submission_count assert len(related_hepsubmissions) == submission_count + related_record_data = get_record_data_list(submission, "related") + related_to_this_record_data = get_record_data_list(submission, "related_to_this") + + assert len(related_record_data) == submission_count + assert len(related_to_this_record_data) == submission_count + + if data['related']: + assert int(related_hepsubmissions[0].publication_recid) == data['related'] + assert int(related_to_this_hepsubmissions[0].publication_recid) == data['related'] + + expected_record_data = [{ + "recid": data['related'], + "title": data['expected_title'], + "version": data['expected_version'] + }] + assert related_record_data == expected_record_data + assert related_to_this_record_data == expected_record_data + for related_table in submission.related_recids: # Get all other RelatedTable entries related to this one # and check against the expected value in `data` assert related_table.related_recid == data['related'] - for related_hepsub in related_hepsubmissions: + for related_hepsub in related_to_this_hepsubmissions: # Get all other submissions related to this one # and check against the expected value in `data` assert related_hepsub.publication_recid == data['related'] @@ -381,11 +421,38 @@ def test_related_records(app, admin_idx): # Check against the expected amount of related objects as defined above assert len(data_submissions) == table_count for s in range(0, len(data_submissions)): - submission = data_submissions[s] - for related in submission.get_related_datasubmissions(): - # Test that the stored DOI is as expected. - check = f"{HEPDATA_DOI_PREFIX}/hepdata.{data['related']}.v1/t{s+1}" - assert check == related.doi + data_submission = data_submissions[s] + # Set the current table number for checking + tablenum = s + 1 + # Generate the test DOI + doi_check = f"{HEPDATA_DOI_PREFIX}/hepdata.{data['related']}.v1/t{tablenum}" + # Generate the expected table data + expected_table_data = [{"name": f"Table {tablenum}", + "doi": doi_check, + "description": f"Test Table {tablenum}"}] + + # Execute the related data functions + # The table data functions generate a dictionary for tooltip data for each contained entry. + # The submission.get_related functions test that the related objects are found as expected. + related_datasubmissions = get_related_datasubmissions(data_submission) + related_to_this_datasubmissions = get_related_to_this_datasubmissions(data_submission) + related_table_data = get_table_data_list(data_submission, "related") + related_to_this_table_data = get_table_data_list(data_submission,"related_to_this") + + # Check that the get related functions are returning the correct amount of objects. + # Based on the current tests, this is either 0, or 3 + assert len(related_datasubmissions) == submission_count + assert len(related_to_this_datasubmissions) == submission_count + assert len(related_table_data) == submission_count + assert len(related_to_this_table_data) == submission_count + + if data['related']: + assert related_table_data == expected_table_data + assert related_to_this_table_data == expected_table_data + + # Test doi_check against the related DOI list. + for related in related_datasubmissions: + assert doi_check == related.doi def test_cleanup_data_related_recid(app, admin_idx): diff --git a/tests/test_data/test_version/test_1_version_1/data1.yaml b/tests/test_data/test_version/test_1_version_1/data1.yaml new file mode 100644 index 00000000..ec9813a1 --- /dev/null +++ b/tests/test_data/test_version/test_1_version_1/data1.yaml @@ -0,0 +1,10 @@ +dependent_variables: +- header: {name: TestData1-dependent-V1, units: NA} + qualifiers: + - {name: TestData1-qualifier-V1, value: 0} + values: + - {value: 0} +independent_variables: +- header: {name: TestData1-independent-V1, units: NA} + values: + - value: 0 \ No newline at end of file diff --git a/tests/test_data/test_version/test_1_version_1/submission.yaml b/tests/test_data/test_version/test_1_version_1/submission.yaml new file mode 100644 index 00000000..688f86b0 --- /dev/null +++ b/tests/test_data/test_version/test_1_version_1/submission.yaml @@ -0,0 +1,9 @@ +--- +description: "TestSubmission1-V1" +comment: TestComment1-V1 +--- +name: "TestTable1-V1" +description: TestTable1-description-V1 +keywords: +- {name: cmenergies, values: [0]} +data_file: data1.yaml diff --git a/tests/test_data/test_version/test_1_version_2/data1.yaml b/tests/test_data/test_version/test_1_version_2/data1.yaml new file mode 100644 index 00000000..4eac4aaf --- /dev/null +++ b/tests/test_data/test_version/test_1_version_2/data1.yaml @@ -0,0 +1,10 @@ +dependent_variables: +- header: {name: TestData1-dependent-V2, units: NA} + qualifiers: + - {name: TestData1-qualifier-V2, value: 0} + values: + - {value: 0} +independent_variables: +- header: {name: TestData1-independent-V2, units: NA} + values: + - value: 0 \ No newline at end of file diff --git a/tests/test_data/test_version/test_1_version_2/submission.yaml b/tests/test_data/test_version/test_1_version_2/submission.yaml new file mode 100644 index 00000000..4526c5b5 --- /dev/null +++ b/tests/test_data/test_version/test_1_version_2/submission.yaml @@ -0,0 +1,9 @@ +--- +description: "TestSubmission1-V2" +comment: TestComment1-V2 +--- +name: "TestTable1-V2" +description: TestTable1-description-V2 +keywords: +- {name: cmenergies, values: [0]} +data_file: data1.yaml diff --git a/tests/test_data/test_version/test_2_version_1/data1.yaml b/tests/test_data/test_version/test_2_version_1/data1.yaml new file mode 100644 index 00000000..23f61ea2 --- /dev/null +++ b/tests/test_data/test_version/test_2_version_1/data1.yaml @@ -0,0 +1,10 @@ +dependent_variables: +- header: {name: TestData2-dependent-V1, units: NA} + qualifiers: + - {name: TestData2-qualifier-V1, value: 0} + values: + - {value: 0} +independent_variables: +- header: {name: TestData2-independent-V1, units: NA} + values: + - value: 0 \ No newline at end of file diff --git a/tests/test_data/test_version/test_2_version_1/submission.yaml b/tests/test_data/test_version/test_2_version_1/submission.yaml new file mode 100644 index 00000000..9391d9ff --- /dev/null +++ b/tests/test_data/test_version/test_2_version_1/submission.yaml @@ -0,0 +1,9 @@ +--- +description: "TestSubmission2-V1" +comment: TestComment2-V1 +--- +name: "TestTable2-V1" +description: TestTable2-description-V1 +keywords: +- {name: cmenergies, values: [0]} +data_file: data1.yaml diff --git a/tests/test_data/test_version/test_2_version_2/data1.yaml b/tests/test_data/test_version/test_2_version_2/data1.yaml new file mode 100644 index 00000000..756f03a4 --- /dev/null +++ b/tests/test_data/test_version/test_2_version_2/data1.yaml @@ -0,0 +1,10 @@ +dependent_variables: +- header: {name: TestData2-dependent-V2, units: NA} + qualifiers: + - {name: TestData2-qualifier-V2, value: 0} + values: + - {value: 0} +independent_variables: +- header: {name: TestData2-independent-V2, units: NA} + values: + - value: 0 \ No newline at end of file diff --git a/tests/test_data/test_version/test_2_version_2/submission.yaml b/tests/test_data/test_version/test_2_version_2/submission.yaml new file mode 100644 index 00000000..d055fe9a --- /dev/null +++ b/tests/test_data/test_version/test_2_version_2/submission.yaml @@ -0,0 +1,9 @@ +--- +description: "TestSubmission2-V2" +comment: TestComment2-V2 +--- +name: "TestTable2-V2" +description: TestTable2-description-V2 +keywords: +- {name: cmenergies, values: [0]} +data_file: data1.yaml