Related data mouseover

hepdata/modules/records/api.py

@@ -27,15 +27,14 @@
 from collections import OrderedDict
 from functools import wraps
 import mimetypes
-import requests
 import time
 
 from celery import shared_task
 from flask import redirect, request, render_template, jsonify, current_app, Response, abort, flash
 from flask_login import current_user
 from invenio_accounts.models import User
 from invenio_db import db
-from sqlalchemy import and_
+from sqlalchemy import and_, func
 from sqlalchemy.orm.exc import NoResultFound
 from werkzeug.utils import secure_filename
 
@@ -57,7 +56,14 @@
 from hepdata.modules.records.utils.workflow import update_action_for_submission_participant
 from hepdata.modules.records.utils.yaml_utils import split_files
 from hepdata.modules.stats.views import increment, get_count
-from hepdata.modules.submission.models import RecordVersionCommitMessage, DataSubmission, HEPSubmission, DataReview
+from hepdata.modules.submission.models import (
+    DataReview,
+    DataSubmission,
+    HEPSubmission,
+    RecordVersionCommitMessage,
+    RelatedRecid,
+    RelatedTable
+)
 from hepdata.utils.file_extractor import extract
 from hepdata.utils.miscellaneous import sanitize_html
 from hepdata.utils.users import get_user_from_id
@@ -262,8 +268,8 @@ def format_resource(resource, contents, content_url):
     ctx['file_mimetype'] = get_resource_mimetype(resource, contents)
     ctx['resource_filename'] = os.path.basename(resource.file_location)
     ctx['resource_filetype'] = f'{resource.file_type} File'
-    ctx['related_recids'] = [r.related_recid for r in hepsubmission.related_recids]
-    ctx['related_to_this_recids'] = [s.publication_recid for s in hepsubmission.get_related_hepsubmissions()]
+    ctx['related_recids'] = get_record_data_list(hepsubmission, "related")
+    ctx['related_to_this_recids'] = get_record_data_list(hepsubmission, "related_to_this")
 
     if resource.file_type in IMAGE_TYPES:
         ctx['display_type'] = 'image'
@@ -401,8 +407,8 @@ def render_record(recid, record, version, output_format, light_mode=False):
         elif not hepdata_submission.overall_status.startswith('sandbox'):
             ctx = format_submission(recid, record, version, version_count, hepdata_submission)
             ctx['record_type'] = 'publication'
-            ctx['related_recids'] = [r.related_recid for r in hepdata_submission.related_recids]
-            ctx['related_to_this_recids'] = [s.publication_recid for s in hepdata_submission.get_related_hepsubmissions()]
+            ctx['related_recids'] = get_record_data_list(hepdata_submission, "related")
+            ctx['related_to_this_recids'] = get_record_data_list(hepdata_submission, "related_to_this")
 
             increment(recid)
 
@@ -457,9 +463,8 @@ def render_record(recid, record, version, output_format, light_mode=False):
             ctx['record_type'] = 'table'
             ctx['related_publication_id'] = publication_recid
             ctx['table_name'] = record['title']
-            ctx['related_recids'] = [r.related_recid for r in hepdata_submission.related_recids]
-            ctx['related_to_this_recids'] = [s.publication_recid for s in
-                                             hepdata_submission.get_related_hepsubmissions()]
+            ctx['related_recids'] = get_record_data_list(hepdata_submission, "related")
+            ctx['related_to_this_recids'] = get_record_data_list(hepdata_submission, "related_to_this")
 
             if output_format == 'html' or output_format == 'json_ld':
                 ctx['json_ld'] = get_json_ld(
@@ -1043,3 +1048,144 @@ def get_all_ids(index=None, id_field='recid', last_updated=None, latest_first=Fa
     else:
         query = query.order_by(HEPSubmission.publication_recid).distinct()
         return [int(x[0]) for x in query.all()]
+
+
+def get_related_hepsubmissions(submission):
+    """
+    Queries the database for all HEPSubmission objects contained in
+    this object's related record ID list.
+    (All submissions this one is relating to)
+
+    :return: [list] A list of HEPSubmission objects
+    """
+    related_submissions = []
+    for related in submission.related_recids:
+        data_submission = get_latest_hepsubmission(
+            publication_recid=related.related_recid,
+            overall_status='finished'
+        )
+        if data_submission:
+            related_submissions.append(data_submission)
+    return related_submissions
+
+
+def get_related_to_this_hepsubmissions(submission):
+    """
+    Queries the database for all records in the RelatedRecId table
+    that have THIS record's id as a related record.
+    Then returns the HEPSubmission object marked in the RelatedRecid table.
+    Returns only submissions marked as 'finished'
+
+    :return: [list] List containing related records.
+    """
+    related_sub_ids = (
+        HEPSubmission.query
+            .join(RelatedRecid, RelatedRecid.this_recid == HEPSubmission.publication_recid)
+            .filter(RelatedRecid.related_recid == submission.publication_recid)
+            .filter(HEPSubmission.overall_status == 'finished')  # Only finished submissions
+            .with_entities(HEPSubmission.publication_recid)
+            .all()
+    )
+
+    # Filter out the non-unique submission results returned by different versions
+    unique_recids =  set(sub[0] for sub in related_sub_ids)
+    return [get_latest_hepsubmission(publication_recid=recid, overall_status='finished') for recid in unique_recids]
+
+
+def get_related_datasubmissions(data_submission):
+    """
+    Queries the database for all DataSubmission objects contained in
+    this object's related DOI list.
+    Only returns an object if associated HEPSubmission status is 'finished'
+    (All submissions this one is relating to)
+
+    :param data_submission: The datasubmission object to find related data for.
+    :return: [list] A list of DataSubmission objects
+    """
+    related_submissions = []
+    for related in data_submission.related_tables:
+        submission = (
+            DataSubmission.query
+                .filter(DataSubmission.doi == related.related_doi)
+                .join(HEPSubmission, HEPSubmission.publication_recid == DataSubmission.publication_recid)
+                .filter(HEPSubmission.overall_status == 'finished')
+                .first()
+        )
+        if submission:
+            related_submissions.append(submission)
+    return related_submissions
+
+
+def get_related_to_this_datasubmissions(data_submission):
+    """
+        Get the DataSubmission Objects with a RelatedTable entry
+        where this doi is referred to in related_doi.
+
+        :param data_submission: The datasubmission to find the related entries for.
+        :return: [List] List of DataSubmission objects.
+    """
+    related_submissions = (
+        DataSubmission.query
+            .join(RelatedTable, RelatedTable.table_doi == DataSubmission.doi)
+            .join(HEPSubmission,(HEPSubmission.publication_recid == DataSubmission.publication_recid))
+            .group_by(DataSubmission.id)
+            .having(func.max(HEPSubmission.version) == DataSubmission.version)
+            .filter(RelatedTable.related_doi == data_submission.doi)
+            .filter(HEPSubmission.overall_status == 'finished')
+            .all()
+    )
+    return related_submissions
+
+
+def get_record_data_list(record, data_type):
+    """
+    Generates a dictionary (title/recid) from a list of record IDs.
+    This must be done as the record contents are not stored within the hepsubmission object.
+
+    :param record: The record used for the query.
+    :param data_type: Either the related, or related to this data.
+    :return: [list] A list of dictionary objects containing record ID and title pairs
+    """
+    # Selects the related data based on the data_type flag
+    data = []
+    if data_type == "related":
+        data = get_related_hepsubmissions(record)
+    elif data_type == "related_to_this":
+        data = get_related_to_this_hepsubmissions(record)
+
+    record_data = []
+    for datum in data:
+        record_data.append(
+        {
+            "recid": datum.publication_recid,
+            "title": get_record_contents(datum.publication_recid)["title"],
+            "version": datum.version
+        })
+    return record_data
+
+
+def get_table_data_list(table, data_type):
+    """
+    Generates a list of general information (name, doi, desc) dictionaries of related DataSubmission objects.
+    Will either use the related data list (get_related_data_submissions)
+    OR the `related to this` list (generated by get_related_to_this_datasubmissions)
+
+    :param table: The DataSubmission object used for querying.
+    :param data_type: The flag to decide which relation data to use.
+    :return: [list] A list of dictionaries with the name, doi and description of the object.
+    """
+    # Selects the related data based on the data_type flag
+    if data_type == "related":
+        data = get_related_datasubmissions(table)
+    elif data_type == "related_to_this":
+        data = get_related_to_this_datasubmissions(table)
+
+    record_data = []
+    if data:
+        for datum in data:
+            record_data.append({
+                "name": datum.name,
+                "doi": datum.doi,
+                "description": datum.description
+            })
+    return record_data

hepdata/modules/records/assets/js/hepdata_tables.js

@@ -195,6 +195,8 @@ HEPDATA.table_renderer = {
       The `placement` argument is used to specify between
         both related record types.
         The records `this is relating to`, and records `that relate to this` record.
+
+      This function also toggles visibility of the related object area.
     */
     var relatedTablesWrapper = $(placement);
     relatedTablesWrapper.find("ul").empty();
@@ -203,13 +205,21 @@ HEPDATA.table_renderer = {
       var relatedList = relatedTablesWrapper.find(".related-list")
 
       for (var i = 0; i < relatedDois.length; i++) {
-        var doi = relatedDois[i];
-        var relatedItem = $('<li>').append($('<a>').text(doi).attr('href', "https://doi.org/" + doi));
+        var related_object = relatedDois[i];
+        // Create the the `li` tag, setting the text to the table name and DOI
+        // Adds a `title` tag containing the table's description
+        // Example: "Table 1"
+        var relatedItem = $('<li>').append($('<a>'
+        ).text(related_object.name
+        ).attr('href', 'https://doi.org/' + related_object.doi).attr('title', related_object.description));
+        // Add the `li` tag to the list
         relatedList.append(relatedItem);
       }
+      // Ensure that the container is visible.
       relatedTablesWrapper.show();
     }
     else {
+      // Hide the container if there is no information.
       relatedTablesWrapper.hide();
     }
   },

hepdata/modules/records/templates/hepdata_records/components/related_recids.html

@@ -4,7 +4,7 @@ <h5>This record is related to:</h5>
     <div class="related-list-container">
         <ul class="related-list">
             {% for related in ctx.related_recids %}
-            <li><a href="{{ related }}">{{ related }}</a></li>
+            <li><a href="{{ related.recid }}" title="{{ related.title }}">{{ related.recid }}</a></li>
             {% endfor %}
           </ul>
     </div>
@@ -17,7 +17,7 @@ <h5>This record is referred to by:</h5>
     <div class="related-list-container">
         <ul class="related-list">
             {% for related in ctx.related_to_this_recids %}
-            <li><a href="{{ related }}">{{ related }}</a></li>
+            <li><a href="{{ related.recid }}" title="{{ related.title }}">{{ related.recid }}</a></li>
             {% endfor %}
         </ul>
     </div>

hepdata/modules/records/utils/common.py

@@ -28,7 +28,7 @@
 import os
 from sqlalchemy.orm.exc import NoResultFound
 
-from hepdata.config import CFG_PUB_TYPE, HISTFACTORY_FILE_TYPE
+from hepdata.config import HISTFACTORY_FILE_TYPE
 from hepdata.ext.opensearch.api import get_record
 from hepdata.modules.submission.models import HEPSubmission, License
 
@@ -251,3 +251,5 @@ def get_record_by_id(recid):
 def record_exists(*args, **kwargs):
     count = HEPSubmission.query.filter_by(**kwargs).count()
     return count > 0
+
+

hepdata/modules/records/utils/submission.py

@@ -218,13 +218,15 @@ def cleanup_data_keywords(data_submission):
 def cleanup_data_related_recid(recid):
     """
     Deletes all related record ID entries of a HEPSubmission object of a given recid
+
     :param recid: The record ID of the HEPSubmission object to be cleaned
     :return:
     """
-    hepsubmission = HEPSubmission.query.filter_by(publication_recid=recid).first()
-    for related in hepsubmission.related_recids:
-        db.session.delete(related)
-    db.session.commit()
+    hepsubmission = get_latest_hepsubmission(publication_recid=recid)
+    if hepsubmission:
+        for related in hepsubmission.related_recids:
+            db.session.delete(related)
+        db.session.commit()
 
 
 def process_data_file(recid, version, basepath, data_obj, datasubmission, main_file_path, tablenum, overall_status):
@@ -270,6 +272,7 @@ def process_data_file(recid, version, basepath, data_obj, datasubmission, main_f
                 datasubmission.keywords.append(keyword)
 
     if overall_status not in ("sandbox", "sandbox_processing"):
+        # Process the related doi data (only if not in sandbox mode)
         if "related_to_table_dois" in data_obj:
             for related_doi in data_obj["related_to_table_dois"]:
                 this_doi = f"{HEPDATA_DOI_PREFIX}/hepdata.{recid}.v{version}/t{tablenum}"

hepdata/modules/records/views.py

@@ -49,7 +49,7 @@
     render_record, current_user, db, jsonify, get_user_from_id, get_record_contents, extract_journal_info, \
     user_allowed_to_perform_action, NoResultFound, OrderedDict, query_messages_for_data_review, returns_json, \
     process_payload, has_upload_permissions, has_coordinator_permissions, create_new_version, format_resource, \
-    should_send_json_ld, JSON_LD_MIMETYPES, get_resource_mimetype
+    should_send_json_ld, JSON_LD_MIMETYPES, get_resource_mimetype, get_table_data_list
 from hepdata.modules.submission.api import get_submission_participants_for_record
 from hepdata.modules.submission.models import HEPSubmission, DataSubmission, \
     DataResource, DataReview, Message, Question
@@ -325,9 +325,8 @@ def get_table_details(recid, data_recid, version):
             table_contents["name"] = datasub_record.name
             table_contents["title"] = datasub_record.description
             table_contents["keywords"] = datasub_record.keywords
-            # The functions used only return the objects themselves.
-            table_contents["related_tables"] = [r.related_doi for r in datasub_record.related_tables]
-            table_contents["related_to_this"] = [r.doi for r in datasub_record.get_related_datasubmissions()]
+            table_contents["related_tables"] = get_table_data_list(datasub_record, "related")
+            table_contents["related_to_this"] = get_table_data_list(datasub_record, "related_to_this")
             table_contents["doi"] = datasub_record.doi
             table_contents["location"] = datasub_record.location_in_publication
 

hepdata/modules/submission/models.py

@@ -121,23 +121,6 @@ class HEPSubmission(db.Model):
                                cascade="all,delete")
 
 
-    def get_related_hepsubmissions(self):
-        """
-        Queries the database for all records in the RelatedRecId table
-        that have THIS record's id as a related record.
-        Then returns the HEPSubmission object marked in the RelatedRecid table.
-        Returns only submissions marked as 'finished'
-        :return: [list] List containing related records.
-        """
-        related_submissions = (
-            HEPSubmission.query
-            .join(RelatedRecid, RelatedRecid.this_recid == HEPSubmission.publication_recid)
-            .filter(RelatedRecid.related_recid == self.publication_recid)
-            .filter(HEPSubmission.overall_status == 'finished') # Only finished submissions
-            .all())
-        return related_submissions
-
-
 # Declarations of the helper tables used to manage many-to-many relationships.
 datafile_identifier = db.Table(
     'datafile_identifier',
@@ -208,22 +191,6 @@ class DataSubmission(db.Model):
     # through a submissions review stages.
     version = db.Column(db.Integer, default=0)
 
-    def get_related_datasubmissions(self):
-        """
-            Get the DataSubmission Objects with a RelatedTable entry
-            where this doi is referred to in related_doi.
-
-            :return: [List] List of DataSubmission objects.
-        """
-        related_submissions = (
-            DataSubmission.query.join(RelatedTable, RelatedTable.table_doi == DataSubmission.doi)
-            .join(HEPSubmission, HEPSubmission.publication_recid == DataSubmission.publication_recid)
-            .filter(RelatedTable.related_doi == self.doi)
-            .filter(HEPSubmission.overall_status == 'finished')
-            .all()
-        )
-        return related_submissions
-
 
 class RelatedTable(db.Model):
     """