Arxivce 1273 browse refactor

.github/workflows/pullreqeust_tests.yaml

@@ -21,7 +21,7 @@ jobs:
           poetry check --lock
           poetry install --with=dev
       - name: Run arxiv-base tests with coverage
-        run: poetry run pytest --cov=arxiv.base fourohfour --cov-fail-under=68 arxiv/base fourohfour
+        run: poetry run pytest --cov=arxiv.base fourohfour --cov-fail-under=67 arxiv/base fourohfour
       # - name: Check Types
       # TODO The types are in bad shape and need to be fixed
       #   run: poetry run mypy --exclude "test*" -p arxiv

README.md

@@ -147,7 +147,7 @@ registering your blueprints. For example:
 ## App tests
 
 Some tests to check app configuration and pattern compliance are provided in
-``arxiv.base.app_tests``. See that module for usage.
+``arxiv.base.tests``. See that module for usage. You can run them with the command ``pytest``
 
 ## Editing and compiling sass
 

arxiv/base/filters.py

@@ -12,8 +12,7 @@
 
 from arxiv.base.urls import urlizer, canonical_url, clickthrough_url
 from arxiv.util.tex2utf import tex2utf
-from arxiv.taxonomy import get_category_display, get_archive_display, \
-    get_group_display
+from arxiv.taxonomy.category import Archive, Category, Group
 
 ET = timezone('US/Eastern')
 
@@ -97,9 +96,6 @@ def register_filters(app: Flask) -> None:
     app.template_filter('tex2utf_no_symbols')(partial(f_tex2utf, greek=False))
     app.template_filter('canonical_url')(canonical_url)
     app.template_filter('clickthrough_url')(clickthrough_url)
-    app.template_filter('get_category_display')(get_category_display)
-    app.template_filter('get_archive_display')(get_archive_display)
-    app.template_filter('get_group_display')(get_group_display)
     app.template_filter('embed_content')(embed_content)
     app.template_filter('tidy_filesize')(tidy_filesize)
     app.template_filter('as_eastern')(as_eastern)

arxiv/base/routes.py

@@ -13,6 +13,7 @@
 from arxiv.base.exceptions import NotFound, Forbidden, Unauthorized, \
     MethodNotAllowed, RequestEntityTooLarge, BadRequest, InternalServerError, \
     default_exceptions, HTTPException
+from arxiv.taxonomy.definitions import CATEGORIES
 
 from . import alerts
 
@@ -60,7 +61,8 @@ def test_macros() -> Response:
         'comments': "This version (physics/9707012v2) was not stored by arXiv."
                     " A subsequent replacement was made before versioning was"
                     " introduced.",
-        'primary_category': 'cond-mat.supr-con',
+        'primary_category': CATEGORIES['cond-mat.supr-con'],
+        "secondary_categories": [CATEGORIES['math.MP'], CATEGORIES['hep-lat']],
         'submitted_date': datetime.now(),
         'submission_history': [
             {'version': 1, 'submitted_date': datetime.now()},

arxiv/base/templates/base/macros.html

@@ -113,10 +113,10 @@
   {%- else -%}
     {%- set vpart = '' -%}
   {% endif %}
-  {%- if primary_category in arxiv_id -%}
+  {%- if primary_category.id in arxiv_id -%}
     <a href="{{ arxiv_id|canonical_url(version) }}">arXiv:{{ arxiv_id }}{{vpart}}</a>
     {%- else -%}
-    <a href="{{ arxiv_id|canonical_url(version) }}">arXiv:{{ arxiv_id }}{{vpart}}</a> [{{ primary_category }}]
+    <a href="{{ arxiv_id|canonical_url(version) }}">arXiv:{{ arxiv_id }}{{vpart}}</a> [{{ primary_category.id }}]
   {%- endif -%}
 {%- endmacro -%}
 
@@ -177,8 +177,8 @@ <h1 class="title mathjax"><span class="descriptor">Title:</span>{{ title|tex2utf
         <tr>
           <td class="tablecell label">Subjects:</td>
           <td class="tablecell subjects">
-            <span class="primary-subject">{{ primary_category|get_category_display }}</span>
-            {%- for category in secondary_categories|sort(attribute='id') -%}; {{ category|get_category_display }}{%- endfor -%}
+            <span class="primary-subject">{{ primary_category.display() }}</span>
+            {%- for category in secondary_categories|sort(attribute='id') -%}; {{ category.display() }}{%- endfor -%}
           </td>
         </tr>
         {%- if msc_class %}
@@ -261,7 +261,7 @@ <h1 class="title mathjax"><span class="descriptor">Title:</span>{{ title|tex2utf
 
 Title: {{ title }}
 Authors: {{ authors }}
-Categories: {{ primary_category }}{%- for category in secondary_categories|sort(attribute='id') -%} {{ category }}{%- endfor %}
+Categories: {{ primary_category.id }}{%- for category in secondary_categories|sort(attribute='id') -%} {{ category }}{%- endfor %}
 {% if comments -%}{{ ("Comments: " + comments)|wordwrap(77, wrapstring="\n  ") }}{%- endif %}
 {% if msc_class -%}MSC classes: {{ msc_class }}{%- endif %}
 {% if acm_class -%}ACM classes: {{ acm_class }}{%- endif %}

arxiv/base/templates/base/testmacros.html

@@ -18,6 +18,8 @@
     version=version,
     submission_history=submission_history,
     comments=comments,
-    doi=doi) }}
+    doi=doi,
+    secondary_categories=secondary_categories
+    ) }}
 
 {% endblock %}

arxiv/base/tests/test_docmeta.py

@@ -0,0 +1,120 @@
+
+from typing import Any, Dict
+from unittest import TestCase,mock
+from datetime import datetime
+
+from arxiv.document.metadata import DocMetadata, AuthorList, Submitter
+from arxiv.taxonomy.definitions import CATEGORIES
+from arxiv.identifier import Identifier
+from arxiv.license import License
+from arxiv.document.version import VersionEntry, SourceFlag
+
+SAMPLE_DOCMETA=DocMetadata(
+        raw_safe="abs text here",
+        arxiv_id='1204.5678',
+        arxiv_id_v="1204.5678v1",
+        arxiv_identifier=Identifier("1204.5678v1"),
+        title="title of paper",
+        abstract="also abs text here",
+        authors=AuthorList("First Name, Second, Name"),
+        submitter=Submitter(name="a submitter", email="[email protected]"),
+        categories="hep-th cs.NA math-ph math.MP",
+        primary_category=CATEGORIES['hep-th'],
+        primary_archive=CATEGORIES['hep-th'].get_archive(),
+        primary_group=CATEGORIES['hep-th'].get_archive().get_group(),
+        secondary_categories=[
+            CATEGORIES["math-ph"],
+            CATEGORIES["math.MP"],
+            CATEGORIES["cs.NA"]
+        ],
+        journal_ref="journal of mystical knowledge",
+        report_num=None,
+        doi=None,
+        acm_class=None,
+        msc_class=None,
+        proxy=None,
+        comments="very insightful comments",
+        version=1,
+        license=License(),
+        version_history=[
+            VersionEntry(
+                version=1,
+                raw="",
+                submitted_date=None, # type: ignore
+                size_kilobytes=30, # type: ignore
+                source_flag=SourceFlag("D"), # type: ignore
+            )
+        ],
+        modified=datetime(year=2011, month=3, day=7)
+    )
+
+class DocMetadataTest(TestCase):
+    fields: Dict[str, Any]
+
+    def __init__(self, *args: str, **kwargs: Dict) -> None:
+        """Set up some common variables."""
+        super().__init__(*args, **kwargs)
+        self.fields = {
+            # TODO: reasonable mock defaults for future tests
+        }
+
+    def test_something(self):
+        """Tests that omission of a required field generates an exception."""
+        fields = self.fields.copy()
+        # TODO: implement a test on a generated DocMetadata
+
+    def test_required_fields(self):
+        """Tests that omission of a required field generates an exception."""
+        fields = self.fields.copy()
+
+        def run_on_empty_args() -> DocMetadata:
+            return DocMetadata(**fields)  # type: ignore
+
+        with self.assertRaises(TypeError) as ctx:
+            run_on_empty_args()
+
+        # Do not indent us or we will not run and be tested!:
+        self.assertTrue('missing 14 required positional arguments' in str(ctx.exception))
+        #
+        self.assertTrue('raw_safe' in str(ctx.exception))
+        self.assertTrue('arxiv_id' in str(ctx.exception))
+        self.assertTrue('arxiv_id_v' in str(ctx.exception))
+        self.assertTrue('arxiv_identifier' in str(ctx.exception))
+        self.assertTrue('modified' in str(ctx.exception))
+        self.assertTrue('title' in str(ctx.exception))
+        self.assertTrue('abstract' in str(ctx.exception))
+        self.assertTrue('authors' in str(ctx.exception))
+        self.assertTrue('submitter' in str(ctx.exception))
+        self.assertTrue('categories' in str(ctx.exception))
+        self.assertTrue('primary_category' in str(ctx.exception))
+        self.assertTrue('primary_archive' in str(ctx.exception))
+        self.assertTrue('primary_group' in str(ctx.exception))
+        self.assertTrue('secondary_categories' in str(ctx.exception))
+        #
+        self.assertTrue('journal_ref' not in str(ctx.exception))
+        self.assertTrue('report_num' not in str(ctx.exception))
+        self.assertTrue('doi' not in str(ctx.exception))
+        self.assertTrue('acm_class' not in str(ctx.exception))
+        self.assertTrue('msc_class' not in str(ctx.exception))
+        self.assertTrue('proxy' not in str(ctx.exception))
+        self.assertTrue('comments' not in str(ctx.exception))
+        self.assertTrue('version' not in str(ctx.exception))
+        self.assertTrue('license' not in str(ctx.exception))
+        self.assertTrue('version_history' not in str(ctx.exception))
+        self.assertTrue('private' not in str(ctx.exception))
+
+    def test_get_seconaries(self):
+        #get secondaries should only return cannonical instance of each secondary category with no duplicates
+
+        self.assertEqual(
+            SAMPLE_DOCMETA.get_secondaries(), 
+            set([CATEGORIES["math-ph"], CATEGORIES["math.NA"]]), 
+            "only retrieves canonical version of each secondary"
+        )
+
+        self.assertEqual(
+            SAMPLE_DOCMETA.display_secondaries(), 
+            ['Mathematical Physics (math-ph)', 'Numerical Analysis (math.NA)'], 
+            "secondary display string must match"
+        )
+

arxiv/base/urls/links.py

@@ -50,7 +50,7 @@
 from flask import url_for, g
 import bleach
 
-from arxiv.taxonomy import CATEGORIES
+from arxiv.taxonomy.definitions import CATEGORIES
 from arxiv import identifier
 from . import clickthrough
 

arxiv/db/__init__.py

@@ -23,6 +23,14 @@
     session.execute(
         select(...)
     )
+
+
+for writing within a transaction in any type of app do:
+
+from arxiv.db import transaction
+
+with transaction() as session:
+    session.add(...)
 """
 from typing import Generator
 import logging

arxiv/document/metadata.py

@@ -4,7 +4,7 @@
 from datetime import datetime
 from typing import Iterator, List, Optional, Set, Literal, Sequence
 
-from ..taxonomy import definitions
+from ..taxonomy.definitions import CATEGORIES
 from ..taxonomy.category import Category, Group, Archive
 from ..identifier import Identifier
 from ..license import License
@@ -157,14 +157,14 @@ def get_browse_context_list(self) -> List[str]:
         if self.primary_category:
             options = {
                 self.primary_category.id: True,
-                definitions.CATEGORIES[self.primary_category.id]['in_archive']: True
+                self.primary_category.in_archive: True
             }
         else:
             options = {}
 
         for category in self.secondary_categories:
             options[category.id] = True
-            in_archive = definitions.CATEGORIES[category.id]['in_archive']
+            in_archive = category.in_archive
             options[in_archive] = True
         return sorted(options.keys())
 
@@ -231,31 +231,27 @@ def get_datetime_of_version(
             return None
         else:
             return versions[0].submitted_date
-
+    
     def get_secondaries(self) -> Set[Category]:
         """Unalias and deduplicate secondary categories."""
         if not self.secondary_categories or not self.primary_category:
             return set()
+
+        result=set()
+        for cat in self.secondary_categories:
+            result.add(cat.get_canonical())
+        result.discard(self.primary_category.get_canonical())
 
-        def unalias(secs: Iterator[Category])->Iterator[Category]:
-            return map(lambda c: Category(c.unalias()), secs)
-        prim = self.primary_category.unalias()
-
-        def de_prim(secs: Iterator[Category])->Iterator[Category]:
-            return filter(lambda c: c.id != prim.id, secs)
-
-        de_primaried = set(de_prim(unalias(iter(self.secondary_categories))))
-        if not de_primaried:
-            return set()
-        return de_primaried
+        return result
 
     def display_secondaries(self) -> List[str]:
         """Unalias, dedup and sort secondaries for display."""
-        de_primaried = self.get_secondaries()
-
-        def to_display(secs: List[Category]) -> List[str]:
-            return list(map(lambda c: str(c.display), secs))
-        return to_display(sorted(de_primaried))
+
+        de_primaried=sorted(self.get_secondaries(), key=lambda cat: cat.id)
+        result=[]
+        for cat in de_primaried:
+            result.append(cat.display())
+        return result
 
     def canonical_url(self, no_version: bool = False) -> str:
         """Return canonical URL for this ID and version."""
@@ -294,7 +290,7 @@ def raw(self) -> str:
             rv += f"Comments: {self.comments}\n"
         # skipping proxy to avoid harvesting of email addresses
         if self.report_num:
-            rv += "Report-no: {self.report_num}\n"
+            rv += f"Report-no: {self.report_num}\n"
         if self.msc_class:
             rv += f"MSC-class: {self.msc_class}\n"
         if self.acm_class: