Merge branch 'release/2.0.0-alpha.4'

.gitignore

@@ -35,6 +35,8 @@ temp_data
 asset/collect/*
 !asset/collect/index.html
 
+apps/static/mirador
+
 # C extensions
 *.so
 

.travis.yml

@@ -12,7 +12,7 @@ before_install:
 - cp config/settings/local.dst config/settings/local.py
 - mkdir logs
 - touch logs/debug.log
-- gem install bundle
+- gem install bundler -v 2.1.0
 - bundle install
 - sudo /etc/init.d/postgresql stop
 - wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key

CHANGELOG.rst

@@ -2,6 +2,22 @@
 
 CHANGELOG
 =========
+Release 2.0.0-alpha.4
+---------------------
+* Full text search of volumes
+* Adds option for partial or exact match searches
+* Adds style information to annotations
+* Adds ability to send to Voyant
+* Fixes bug for links to current volume, page, etc.
+
+Release 2.0.0-alpha.3
+---------------------
+* Add Zenodo integration
+
+Release 2.0.0-alpha.2
+---------------------
+* Add branch option for deployment.
+
 Release 2.0.0-alpha.1
 ---------------------
 Readux 2 is a total rewrite centered on using IIIF.

Gemfile.lock

@@ -38,4 +38,4 @@ DEPENDENCIES
   iiif_to_jekyll (= 0.3.0)!
 
 BUNDLED WITH
-   2.0.2
+   2.1.0

apps/iiif/annotations/migrations/0005_annotation_style.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.1.2 on 2019-12-16 13:35
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('annotations', '0004_auto_20190920_1313'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='annotation',
+            name='style',
+            field=models.CharField(blank=True, max_length=1000, null=True),
+        ),
+    ]

apps/iiif/annotations/models.py

@@ -49,8 +49,9 @@ class AbstractAnnotation(models.Model):
     language = models.CharField(max_length=10, default='en')
     owner = models.ForeignKey(get_user_model(), on_delete=models.CASCADE, blank=True, null=True)
     oa_annotation = JSONField(default=dict, blank=False)
-    # TODO Should we keep this for annotations from Mirador, or just get rid of it?
+    # TODO: Should we keep this for annotations from Mirador, or just get rid of it?
     svg = models.TextField(blank=True, null=True)
+    style = models.CharField(max_length=1000, blank=True, null=True)
     item = None
 
     ordering = ['order']
@@ -86,7 +87,8 @@ def set_span_element(sender, instance, **kwargs):
             # This is used by OpenSeadragon. OSD will update the letter spacing relative to
             # the width of the overlayed element when someone zooms in and out.
             relative_letter_spacing = letter_spacing / instance.w
-            instance.content = "<span id='{pk}' style='height: {h}px; width: {w}px; font-size: {f}px; letter-spacing: {s}px' data-letter-spacing='{p}'>{content}</span>".format(pk=instance.pk, h=str(instance.h), w=str(instance.w), content=instance.content, f=str(font_size), s=str(letter_spacing), p=str(relative_letter_spacing))
+            instance.content = "<span id='{pk}' data-letter-spacing='{p}'>{content}</span>".format(pk=instance.pk, content=instance.content, p=str(relative_letter_spacing))
+            instance.style = ".anno-{c}: {{ height: {h}px; width: {w}px; font-size: {f}px; }}".format(c=(instance.pk), h=str(instance.h), w=str(instance.w), f=str(font_size))
         # TODO Is this actually how this should be handeled? If not, remove the import of `IntegrityError`
         except (ValueError, ZeroDivisionError, IntegrityError) as error:
             instance.content = ''

apps/iiif/annotations/tests.py

@@ -53,7 +53,7 @@ def test_ocr_span(self):
         ocr.h = 10
         ocr.content = "Obviously you're not a golfer"
         ocr.save()
-        assert ocr.content == "<span id='{pk}' style='height: 10px; width: 100px; font-size: 6.25px; letter-spacing: 0.3232758620689655px' data-letter-spacing='0.003232758620689655'>Obviously you're not a golfer</span>".format(pk=ocr.pk)
+        assert ocr.content == "<span id='{pk}' data-letter-spacing='0.003232758620689655'>Obviously you're not a golfer</span>".format(pk=ocr.pk)
         assert ocr.owner == User.objects.get(username='ocr')
 
     def test_invalid_ocr(self):
@@ -96,4 +96,16 @@ def test_ocr_for_page(self):
         response = self.client.get(url)
         annotations = json.loads(response.content.decode('UTF-8-sig'))['resources']
         assert len(annotations) == self.canvas.annotation_set.filter(resource_type='cnt:ContentAsText', canvas=self.canvas).count()
-        assert response.status_code == 200
+        assert response.status_code == 200
+
+    def test_annotation_style(self):
+        anno = Annotation.objects.all().first()
+        assert anno.style == ".anno-{c}: {{ height: {h}px; width: {w}px; font-size: {f}px; }}".format(c=(anno.pk), h=str(anno.h), w=str(anno.w), f=str(anno.h / 1.6))
+
+    def test_annotation_style_serialization(self):
+        kwargs = {'vol': self.volume.pid, 'page': self.canvas.pid, 'version': 'v2'}
+        url = reverse('ocr', kwargs=kwargs)
+        response = self.client.get(url)
+        serialized_anno = json.loads(response.content.decode('UTF-8-sig'))['resources'][0]
+        assert serialized_anno['stylesheet']['type'] == 'CssStylesheet'
+        assert serialized_anno['stylesheet']['value'].startswith(".anno-{id}".format(id=serialized_anno['@id']))

apps/iiif/canvases/models.py

@@ -8,12 +8,13 @@
 from django.apps import apps
 from . import services
 import uuid
+from bs4 import BeautifulSoup
 
-# TODO add a test fixture that calls this.
+# TODO: add a test fixture that calls this.
 def get_default_iiif_setting():
   return "%s" % (settings.IIIF_IMAGE_SERVER_BASE)
 
-# TODO move this to the manifest model
+# TODO: move this to the manifest model
 class IServer(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     IIIF_IMAGE_SERVER_BASE = models.CharField(max_length=255, default=get_default_iiif_setting)
@@ -37,7 +38,7 @@ class Canvas(models.Model):
         ('line', 'line'),
         ('both', 'both')
     )
-    # TODO move this to the mainfest level.
+    # TODO: move this to the mainfest level.
     default_ocr = models.CharField(max_length=30, choices=preferred_ocr, default="word")
 
     @property
@@ -67,12 +68,12 @@ def social_media(self):
     # @property
     # def get_IIIF_IMAGE_SERVER_BASE(self):
     #     return self.IIIF_IMAGE_SERVER_BASE
-        
+
     @property
     def twitter_media1(self):
         # TODO shouldn't this use `self.IIIF_IMAGE_SERVER_BASE`
         return "http://images.readux.ecds.emory.edu/cantaloupe/iiif/2/%s/full/600,/0/default.jpg" % (self.pid)
-        
+
     @property
     def twitter_media2(self):
         # TODO shouldn't this use `self.IIIF_IMAGE_SERVER_BASE`
@@ -110,10 +111,17 @@ def thumbnail_crop_volume(self):
             # TODO add a landscape image to tests
             return "%s/%s/pct:25,15,50,85/,600/0/default.jpg" % (self.IIIF_IMAGE_SERVER_BASE, self.pid)
 
-    # @property
-    # def result(self):
+    @property
+    def result(self):
     #     "Empty attribute to hold the result of requests to get OCR data."
     #     return None
+        words = Annotation.objects.filter(canvas=self.id).order_by('order')
+        # NOTE: The above query really should have a filter for resource_type=OCR but it currently returns an empty set with this parameter set.
+        clean_words = []
+        for word in words:
+            clean_word = BeautifulSoup(word.content).text
+            clean_words.append(clean_word)
+        return ' '.join(clean_words)
 
     def __str__(self):
         return str(self.pid)
@@ -153,7 +161,7 @@ def add_ocr(sender, instance, **kwargs):
             a.content = word['content']
             a.order = word_order
             a.save()
-            word_order += 1           
+            word_order += 1
 
 class Meta:
     # Translators: admin:skip

apps/iiif/canvases/views.py

@@ -20,7 +20,7 @@ def get(self, request, *args, **kwargs):
 class IIIFV2Detail(View):
     def get_queryset(self):
         return Canvas.objects.filter(pid=self.kwargs['pid'])
-    
+
     def get(self, request, *args, **kwargs):
         return JsonResponse(json.loads(serialize('canvas', self.get_queryset())))
 

apps/iiif/manifests/export.py

@@ -564,6 +564,7 @@ def github_export(self):
                 ghpages_url = 'https://%s.github.io/%s/' % (self.github_username, self.github_repo)
             except GithubExportException as err:
                 notify_msg('Export failed: %s' % err, 'error')
+
         return [repo_url, ghpages_url, pr_url]
 
 

apps/iiif/manifests/migrations/0005_auto_20191115_1527.py

@@ -0,0 +1,19 @@
+# Generated by Django 2.1.2 on 2019-11-15 15:27
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('manifests', '0004_auto_20190924_1947'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='manifest',
+            name='metadata',
+            field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict),
+        ),
+    ]

apps/iiif/manifests/migrations/0006_auto_20191217_1952.py

@@ -0,0 +1,24 @@
+# Generated by Django 2.1.2 on 2019-12-17 19:52
+
+import django.contrib.postgres.indexes
+import django.contrib.postgres.search
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('manifests', '0005_auto_20191115_1527'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='manifest',
+            name='search_vector',
+            field=django.contrib.postgres.search.SearchVectorField(null=True),
+        ),
+        migrations.AddIndex(
+            model_name='manifest',
+            index=django.contrib.postgres.indexes.GinIndex(fields=['search_vector'], name='manifests_m_search__bd83b2_gin'),
+        ),
+    ]

apps/iiif/manifests/models.py

@@ -8,6 +8,8 @@
 import urllib.request
 from modelcluster.models import ClusterableModel
 from wagtailautocomplete.edit_handlers import AutocompletePanel
+from django.contrib.postgres.search import SearchVectorField
+from django.contrib.postgres.indexes import GinIndex
 from json import JSONEncoder
 import uuid
 from uuid import UUID
@@ -21,6 +23,11 @@ def JSONEncoder_newdefault(self, o):
     return JSONEncoder_olddefault(self, o)
 JSONEncoder.default = JSONEncoder_newdefault
 
+class ManifestManager(models.Manager):
+    def with_documents(self):
+        vector = SearchVector('canvas__annotation__content')
+        return self.get_queryset().annotate(document=vector)
+
 class Manifest(ClusterableModel):
     DIRECTIONS = (
         ('left-to-right', 'Left to Right'),
@@ -40,11 +47,12 @@ class Manifest(ClusterableModel):
     #collection = models.ForeignKey(Collection, on_delete=models.CASCADE, related_name = 'manifest')
     collections = models.ManyToManyField(Collection, null=True, blank=True, related_name = 'manifests')
     pdf = models.URLField()
-    metadata = JSONField(default=dict, blank=False)
+    metadata = JSONField(default=dict, blank=True)
     viewingDirection = models.CharField(max_length=13, choices=DIRECTIONS, default="left-to-right")
     created_at = models.DateTimeField(auto_now_add=True)
     #starting_page = models.ForeignKey('canvases.Canvas', related_name="first", on_delete=models.SET_NULL, null=True, blank=True, help_text="Choose the page that will show on loading.")
     autocomplete_search_field = 'label'
+    search_vector = SearchVectorField(null=True)
 
     def get_absolute_url(self):
         return "%s/volume/%s" % (settings.HOSTNAME, self.pid)
@@ -54,6 +62,7 @@ def get_volume_url(self):
 
     class Meta:
         ordering = ['published_date']
+        indexes = [GinIndex(fields=['search_vector'])]
 
     # TODO is this needed?
     # @property
@@ -82,6 +91,14 @@ def autocomplete_label(self):
 
     def __str__(self):
         return self.label
+
+    #update search_vector every time the entry updates
+    def save(self, *args, **kwargs):
+        super().save(*args, **kwargs)
+        if 'update_fields' not in kwargs or 'search_vector' not in kwargs['update_fields']:
+            instance = self._meta.default_manager.with_documents().get(pk=self.pk)
+            instance.search_vector = instance.document
+            instance.save(update_fields=['search_vector'])
 
 
 #     def volume_annotation_count(request):

apps/iiif/manifests/tests.py

@@ -3,14 +3,18 @@
 from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.urls import reverse
-from .views import ManifestDetail, ManifestSitemap, ManifestRis, ManifestExport, JekyllExport
+from .views import ManifestDetail, ManifestSitemap, ManifestRis
 from .models import Manifest
 from ..canvases.models import Canvas
 from .export import IiifManifestExport, JekyllSiteExport
 from iiif_prezi.loader import ManifestReader
+import io
 import json
 import logging
+import os
+import re
 import tempfile
+import zipfile
 
 User = get_user_model()
 
@@ -27,8 +31,6 @@ def setUp(self):
         self.default_start_canvas = self.volume.canvas_set.filter(is_starting_page=False).first()
         self.assumed_label = ' Descrizione del Palazzo Apostolico Vaticano '
         self.assumed_pid = 'readux:st7r6'
-        self.manifest_export_view = ManifestExport.as_view()
-        self.jekyll_export_view = JekyllExport.as_view()
 
     def test_validate_iiif(self):
         view = ManifestDetail.as_view()
@@ -72,50 +74,3 @@ def test_sitemap(self):
     def test_ris_view(self):
         ris = ManifestRis()
         assert ris.get_context_data(volume=self.assumed_pid)['volume'] == self.volume
-
-    def test_zip_creation(self):
-        zip = IiifManifestExport.get_zip(self.volume, 'v2', owners=[self.user.id])
-        assert isinstance(zip, bytes)
-
-    def test_jekyll_site_export(self):
-        j = JekyllSiteExport(self.volume, 'v2', owners=[self.user.id])
-        zip = j.get_zip()
-        tempdir = j.generate_website()
-        web_zip = j.website_zip()
-        # j.import_iiif_jekyll(j.manifest, j.jekyll_site_dir)
-        assert isinstance(zip, tempfile._TemporaryFileWrapper)
-        assert "%s_annotated_site_" % (str(self.volume.pk)) in zip.name
-        assert zip.name.endswith('.zip')
-        assert isinstance(web_zip, tempfile._TemporaryFileWrapper)
-        assert "%s_annotated_site_" % (str(self.volume.pk)) in web_zip.name
-        assert web_zip.name.endswith('.zip')
-        assert 'tmp-rdx-export' in tempdir
-        assert tempdir.endswith('/export')
-
-    def test_maifest_export(self):
-        kwargs = { 'pid': self.volume.pid, 'version': 'v2' }
-        url = reverse('ManifestExport', kwargs=kwargs)
-        request = self.factory.post(url, kwargs=kwargs)
-        request.user = self.user
-        response = self.manifest_export_view(request, pid=self.volume.pid, version='v2')
-        assert isinstance(response.getvalue(), bytes)
-
-    def test_jekyll_export_exclude_download(self):
-        kwargs = { 'pid': self.volume.pid, 'version': 'v2' }
-        url = reverse('JekyllExport', kwargs=kwargs)
-        kwargs['deep_zoom'] = 'exclude'
-        kwargs['mode'] = 'download'
-        request = self.factory.post(url, data=kwargs) 
-        request.user = self.user       
-        response = self.jekyll_export_view(request, pid=self.volume.pid, version='v2', content_type="application/x-www-form-urlencoded")
-        assert isinstance(response.getvalue(), bytes)
-
-    def test_jekyll_export_include_download(self):
-        kwargs = { 'pid': self.volume.pid, 'version': 'v2' }
-        url = reverse('JekyllExport', kwargs=kwargs)
-        kwargs['deep_zoom'] = 'include'
-        kwargs['mode'] = 'download'
-        request = self.factory.post(url, data=kwargs) 
-        request.user = self.user       
-        response = self.jekyll_export_view(request, pid=self.volume.pid, version='v2', content_type="application/x-www-form-urlencoded")
-        assert isinstance(response.getvalue(), bytes)

apps/iiif/manifests/urls.py

@@ -3,8 +3,9 @@
 from . import views
 
 urlpatterns = [
+  path('iiif/<version>/<pid>/plain', views.PlainExport.as_view(), name="PlainExport"),
   path('iiif/<version>/<pid>/manifest', views.ManifestDetail.as_view(), name="ManifestRender"),
   path('iiif/<version>/<pid>/export', views.ManifestExport.as_view(), name="ManifestExport"),
   path('iiif/<version>/<pid>/jekyllexport', views.JekyllExport.as_view(), name="JekyllExport"),
   path('volume/<volume>/citation.ris', views.ManifestRis.as_view(), name='ris' )
-]
+]