diff --git a/.gitignore b/.gitignore index 8a95b55..57b0b3d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ .idea/ *.pyc +__pycache__ .vscode/* **/.mypy_cache +build/ dist/ *.egg-info/ diff --git a/README.md b/README.md index 0a0aa78..758d455 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Content: ## 1. Requirements -- Wagtail > 2.0 and WagtailMedia +- Wagtail >= 2.12 and WagtailMedia - Django > 2.2 - Celery - RestFramework diff --git a/setup.py b/setup.py index d6d64cc..871f2da 100644 --- a/setup.py +++ b/setup.py @@ -80,7 +80,7 @@ def package_data_with_recursive_dirs(package_data_spec): keywords="ION Wagtail API Adapter", install_requires=[ "django>=2.2", - "wagtail>2.0", + "wagtail>=2.12", "celery[redis]>=4.3", "djangorestframework>=3.9", "beautifulsoup4>=4.6", diff --git a/wagtail_to_ion/admin.py b/wagtail_to_ion/admin.py index 82e58e9..c6b55f8 100644 --- a/wagtail_to_ion/admin.py +++ b/wagtail_to_ion/admin.py @@ -4,13 +4,30 @@ from wagtail_to_ion.models import get_ion_media_rendition_model -class AbstractIonImageAdmin(admin.ModelAdmin): +class ProtectInUseModelAdmin(admin.ModelAdmin): + protect_objects_in_use: bool = True + + def get_deleted_objects(self, objs, request): + if self.protect_objects_in_use: + protected = [] + + for obj in objs: + usage = obj.get_usage() + if usage: + protected += list(usage) + if protected: + return [], {}, set(), protected + + return super().get_deleted_objects(objs, request) + + +class AbstractIonImageAdmin(ProtectInUseModelAdmin): list_display = ('title', 'collection', 'include_in_archive') list_filter = ('collection', 'include_in_archive') search_fields = ('title',) -class AbstractIonDocumentAdmin(admin.ModelAdmin): +class AbstractIonDocumentAdmin(ProtectInUseModelAdmin): list_display = ('title', 'collection', 'include_in_archive') list_filter = ('collection', 'include_in_archive') search_fields = ('title',) @@ -21,7 +38,7 @@ class IonMediaRenditionInlineAdmin(admin.TabularInline): extra = 0 -class AbstractIonMediaAdmin(admin.ModelAdmin): +class AbstractIonMediaAdmin(ProtectInUseModelAdmin): list_display = ('title', 'collection', 'include_in_archive') list_filter = ('collection', 'include_in_archive') search_fields = ('title',) diff --git a/wagtail_to_ion/blocks.py b/wagtail_to_ion/blocks.py new file mode 100644 index 0000000..155af00 --- /dev/null +++ b/wagtail_to_ion/blocks.py @@ -0,0 +1,17 @@ +from django.utils.functional import cached_property +from wagtailmedia.blocks import AbstractMediaChooserBlock + + +class IonMediaBlock(AbstractMediaChooserBlock): + @cached_property + def target_model(self): + from wagtailmedia.models import get_media_model + return get_media_model() + + @cached_property + def widget(self): + from wagtailmedia.widgets import AdminMediaChooser + return AdminMediaChooser + + def render_basic(self, value, context=None): + raise NotImplementedError('You need to implement %s.render_basic' % self.__class__.__name__) diff --git a/wagtail_to_ion/conf.py b/wagtail_to_ion/conf.py index 9a5ffc8..45f9db8 100644 --- a/wagtail_to_ion/conf.py +++ b/wagtail_to_ion/conf.py @@ -19,3 +19,9 @@ False ) + +settings.ION_ARCHIVE_BUILD_URL_FUNCTION = getattr( + settings, + 'ION_ARCHIVE_BUILD_URL_FUNCTION', + None +) diff --git a/wagtail_to_ion/models/abstract.py b/wagtail_to_ion/models/abstract.py index 313f6ab..074269a 100644 --- a/wagtail_to_ion/models/abstract.py +++ b/wagtail_to_ion/models/abstract.py @@ -1,76 +1,109 @@ -# Copyright © 2017 anfema GmbH. All rights reserved. -from typing import Optional - -from django.conf import settings -from django.utils.text import slugify - -from wagtail.core.models import Page, Site - - -class AbstractIonPage(Page): - - @classmethod - def ion_metadata(cls): - """ - Returns additional metadata for the page. - - See `DynamicPageSerializer.get_meta()` - """ - return () - - @classmethod - def ion_extra_fields(cls): - """ - Add extra fields (not part of `content_panels`) for serialization with ION. - - Returns an iterable of tuples containing two strings: - - the outlet name - - the path to the extra field - - Example: to add the value of `self.some_related_model.some_field` use: - return ( - ('outlet_name', 'some_related_model.some_field'), - ) - """ - return () - - @classmethod - def get_layout_name(cls, api_version: Optional[int] = None): # TODO: rename to `get_ion_layout_name`? - return cls.__name__.lower() # TODO: use `cls._meta.model_name`? - - # - # wagtail attributes/methods - # - - # ion pages have no preview currently; clear wagtail preview modes - preview_modes = () - - # ion pages have no public url (disables wagtail "live view" buttons too) - def get_url_parts(self, request=None): - site = Site.find_for_request(request) - return site, None, None # return only the site to silence the "no site configured" warning - - # - # django model setup - # - - class Meta: - abstract = True - - def __str__(self): - return self.title - - def save(self, *args, **kwargs): - if self.slug.startswith('to-be-filled'): - self.slug = slugify(self.title) - super().save(*args, **kwargs) - - -class AbstractIonCollection(AbstractIonPage): - ion_api_object_name = 'collection' - - parent_page_types = ['wagtailcore.Page'] - subpage_types = [settings.ION_LANGUAGE_MODEL] - - class Meta: - abstract = True +# Copyright © 2017 anfema GmbH. All rights reserved. +from typing import Optional +from uuid import uuid4 + +from django.conf import settings +from django.utils.text import slugify + +from wagtail.core.models import Page, Site + + +class AbstractIonPage(Page): + ion_generate_page_title = True + + @classmethod + def generate_page_title(cls): + """Generate ION specific page title""" + return '{class_name}{uuid}'.format(class_name=cls.__name__, uuid=uuid4()) + + @classmethod + def ion_metadata(cls): + """ + Returns additional metadata for the page. + + See `DynamicPageSerializer.get_meta()` + """ + return () + + @classmethod + def ion_extra_fields(cls): + """ + Add extra fields (not part of `content_panels`) for serialization with ION. + + Returns an iterable of tuples containing two strings: + - the outlet name + - the path to the extra field + + Example: to add the value of `self.some_related_model.some_field` use: + return ( + ('outlet_name', 'some_related_model.some_field'), + ) + """ + return () + + @classmethod + def get_layout_name(cls, api_version: Optional[int] = None): # TODO: rename to `get_ion_layout_name`? + return cls.__name__.lower() # TODO: use `cls._meta.model_name`? + + # + # wagtail attributes/methods + # + + # ion pages have no preview currently; clear wagtail preview modes + preview_modes = () + + # ion pages have no public url (disables wagtail "live view" buttons too) + def get_url_parts(self, request=None): + site = Site.find_for_request(request) + return site, None, None # return only the site to silence the "no site configured" warning + + # overwrite Page.copy() to automatically replace the page title & slug if `ion_generate_page_title` flag is set. + # this method is called for every child page too (on recursive copy). + def copy(self, *args, **kwargs): + new_update_attrs = kwargs.get('update_attrs', None) or {} + + if self.ion_generate_page_title: + new_page_title = self.generate_page_title() + new_update_attrs.update({ + 'title': new_page_title, + 'slug': slugify(new_page_title), + }) + + # Backport: Fix crash when copying an alias page + # TODO: remove once https://github.com/wagtail/wagtail/pull/6854 is merged & released + new_update_attrs.update({ + 'alias_of': None, + }) + kwargs['update_attrs'] = new_update_attrs + + return super().copy(*args, **kwargs) + + # + # django model setup + # + + class Meta: + abstract = True + + def __str__(self): + return self.title + + # overwrite Page.full_clean() to implement ION specific title & slug handling + def full_clean(self, *args, **kwargs): + # most ION pages have an auto-generated title (indicated by the `ion_generate_page_title` flag) + if not self.title and self.ion_generate_page_title: + self.title = self.generate_page_title() + self.slug = slugify(self.title) + + super().full_clean(*args, **kwargs) + + +class AbstractIonCollection(AbstractIonPage): + ion_api_object_name = 'collection' + ion_generate_page_title = False + + parent_page_types = ['wagtailcore.Page'] + subpage_types = [settings.ION_LANGUAGE_MODEL] + + class Meta: + abstract = True diff --git a/wagtail_to_ion/models/file_based_models.py b/wagtail_to_ion/models/file_based_models.py index c1292e0..a876975 100644 --- a/wagtail_to_ion/models/file_based_models.py +++ b/wagtail_to_ion/models/file_based_models.py @@ -3,18 +3,22 @@ import os from django.db import models -from django.db.utils import cached_property +from django.db.models import ProtectedError +from django.db.models.signals import post_delete, pre_delete +from django.dispatch import receiver from django.utils.translation import ugettext_lazy as _ from magic import from_buffer as magic_from_buffer +from wagtail.documents.blocks import DocumentChooserBlock from wagtail.documents.models import AbstractDocument -from wagtail.images.models import AbstractImage, AbstractRendition, SourceImageIOError -from wagtailmedia.blocks import AbstractMediaChooserBlock +from wagtail.images.blocks import ImageChooserBlock +from wagtail.images.models import AbstractImage, AbstractRendition from wagtailmedia.models import AbstractMedia +from wagtail_to_ion.blocks import IonMediaBlock from wagtail_to_ion.conf import settings from wagtail_to_ion.models import get_ion_media_rendition_model -from wagtail_to_ion.tasks import generate_media_rendition +from wagtail_to_ion.tasks import generate_media_rendition, get_audio_metadata BUFFER_SIZE = 64 * 1024 @@ -32,6 +36,7 @@ class AbstractIonDocument(AbstractDocument): 'tags', 'include_in_archive', ) + check_usage_block_types = (DocumentChooserBlock,) class Meta: abstract = True @@ -52,10 +57,15 @@ def save(self, *args, **kwargs): super().save(*args, **kwargs) os.chmod(self.file.path, 0o644) + def get_usage(self): + from wagtail_to_ion.utils import get_object_block_usage + return super().get_usage().union(get_object_block_usage(self, block_types=self.check_usage_block_types)) + class AbstractIonImage(AbstractImage): checksum = models.CharField(max_length=255) mime_type = models.CharField(max_length=128) + rendition_type = models.CharField(max_length=128, default='jpegquality-70') include_in_archive = models.BooleanField(default=True) updated_at = models.DateTimeField(auto_now=True) admin_form_fields = ( @@ -69,6 +79,7 @@ class AbstractIonImage(AbstractImage): 'focal_point_height', 'include_in_archive', ) + check_usage_block_types = (ImageChooserBlock,) class Meta: abstract = True @@ -89,14 +100,13 @@ def save(self, *args, **kwargs): super().save(*args, **kwargs) os.chmod(self.file.path, 0o644) + def get_usage(self): + from wagtail_to_ion.utils import get_object_block_usage + return super().get_usage().union(get_object_block_usage(self, block_types=self.check_usage_block_types)) + @property def archive_rendition(self): - try: - result = self.get_rendition('jpegquality-70') - except SourceImageIOError as e: - if not settings.ION_ALLOW_MISSING_FILES: - raise e - return None + result = self.get_rendition(self.rendition_type) h = hashlib.new('sha256') try: @@ -148,8 +158,8 @@ class AbstractIonMedia(AbstractMedia): width = models.PositiveIntegerField(null=True, blank=True, verbose_name=_('width')) height = models.PositiveIntegerField(null=True, blank=True, verbose_name=_('height')) - thumbnail_checksum = models.CharField(max_length=255) - thumbnail_mime_type = models.CharField(max_length=128) + thumbnail_checksum = models.CharField(blank=True, max_length=255) + thumbnail_mime_type = models.CharField(blank=True, max_length=128) include_in_archive = models.BooleanField( default=False, help_text="If enabled, the file will be included in the ION archive " @@ -162,11 +172,19 @@ class AbstractIonMedia(AbstractMedia): 'collection', 'tags', ) + check_usage_block_types = (IonMediaBlock,) class Meta: abstract = True def save(self, *args, **kwargs): + # handle audio files + if self.type == 'audio': + self.set_media_metadata() + super().save(*args, **kwargs) + return + + # handle video files needs_transcode = False needs_thumbnail = False try: @@ -184,33 +202,10 @@ def save(self, *args, **kwargs): pass if needs_thumbnail: - try: - self.thumbnail.open() - h = hashlib.new('sha256') - buffer = self.thumbnail.read(BUFFER_SIZE) - if not self.thumbnail_mime_type: - self.thumbnail_mime_type = magic_from_buffer(buffer, mime=True) - while len(buffer) > 0: - h.update(buffer) - buffer = self.thumbnail.read(BUFFER_SIZE) - self.thumbnail_checksum = 'sha256:' + h.hexdigest() - except FileNotFoundError as exception: - raise exception - except ValueError: - pass + self.set_thumbnail_metadata() if needs_transcode: - try: - self.file.open() - h = hashlib.new('sha256') - buffer = self.file.read(BUFFER_SIZE) - self.mime_type = magic_from_buffer(buffer, mime=True) - while len(buffer) > 0: - h.update(buffer) - buffer = self.file.read(BUFFER_SIZE) - self.checksum = 'sha256:' + h.hexdigest() - except FileNotFoundError as exception: - raise exception + self.set_media_metadata() super().save(*args, **kwargs) try: @@ -221,37 +216,59 @@ def save(self, *args, **kwargs): # remove all renditions and generate new ones if needs_transcode: - renditions = [] - for rendition in self.renditions.all(): - rendition.delete() - for key, config in settings.ION_VIDEO_RENDITIONS.items(): - rendition = get_ion_media_rendition_model().objects.create( - name=key, - media_item=self - ) - if not self.mime_type.startswith('video/'): - rendition.transcode_errors = 'Not a video file' - rendition.save() - renditions.append(rendition) - - if self.mime_type.startswith('video/'): - # Run transcode in background - for rendition in renditions: - generate_media_rendition.delay(rendition.id) - - def delete(self, *args, **kwargs): - try: - self.file.delete() - except ValueError: - pass + self.create_renditions() + + def get_usage(self): + from wagtail_to_ion.utils import get_object_block_usage + return super().get_usage().union(get_object_block_usage(self, block_types=self.check_usage_block_types)) + + def set_media_metadata(self): + self.file.open() + h = hashlib.new('sha256') + buffer = self.file.read(BUFFER_SIZE) + self.mime_type = magic_from_buffer(buffer, mime=True) + while len(buffer) > 0: + h.update(buffer) + buffer = self.file.read(BUFFER_SIZE) + self.checksum = 'sha256:' + h.hexdigest() + self.file.seek(0) + + if self.type == 'audio': + metadata = get_audio_metadata(self.file) + self.duration = round(float(metadata.get('duration'))) + + def set_thumbnail_metadata(self): try: - self.thumbnail.delete() + self.thumbnail.open() + h = hashlib.new('sha256') + buffer = self.thumbnail.read(BUFFER_SIZE) + if not self.thumbnail_mime_type: + self.thumbnail_mime_type = magic_from_buffer(buffer, mime=True) + while len(buffer) > 0: + h.update(buffer) + buffer = self.thumbnail.read(BUFFER_SIZE) + self.thumbnail_checksum = 'sha256:' + h.hexdigest() except ValueError: pass - # delete one by one to make sure files are deleted + + def create_renditions(self): + renditions = [] for rendition in self.renditions.all(): rendition.delete() - super().delete(*args, **kwargs) + for key, config in settings.ION_VIDEO_RENDITIONS.items(): + rendition = get_ion_media_rendition_model().objects.create( + name=key, + media_item=self, + ) + if not self.mime_type.startswith('video/'): + rendition.transcode_errors = 'Not a video file' + rendition.save() + renditions.append(rendition) + + if self.mime_type.startswith('video/'): + # Run transcode in background + for rendition in renditions: + generate_media_rendition.delay(rendition.id) class AbstractIonMediaRendition(models.Model): @@ -285,28 +302,27 @@ class Meta: def __str__(self): return "IonMediaRendition {} for {}".format(self.name, self.media_item) - def delete(self, *args, **kwargs): + +@receiver(pre_delete) +def prevent_deletion_if_in_use(sender, instance, **kwargs): + if isinstance(instance, (AbstractIonDocument, AbstractIonImage, AbstractIonMedia)): + usage = instance.get_usage() + if usage: + model_name = instance.__class__.__name__ + raise ProtectedError( + f"Cannot delete instance of model '{model_name}' because it is referenced in stream field blocks", + usage, + ) + + +@receiver(post_delete) +def remove_media_files(sender, instance, **kwargs): + if isinstance(instance, (AbstractIonMedia, AbstractIonMediaRendition)): try: - self.file.delete() + instance.file.delete(save=False) except ValueError: pass try: - self.thumbnail.delete() + instance.thumbnail.delete(save=False) except ValueError: pass - super().delete(*args, **kwargs) - - -class IonMediaBlock(AbstractMediaChooserBlock): - @cached_property - def target_model(self): - from wagtailmedia.models import get_media_model - return get_media_model() - - @cached_property - def widget(self): - from wagtailmedia.widgets import AdminMediaChooser - return AdminMediaChooser - - def render_basic(self, value, context=None): - raise NotImplementedError('You need to implement %s.render_basic' % self.__class__.__name__) diff --git a/wagtail_to_ion/models/page_models.py b/wagtail_to_ion/models/page_models.py index 0a0d1d4..812d715 100644 --- a/wagtail_to_ion/models/page_models.py +++ b/wagtail_to_ion/models/page_models.py @@ -22,6 +22,7 @@ class Meta: abstract = True ion_api_object_name = 'language' + ion_generate_page_title = False parent_page_types = [settings.ION_COLLECTION_MODEL] diff --git a/wagtail_to_ion/serializers/pages.py b/wagtail_to_ion/serializers/pages.py index 312dc1a..1862a3b 100644 --- a/wagtail_to_ion/serializers/pages.py +++ b/wagtail_to_ion/serializers/pages.py @@ -4,7 +4,7 @@ from datetime import datetime, date from typing import Iterable, Tuple - +from django.core.exceptions import FieldDoesNotExist from django.db import models from django.urls import reverse @@ -81,19 +81,19 @@ def parse_data(content_data, content, fieldname, *, content_field_meta=None, blo content['is_multiline'] = False content['mime_type'] = 'text/plain' content['outlet'] = fieldname - elif content_field_meta is not None and hasattr(content_field_meta, 'choices') and content_field_meta.choices is not None: - # Choicefield - content['type'] = 'optioncontent' - data = content_data - for key, display in content_field_meta.choices: - if key == content_data: - data = display - break - if data.__class__.__name__ == 'str': - content['value'] = data.strip() - else: - content['value'] = data - content['outlet'] = fieldname + # elif content_field_meta is not None and hasattr(content_field_meta, 'choices') and content_field_meta.choices is not None: + # # Choicefield + # content['type'] = 'optioncontent' + # data = content_data + # for key, display in content_field_meta.choices: + # if key == content_data: + # data = display + # break + # if data.__class__.__name__ == 'str': + # content['value'] = data.strip() + # else: + # content['value'] = data + # content['outlet'] = fieldname elif content_data.__class__.__name__ in ['str', 'RichText']: try: # check if text is html @@ -215,42 +215,56 @@ def parse_data(content_data, content, fieldname, *, content_field_meta=None, blo thumbnail_slot['type'] = 'imagecontent' if os.path.exists(content_data.file.path): - rendition = content_data.renditions.filter(transcode_finished=True).first() - if rendition is None: - rendition = content_data - media_slot['mime_type'] = content_data.mime_type - media_slot['file'] = settings.BASE_URL + rendition.file.url - media_slot['checksum'] = rendition.checksum - media_slot['width'] = rendition.width if rendition.width else 0 - media_slot['height'] = rendition.height if rendition.height else 0 - media_slot['length'] = content_data.duration - media_slot['file_size'] = rendition.file.size - media_slot['name'] = content_data.title - media_slot['original_mime_type'] = content_data.mime_type - media_slot['original_file'] = settings.BASE_URL + content_data.file.url - media_slot['original_checksum'] = content_data.checksum - media_slot['original_width'] = content_data.width if content_data.width else 0 - media_slot['original_height'] = content_data.height if content_data.height else 0 - media_slot['original_length'] = content_data.duration - media_slot['original_file_size'] = content_data.file.size - media_slot['outlet'] = 'video' - - thumbnail_slot['mime_type'] = content_data.thumbnail_mime_type - thumbnail_slot['image'] = settings.BASE_URL + rendition.thumbnail.url - thumbnail_slot['checksum'] = rendition.thumbnail_checksum - thumbnail_slot['width'] = rendition.width - thumbnail_slot['height'] = rendition.height - thumbnail_slot['file_size'] = rendition.thumbnail.size - thumbnail_slot['original_mime_type'] = content_data.thumbnail_mime_type - thumbnail_slot['original_image'] = settings.BASE_URL + content_data.thumbnail.url - thumbnail_slot['original_checksum'] = content_data.thumbnail_checksum - thumbnail_slot['original_width'] = content_data.width - thumbnail_slot['original_height'] = content_data.height - thumbnail_slot['original_file_size'] = content_data.thumbnail.size - thumbnail_slot['translation_x'] = 0 - thumbnail_slot['translation_y'] = 0 - thumbnail_slot['scale'] = 1.0 - thumbnail_slot['outlet'] = "video_thumbnail" + if content_data.type == 'audio': + media_slot['mime_type'] = content_data.mime_type + media_slot['file'] = settings.BASE_URL + content_data.file.url + media_slot['checksum'] = content_data.checksum + media_slot['length'] = content_data.duration + media_slot['file_size'] = content_data.file.size + media_slot['name'] = content_data.title + media_slot['original_mime_type'] = content_data.mime_type + media_slot['original_file'] = settings.BASE_URL + content_data.file.url + media_slot['original_checksum'] = content_data.checksum + media_slot['original_length'] = content_data.duration + media_slot['original_file_size'] = content_data.file.size + media_slot['outlet'] = 'audio' + else: + rendition = content_data.renditions.filter(transcode_finished=True).first() + if rendition is None: + rendition = content_data + media_slot['mime_type'] = content_data.mime_type + media_slot['file'] = settings.BASE_URL + rendition.file.url + media_slot['checksum'] = rendition.checksum + media_slot['width'] = rendition.width if rendition.width else 0 + media_slot['height'] = rendition.height if rendition.height else 0 + media_slot['length'] = content_data.duration + media_slot['file_size'] = rendition.file.size + media_slot['name'] = content_data.title + media_slot['original_mime_type'] = content_data.mime_type + media_slot['original_file'] = settings.BASE_URL + content_data.file.url + media_slot['original_checksum'] = content_data.checksum + media_slot['original_width'] = content_data.width if content_data.width else 0 + media_slot['original_height'] = content_data.height if content_data.height else 0 + media_slot['original_length'] = content_data.duration + media_slot['original_file_size'] = content_data.file.size + media_slot['outlet'] = 'video' + + thumbnail_slot['mime_type'] = content_data.thumbnail_mime_type + thumbnail_slot['image'] = settings.BASE_URL + rendition.thumbnail.url + thumbnail_slot['checksum'] = rendition.thumbnail_checksum + thumbnail_slot['width'] = rendition.width + thumbnail_slot['height'] = rendition.height + thumbnail_slot['file_size'] = rendition.thumbnail.size + thumbnail_slot['original_mime_type'] = content_data.thumbnail_mime_type + thumbnail_slot['original_image'] = settings.BASE_URL + content_data.thumbnail.url + thumbnail_slot['original_checksum'] = content_data.thumbnail_checksum + thumbnail_slot['original_width'] = content_data.width + thumbnail_slot['original_height'] = content_data.height + thumbnail_slot['original_file_size'] = content_data.thumbnail.size + thumbnail_slot['translation_x'] = 0 + thumbnail_slot['translation_y'] = 0 + thumbnail_slot['scale'] = 1.0 + thumbnail_slot['outlet'] = "video_thumbnail" fill_contents(media_slot, media_container) fill_contents(thumbnail_slot, media_container) @@ -271,7 +285,7 @@ def parse_data(content_data, content, fieldname, *, content_field_meta=None, blo content['outlet'] = get_stream_field_outlet_name(fieldname, block_type, count) else: content['outlet'] = fieldname - elif isinstance(content_data, AbstractIonPage): + elif isinstance(content_data, AbstractIonPage) or isinstance(content_data, Page): content['type'] = 'connectioncontent' content['connection_string'] = '//{}/{}'.format(get_collection_for_page(content_data), content_data.slug) if streamfield: @@ -455,7 +469,10 @@ def get_contents(self, obj): if page_filled: for outlet_name, field_name, instance in get_wagtail_panels_and_extra_fields(obj): field_data = getattr(instance, field_name) - field_type = instance._meta.get_field(field_name) + try: + field_type = instance._meta.get_field(field_name) + except FieldDoesNotExist: + field_type = None content = {} # parse content for all standard django and wagtail fields diff --git a/wagtail_to_ion/serializers/tar.py b/wagtail_to_ion/serializers/tar.py index 75382b3..6579776 100644 --- a/wagtail_to_ion/serializers/tar.py +++ b/wagtail_to_ion/serializers/tar.py @@ -2,7 +2,10 @@ import json import os +from django.core.exceptions import ImproperlyConfigured from django.urls import reverse +from django.utils.module_loading import import_string + from rest_framework.renderers import JSONRenderer from wagtail_to_ion.tar import TarWriter @@ -12,13 +15,19 @@ from wagtail_to_ion.utils import get_collection_for_page -def build_url(request, locale_code, page, variation='default'): - url = reverse('v1:page-detail', kwargs={ - 'locale': locale_code, - 'collection': get_collection_for_page(page), - 'slug': page.slug, - }) - return request.build_absolute_uri(url) + "?variation={}".format(variation) +if settings.ION_ARCHIVE_BUILD_URL_FUNCTION is not None: + try: + build_url = import_string(settings.ION_ARCHIVE_BUILD_URL_FUNCTION) + except ImportError: + raise ImproperlyConfigured(f"The function {settings.ION_ARCHIVE_BUILD_URL_FUNCTION} couldn't be imported.") +else: + def build_url(request, locale_code, page, variation='default'): + url = reverse('v1:page-detail', kwargs={ + 'locale': locale_code, + 'collection': get_collection_for_page(page), + 'slug': page.slug, + }) + return request.build_absolute_uri(url) + "?variation={}".format(variation) def collect_files(request, self, page, collected_files, user): diff --git a/wagtail_to_ion/tasks.py b/wagtail_to_ion/tasks.py index 0d52cd5..ffd9534 100644 --- a/wagtail_to_ion/tasks.py +++ b/wagtail_to_ion/tasks.py @@ -3,11 +3,16 @@ import subprocess import json import hashlib +from tempfile import NamedTemporaryFile + +from django.core.files.uploadedfile import InMemoryUploadedFile, TemporaryUploadedFile +from django.db.models.fields.files import FieldFile from wagtail_to_ion.conf import settings from celery import shared_task + BUFFER_SIZE = 64 * 1024 # Use ffmpeg from user's bin if it exists, global ffmpeg otherwise @@ -345,3 +350,48 @@ def generate_media_rendition(rendition_id: int): else: rendition.transcode_errors += str(e) rendition.save() + + +def extract_media_format(file_path): + ffprobe = os.path.expanduser('~/bin/ffprobe') + if not os.path.exists(ffprobe): + ffprobe = 'ffprobe' + + probe = [ + ffprobe, + '-print_format', 'json', + '-show_format', + '-i', file_path, + ] + + try: + result = subprocess.run( + probe, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + ) + except subprocess.CalledProcessError as e: + print(e.stderr) + raise + + return json.loads(result.stdout.decode('utf-8')) + + +def get_audio_metadata(media_item: FieldFile): + if isinstance(media_item.file, InMemoryUploadedFile): + with NamedTemporaryFile(delete=False) as temp_file: + media_item.file.open() + temp_file.write(media_item.file.read()) + temp_file.close() + mediainfo = extract_media_format(temp_file.name) + try: + os.remove(temp_file.name) + except OSError: + pass + elif isinstance(media_item.file, TemporaryUploadedFile): + mediainfo = extract_media_format(media_item.file.temporary_file_path()) + else: + mediainfo = extract_media_format(media_item.path) + + return mediainfo.get('format', {}) diff --git a/wagtail_to_ion/templates/wagtailadmin/pages/copy_auto_title.html b/wagtail_to_ion/templates/wagtailadmin/pages/copy_auto_title.html new file mode 100644 index 0000000..3eff7d5 --- /dev/null +++ b/wagtail_to_ion/templates/wagtailadmin/pages/copy_auto_title.html @@ -0,0 +1,39 @@ +{% extends "wagtailadmin/pages/copy.html" %} +{% comment %} +Custom copy form (extends wagtails `wagtailadmin/pages/copy.html` template) + +Adjustments: + - remove `new_title` & `new_slug` fields +{% endcomment %} +{% load i18n %} +{% block content %} + {% trans "Copy" as copy_str %} + {% include "wagtailadmin/shared/header.html" with title=copy_str subtitle=page.specific_deferred.get_admin_display_title icon="doc-empty-inverse" %} + +
+
+ {% csrf_token %} + + + + + +
+
+{% endblock %} diff --git a/wagtail_to_ion/templates/wagtailadmin/pages/publish_with_children.html b/wagtail_to_ion/templates/wagtailadmin/pages/publish_with_children.html index 6386be3..d67a2bc 100644 --- a/wagtail_to_ion/templates/wagtailadmin/pages/publish_with_children.html +++ b/wagtail_to_ion/templates/wagtailadmin/pages/publish_with_children.html @@ -3,7 +3,7 @@ {% load i18n %} {% load l10n %} -{% block titletag %}{% blocktrans with title=page.get_admin_display_title page_type=content_type.model_class.get_verbose_name %}Publish {{ page_type }}: {{ title }}{% endblocktrans %}{% endblock %} +{% block titletag %}{% blocktrans with title=page.specific_deferred.get_admin_display_title page_type=content_type.model_class.get_verbose_name %}Publish {{ page_type }}: {{ title }}{% endblocktrans %}{% endblock %} {% block content %} {% page_permissions page as page_perms %} @@ -11,7 +11,7 @@

- {% blocktrans with title=page.get_admin_display_title page_type=content_type.model_class.get_verbose_name %}Publish {{ page_type }} {{ title }}{% endblocktrans %}

+ {% blocktrans with title=page.specific_deferred.get_admin_display_title page_type=content_type.model_class.get_verbose_name %}Publish {{ page_type }} {{ title }}{% endblocktrans %}
@@ -38,7 +38,7 @@

{% trans "Are you sure you want to publish this page and all of its children?" %}

  • - + {% trans "No, don't publish." %}
  • diff --git a/wagtail_to_ion/utils.py b/wagtail_to_ion/utils.py index 54eb7bc..a995a57 100644 --- a/wagtail_to_ion/utils.py +++ b/wagtail_to_ion/utils.py @@ -1,6 +1,11 @@ -from django.db.models import Q +import functools +from typing import Any, Dict, Generator, List, NamedTuple, Tuple, Type, Union -from wagtail.core.models import Collection, Page, PageViewRestriction +from django.db.models import Q, Model + +from wagtail.core.blocks import Block, BoundBlock, StreamValue, StructBlock, StructValue +from wagtail.core.fields import StreamField +from wagtail.core.models import Collection, Page, PageViewRestriction, get_page_models from wagtail.images import get_image_model from wagtail.documents import get_document_model @@ -61,7 +66,7 @@ def get_collection_for_page(page): # TODO: might be obsolete once https://github.com/wagtail/wagtail/pull/6300 has been merged def visible_tree_by_user(root, user): collection = IonCollection.objects.get(live=True, slug=get_collection_for_page(root)) - + if collection.view_restrictions.exists(): restrictions = collection.view_restrictions.filter( restriction_type=PageViewRestriction.GROUPS, @@ -115,3 +120,110 @@ def isoDate(d): return d.replace(microsecond=0, tzinfo=None).strftime("%Y-%m-%dT%H:%M:%SZ") else: return 'None' + + +def get_object_block_usage(obj, block_types: Union[Type[Block], Tuple[Type[Block]]]): + """ + Returns a queryset of pages that contain a block linked to a particular object. + + Works like `wagtail.admin.models.get_object_usage` but inspects all pages + which might contain a block of the specified type(s). + + To avoid inspecting all pages the following optimizations are applied: + 1. operate only on page models with a StreamField containing the specified block type(s) + 2. filter pages by expected JSON string in StreamField column(s) + """ + block_usage = get_page_models_using_blocks(block_types=block_types) + page_ptr_ids = set() + + for page_model in block_usage.keys(): + stream_field_filter_q = Q() + stream_fields = set() + + # create a (postgres specific) filter for every block; look for: + # - `"value": ` string if block is in a StreamValue + # - `"": ` if block is in a StructValue + for block in block_usage[page_model]: + filter_att = block.block_name if block.in_struct else 'value' + stream_field_filter_q |= Q(**{f'{block.stream_field_name}__regex': rf'"{filter_att}":\s*{obj.pk}\M'}) + stream_fields.add(block.stream_field_name) + + for page_with_obj_pk_in_blocks in page_model.objects.filter(stream_field_filter_q): + for field_name in stream_fields: + for bound_block, _ in get_stream_value_blocks(getattr(page_with_obj_pk_in_blocks, field_name)): + if isinstance(bound_block.block, block_types) and bound_block.value == obj: + page_ptr_ids.add(page_with_obj_pk_in_blocks.page_ptr_id) + + return Page.objects.filter(pk__in=page_ptr_ids) + + +class StreamFieldBlockInfo(NamedTuple): + block_name: str + block_type: Block + in_struct: bool + + +def get_stream_field_blocks(stream_field) -> Generator[StreamFieldBlockInfo, None, None]: + """Generates an un-nested list of blocks of a `StreamField`.""" + def unnest_blocks(blocks: dict, in_struct: bool = False): + for block in blocks.items(): + yield StreamFieldBlockInfo(block[0], block[1], in_struct) + if hasattr(block[1], 'child_blocks'): + yield from unnest_blocks(block[1].child_blocks, in_struct=isinstance(block[1], StructBlock)) + + return unnest_blocks(stream_field.stream_block.child_blocks) + + +class StreamValueBlockInfo(NamedTuple): + bound_block: BoundBlock + json_value_field_name: str + + +def get_stream_value_blocks(stream_value: StreamValue) -> Generator[StreamValueBlockInfo, None, None]: + """Generates an un-nested list of blocks of a `StreamValue`.""" + def unnest_blocks(value: Union[StreamValue, StructValue]): + if isinstance(value, StreamValue): + for stream_block in value: + assert isinstance(stream_block, BoundBlock) + if isinstance(stream_block.value, StructValue): + yield from unnest_blocks(stream_block.value) + else: + yield StreamValueBlockInfo(stream_block, 'value') + elif isinstance(value, StructValue): + for block_name, bound_block in value.bound_blocks.items(): + assert isinstance(bound_block, BoundBlock) + if isinstance(bound_block.value, StreamValue): + yield from unnest_blocks(bound_block.value) + else: + yield StreamValueBlockInfo(bound_block, block_name) + else: + raise RuntimeError(f'Unexpected type: {type(value)}') + + return unnest_blocks(stream_value) + + +class ModelStreamFieldBlockInfo(NamedTuple): + stream_field_name: str + block_name: str + block_type: Block + in_struct: bool + + +@functools.lru_cache() +def get_page_models_using_blocks( + block_types: Union[Type[Block], Tuple[Type[Block]]], +) -> Dict[Type[Model], List[ModelStreamFieldBlockInfo]]: + """Returns information about all page models which use the specified block type(s) in a StreamField.""" + models_with_block: Dict[Type[Model], List[ModelStreamFieldBlockInfo]] = {} + + for page_model in get_page_models(): + for stream_field in [field for field in page_model._meta.get_fields() if isinstance(field, StreamField)]: + for stream_field_block in get_stream_field_blocks(stream_field): + if isinstance(stream_field_block.block_type, block_types): + if page_model not in models_with_block: + models_with_block[page_model] = [] + models_with_block[page_model].append( + ModelStreamFieldBlockInfo(stream_field.attname, *stream_field_block), + ) + + return models_with_block diff --git a/wagtail_to_ion/views/wagtail_override/__init__.py b/wagtail_to_ion/views/wagtail_override/__init__.py index fdabaa0..f12ae1e 100644 --- a/wagtail_to_ion/views/wagtail_override/__init__.py +++ b/wagtail_to_ion/views/wagtail_override/__init__.py @@ -1 +1,2 @@ from .pages import ion_unpublish, ion_publish_with_children +from .file_based_models import document_safe_delete, image_safe_delete, media_safe_delete diff --git a/wagtail_to_ion/views/wagtail_override/file_based_models.py b/wagtail_to_ion/views/wagtail_override/file_based_models.py new file mode 100644 index 0000000..8a6bd59 --- /dev/null +++ b/wagtail_to_ion/views/wagtail_override/file_based_models.py @@ -0,0 +1,48 @@ +from django.shortcuts import get_object_or_404, redirect +from django.utils.translation import gettext_lazy + +from wagtail.admin import messages +from wagtail.admin.auth import PermissionPolicyChecker +from wagtail.documents import get_document_model +from wagtail.documents.permissions import permission_policy as document_permission_policy +from wagtail.documents.views.documents import delete as document_delete_view +from wagtail.images import get_image_model +from wagtail.images.permissions import permission_policy as image_permission_policy +from wagtail.images.views.images import delete as image_delete_view +from wagtailmedia.models import get_media_model +from wagtailmedia.permissions import permission_policy as media_permission_policy +from wagtailmedia.views.media import delete as media_delete_view + + +document_permission_checker = PermissionPolicyChecker(document_permission_policy) +image_permission_checker = PermissionPolicyChecker(image_permission_policy) +media_permission_checker = PermissionPolicyChecker(media_permission_policy) + +MESSAGE = gettext_lazy('Cannot delete object if used by any page') + + +@image_permission_checker.require('delete') +def image_safe_delete(request, image_id): + image = get_object_or_404(get_image_model(), pk=image_id) + if image.get_usage().exists(): + messages.error(request, MESSAGE) + return redirect('wagtailimages:edit', image_id=image_id) + return image_delete_view(request, image_id) + + +@document_permission_checker.require('delete') +def document_safe_delete(request, document_id): + document = get_object_or_404(get_document_model(), pk=document_id) + if document.get_usage().exists(): + messages.error(request, MESSAGE) + return redirect('wagtaildocs:edit', document_id=document_id) + return document_delete_view(request, document_id) + + +@media_permission_checker.require('delete') +def media_safe_delete(request, media_id): + media = get_object_or_404(get_media_model(), pk=media_id) + if media.get_usage().exists(): + messages.error(request, MESSAGE) + return redirect('wagtailmedia:edit', media_id) + return media_delete_view(request, media_id) diff --git a/wagtail_to_ion/views/wagtail_override/pages.py b/wagtail_to_ion/views/wagtail_override/pages.py index 34aaca6..3529d70 100644 --- a/wagtail_to_ion/views/wagtail_override/pages.py +++ b/wagtail_to_ion/views/wagtail_override/pages.py @@ -1,14 +1,16 @@ # Copyright © 2017 anfema GmbH. All rights reserved. from django.shortcuts import get_object_or_404, redirect from django.template.response import TemplateResponse +from django.utils.text import slugify from django.utils.translation import ugettext as _ from django.urls import reverse from django.core.exceptions import PermissionDenied from wagtail.core import hooks from wagtail.admin import messages +from wagtail.admin.forms.pages import CopyForm from wagtail.admin.widgets import Button -from wagtail.admin.auth import PermissionPolicyChecker +from wagtail.admin.auth import PermissionPolicyChecker, user_has_any_page_permission, user_passes_test from wagtail.images.permissions import permission_policy from wagtail.core.models import Page from wagtail.admin.views.pages.utils import get_valid_next_url_from_request @@ -19,7 +21,7 @@ # add publish button to dropdown menu @hooks.register('register_page_listing_more_buttons') -def page_listing_more_buttons(page, page_perms, is_parent=False): +def page_listing_more_buttons(page, page_perms, is_parent=False, next_url=None): if page_perms.can_publish(): yield Button( 'Publish with children', @@ -100,16 +102,23 @@ def ion_unpublish(request, page_id): def ion_publish_with_children(request, page_id): page = get_object_or_404(Page, id=page_id) page_perms = page.permissions_for_user(request.user) - + if not page_perms.can_publish(): raise PermissionDenied revision = page.get_latest_revision() - unpublished_descendant_pages = page.get_descendants().filter(live=False).specific() + unpublished_descendant_pages = page.get_descendants().not_live() + has_unpublished_alias_pages = unpublished_descendant_pages.filter(alias_of__isnull=False).exists() + + if has_unpublished_alias_pages: + messages.error(request, "The page has unpublished alias subpages. Please publish the original page(s) first.") next_url = get_valid_next_url_from_request(request) if request.method == 'POST': + if has_unpublished_alias_pages: + return redirect(next_url) if next_url else redirect('wagtailadmin_explore', page.get_parent().id) + revision.publish() for unpublished_descendant_page in unpublished_descendant_pages: @@ -126,5 +135,134 @@ def ion_publish_with_children(request, page_id): 'page': page, 'next': next_url, 'unpublished_descendants': unpublished_descendant_pages, - 'unpublished_descendant_count': page.get_descendants().not_live().count(), + 'unpublished_descendant_count': unpublished_descendant_pages.count(), + 'has_unpublished_alias_pages': has_unpublished_alias_pages, }) + + +class AutoTitleCopyForm(CopyForm): + """ + Copy form for pages with auto-generated title & slug fields. + + The `new_title` & `new_slug` fields are marked as optional and can be omitted from the form. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + for omitted_field in ('new_title', 'new_slug'): + self.fields[omitted_field].required = False + self.fields[omitted_field].initial = '' + + def clean(self): + # generate `new_title` & `new_slug`: + # only required for copy as alias functionality; new title & slug for standard copy operations are handled + # in `AbstractIonPage.copy()` + new_title = self.page.specific_class.generate_page_title() + return { + **super().clean(), + 'new_title': new_title, + 'new_slug': slugify(new_title), + } + + +# Copy of the wagtail `wagtail.admin.views.pages.copy.copy()` view +# +# Adjustments: +# - `CopyForm` replaced by `AutoTitleCopyForm` +# - disabled `before_copy_page` hook handling +# - replaced template +# - fix crash when alias field is not available in form: +# replaced `form.cleaned_data['alias']` with `form.cleaned_data.get('alias')` +# +@user_passes_test(user_has_any_page_permission) +def ion_copy_auto_title(request, page_id): + page = Page.objects.get(id=page_id) + + # Parent page defaults to parent of source page + parent_page = page.get_parent() + + # Check if the user has permission to publish subpages on the parent + can_publish = parent_page.permissions_for_user(request.user).can_publish_subpage() + + # Create the form + form = AutoTitleCopyForm(request.POST or None, user=request.user, page=page, can_publish=can_publish) + + next_url = get_valid_next_url_from_request(request) + + # for fn in hooks.get_hooks('before_copy_page'): + # result = fn(request, page) + # if hasattr(result, 'status_code'): + # return result + + # Check if user is submitting + if request.method == 'POST': + # Prefill parent_page in case the form is invalid (as prepopulated value for the form field, + # because ModelChoiceField seems to not fall back to the user given value) + parent_page = Page.objects.get(id=request.POST['new_parent_page']) + + if form.is_valid(): + # Receive the parent page (this should never be empty) + if form.cleaned_data['new_parent_page']: + parent_page = form.cleaned_data['new_parent_page'] + + if not page.permissions_for_user(request.user).can_copy_to(parent_page, + form.cleaned_data.get('copy_subpages')): + raise PermissionDenied + + # Re-check if the user has permission to publish subpages on the new parent + can_publish = parent_page.permissions_for_user(request.user).can_publish_subpage() + keep_live = can_publish and form.cleaned_data.get('publish_copies') + + # Copy the page + # Note that only users who can publish in the new parent page can create an alias. + # This is because alias pages must always match their original page's state. + if can_publish and form.cleaned_data.get('alias'): + new_page = page.specific.create_alias( + recursive=form.cleaned_data.get('copy_subpages'), + parent=parent_page, + update_slug=form.cleaned_data['new_slug'], + user=request.user, + ) + else: + new_page = page.specific.copy( + recursive=form.cleaned_data.get('copy_subpages'), + to=parent_page, + update_attrs={ + 'title': form.cleaned_data['new_title'], + 'slug': form.cleaned_data['new_slug'], + }, + keep_live=keep_live, + user=request.user, + ) + + # Give a success message back to the user + if form.cleaned_data.get('copy_subpages'): + messages.success( + request, + _("Page '{0}' and {1} subpages copied.").format(page.get_admin_display_title(), + new_page.get_descendants().count()) + ) + else: + messages.success(request, _("Page '{0}' copied.").format(page.get_admin_display_title())) + + for fn in hooks.get_hooks('after_copy_page'): + result = fn(request, page, new_page) + if hasattr(result, 'status_code'): + return result + + # Redirect to explore of parent page + if next_url: + return redirect(next_url) + return redirect('wagtailadmin_explore', parent_page.id) + + return TemplateResponse(request, 'wagtailadmin/pages/copy_auto_title.html', { + 'page': page, + 'form': form, + 'next': next_url, + }) + + +@hooks.register('before_copy_page') +def use_auto_title_copy_form(request, page): + if getattr(page.specific_class, 'ion_generate_page_title', None): + return ion_copy_auto_title(request, page.pk) diff --git a/wagtail_to_ion/wagtail_hooks/__init__.py b/wagtail_to_ion/wagtail_hooks/__init__.py index 1a62fdd..3dcccac 100644 --- a/wagtail_to_ion/wagtail_hooks/__init__.py +++ b/wagtail_to_ion/wagtail_hooks/__init__.py @@ -1,3 +1,4 @@ from .show_only_user_images import show_only_user_images from .show_only_user_documents import show_only_user_documents from .dual_range_slider import global_admin_css, global_admin_js +from .wagtail_admin_urls import register_admin_urls diff --git a/wagtail_to_ion/wagtail_hooks/wagtail_admin_urls.py b/wagtail_to_ion/wagtail_hooks/wagtail_admin_urls.py new file mode 100644 index 0000000..da16764 --- /dev/null +++ b/wagtail_to_ion/wagtail_hooks/wagtail_admin_urls.py @@ -0,0 +1,14 @@ +from django.urls import path + +from wagtail.core import hooks + +from wagtail_to_ion.views.wagtail_override import image_safe_delete, document_safe_delete, media_safe_delete + + +@hooks.register('register_admin_urls') +def register_admin_urls(): + return [ + path('documents/delete//', document_safe_delete, name='documents-safe-delete'), + path('images//delete/', image_safe_delete, name='images-safe-delete'), + path('media/delete//', media_safe_delete, name='media-safe-delete'), + ]