diff --git a/.gitignore b/.gitignore index 8a95b55..57b0b3d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ .idea/ *.pyc +__pycache__ .vscode/* **/.mypy_cache +build/ dist/ *.egg-info/ diff --git a/README.md b/README.md index 0a0aa78..758d455 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Content: ## 1. Requirements -- Wagtail > 2.0 and WagtailMedia +- Wagtail >= 2.12 and WagtailMedia - Django > 2.2 - Celery - RestFramework diff --git a/setup.py b/setup.py index d6d64cc..871f2da 100644 --- a/setup.py +++ b/setup.py @@ -80,7 +80,7 @@ def package_data_with_recursive_dirs(package_data_spec): keywords="ION Wagtail API Adapter", install_requires=[ "django>=2.2", - "wagtail>2.0", + "wagtail>=2.12", "celery[redis]>=4.3", "djangorestframework>=3.9", "beautifulsoup4>=4.6", diff --git a/wagtail_to_ion/admin.py b/wagtail_to_ion/admin.py index 82e58e9..c6b55f8 100644 --- a/wagtail_to_ion/admin.py +++ b/wagtail_to_ion/admin.py @@ -4,13 +4,30 @@ from wagtail_to_ion.models import get_ion_media_rendition_model -class AbstractIonImageAdmin(admin.ModelAdmin): +class ProtectInUseModelAdmin(admin.ModelAdmin): + protect_objects_in_use: bool = True + + def get_deleted_objects(self, objs, request): + if self.protect_objects_in_use: + protected = [] + + for obj in objs: + usage = obj.get_usage() + if usage: + protected += list(usage) + if protected: + return [], {}, set(), protected + + return super().get_deleted_objects(objs, request) + + +class AbstractIonImageAdmin(ProtectInUseModelAdmin): list_display = ('title', 'collection', 'include_in_archive') list_filter = ('collection', 'include_in_archive') search_fields = ('title',) -class AbstractIonDocumentAdmin(admin.ModelAdmin): +class AbstractIonDocumentAdmin(ProtectInUseModelAdmin): list_display = ('title', 'collection', 'include_in_archive') list_filter = ('collection', 'include_in_archive') search_fields = ('title',) @@ -21,7 +38,7 @@ class IonMediaRenditionInlineAdmin(admin.TabularInline): extra = 0 -class AbstractIonMediaAdmin(admin.ModelAdmin): +class AbstractIonMediaAdmin(ProtectInUseModelAdmin): list_display = ('title', 'collection', 'include_in_archive') list_filter = ('collection', 'include_in_archive') search_fields = ('title',) diff --git a/wagtail_to_ion/blocks.py b/wagtail_to_ion/blocks.py new file mode 100644 index 0000000..155af00 --- /dev/null +++ b/wagtail_to_ion/blocks.py @@ -0,0 +1,17 @@ +from django.utils.functional import cached_property +from wagtailmedia.blocks import AbstractMediaChooserBlock + + +class IonMediaBlock(AbstractMediaChooserBlock): + @cached_property + def target_model(self): + from wagtailmedia.models import get_media_model + return get_media_model() + + @cached_property + def widget(self): + from wagtailmedia.widgets import AdminMediaChooser + return AdminMediaChooser + + def render_basic(self, value, context=None): + raise NotImplementedError('You need to implement %s.render_basic' % self.__class__.__name__) diff --git a/wagtail_to_ion/conf.py b/wagtail_to_ion/conf.py index 9a5ffc8..45f9db8 100644 --- a/wagtail_to_ion/conf.py +++ b/wagtail_to_ion/conf.py @@ -19,3 +19,9 @@ False ) + +settings.ION_ARCHIVE_BUILD_URL_FUNCTION = getattr( + settings, + 'ION_ARCHIVE_BUILD_URL_FUNCTION', + None +) diff --git a/wagtail_to_ion/models/abstract.py b/wagtail_to_ion/models/abstract.py index 313f6ab..074269a 100644 --- a/wagtail_to_ion/models/abstract.py +++ b/wagtail_to_ion/models/abstract.py @@ -1,76 +1,109 @@ -# Copyright © 2017 anfema GmbH. All rights reserved. -from typing import Optional - -from django.conf import settings -from django.utils.text import slugify - -from wagtail.core.models import Page, Site - - -class AbstractIonPage(Page): - - @classmethod - def ion_metadata(cls): - """ - Returns additional metadata for the page. - - See `DynamicPageSerializer.get_meta()` - """ - return () - - @classmethod - def ion_extra_fields(cls): - """ - Add extra fields (not part of `content_panels`) for serialization with ION. - - Returns an iterable of tuples containing two strings: - - the outlet name - - the path to the extra field - - Example: to add the value of `self.some_related_model.some_field` use: - return ( - ('outlet_name', 'some_related_model.some_field'), - ) - """ - return () - - @classmethod - def get_layout_name(cls, api_version: Optional[int] = None): # TODO: rename to `get_ion_layout_name`? - return cls.__name__.lower() # TODO: use `cls._meta.model_name`? - - # - # wagtail attributes/methods - # - - # ion pages have no preview currently; clear wagtail preview modes - preview_modes = () - - # ion pages have no public url (disables wagtail "live view" buttons too) - def get_url_parts(self, request=None): - site = Site.find_for_request(request) - return site, None, None # return only the site to silence the "no site configured" warning - - # - # django model setup - # - - class Meta: - abstract = True - - def __str__(self): - return self.title - - def save(self, *args, **kwargs): - if self.slug.startswith('to-be-filled'): - self.slug = slugify(self.title) - super().save(*args, **kwargs) - - -class AbstractIonCollection(AbstractIonPage): - ion_api_object_name = 'collection' - - parent_page_types = ['wagtailcore.Page'] - subpage_types = [settings.ION_LANGUAGE_MODEL] - - class Meta: - abstract = True +# Copyright © 2017 anfema GmbH. All rights reserved. +from typing import Optional +from uuid import uuid4 + +from django.conf import settings +from django.utils.text import slugify + +from wagtail.core.models import Page, Site + + +class AbstractIonPage(Page): + ion_generate_page_title = True + + @classmethod + def generate_page_title(cls): + """Generate ION specific page title""" + return '{class_name}{uuid}'.format(class_name=cls.__name__, uuid=uuid4()) + + @classmethod + def ion_metadata(cls): + """ + Returns additional metadata for the page. + + See `DynamicPageSerializer.get_meta()` + """ + return () + + @classmethod + def ion_extra_fields(cls): + """ + Add extra fields (not part of `content_panels`) for serialization with ION. + + Returns an iterable of tuples containing two strings: + - the outlet name + - the path to the extra field + + Example: to add the value of `self.some_related_model.some_field` use: + return ( + ('outlet_name', 'some_related_model.some_field'), + ) + """ + return () + + @classmethod + def get_layout_name(cls, api_version: Optional[int] = None): # TODO: rename to `get_ion_layout_name`? + return cls.__name__.lower() # TODO: use `cls._meta.model_name`? + + # + # wagtail attributes/methods + # + + # ion pages have no preview currently; clear wagtail preview modes + preview_modes = () + + # ion pages have no public url (disables wagtail "live view" buttons too) + def get_url_parts(self, request=None): + site = Site.find_for_request(request) + return site, None, None # return only the site to silence the "no site configured" warning + + # overwrite Page.copy() to automatically replace the page title & slug if `ion_generate_page_title` flag is set. + # this method is called for every child page too (on recursive copy). + def copy(self, *args, **kwargs): + new_update_attrs = kwargs.get('update_attrs', None) or {} + + if self.ion_generate_page_title: + new_page_title = self.generate_page_title() + new_update_attrs.update({ + 'title': new_page_title, + 'slug': slugify(new_page_title), + }) + + # Backport: Fix crash when copying an alias page + # TODO: remove once https://github.com/wagtail/wagtail/pull/6854 is merged & released + new_update_attrs.update({ + 'alias_of': None, + }) + kwargs['update_attrs'] = new_update_attrs + + return super().copy(*args, **kwargs) + + # + # django model setup + # + + class Meta: + abstract = True + + def __str__(self): + return self.title + + # overwrite Page.full_clean() to implement ION specific title & slug handling + def full_clean(self, *args, **kwargs): + # most ION pages have an auto-generated title (indicated by the `ion_generate_page_title` flag) + if not self.title and self.ion_generate_page_title: + self.title = self.generate_page_title() + self.slug = slugify(self.title) + + super().full_clean(*args, **kwargs) + + +class AbstractIonCollection(AbstractIonPage): + ion_api_object_name = 'collection' + ion_generate_page_title = False + + parent_page_types = ['wagtailcore.Page'] + subpage_types = [settings.ION_LANGUAGE_MODEL] + + class Meta: + abstract = True diff --git a/wagtail_to_ion/models/file_based_models.py b/wagtail_to_ion/models/file_based_models.py index c1292e0..a876975 100644 --- a/wagtail_to_ion/models/file_based_models.py +++ b/wagtail_to_ion/models/file_based_models.py @@ -3,18 +3,22 @@ import os from django.db import models -from django.db.utils import cached_property +from django.db.models import ProtectedError +from django.db.models.signals import post_delete, pre_delete +from django.dispatch import receiver from django.utils.translation import ugettext_lazy as _ from magic import from_buffer as magic_from_buffer +from wagtail.documents.blocks import DocumentChooserBlock from wagtail.documents.models import AbstractDocument -from wagtail.images.models import AbstractImage, AbstractRendition, SourceImageIOError -from wagtailmedia.blocks import AbstractMediaChooserBlock +from wagtail.images.blocks import ImageChooserBlock +from wagtail.images.models import AbstractImage, AbstractRendition from wagtailmedia.models import AbstractMedia +from wagtail_to_ion.blocks import IonMediaBlock from wagtail_to_ion.conf import settings from wagtail_to_ion.models import get_ion_media_rendition_model -from wagtail_to_ion.tasks import generate_media_rendition +from wagtail_to_ion.tasks import generate_media_rendition, get_audio_metadata BUFFER_SIZE = 64 * 1024 @@ -32,6 +36,7 @@ class AbstractIonDocument(AbstractDocument): 'tags', 'include_in_archive', ) + check_usage_block_types = (DocumentChooserBlock,) class Meta: abstract = True @@ -52,10 +57,15 @@ def save(self, *args, **kwargs): super().save(*args, **kwargs) os.chmod(self.file.path, 0o644) + def get_usage(self): + from wagtail_to_ion.utils import get_object_block_usage + return super().get_usage().union(get_object_block_usage(self, block_types=self.check_usage_block_types)) + class AbstractIonImage(AbstractImage): checksum = models.CharField(max_length=255) mime_type = models.CharField(max_length=128) + rendition_type = models.CharField(max_length=128, default='jpegquality-70') include_in_archive = models.BooleanField(default=True) updated_at = models.DateTimeField(auto_now=True) admin_form_fields = ( @@ -69,6 +79,7 @@ class AbstractIonImage(AbstractImage): 'focal_point_height', 'include_in_archive', ) + check_usage_block_types = (ImageChooserBlock,) class Meta: abstract = True @@ -89,14 +100,13 @@ def save(self, *args, **kwargs): super().save(*args, **kwargs) os.chmod(self.file.path, 0o644) + def get_usage(self): + from wagtail_to_ion.utils import get_object_block_usage + return super().get_usage().union(get_object_block_usage(self, block_types=self.check_usage_block_types)) + @property def archive_rendition(self): - try: - result = self.get_rendition('jpegquality-70') - except SourceImageIOError as e: - if not settings.ION_ALLOW_MISSING_FILES: - raise e - return None + result = self.get_rendition(self.rendition_type) h = hashlib.new('sha256') try: @@ -148,8 +158,8 @@ class AbstractIonMedia(AbstractMedia): width = models.PositiveIntegerField(null=True, blank=True, verbose_name=_('width')) height = models.PositiveIntegerField(null=True, blank=True, verbose_name=_('height')) - thumbnail_checksum = models.CharField(max_length=255) - thumbnail_mime_type = models.CharField(max_length=128) + thumbnail_checksum = models.CharField(blank=True, max_length=255) + thumbnail_mime_type = models.CharField(blank=True, max_length=128) include_in_archive = models.BooleanField( default=False, help_text="If enabled, the file will be included in the ION archive " @@ -162,11 +172,19 @@ class AbstractIonMedia(AbstractMedia): 'collection', 'tags', ) + check_usage_block_types = (IonMediaBlock,) class Meta: abstract = True def save(self, *args, **kwargs): + # handle audio files + if self.type == 'audio': + self.set_media_metadata() + super().save(*args, **kwargs) + return + + # handle video files needs_transcode = False needs_thumbnail = False try: @@ -184,33 +202,10 @@ def save(self, *args, **kwargs): pass if needs_thumbnail: - try: - self.thumbnail.open() - h = hashlib.new('sha256') - buffer = self.thumbnail.read(BUFFER_SIZE) - if not self.thumbnail_mime_type: - self.thumbnail_mime_type = magic_from_buffer(buffer, mime=True) - while len(buffer) > 0: - h.update(buffer) - buffer = self.thumbnail.read(BUFFER_SIZE) - self.thumbnail_checksum = 'sha256:' + h.hexdigest() - except FileNotFoundError as exception: - raise exception - except ValueError: - pass + self.set_thumbnail_metadata() if needs_transcode: - try: - self.file.open() - h = hashlib.new('sha256') - buffer = self.file.read(BUFFER_SIZE) - self.mime_type = magic_from_buffer(buffer, mime=True) - while len(buffer) > 0: - h.update(buffer) - buffer = self.file.read(BUFFER_SIZE) - self.checksum = 'sha256:' + h.hexdigest() - except FileNotFoundError as exception: - raise exception + self.set_media_metadata() super().save(*args, **kwargs) try: @@ -221,37 +216,59 @@ def save(self, *args, **kwargs): # remove all renditions and generate new ones if needs_transcode: - renditions = [] - for rendition in self.renditions.all(): - rendition.delete() - for key, config in settings.ION_VIDEO_RENDITIONS.items(): - rendition = get_ion_media_rendition_model().objects.create( - name=key, - media_item=self - ) - if not self.mime_type.startswith('video/'): - rendition.transcode_errors = 'Not a video file' - rendition.save() - renditions.append(rendition) - - if self.mime_type.startswith('video/'): - # Run transcode in background - for rendition in renditions: - generate_media_rendition.delay(rendition.id) - - def delete(self, *args, **kwargs): - try: - self.file.delete() - except ValueError: - pass + self.create_renditions() + + def get_usage(self): + from wagtail_to_ion.utils import get_object_block_usage + return super().get_usage().union(get_object_block_usage(self, block_types=self.check_usage_block_types)) + + def set_media_metadata(self): + self.file.open() + h = hashlib.new('sha256') + buffer = self.file.read(BUFFER_SIZE) + self.mime_type = magic_from_buffer(buffer, mime=True) + while len(buffer) > 0: + h.update(buffer) + buffer = self.file.read(BUFFER_SIZE) + self.checksum = 'sha256:' + h.hexdigest() + self.file.seek(0) + + if self.type == 'audio': + metadata = get_audio_metadata(self.file) + self.duration = round(float(metadata.get('duration'))) + + def set_thumbnail_metadata(self): try: - self.thumbnail.delete() + self.thumbnail.open() + h = hashlib.new('sha256') + buffer = self.thumbnail.read(BUFFER_SIZE) + if not self.thumbnail_mime_type: + self.thumbnail_mime_type = magic_from_buffer(buffer, mime=True) + while len(buffer) > 0: + h.update(buffer) + buffer = self.thumbnail.read(BUFFER_SIZE) + self.thumbnail_checksum = 'sha256:' + h.hexdigest() except ValueError: pass - # delete one by one to make sure files are deleted + + def create_renditions(self): + renditions = [] for rendition in self.renditions.all(): rendition.delete() - super().delete(*args, **kwargs) + for key, config in settings.ION_VIDEO_RENDITIONS.items(): + rendition = get_ion_media_rendition_model().objects.create( + name=key, + media_item=self, + ) + if not self.mime_type.startswith('video/'): + rendition.transcode_errors = 'Not a video file' + rendition.save() + renditions.append(rendition) + + if self.mime_type.startswith('video/'): + # Run transcode in background + for rendition in renditions: + generate_media_rendition.delay(rendition.id) class AbstractIonMediaRendition(models.Model): @@ -285,28 +302,27 @@ class Meta: def __str__(self): return "IonMediaRendition {} for {}".format(self.name, self.media_item) - def delete(self, *args, **kwargs): + +@receiver(pre_delete) +def prevent_deletion_if_in_use(sender, instance, **kwargs): + if isinstance(instance, (AbstractIonDocument, AbstractIonImage, AbstractIonMedia)): + usage = instance.get_usage() + if usage: + model_name = instance.__class__.__name__ + raise ProtectedError( + f"Cannot delete instance of model '{model_name}' because it is referenced in stream field blocks", + usage, + ) + + +@receiver(post_delete) +def remove_media_files(sender, instance, **kwargs): + if isinstance(instance, (AbstractIonMedia, AbstractIonMediaRendition)): try: - self.file.delete() + instance.file.delete(save=False) except ValueError: pass try: - self.thumbnail.delete() + instance.thumbnail.delete(save=False) except ValueError: pass - super().delete(*args, **kwargs) - - -class IonMediaBlock(AbstractMediaChooserBlock): - @cached_property - def target_model(self): - from wagtailmedia.models import get_media_model - return get_media_model() - - @cached_property - def widget(self): - from wagtailmedia.widgets import AdminMediaChooser - return AdminMediaChooser - - def render_basic(self, value, context=None): - raise NotImplementedError('You need to implement %s.render_basic' % self.__class__.__name__) diff --git a/wagtail_to_ion/models/page_models.py b/wagtail_to_ion/models/page_models.py index 0a0d1d4..812d715 100644 --- a/wagtail_to_ion/models/page_models.py +++ b/wagtail_to_ion/models/page_models.py @@ -22,6 +22,7 @@ class Meta: abstract = True ion_api_object_name = 'language' + ion_generate_page_title = False parent_page_types = [settings.ION_COLLECTION_MODEL] diff --git a/wagtail_to_ion/serializers/pages.py b/wagtail_to_ion/serializers/pages.py index 312dc1a..1862a3b 100644 --- a/wagtail_to_ion/serializers/pages.py +++ b/wagtail_to_ion/serializers/pages.py @@ -4,7 +4,7 @@ from datetime import datetime, date from typing import Iterable, Tuple - +from django.core.exceptions import FieldDoesNotExist from django.db import models from django.urls import reverse @@ -81,19 +81,19 @@ def parse_data(content_data, content, fieldname, *, content_field_meta=None, blo content['is_multiline'] = False content['mime_type'] = 'text/plain' content['outlet'] = fieldname - elif content_field_meta is not None and hasattr(content_field_meta, 'choices') and content_field_meta.choices is not None: - # Choicefield - content['type'] = 'optioncontent' - data = content_data - for key, display in content_field_meta.choices: - if key == content_data: - data = display - break - if data.__class__.__name__ == 'str': - content['value'] = data.strip() - else: - content['value'] = data - content['outlet'] = fieldname + # elif content_field_meta is not None and hasattr(content_field_meta, 'choices') and content_field_meta.choices is not None: + # # Choicefield + # content['type'] = 'optioncontent' + # data = content_data + # for key, display in content_field_meta.choices: + # if key == content_data: + # data = display + # break + # if data.__class__.__name__ == 'str': + # content['value'] = data.strip() + # else: + # content['value'] = data + # content['outlet'] = fieldname elif content_data.__class__.__name__ in ['str', 'RichText']: try: # check if text is html @@ -215,42 +215,56 @@ def parse_data(content_data, content, fieldname, *, content_field_meta=None, blo thumbnail_slot['type'] = 'imagecontent' if os.path.exists(content_data.file.path): - rendition = content_data.renditions.filter(transcode_finished=True).first() - if rendition is None: - rendition = content_data - media_slot['mime_type'] = content_data.mime_type - media_slot['file'] = settings.BASE_URL + rendition.file.url - media_slot['checksum'] = rendition.checksum - media_slot['width'] = rendition.width if rendition.width else 0 - media_slot['height'] = rendition.height if rendition.height else 0 - media_slot['length'] = content_data.duration - media_slot['file_size'] = rendition.file.size - media_slot['name'] = content_data.title - media_slot['original_mime_type'] = content_data.mime_type - media_slot['original_file'] = settings.BASE_URL + content_data.file.url - media_slot['original_checksum'] = content_data.checksum - media_slot['original_width'] = content_data.width if content_data.width else 0 - media_slot['original_height'] = content_data.height if content_data.height else 0 - media_slot['original_length'] = content_data.duration - media_slot['original_file_size'] = content_data.file.size - media_slot['outlet'] = 'video' - - thumbnail_slot['mime_type'] = content_data.thumbnail_mime_type - thumbnail_slot['image'] = settings.BASE_URL + rendition.thumbnail.url - thumbnail_slot['checksum'] = rendition.thumbnail_checksum - thumbnail_slot['width'] = rendition.width - thumbnail_slot['height'] = rendition.height - thumbnail_slot['file_size'] = rendition.thumbnail.size - thumbnail_slot['original_mime_type'] = content_data.thumbnail_mime_type - thumbnail_slot['original_image'] = settings.BASE_URL + content_data.thumbnail.url - thumbnail_slot['original_checksum'] = content_data.thumbnail_checksum - thumbnail_slot['original_width'] = content_data.width - thumbnail_slot['original_height'] = content_data.height - thumbnail_slot['original_file_size'] = content_data.thumbnail.size - thumbnail_slot['translation_x'] = 0 - thumbnail_slot['translation_y'] = 0 - thumbnail_slot['scale'] = 1.0 - thumbnail_slot['outlet'] = "video_thumbnail" + if content_data.type == 'audio': + media_slot['mime_type'] = content_data.mime_type + media_slot['file'] = settings.BASE_URL + content_data.file.url + media_slot['checksum'] = content_data.checksum + media_slot['length'] = content_data.duration + media_slot['file_size'] = content_data.file.size + media_slot['name'] = content_data.title + media_slot['original_mime_type'] = content_data.mime_type + media_slot['original_file'] = settings.BASE_URL + content_data.file.url + media_slot['original_checksum'] = content_data.checksum + media_slot['original_length'] = content_data.duration + media_slot['original_file_size'] = content_data.file.size + media_slot['outlet'] = 'audio' + else: + rendition = content_data.renditions.filter(transcode_finished=True).first() + if rendition is None: + rendition = content_data + media_slot['mime_type'] = content_data.mime_type + media_slot['file'] = settings.BASE_URL + rendition.file.url + media_slot['checksum'] = rendition.checksum + media_slot['width'] = rendition.width if rendition.width else 0 + media_slot['height'] = rendition.height if rendition.height else 0 + media_slot['length'] = content_data.duration + media_slot['file_size'] = rendition.file.size + media_slot['name'] = content_data.title + media_slot['original_mime_type'] = content_data.mime_type + media_slot['original_file'] = settings.BASE_URL + content_data.file.url + media_slot['original_checksum'] = content_data.checksum + media_slot['original_width'] = content_data.width if content_data.width else 0 + media_slot['original_height'] = content_data.height if content_data.height else 0 + media_slot['original_length'] = content_data.duration + media_slot['original_file_size'] = content_data.file.size + media_slot['outlet'] = 'video' + + thumbnail_slot['mime_type'] = content_data.thumbnail_mime_type + thumbnail_slot['image'] = settings.BASE_URL + rendition.thumbnail.url + thumbnail_slot['checksum'] = rendition.thumbnail_checksum + thumbnail_slot['width'] = rendition.width + thumbnail_slot['height'] = rendition.height + thumbnail_slot['file_size'] = rendition.thumbnail.size + thumbnail_slot['original_mime_type'] = content_data.thumbnail_mime_type + thumbnail_slot['original_image'] = settings.BASE_URL + content_data.thumbnail.url + thumbnail_slot['original_checksum'] = content_data.thumbnail_checksum + thumbnail_slot['original_width'] = content_data.width + thumbnail_slot['original_height'] = content_data.height + thumbnail_slot['original_file_size'] = content_data.thumbnail.size + thumbnail_slot['translation_x'] = 0 + thumbnail_slot['translation_y'] = 0 + thumbnail_slot['scale'] = 1.0 + thumbnail_slot['outlet'] = "video_thumbnail" fill_contents(media_slot, media_container) fill_contents(thumbnail_slot, media_container) @@ -271,7 +285,7 @@ def parse_data(content_data, content, fieldname, *, content_field_meta=None, blo content['outlet'] = get_stream_field_outlet_name(fieldname, block_type, count) else: content['outlet'] = fieldname - elif isinstance(content_data, AbstractIonPage): + elif isinstance(content_data, AbstractIonPage) or isinstance(content_data, Page): content['type'] = 'connectioncontent' content['connection_string'] = '//{}/{}'.format(get_collection_for_page(content_data), content_data.slug) if streamfield: @@ -455,7 +469,10 @@ def get_contents(self, obj): if page_filled: for outlet_name, field_name, instance in get_wagtail_panels_and_extra_fields(obj): field_data = getattr(instance, field_name) - field_type = instance._meta.get_field(field_name) + try: + field_type = instance._meta.get_field(field_name) + except FieldDoesNotExist: + field_type = None content = {} # parse content for all standard django and wagtail fields diff --git a/wagtail_to_ion/serializers/tar.py b/wagtail_to_ion/serializers/tar.py index 75382b3..6579776 100644 --- a/wagtail_to_ion/serializers/tar.py +++ b/wagtail_to_ion/serializers/tar.py @@ -2,7 +2,10 @@ import json import os +from django.core.exceptions import ImproperlyConfigured from django.urls import reverse +from django.utils.module_loading import import_string + from rest_framework.renderers import JSONRenderer from wagtail_to_ion.tar import TarWriter @@ -12,13 +15,19 @@ from wagtail_to_ion.utils import get_collection_for_page -def build_url(request, locale_code, page, variation='default'): - url = reverse('v1:page-detail', kwargs={ - 'locale': locale_code, - 'collection': get_collection_for_page(page), - 'slug': page.slug, - }) - return request.build_absolute_uri(url) + "?variation={}".format(variation) +if settings.ION_ARCHIVE_BUILD_URL_FUNCTION is not None: + try: + build_url = import_string(settings.ION_ARCHIVE_BUILD_URL_FUNCTION) + except ImportError: + raise ImproperlyConfigured(f"The function {settings.ION_ARCHIVE_BUILD_URL_FUNCTION} couldn't be imported.") +else: + def build_url(request, locale_code, page, variation='default'): + url = reverse('v1:page-detail', kwargs={ + 'locale': locale_code, + 'collection': get_collection_for_page(page), + 'slug': page.slug, + }) + return request.build_absolute_uri(url) + "?variation={}".format(variation) def collect_files(request, self, page, collected_files, user): diff --git a/wagtail_to_ion/tasks.py b/wagtail_to_ion/tasks.py index 0d52cd5..ffd9534 100644 --- a/wagtail_to_ion/tasks.py +++ b/wagtail_to_ion/tasks.py @@ -3,11 +3,16 @@ import subprocess import json import hashlib +from tempfile import NamedTemporaryFile + +from django.core.files.uploadedfile import InMemoryUploadedFile, TemporaryUploadedFile +from django.db.models.fields.files import FieldFile from wagtail_to_ion.conf import settings from celery import shared_task + BUFFER_SIZE = 64 * 1024 # Use ffmpeg from user's bin if it exists, global ffmpeg otherwise @@ -345,3 +350,48 @@ def generate_media_rendition(rendition_id: int): else: rendition.transcode_errors += str(e) rendition.save() + + +def extract_media_format(file_path): + ffprobe = os.path.expanduser('~/bin/ffprobe') + if not os.path.exists(ffprobe): + ffprobe = 'ffprobe' + + probe = [ + ffprobe, + '-print_format', 'json', + '-show_format', + '-i', file_path, + ] + + try: + result = subprocess.run( + probe, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + ) + except subprocess.CalledProcessError as e: + print(e.stderr) + raise + + return json.loads(result.stdout.decode('utf-8')) + + +def get_audio_metadata(media_item: FieldFile): + if isinstance(media_item.file, InMemoryUploadedFile): + with NamedTemporaryFile(delete=False) as temp_file: + media_item.file.open() + temp_file.write(media_item.file.read()) + temp_file.close() + mediainfo = extract_media_format(temp_file.name) + try: + os.remove(temp_file.name) + except OSError: + pass + elif isinstance(media_item.file, TemporaryUploadedFile): + mediainfo = extract_media_format(media_item.file.temporary_file_path()) + else: + mediainfo = extract_media_format(media_item.path) + + return mediainfo.get('format', {}) diff --git a/wagtail_to_ion/templates/wagtailadmin/pages/copy_auto_title.html b/wagtail_to_ion/templates/wagtailadmin/pages/copy_auto_title.html new file mode 100644 index 0000000..3eff7d5 --- /dev/null +++ b/wagtail_to_ion/templates/wagtailadmin/pages/copy_auto_title.html @@ -0,0 +1,39 @@ +{% extends "wagtailadmin/pages/copy.html" %} +{% comment %} +Custom copy form (extends wagtails `wagtailadmin/pages/copy.html` template) + +Adjustments: + - remove `new_title` & `new_slug` fields +{% endcomment %} +{% load i18n %} +{% block content %} + {% trans "Copy" as copy_str %} + {% include "wagtailadmin/shared/header.html" with title=copy_str subtitle=page.specific_deferred.get_admin_display_title icon="doc-empty-inverse" %} + +
Published: Yes
{% else %} @@ -49,7 +49,7 @@{% trans "Are you sure you want to publish this page and all of its children?" %}