Merge branch 'dev' into dependabot/pip/dev/django-filter-23.5

docs/managing_data/customizing_data_processing.rst

@@ -48,6 +48,20 @@ In order to add new data product types, simply add a new key/value pair,
 with the value being a 2-tuple. The first tuple item is the database
 value, and the second is the display value.
 
+When a ``DataProduct`` is uploaded, its path is set by a `data_product_path`. By default this is set to
+``{target}/{facility}/{filename}``, but can be overridden by setting the ``DATA_PRODUCT_PATH`` in
+``settings.py`` to a dot-separated method path that takes a ``DataProduct`` and filename as arguments. For example, if
+you wanted to set the `data_product_path` to ``{target}/{facility}/{observation_id}/{filename}``, you would point the
+``DATA_PRODUCT_PATH`` to a custom method that looks something like this:
+
+.. code:: python
+
+   def custom_data_product_path(data_product, filename):
+       return f'{data_product.target.name}/' \
+               f'{data_product.observation_record.facility}/' \
+               f'{data_product.observation_record.observation_id}/' \
+               f'{filename}'
+
 All data products are automatically “processed” on upload, as well. Of
 course, that can mean different things to different TOMs! The TOM has
 two built-in data processors, both of which simply ingest the data into

tom_dataproducts/models.py

@@ -10,6 +10,7 @@
 from django.core.exceptions import ValidationError
 from fits2image.conversions import fits_to_jpg
 from PIL import Image
+from importlib import import_module
 
 from tom_targets.models import Target
 from tom_alerts.models import AlertStreamMessage
@@ -73,7 +74,14 @@ def is_fits_image_file(file):
 
 def data_product_path(instance, filename):
     """
-    Returns the TOM-style path for a ``DataProduct`` file. Structure is <target identifier>/<facility>/<filename>.
+    Returns the TOM-style path for a ``DataProduct`` file.
+    Default behavior can be overridden by user in settings.DATA_PRODUCT_PATH
+    DATA_PRODUCT_PATH must be a dot separated method name pointing to a method that takes two arguments:
+    instance: The specific instance of the ``DataProduct`` class.
+    filename: The filename to add to the path.
+    The method must return a string representing the path to the file.
+
+    The default structure is <target identifier>/<facility>/<filename>.
     ``DataProduct`` objects not associated with a facility will save with 'None' as the facility.
 
     :param instance: The specific instance of the ``DataProduct`` class.
@@ -85,11 +93,21 @@ def data_product_path(instance, filename):
     :returns: The TOM-style path of the file
     :rtype: str
     """
-    # Uploads go to MEDIA_ROOT
-    if instance.observation_record is not None:
-        return '{0}/{1}/{2}'.format(instance.target.name, instance.observation_record.facility, filename)
-    else:
-        return '{0}/none/{1}'.format(instance.target.name, filename)
+    try:
+        path_class = settings.DATA_PRODUCT_PATH
+        try:
+            mod_name, class_name = path_class.rsplit('.', 1)
+            mod = import_module(mod_name)
+            clazz = getattr(mod, class_name)
+        except (ImportError, AttributeError):
+            raise ImportError(f'Could not import {path_class}. Did you provide the correct path?')
+        return clazz(instance, filename)
+    except AttributeError:
+        # Uploads go to MEDIA_ROOT
+        if instance.observation_record is not None:
+            return f'{instance.target.name}/{instance.observation_record.facility}/{filename}'
+        else:
+            return f'{instance.target.name}/none/{filename}'
 
 
 class DataProductGroup(models.Model):

tom_dataproducts/tests/tests.py

@@ -19,7 +19,7 @@
 
 from tom_dataproducts.exceptions import InvalidFileFormatException
 from tom_dataproducts.forms import DataProductUploadForm
-from tom_dataproducts.models import DataProduct, is_fits_image_file, ReducedDatum
+from tom_dataproducts.models import DataProduct, is_fits_image_file, ReducedDatum, data_product_path
 from tom_dataproducts.processors.data_serializers import SpectrumSerializer
 from tom_dataproducts.processors.photometry_processor import PhotometryProcessor
 from tom_dataproducts.processors.spectroscopy_processor import SpectroscopyProcessor
@@ -40,6 +40,10 @@ def mock_is_fits_image_file(filename):
     return True
 
 
+def dp_path(instance, filename):
+    return f'new_path/{filename}'
+
+
 @override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'],
                    TARGET_PERMISSIONS_ONLY=True)
 @patch('tom_dataproducts.models.DataProduct.get_preview', return_value='/no-image.jpg')
@@ -144,6 +148,35 @@ def test_create_jpeg(self, dp_mock):
         self.assertEqual(products.count(), 1)
 
 
+class TestModels(TestCase):
+    def setUp(self):
+        self.overide_path = 'tom_dataproducts.tests.tests.dp_path'
+        self.target = SiderealTargetFactory.create()
+        self.observation_record = ObservingRecordFactory.create(
+            target_id=self.target.id,
+            facility=FakeRoboticFacility.name,
+            parameters={}
+        )
+        self.data_product = DataProduct.objects.create(
+            product_id='testproductid',
+            target=self.target,
+            observation_record=self.observation_record,
+            data=SimpleUploadedFile('afile.fits', b'somedata')
+        )
+
+    def test_no_path_overide(self):
+        """Test that the default path is used if no overide is set"""
+        filename = 'afile.fits'
+        path = data_product_path(self.data_product, filename)
+        self.assertIn(f'FakeRoboticFacility/{filename}', path)
+
+    @override_settings(DATA_PRODUCT_PATH='tom_dataproducts.tests.tests.dp_path')
+    def test_path_overide(self):
+        """Test that the overide path is used if set"""
+        path = data_product_path(self.data_product, 'afile.fits')
+        self.assertIn('new_path/afile.fits', path)
+
+
 @override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'],
                    TARGET_PERMISSIONS_ONLY=False)
 @patch('tom_dataproducts.models.DataProduct.get_preview', return_value='/no-image.jpg')