Add S3 bucket integration for currency logos (safe-global#217)

- Add S3 storage integration using django-storages
- Native currency logos are stored in the following path: safe-config/chains/{chain_id}_currency_logo.{ext}
- Add the following environment variables:
  * DEFAULT_FILE_STORAGE (default is S3Boto3Storage)
  * AWS_ACCESS_KEY_ID (no default)
  * AWS_SECRET_ACCESS_KEY (no default)
  * AWS_STORAGE_BUCKET_NAME (no default)
  * AWS_S3_CUSTOM_DOMAIN (no default)
- If S3Boto3Storage is set, the AWS environment variables also need to be set (an exception is thrown otherwise)
- For local development FileSystemStorage can be used and files will be stored under {repoRoot}/media/ (see MEDIA_ROOT)
- For testing purposes, FileSystemStorage is automatically set for all the tests and a temporary directory is created and removed on test execution

.env.example

@@ -83,3 +83,26 @@ GUNICORN_WEB_RELOAD=false
 #DOCKER_WEB_MEMORY=0
 #DOCKER_WORKER_CPUS=0
 #DOCKER_WORKER_MEMORY=0
+
+# Default file storage class to be used for any file-related operations that don’t specify a particular storage system.
+# storages.backends.s3boto3.S3Boto3Storage – Used for S3 configuration (AWS needs to be configured)
+# django.core.files.storage.FileSystemStorage – Django default
+#DEFAULT_FILE_STORAGE=django.core.files.storage.FileSystemStorage
+DEFAULT_FILE_STORAGE=storages.backends.s3boto3.S3Boto3Storage
+
+# django-storages – Amazon S3 configuration
+# See https://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html
+
+# Your Amazon Web Services access key, as a string.
+#AWS_ACCESS_KEY_ID=
+
+# Your Amazon Web Services secret access key, as a string.
+#AWS_SECRET_ACCESS_KEY=
+
+# Your Amazon Web Services storage bucket name, as a string.
+#AWS_STORAGE_BUCKET_NAME=
+
+# If you're using S3 as a CDN (via CloudFront), you'll probably want this storage to serve those files using tha
+# This is just the domain name ie.: no protocol should be set here and it should not end in a slash `/`
+# eg.: AWS_S3_CUSTOM_DOMAIN=cdn.mydomain.com
+#AWS_S3_CUSTOM_DOMAIN=

.github/workflows/test.yml

@@ -77,6 +77,9 @@ jobs:
       SECRET_KEY: 'insecure_key_for_dev'
       POSTGRES_HOST: localhost
       POSTGRES_PORT: 5432
+      AWS_ACCESS_KEY_ID: 'example-aws-access-key-id'
+      AWS_SECRET_ACCESS_KEY: 'example-aws-secret-access-key'
+      AWS_STORAGE_BUCKET_NAME: 'example-aws-storage-bucket-name'
     steps:
       - name: Check out repository code
         uses: actions/[email protected]

.gitignore

@@ -134,3 +134,4 @@ dmypy.json
 data/
 docker-compose.override.yml
 .vscode/
+src/media/

requirements.txt

@@ -1,9 +1,12 @@
+boto3==1.18.36
 Django==3.2.7
 django-cors-headers==3.8.0
 djangorestframework==3.12.4
 djangorestframework-camel-case==1.2.0
+django-storages==1.11.1
 drf-yasg[validation]==1.20.0
 gnosis-py==3.2.2
 gunicorn==20.1.0
+Pillow==8.3.2
 psycopg2-binary==2.9.1
 requests==2.26.0

src/chains/apps.py

@@ -1,4 +1,16 @@
 from django.apps import AppConfig
+from django.conf import settings
+from django.core.exceptions import ImproperlyConfigured
+
+
+def _validate_storage_setup():
+    if (
+        settings.DEFAULT_FILE_STORAGE == "storages.backends.s3boto3.S3Boto3Storage"
+        and settings.AWS_ACCESS_KEY_ID is None
+        and settings.AWS_SECRET_ACCESS_KEY is None
+        and settings.AWS_STORAGE_BUCKET_NAME is None
+    ):
+        raise ImproperlyConfigured("Storage set to S3 but AWS is not configured")
 
 
 class AppsConfig(AppConfig):
@@ -7,3 +19,7 @@ class AppsConfig(AppConfig):
 
     def ready(self):
         import chains.signals  # noqa: F401
+
+        # This application depends on S3 configuration (if set)
+        # so we validate if its django.conf.settings contains the required parameters
+        _validate_storage_setup()

src/chains/migrations/0025_alter_chain_currency_logo_uri.py

@@ -0,0 +1,20 @@
+# Generated by Django 3.2.7 on 2021-09-06 15:05
+
+from django.db import migrations, models
+
+import chains.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("chains", "0024_remove_gas_price_fields"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="chain",
+            name="currency_logo_uri",
+            field=models.ImageField(upload_to=chains.models.native_currency_path),
+        ),
+    ]

src/chains/models.py

@@ -1,3 +1,4 @@
+import os
 import re
 
 from django.core.exceptions import ValidationError
@@ -17,6 +18,11 @@
 sem_ver_validator = RegexValidator(SEM_VER_REGEX, "Invalid version (semver)", "invalid")
 
 
+def native_currency_path(instance: "Chain", filename):
+    _, file_extension = os.path.splitext(filename)  # file_extension includes the dot
+    return f"chains/{instance.id}/currency_logo{file_extension}"
+
+
 class Chain(models.Model):
     class RpcAuthentication(models.TextChoices):
         API_KEY_PATH = "API_KEY_PATH"
@@ -42,7 +48,7 @@ class RpcAuthentication(models.TextChoices):
     currency_name = models.CharField(max_length=255)
     currency_symbol = models.CharField(max_length=255)
     currency_decimals = models.IntegerField(default=18)
-    currency_logo_uri = models.URLField()
+    currency_logo_uri = models.ImageField(upload_to=native_currency_path)
     transaction_service_uri = models.URLField()
     theme_text_color = models.CharField(
         validators=[color_validator],

src/chains/serializers.py

@@ -41,7 +41,7 @@ class CurrencySerializer(serializers.Serializer):
     name = serializers.CharField(source="currency_name")
     symbol = serializers.CharField(source="currency_symbol")
     decimals = serializers.IntegerField(source="currency_decimals")
-    logo_uri = serializers.URLField(source="currency_logo_uri")
+    logo_uri = serializers.ImageField(use_url=True, source="currency_logo_uri")
 
 
 class BaseRpcUriSerializer(serializers.Serializer):

src/chains/tests/factories.py

@@ -27,7 +27,7 @@ class Meta:
     currency_name = factory.Faker("cryptocurrency_name")
     currency_symbol = factory.Faker("cryptocurrency_code")
     currency_decimals = factory.Faker("pyint")
-    currency_logo_uri = factory.Faker("url")
+    currency_logo_uri = factory.django.ImageField()
     transaction_service_uri = factory.Faker("url")
     theme_text_color = factory.Faker("hex_color")
     theme_background_color = factory.Faker("hex_color")

src/chains/tests/test_apps.py

@@ -0,0 +1,28 @@
+from django.core.exceptions import ImproperlyConfigured
+from pytest_django.asserts import assertRaisesMessage
+
+from chains.apps import _validate_storage_setup
+
+
+# Overriding settings on app configuration seems to be quite complex
+# ie.: using @override_settings in a typical TestCase might not have
+# the intended effect due to the other on which Django initializes
+# some internals.
+#
+# Therefore a settings fixture is used:
+# https://pytest-django.readthedocs.io/en/latest/helpers.html#settings
+#
+# More reading:
+# https://stackoverflow.com/questions/31148172/django-override-setting-used-in-appconfig-ready-function
+# https://code.djangoproject.com/ticket/22002
+def test_validate_storage_setup(settings):
+    settings.DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
+    settings.AWS_ACCESS_KEY_ID = None
+    settings.AWS_SECRET_ACCESS_KEY = None
+    settings.AWS_STORAGE_BUCKET_NAME = None
+
+    assertRaisesMessage(
+        ImproperlyConfigured,
+        "Storage set to S3 but AWS is not configured",
+        _validate_storage_setup,
+    )

src/chains/tests/test_models.py

@@ -231,3 +231,12 @@ def test_valid_versions(self):
                 )
                 # run validators
                 chain.full_clean()
+
+
+class ChainCurrencyLogoTestCase(TestCase):
+    def test_currency_logo_upload_path(self):
+        chain = ChainFactory.create(id=12)
+
+        self.assertEqual(
+            chain.currency_logo_uri.url, "/media/chains/12/currency_logo.jpg"
+        )

src/chains/tests/test_views.py

@@ -44,7 +44,7 @@ def test_json_payload_format(self):
                         "name": chain.currency_name,
                         "symbol": chain.currency_symbol,
                         "decimals": chain.currency_decimals,
-                        "logoUri": chain.currency_logo_uri,
+                        "logoUri": chain.currency_logo_uri.url,
                     },
                     "transactionService": chain.transaction_service_uri,
                     "theme": {
@@ -148,7 +148,7 @@ def test_json_payload_format(self):
                 "name": chain.currency_name,
                 "symbol": chain.currency_symbol,
                 "decimals": chain.currency_decimals,
-                "logoUri": chain.currency_logo_uri,
+                "logoUri": chain.currency_logo_uri.url,
             },
             "transactionService": chain.transaction_service_uri,
             "theme": {
@@ -201,7 +201,7 @@ def test_match(self):
                 "name": chain.currency_name,
                 "symbol": chain.currency_symbol,
                 "decimals": chain.currency_decimals,
-                "logo_uri": chain.currency_logo_uri,
+                "logo_uri": chain.currency_logo_uri.url,
             },
             "transaction_service": chain.transaction_service_uri,
             "theme": {

src/config/settings.py

@@ -14,6 +14,7 @@
 from pathlib import Path
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
+
 BASE_DIR = Path(__file__).resolve().parent.parent
 
 # Quick-start development settings - unsuitable for production
@@ -193,3 +194,23 @@
 
 CGW_URL = os.environ.get("CGW_URL")
 CGW_FLUSH_TOKEN = os.environ.get("CGW_FLUSH_TOKEN")
+
+# By default, Django stores files locally, using the MEDIA_ROOT and MEDIA_URL settings.
+# (using the default the default FileSystemStorage)
+# https://docs.djangoproject.com/en/dev/ref/settings/#media-root
+MEDIA_ROOT = f"{BASE_DIR}/media/"
+# https://docs.djangoproject.com/en/dev/ref/settings/#media-url
+MEDIA_URL = "/media/"
+
+AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID")
+AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY")
+AWS_STORAGE_BUCKET_NAME = os.getenv("AWS_STORAGE_BUCKET_NAME")
+AWS_S3_CUSTOM_DOMAIN = os.getenv("AWS_S3_CUSTOM_DOMAIN")
+# By default files with the same name will overwrite each other. Set this to False to have extra characters appended.
+AWS_S3_FILE_OVERWRITE = True
+# Setting AWS_QUERYSTRING_AUTH to False to remove query parameter authentication from generated URLs.
+# This can be useful if your S3 buckets are public.
+AWS_QUERYSTRING_AUTH = False
+DEFAULT_FILE_STORAGE = os.getenv(
+    "DEFAULT_FILE_STORAGE", "storages.backends.s3boto3.S3Boto3Storage"
+)

src/config/urls.py

@@ -1,9 +1,12 @@
+from django.conf.urls.static import static
 from django.contrib import admin
 from django.http import HttpResponse
 from django.urls import include, path, re_path
 from drf_yasg.views import get_schema_view
 from rest_framework import permissions
 
+from config import settings
+
 schema_view = get_schema_view(
     validators=["flex", "ssv"],
     public=True,
@@ -30,4 +33,4 @@
         schema_view.with_ui("swagger", cache_timeout=0),
         name="schema-swagger-ui",
     ),
-]
+] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)

src/conftest.py

@@ -0,0 +1,20 @@
+import shutil
+import tempfile
+
+import pytest
+
+
+@pytest.fixture(autouse=True)
+def use_file_system_storage(settings):
+    settings.DEFAULT_FILE_STORAGE = "django.core.files.storage.FileSystemStorage"
+
+
+@pytest.fixture(autouse=True)
+def use_tmp_media_root(settings):
+    # Creates tmp directory for tests
+    settings.MEDIA_ROOT = tempfile.mkdtemp()
+
+    yield  # run test
+
+    # After running each test remove the tmp directory
+    shutil.rmtree(settings.MEDIA_ROOT)