Full squash support (#13)

* introduce project settings

in case you're interested have a read here[^1]
- [^1] https://wiki.godesteem.de/wiki/configure-settings-for-a-django-package/
* only set default config for django < 3.2
* restructure file test case and share some code
* use new setting in tests
* add new command to allow replacing graphs
* bump django version for dev
* extend readme
* drop unused/untested code
* build more test coverage

README.rst

@@ -28,7 +28,7 @@ Install package:
 
 | ``pip install django-data-migrations``
 
-Configure package in Django settings:
+| Configure package in Django settings:
 
 .. code:: python
 
@@ -38,16 +38,35 @@ Configure package in Django settings:
         # your apps
     ]
 
+Configuration
+=============
+
+| The package is configurable using the
+
+.. code:: python
+
+    DATA_MIGRATION = {}
+
+setting.
+
+Currently supported attributes:
+
+- ``SQUASHABLE_APPS``: a list of app(-label) names which allow squashing, you should only provide your own apps here
+
+
 Usage
 =====
 
-Extended management commands: - ``makemigrations`` - ``migrate`` -
-``squashmigrations``
+Extended management commands:
+- ``makemigrations``
+- ``migrate``
+- ``squashmigrations``
+- ``data_migrate``
 
 ``makemigrations``
 ~~~~~~~~~~~~~~~~~~
 
-.. code:: python
+.. code:: shell
 
     # generate data migration file
     ./manage.py makemigrations --data-only [app_name]
@@ -76,7 +95,7 @@ The ``makemigrations`` command generates a file
 ``migrate``
 ~~~~~~~~~~~
 
-::
+.. code:: shell
 
     # apply data migration file
     ./manage.py migrate --data-only
@@ -87,6 +106,35 @@ The ``makemigrations`` command generates a file
     # revert partial data migration state
     ./manage.py migrate --data-only 0002_some_big_change
 
+
+
+``squashmigrations``
+~~~~~~~~~~~~~~~~~~~~
+
+| App-wise squashing of data/regular migrations.
+
+.. code:: shell
+
+    # regular squashing of test_app migrations 0001-0015
+    ./manage.py squashmigrations test_app 0001 0015
+
+    # squash and replace test_app migrations 0001-0015 and extract data_migrations
+    ./manage.py squashmigrations --extract-data-migrations test_app 0001 0015
+
+``data_migrate``
+~~~~~~~~~~~~~~~~
+
+| Extended squashing. Allows squashing a single app, a list of apps, or all apps at once.
+
+.. code:: shell
+
+    # squash and replace all migrations at once
+    ./manage.py data_migrate --all
+
+    # squash and replace migrations app-wise
+    ./manage.py data_migrate test_app
+
+
 Development
 ===========
 

data_migration/__init__.py

@@ -1 +1,5 @@
-default_apps_config = 'data_migration.apps.DataMigrationsConfig'
+import django
+
+
+if django.VERSION < (3, 2):
+    default_app_config = 'data_migration.apps.DataMigrationsConfig'

data_migration/apps.py

@@ -0,0 +1,12 @@
+from django.conf import settings
+from django.apps import AppConfig
+
+from data_migration.settings import internal_settings
+
+
+class DataMigrationsConfig(AppConfig):
+    name = 'data_migration'
+    verbose_name = 'Django data migrations'
+
+    def ready(self):
+        internal_settings.update(settings)

data_migration/management/commands/data_migrate.py

@@ -0,0 +1,31 @@
+from django.core.management import BaseCommand
+
+from data_migration.services.squasher import MigrationSquash
+from data_migration.settings import internal_settings as data_migration_settings
+
+
+class Command(BaseCommand):
+    """
+    Extended migrate command.
+
+    Allows forward and backward migration of data/regular migrations
+    """
+
+    def add_arguments(self, parser):  # noqa D102
+        parser.add_argument(
+            '--app_labels', nargs='?', dest='app_labels',
+            help='App label of an application to synchronize the state.',
+        )
+        parser.add_argument(
+            '--all', '-a', action='store_true', dest='squash_all',
+            help='Squash all apps.',
+        )
+        super().add_arguments(parser)
+
+    def handle(self, *args, **options):  # noqa D102
+        # extract parameters
+        apps_to_squash = options.get('app_labels')
+        if apps_to_squash is None and options['squash_all']:
+            apps_to_squash = data_migration_settings.SQUASHABLE_APPS
+
+        MigrationSquash(apps_to_squash).squash()

data_migration/services/node.py

@@ -9,10 +9,6 @@ def __init__(self, method=None):
     def __get__(self, instance, cls=None):
         return self.fget(cls)
 
-    def getter(self, method):
-        self.fget = method
-        return self
-
 
 class AlreadyAppliedError(Exception):
     def __init__(self, node: 'Node.Node'):
@@ -41,14 +37,20 @@ def Node(cls):
 
             class NodeClass(models.Model):
                 app_name = models.CharField(max_length=255)
-                name = models.CharField(max_length=255, unique=True)
+                name = models.CharField(max_length=255)
                 created_at = models.DateTimeField()
 
                 class Meta:
                     apps = Apps()
                     app_label = 'data_migration'
                     db_table = 'data_migrations'
                     get_latest_by = 'created_at'
+                    constraints = [
+                        models.UniqueConstraint(
+                            fields=['app_name', 'name'],
+                            name='unique_name_for_app'
+                        )
+                    ]
 
             cls._node_model = NodeClass
         return cls._node_model

data_migration/settings.py

@@ -0,0 +1,44 @@
+from typing import Dict, Optional
+
+from django.core.signals import setting_changed
+
+DATA_MIGRATION_DEFAULTS = {
+    "SQUASHABLE_APPS": []
+}
+
+
+class DataMigrationSettings(object):
+    def __init__(self, user_settings=None, defaults: Optional[Dict] = None):
+        if defaults is None:
+            defaults = DATA_MIGRATION_DEFAULTS
+
+        self.settings = defaults.copy()
+        if user_settings:
+            self.update(user_settings)
+
+    def update(self, settings):
+        try:
+            self.settings.update(getattr(settings, 'DATA_MIGRATION'))
+        except AttributeError:
+            self.settings.update(settings.get('DATA_MIGRATION', {}))
+
+    def reload(self, settings):
+        try:
+            _user_settings = getattr(settings, 'DATA_MIGRATION')
+            self.settings = _user_settings
+        except AttributeError:
+            pass
+
+    def __getattr__(self, item):
+        return self.settings[item]
+
+
+internal_settings = DataMigrationSettings(None)
+
+
+def reload(sender, setting, value, *args, **kwargs):
+    if setting == 'DATA_MIGRATION':
+        internal_settings.update(value)
+
+
+setting_changed.connect(reload)

dev-requirements.txt

@@ -1,7 +1,7 @@
 asgiref==3.3.4
 attrs==21.2.0
 coverage==5.5
-Django==3.2.4
+Django==3.2.5
 iniconfig==1.1.1
 packaging==20.9
 pluggy==0.13.1

tests/unittests/commands/test_data_migrate.py

@@ -0,0 +1,35 @@
+import os
+from unittest import TestCase, mock
+
+from data_migration.settings import internal_settings
+from tests.unittests.test_app_2.helper import ResetDirectory2Context
+from tests.utils import FileTestCase
+
+from django.core.management import call_command
+
+this_dir = os.path.dirname(__file__)
+
+
+class DataMigrateCommandTestCase(FileTestCase):
+    internal_target = os.path.join(this_dir, '../test_app_2/')
+    needs_cleanup = False
+
+    def test_explicit_app(self):
+        with ResetDirectory2Context():
+            call_command('data_migrate', app_labels=['test_app_2'])
+            prev_target = self.target
+            self.target = os.path.join(prev_target, 'data_migrations/')
+            self.assertTrue(self.has_file('0001_0002_split_name.py'))
+            self.target = os.path.join(prev_target, 'migrations/')
+            self.assertTrue(self.has_file('0001_squashed_0008.py'))
+
+    def test_all_apps(self):
+        self.assertIsNotNone(internal_settings.SQUASHABLE_APPS)
+        with ResetDirectory2Context():
+            call_command('data_migrate', squash_all=True)
+            prev_target = self.target
+            self.target = os.path.join(prev_target, 'data_migrations/')
+            self.assertTrue(self.has_file('0001_0002_split_name.py'))
+            self.assertTrue(self.has_file('0002_0006_address_line_split.py'))
+            self.target = os.path.join(prev_target, 'migrations/')
+            self.assertTrue(self.has_file('0001_squashed_0008.py'))

tests/unittests/commands/test_makemigrations.py

@@ -20,7 +20,7 @@ def inner(*args, **kwargs):
 
 
 class MakemigrationsCommandTestCase(FileTestCase):
-    target = os.path.join(this_dir, 'out/data_migrations')
+    internal_target = os.path.join(this_dir, 'out/data_migrations')
     needs_cleanup = False
 
     def tearDown(self) -> None:

tests/unittests/commands/test_squashmigrations.py

@@ -23,7 +23,7 @@ def inner(*args, **kwargs):
 
 
 class SquashmigrationsCommandTestCase(FileTestCase):
-    target = os.path.join(this_dir, '../test_app_2/data_migrations')
+    internal_target = os.path.join(this_dir, '../test_app_2/data_migrations')
     needs_cleanup = False
 
     def tearDown(self) -> None:

tests/unittests/services/test_file_generator.py

@@ -24,7 +24,7 @@ def inner(*args, **kwargs):
 
 
 class DataMigrationGeneratorTestCase(FileTestCase):
-    target = os.path.join(this_dir, 'out/data_migrations')
+    internal_target = os.path.join(this_dir, 'out/data_migrations')
 
     def tearDown(self) -> None:
         self.clean_directory()
@@ -108,3 +108,13 @@ def test_with_dependencies(self):
             set(migration_dependencies)
         )
         self.assertTrue(hasattr(node, 'dependencies'))
+
+    @mock.patch('django.apps.apps.get_app_config')
+    @mock.patch('importlib.import_module')
+    def test_set_applied_fails_gracefully(self, import_mock, app_config_mock):
+        app_config_mock.return_value = mock.Mock(
+            path=os.path.join(this_dir, 'out'), module=mock.Mock(__name__='foo')
+        )
+        import_mock.side_effect = ModuleNotFoundError()
+        # works without issue
+        DataMigrationGenerator(['foo']).set_applied()

tests/unittests/test_app/helper.py

@@ -1,27 +1,11 @@
 import os.path
 
+from tests.utils import ResetDirectoryMixin
 
 this_dir = os.path.dirname(__file__)
 
 
-class ResetDirectoryContext:
+class ResetDirectoryContext(ResetDirectoryMixin):
     targets = ['migrations', 'data_migrations']
     protected_files = ['__init__.py', '0001_first.py', '0002_add_name.py']
-
-    def __enter__(self):
-        return True
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        for target in self.targets:
-            dir_path = os.path.join(this_dir, target)
-            try:
-                files = [
-                    os.path.join(dir_path, f)
-                    for f in os.listdir(dir_path)
-                    if f not in self.protected_files
-                    and os.path.isfile(os.path.join(dir_path, f))
-                ]
-            except FileNotFoundError:
-                continue
-            for file in files:
-                os.remove(file)
+    this_dir = this_dir

tests/unittests/test_app_2/helper.py

@@ -1,9 +1,10 @@
 import os.path
+from tests.utils import ResetDirectoryMixin
 
 this_dir = os.path.dirname(__file__)
 
 
-class ResetDirectory2Context:
+class ResetDirectory2Context(ResetDirectoryMixin):
     targets = ['migrations', 'data_migrations']
     protected_files = [
         '0001_initial.py', '0004_customer_is_active.py',
@@ -12,21 +13,4 @@ class ResetDirectory2Context:
         '0008_customer_is_business.py', '0003_mmodel.py',
         '0006_address_line_split.py', '__init__.py',
     ]
-
-    def __enter__(self):
-        return True
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        for target in self.targets:
-            dir_path = os.path.join(this_dir, target)
-            try:
-                files = [
-                    os.path.join(dir_path, f)
-                    for f in os.listdir(dir_path)
-                    if f not in self.protected_files
-                    and os.path.isfile(os.path.join(dir_path, f))
-                ]
-            except FileNotFoundError:
-                continue
-            for file in files:
-                os.remove(file)
+    this_dir = this_dir