From 6bc996b969544bebc38f4b86de75b4538f963895 Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Wed, 6 Dec 2023 16:37:06 +0300 Subject: [PATCH 1/3] Add command to organize old plugin version packages --- dockerize/docker-compose.yml | 226 +++++++++--------- .../commands/organize_old_package.py | 40 ++++ .../migrations/0004_merge_20231201_0621.py | 14 ++ setup.cfg | 1 + 4 files changed, 169 insertions(+), 112 deletions(-) create mode 100644 qgis-app/plugins/management/commands/organize_old_package.py create mode 100644 qgis-app/plugins/migrations/0004_merge_20231201_0621.py diff --git a/dockerize/docker-compose.yml b/dockerize/docker-compose.yml index 2ee672fb..eb50ecfd 100644 --- a/dockerize/docker-compose.yml +++ b/dockerize/docker-compose.yml @@ -1,117 +1,119 @@ -db: - container_name: qgis-plugins-db - image: kartoza/postgis:9.6-2.4 - environment: - - ALLOW_IP_RANGE=0.0.0.0/0 - - POSTGRES_USER=docker - - POSTGRES_PASS=docker - volumes: - - ./backups:/backups - restart: unless-stopped +version: "3.8" +services: + db: + container_name: qgis-plugins-db + image: kartoza/postgis:9.6-2.4 + environment: + - ALLOW_IP_RANGE=0.0.0.0/0 + - POSTGRES_USER=docker + - POSTGRES_PASS=docker + volumes: + - ./backups:/backups + restart: unless-stopped -web: - # Note you cannot scale if you use container_name - container_name: qgis-plugins-web - build: docker - hostname: uwsgi - environment: - - DATABASE_NAME=gis - - DATABASE_USERNAME=docker - - DATABASE_PASSWORD=docker - - DATABASE_HOST=db - - DJANGO_SETTINGS_MODULE=settings_docker - - VIRTUAL_HOST=plugins.kartoza.com - - VIRTUAL_PORT=8080 - - DEBUG=False - - RABBITMQ_HOST=rabbitmq - volumes: - - ../qgis-app:/home/web/django_project - - ./static:/home/web/static:rw - - ./static:/home/web/media:rw - links: - - db:db - - rabbitmq:rabbitmq - - worker:worker - restart: unless-stopped - user: root - command: uwsgi --ini /uwsgi.conf + web: + # Note you cannot scale if you use container_name + container_name: qgis-plugins-web + build: docker + hostname: uwsgi + environment: + - DATABASE_NAME=gis + - DATABASE_USERNAME=docker + - DATABASE_PASSWORD=docker + - DATABASE_HOST=db + - DJANGO_SETTINGS_MODULE=settings_docker + - VIRTUAL_HOST=plugins.kartoza.com + - VIRTUAL_PORT=8080 + - DEBUG=False + - RABBITMQ_HOST=rabbitmq + volumes: + - ../qgis-app:/home/web/django_project + - ./static:/home/web/static:rw + - ./static:/home/web/media:rw + links: + - db:db + - rabbitmq:rabbitmq + - worker:worker + restart: unless-stopped + user: root + command: uwsgi --ini /uwsgi.conf -devweb: - # Note you cannot scale if you use container_name - container_name: qgis-plugins-devweb - build: docker - hostname: uwsgi - environment: - - DATABASE_NAME=gis - - DATABASE_USERNAME=docker - - DATABASE_PASSWORD=docker - - DATABASE_HOST=db - - DJANGO_SETTINGS_MODULE=settings_docker - - VIRTUAL_HOST=plugins.kartoza.com - - VIRTUAL_PORT=8080 - - RABBITMQ_HOST=rabbitmq - volumes: - - ../qgis-app:/home/web/django_project - - ./static:/home/web/static:rw - - ./static:/home/web/media:rw - links: - - db:db - - rabbitmq:rabbitmq - - worker:worker - restart: unless-stopped - user: root - ports: - # for django test server - - "62202:8080" - # for ssh - - "62203:22" + devweb: + # Note you cannot scale if you use container_name + container_name: qgis-plugins-devweb + build: docker + hostname: uwsgi + environment: + - DATABASE_NAME=gis + - DATABASE_USERNAME=docker + - DATABASE_PASSWORD=docker + - DATABASE_HOST=db + - DJANGO_SETTINGS_MODULE=settings_docker + - VIRTUAL_HOST=plugins.kartoza.com + - VIRTUAL_PORT=8080 + - RABBITMQ_HOST=rabbitmq + volumes: + - ../qgis-app:/home/web/django_project + - ./static:/home/web/static:rw + - ./static:/home/web/media:rw + links: + - db:db + - rabbitmq:rabbitmq + - worker:worker + restart: unless-stopped + user: root + ports: + # for django test server + - "62202:8080" + # for ssh + - "62203:22" -rabbitmq: - image: library/rabbitmq:3.6 - hostname: rabbitmq - environment: - - RABBIT_PASSWORD=rabbit_test_password - - USER=rabbit_user - - RABBITMQ_NODENAME=rabbit - restart: unless-stopped + rabbitmq: + image: library/rabbitmq:3.6 + hostname: rabbitmq + environment: + - RABBIT_PASSWORD=rabbit_test_password + - USER=rabbit_user + - RABBITMQ_NODENAME=rabbit + restart: unless-stopped -worker: - # Note you cannot scale if you use container_name - container_name: qgis-plugins-worker - build: docker - hostname: uwsgi - working_dir: /home/web/django_project - command: celery -A plugins worker -l info - environment: - - DATABASE_NAME=gis - - DATABASE_USERNAME=docker - - DATABASE_PASSWORD=docker - - DATABASE_HOST=db - - DJANGO_SETTINGS_MODULE=settings_docker - - VIRTUAL_HOST=plugins.kartoza.com - - VIRTUAL_PORT=8080 - - RABBITMQ_HOST=rabbitmq - volumes: - - ../qgis-app:/home/web/django_project - - ./static:/home/web/static:rw - - ./static:/home/web/media:rw - links: - - db:db - - rabbitmq:rabbitmq + worker: + # Note you cannot scale if you use container_name + container_name: qgis-plugins-worker + build: docker + hostname: uwsgi + working_dir: /home/web/django_project + command: celery -A plugins worker -l info + environment: + - DATABASE_NAME=gis + - DATABASE_USERNAME=docker + - DATABASE_PASSWORD=docker + - DATABASE_HOST=db + - DJANGO_SETTINGS_MODULE=settings_docker + - VIRTUAL_HOST=plugins.kartoza.com + - VIRTUAL_PORT=8080 + - RABBITMQ_HOST=rabbitmq + volumes: + - ../qgis-app:/home/web/django_project + - ./static:/home/web/static:rw + - ./static:/home/web/media:rw + links: + - db:db + - rabbitmq:rabbitmq -nginx: - # Note you cannot scale if you use container_name - container_name: qgis-plugins-nginx - image: nginx - hostname: nginx - volumes: - - ./sites-enabled:/etc/nginx/conf.d:ro - # I dont use volumes_from as I want to use the ro modifier - - ./static:/home/web/static:ro - - ./static:/home/web/media:ro - - ./logs:/var/log/nginx - links: - - web:uwsgi - ports: - - "62201:8080" - restart: unless-stopped + nginx: + # Note you cannot scale if you use container_name + container_name: qgis-plugins-nginx + image: nginx + hostname: nginx + volumes: + - ./sites-enabled:/etc/nginx/conf.d:ro + # I dont use volumes_from as I want to use the ro modifier + - ./static:/home/web/static:ro + - ./static:/home/web/media:ro + - ./logs:/var/log/nginx + links: + - web:uwsgi + ports: + - "62201:8080" + restart: unless-stopped diff --git a/qgis-app/plugins/management/commands/organize_old_package.py b/qgis-app/plugins/management/commands/organize_old_package.py new file mode 100644 index 00000000..bcda2c58 --- /dev/null +++ b/qgis-app/plugins/management/commands/organize_old_package.py @@ -0,0 +1,40 @@ +# myapp/management/commands/organize_packages.py +import os +import shutil +from django.core.management.base import BaseCommand +from django.conf import settings +from plugins.models import PluginVersion + +PLUGINS_STORAGE_PATH = getattr(settings, "PLUGINS_STORAGE_PATH", "packages") +class Command(BaseCommand): + help = 'Organize packages created before 2014 into folders by year' + + def handle(self, *args, **options): + packages_dir = os.path.join(settings.MEDIA_ROOT, PLUGINS_STORAGE_PATH) + + # Some of the packages created on 2014 also need to be organized + versions = PluginVersion.objects.filter(created_on__lt='2014-12-31').exclude(package__icontains='2014/') + self.stdout.write(self.style.NOTICE(f'{versions.count()} packages will be organized.')) + + for version in versions: + year_folder = os.path.join(packages_dir, str(version.created_on.year)) + + # Create the year folder if it doesn't exist + os.makedirs(year_folder, exist_ok=True) + + # Move the package file to the year folder + old_path = version.package.path + if os.path.exists(old_path): + new_path = os.path.join(year_folder, os.path.basename(old_path)) + if not os.path.exists(new_path): + shutil.move(old_path, year_folder) + + # Update the model with the new package path + version.package.name = os.path.relpath(new_path, settings.MEDIA_ROOT) + version.save() + else: + self.stdout.write(self.style.WARNING(f'Plugin version id {version.pk} ignored: {new_path} already exists.')) + else: + self.stdout.write(self.style.WARNING(f'Plugin version id {version.pk} ignored: {old_path} is not found.')) + + self.stdout.write(self.style.SUCCESS('Packages organized successfully')) diff --git a/qgis-app/plugins/migrations/0004_merge_20231201_0621.py b/qgis-app/plugins/migrations/0004_merge_20231201_0621.py new file mode 100644 index 00000000..d0bc7d66 --- /dev/null +++ b/qgis-app/plugins/migrations/0004_merge_20231201_0621.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.25 on 2023-12-01 06:21 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('plugins', '0002_plugins_feedback'), + ('plugins', '0003_plugin_allow_update_name'), + ] + + operations = [ + ] diff --git a/setup.cfg b/setup.cfg index fae25c68..2fa09d19 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,6 +3,7 @@ exclude = .git, .venv/, ./.venv/, + venv/, vagrant_assets, qgis-app/*/migrations/, qgis-app/plugins/tests/HelloWorld/, From a64eb5d1d5926d4543da79f5d1d73630ed3078e3 Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Wed, 6 Dec 2023 17:44:29 +0300 Subject: [PATCH 2/3] Revert changes for other files --- dockerize/docker-compose.yml | 226 +++++++++--------- .../migrations/0004_merge_20231201_0621.py | 14 -- setup.cfg | 1 - 3 files changed, 112 insertions(+), 129 deletions(-) delete mode 100644 qgis-app/plugins/migrations/0004_merge_20231201_0621.py diff --git a/dockerize/docker-compose.yml b/dockerize/docker-compose.yml index eb50ecfd..2ee672fb 100644 --- a/dockerize/docker-compose.yml +++ b/dockerize/docker-compose.yml @@ -1,119 +1,117 @@ -version: "3.8" -services: - db: - container_name: qgis-plugins-db - image: kartoza/postgis:9.6-2.4 - environment: - - ALLOW_IP_RANGE=0.0.0.0/0 - - POSTGRES_USER=docker - - POSTGRES_PASS=docker - volumes: - - ./backups:/backups - restart: unless-stopped +db: + container_name: qgis-plugins-db + image: kartoza/postgis:9.6-2.4 + environment: + - ALLOW_IP_RANGE=0.0.0.0/0 + - POSTGRES_USER=docker + - POSTGRES_PASS=docker + volumes: + - ./backups:/backups + restart: unless-stopped - web: - # Note you cannot scale if you use container_name - container_name: qgis-plugins-web - build: docker - hostname: uwsgi - environment: - - DATABASE_NAME=gis - - DATABASE_USERNAME=docker - - DATABASE_PASSWORD=docker - - DATABASE_HOST=db - - DJANGO_SETTINGS_MODULE=settings_docker - - VIRTUAL_HOST=plugins.kartoza.com - - VIRTUAL_PORT=8080 - - DEBUG=False - - RABBITMQ_HOST=rabbitmq - volumes: - - ../qgis-app:/home/web/django_project - - ./static:/home/web/static:rw - - ./static:/home/web/media:rw - links: - - db:db - - rabbitmq:rabbitmq - - worker:worker - restart: unless-stopped - user: root - command: uwsgi --ini /uwsgi.conf +web: + # Note you cannot scale if you use container_name + container_name: qgis-plugins-web + build: docker + hostname: uwsgi + environment: + - DATABASE_NAME=gis + - DATABASE_USERNAME=docker + - DATABASE_PASSWORD=docker + - DATABASE_HOST=db + - DJANGO_SETTINGS_MODULE=settings_docker + - VIRTUAL_HOST=plugins.kartoza.com + - VIRTUAL_PORT=8080 + - DEBUG=False + - RABBITMQ_HOST=rabbitmq + volumes: + - ../qgis-app:/home/web/django_project + - ./static:/home/web/static:rw + - ./static:/home/web/media:rw + links: + - db:db + - rabbitmq:rabbitmq + - worker:worker + restart: unless-stopped + user: root + command: uwsgi --ini /uwsgi.conf - devweb: - # Note you cannot scale if you use container_name - container_name: qgis-plugins-devweb - build: docker - hostname: uwsgi - environment: - - DATABASE_NAME=gis - - DATABASE_USERNAME=docker - - DATABASE_PASSWORD=docker - - DATABASE_HOST=db - - DJANGO_SETTINGS_MODULE=settings_docker - - VIRTUAL_HOST=plugins.kartoza.com - - VIRTUAL_PORT=8080 - - RABBITMQ_HOST=rabbitmq - volumes: - - ../qgis-app:/home/web/django_project - - ./static:/home/web/static:rw - - ./static:/home/web/media:rw - links: - - db:db - - rabbitmq:rabbitmq - - worker:worker - restart: unless-stopped - user: root - ports: - # for django test server - - "62202:8080" - # for ssh - - "62203:22" +devweb: + # Note you cannot scale if you use container_name + container_name: qgis-plugins-devweb + build: docker + hostname: uwsgi + environment: + - DATABASE_NAME=gis + - DATABASE_USERNAME=docker + - DATABASE_PASSWORD=docker + - DATABASE_HOST=db + - DJANGO_SETTINGS_MODULE=settings_docker + - VIRTUAL_HOST=plugins.kartoza.com + - VIRTUAL_PORT=8080 + - RABBITMQ_HOST=rabbitmq + volumes: + - ../qgis-app:/home/web/django_project + - ./static:/home/web/static:rw + - ./static:/home/web/media:rw + links: + - db:db + - rabbitmq:rabbitmq + - worker:worker + restart: unless-stopped + user: root + ports: + # for django test server + - "62202:8080" + # for ssh + - "62203:22" - rabbitmq: - image: library/rabbitmq:3.6 - hostname: rabbitmq - environment: - - RABBIT_PASSWORD=rabbit_test_password - - USER=rabbit_user - - RABBITMQ_NODENAME=rabbit - restart: unless-stopped +rabbitmq: + image: library/rabbitmq:3.6 + hostname: rabbitmq + environment: + - RABBIT_PASSWORD=rabbit_test_password + - USER=rabbit_user + - RABBITMQ_NODENAME=rabbit + restart: unless-stopped - worker: - # Note you cannot scale if you use container_name - container_name: qgis-plugins-worker - build: docker - hostname: uwsgi - working_dir: /home/web/django_project - command: celery -A plugins worker -l info - environment: - - DATABASE_NAME=gis - - DATABASE_USERNAME=docker - - DATABASE_PASSWORD=docker - - DATABASE_HOST=db - - DJANGO_SETTINGS_MODULE=settings_docker - - VIRTUAL_HOST=plugins.kartoza.com - - VIRTUAL_PORT=8080 - - RABBITMQ_HOST=rabbitmq - volumes: - - ../qgis-app:/home/web/django_project - - ./static:/home/web/static:rw - - ./static:/home/web/media:rw - links: - - db:db - - rabbitmq:rabbitmq +worker: + # Note you cannot scale if you use container_name + container_name: qgis-plugins-worker + build: docker + hostname: uwsgi + working_dir: /home/web/django_project + command: celery -A plugins worker -l info + environment: + - DATABASE_NAME=gis + - DATABASE_USERNAME=docker + - DATABASE_PASSWORD=docker + - DATABASE_HOST=db + - DJANGO_SETTINGS_MODULE=settings_docker + - VIRTUAL_HOST=plugins.kartoza.com + - VIRTUAL_PORT=8080 + - RABBITMQ_HOST=rabbitmq + volumes: + - ../qgis-app:/home/web/django_project + - ./static:/home/web/static:rw + - ./static:/home/web/media:rw + links: + - db:db + - rabbitmq:rabbitmq - nginx: - # Note you cannot scale if you use container_name - container_name: qgis-plugins-nginx - image: nginx - hostname: nginx - volumes: - - ./sites-enabled:/etc/nginx/conf.d:ro - # I dont use volumes_from as I want to use the ro modifier - - ./static:/home/web/static:ro - - ./static:/home/web/media:ro - - ./logs:/var/log/nginx - links: - - web:uwsgi - ports: - - "62201:8080" - restart: unless-stopped +nginx: + # Note you cannot scale if you use container_name + container_name: qgis-plugins-nginx + image: nginx + hostname: nginx + volumes: + - ./sites-enabled:/etc/nginx/conf.d:ro + # I dont use volumes_from as I want to use the ro modifier + - ./static:/home/web/static:ro + - ./static:/home/web/media:ro + - ./logs:/var/log/nginx + links: + - web:uwsgi + ports: + - "62201:8080" + restart: unless-stopped diff --git a/qgis-app/plugins/migrations/0004_merge_20231201_0621.py b/qgis-app/plugins/migrations/0004_merge_20231201_0621.py deleted file mode 100644 index d0bc7d66..00000000 --- a/qgis-app/plugins/migrations/0004_merge_20231201_0621.py +++ /dev/null @@ -1,14 +0,0 @@ -# Generated by Django 2.2.25 on 2023-12-01 06:21 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('plugins', '0002_plugins_feedback'), - ('plugins', '0003_plugin_allow_update_name'), - ] - - operations = [ - ] diff --git a/setup.cfg b/setup.cfg index 2fa09d19..fae25c68 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,7 +3,6 @@ exclude = .git, .venv/, ./.venv/, - venv/, vagrant_assets, qgis-app/*/migrations/, qgis-app/plugins/tests/HelloWorld/, From e8592d88be072e08400caf451e5d8c2d9021af07 Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Tue, 26 Dec 2023 08:40:14 +0300 Subject: [PATCH 3/3] Copy old packages to year folders --- qgis-app/plugins/management/commands/organize_old_package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qgis-app/plugins/management/commands/organize_old_package.py b/qgis-app/plugins/management/commands/organize_old_package.py index bcda2c58..991ed958 100644 --- a/qgis-app/plugins/management/commands/organize_old_package.py +++ b/qgis-app/plugins/management/commands/organize_old_package.py @@ -22,12 +22,12 @@ def handle(self, *args, **options): # Create the year folder if it doesn't exist os.makedirs(year_folder, exist_ok=True) - # Move the package file to the year folder + # Copy the package file to the year folder old_path = version.package.path if os.path.exists(old_path): new_path = os.path.join(year_folder, os.path.basename(old_path)) if not os.path.exists(new_path): - shutil.move(old_path, year_folder) + shutil.copy(old_path, year_folder) # Update the model with the new package path version.package.name = os.path.relpath(new_path, settings.MEDIA_ROOT)