Feature/1865

.circleci/config.yml

@@ -147,7 +147,7 @@ jobs:
           command: |
             mkdir -p $HOME/bin
             export PATH=$HOME/bin:$PATH
-            curl -L "https://cli.run.pivotal.io/stable?release=linux64-binary&version=7.1.0" | tar xzv -C $HOME/bin
+            curl -L "https://cli.run.pivotal.io/stable?release=linux64-binary&version=7.3.0" | tar xzv -C $HOME/bin
 
       - run:
           name: Deploy API

bin/run-api.sh

@@ -3,8 +3,6 @@ cd django-backend
 # Only Instance 0 runs migrations and creates views
 echo "------ Starting APP ------"
 if [ $CF_INSTANCE_INDEX = "0" ]; then
-    echo "----- Migrating Database -----"
-    python manage.py migrate --no-input --traceback --verbosity 3
     echo "----- Creating committee views -----"
     python manage.py create_committee_views
 fi

manifests/manifest-dev-migrator.yml

@@ -0,0 +1,35 @@
+---
+applications:
+  - name: fecfile-api-migrator
+    instances: 1
+    stack: cflinuxfs4
+    buildpacks:
+      - python_buildpack
+    command: bin/run-api.sh
+    memory: 2G
+    services:
+      - fecfile-api-rds
+      - fecfile-api-s3
+      - fecfile-api-redis
+      - fecfile-api-creds-dev
+    env:
+      DISABLE_COLLECTSTATIC: 1
+      DJANGO_SETTINGS_MODULE: fecfiler.settings.production
+      FFAPI_COOKIE_DOMAIN: fecfile.fec.gov
+      LOGIN_REDIRECT_CLIENT_URL: https://dev.fecfile.fec.gov
+      LOGIN_REDIRECT_SERVER_URL: https://dev-api.fecfile.fec.gov/api/v1/oidc/login-redirect
+      LOGOUT_REDIRECT_URL: https://dev-api.fecfile.fec.gov/api/v1/oidc/logout-redirect
+      PRODUCTION_OPEN_FEC_API: https://api.open.fec.gov/v1/
+      STAGE_OPEN_FEC_API: https://api-stage.open.fec.gov/v1/
+      SESSION_COOKIE_AGE: 64800
+      BP_PIP_VERSION: latest
+      LOG_FORMAT: KEY_VALUE
+      MOCK_EFO: True
+      SYSTEM_STATUS_CACHE_AGE: 60
+      INITIAL_POLLING_INTERVAL: 10
+      INITIAL_POLLING_DURATION: 900
+      SECONDARY_POLLING_INTERVAL: 3600
+      SECONDARY_POLLING_DURATION: 86400
+      # ---- FEATURE FLAGS ----
+      FLAG__COMMITTEE_DATA_SOURCE: TEST # Values are PRODUCTION, TEST, MOCKED
+      ENABLE_DEVELOPER_COMMANDS: True

manifests/manifest-prod-migrator.yml

@@ -0,0 +1,35 @@
+---
+applications:
+  - name: fecfile-api-migrator
+    instances: 1
+    stack: cflinuxfs4
+    buildpacks:
+      - python_buildpack
+    command: bin/run-api.sh
+    memory: 2G
+    services:
+      - fecfile-api-rds
+      - fecfile-api-s3
+      - fecfile-api-redis
+      - fecfile-api-creds-prod
+    env:
+      DISABLE_COLLECTSTATIC: 1
+      DJANGO_SETTINGS_MODULE: fecfiler.settings.production
+      FFAPI_COOKIE_DOMAIN: fecfile.fec.gov
+      LOGIN_REDIRECT_CLIENT_URL: https://fecfile.fec.gov
+      LOGIN_REDIRECT_SERVER_URL: https://api.fecfile.fec.gov/api/v1/oidc/login-redirect
+      LOGOUT_REDIRECT_URL: https://api.fecfile.fec.gov/api/v1/oidc/logout-redirect
+      PRODUCTION_OPEN_FEC_API: https://api.open.fec.gov/v1/
+      STAGE_OPEN_FEC_API: https://api-stage.open.fec.gov/v1/
+      FEC_FILING_API: https://efoservices.stage.efo.fec.gov
+      SESSION_COOKIE_AGE: 64800
+      BP_PIP_VERSION: latest
+      LOG_FORMAT: KEY_VALUE
+      SYSTEM_STATUS_CACHE_AGE: 60
+      INITIAL_POLLING_INTERVAL: 10
+      INITIAL_POLLING_DURATION: 900
+      SECONDARY_POLLING_INTERVAL: 3600
+      SECONDARY_POLLING_DURATION: 86400
+      # ---- FEATURE FLAGS ----
+      FLAG__COMMITTEE_DATA_SOURCE: PRODUCTION # Values are PRODUCTION, TEST, and MOCKED
+      ENABLE_DEVELOPER_COMMANDS: False

manifests/manifest-stage-migrator.yml

@@ -0,0 +1,35 @@
+---
+applications:
+  - name: fecfile-api-migrator
+    instances: 1
+    stack: cflinuxfs4
+    buildpacks:
+      - python_buildpack
+    command: bin/run-api.sh
+    memory: 2G
+    services:
+      - fecfile-api-rds
+      - fecfile-api-s3
+      - fecfile-api-redis
+      - fecfile-api-creds-stage
+    env:
+      DISABLE_COLLECTSTATIC: 1
+      DJANGO_SETTINGS_MODULE: fecfiler.settings.production
+      FFAPI_COOKIE_DOMAIN: fecfile.fec.gov
+      LOGIN_REDIRECT_CLIENT_URL: https://stage.fecfile.fec.gov
+      LOGIN_REDIRECT_SERVER_URL: https://stage-api.fecfile.fec.gov/api/v1/oidc/login-redirect
+      LOGOUT_REDIRECT_URL: https://stage-api.fecfile.fec.gov/api/v1/oidc/logout-redirect
+      PRODUCTION_OPEN_FEC_API: https://api.open.fec.gov/v1/
+      STAGE_OPEN_FEC_API: https://api-stage.open.fec.gov/v1/
+      FEC_FILING_API: https://efoservices.stage.efo.fec.gov
+      SESSION_COOKIE_AGE: 64800
+      BP_PIP_VERSION: latest
+      LOG_FORMAT: KEY_VALUE
+      SYSTEM_STATUS_CACHE_AGE: 60
+      INITIAL_POLLING_INTERVAL: 10
+      INITIAL_POLLING_DURATION: 900
+      SECONDARY_POLLING_INTERVAL: 3600
+      SECONDARY_POLLING_DURATION: 86400
+      # ---- FEATURE FLAGS ----
+      FLAG__COMMITTEE_DATA_SOURCE: TEST # Values are TEST and PRODUCTION
+      ENABLE_DEVELOPER_COMMANDS: True

manifests/manifest-test-migrator.yml

@@ -0,0 +1,35 @@
+---
+applications:
+  - name: fecfile-api-migrator
+    instances: 1
+    stack: cflinuxfs4
+    buildpacks:
+      - python_buildpack
+    command: bin/run-api.sh
+    memory: 2G
+    services:
+      - fecfile-api-rds
+      - fecfile-api-s3
+      - fecfile-api-redis
+      - fecfile-api-creds-test
+    env:
+      DISABLE_COLLECTSTATIC: 1
+      DJANGO_SETTINGS_MODULE: fecfiler.settings.production
+      FFAPI_COOKIE_DOMAIN: fecfile.fec.gov
+      LOGIN_REDIRECT_CLIENT_URL: https://test.fecfile.fec.gov
+      LOGIN_REDIRECT_SERVER_URL: https://test-api.fecfile.fec.gov/api/v1/oidc/login-redirect
+      LOGOUT_REDIRECT_URL: https://test-api.fecfile.fec.gov/api/v1/oidc/logout-redirect
+      PRODUCTION_OPEN_FEC_API: https://api.open.fec.gov/v1/
+      STAGE_OPEN_FEC_API: https://api-stage.open.fec.gov/v1/
+      FEC_FILING_API: https://efoservices.stage.efo.fec.gov
+      SESSION_COOKIE_AGE: 64800
+      BP_PIP_VERSION: latest
+      LOG_FORMAT: KEY_VALUE
+      SYSTEM_STATUS_CACHE_AGE: 60
+      INITIAL_POLLING_INTERVAL: 10
+      INITIAL_POLLING_DURATION: 900
+      SECONDARY_POLLING_INTERVAL: 3600
+      SECONDARY_POLLING_DURATION: 86400
+      # ---- FEATURE FLAGS ----
+      FLAG__COMMITTEE_DATA_SOURCE: TEST # Values are PRODUCTION, TEST, and MOCKED
+      ENABLE_DEVELOPER_COMMANDS: True

tasks.py

@@ -89,7 +89,6 @@ def _login_to_cf(ctx, space):
 
 
 def _do_deploy(ctx, space, app):
-
     manifest_filename = f"manifests/manifest-{space}-{MANIFEST_LABEL.get(app)}.yml"
     existing_deploy = ctx.run(f"cf app {app}", echo=True, warn=True)
     print("\n")
@@ -151,6 +150,61 @@ def _rollback(ctx, app):
             print("Unable to cancel deploy. Check logs.")
 
 
+def _delete_migrator_app(ctx, space, migrator_app):
+    print("Deleting migrator app...")
+
+    existing_migrator_app = ctx.run(f"cf app {migrator_app}", echo=True, warn=True)
+    if not existing_migrator_app.ok:
+        print("No migrator app detected.  There is nothing to delete.")
+        return True
+
+    if migrator_app == APP_NAME:
+        print(f"Possible error: could result in deleting main app - {APP_NAME}")
+        print("Canceling migrator app deletion attempt.")
+        return False
+
+    delete_app = ctx.run(
+        f"cf delete {migrator_app} -f",
+        echo=True,
+        warn=True
+    )
+    if not delete_app.ok:
+        print('Failed to delete migrator app.')
+        print(f'Stray migrator app remains on {space}: "{migrator_app}"')
+        return False
+    print("Migrator app deleted successfully.")
+    return True
+
+
+def _run_migrations(ctx, space, migrator_app):
+    print("Running migrations...")
+
+    # Start migrator app
+    manifest_filename = f"manifests/manifest-{space}-migrator.yml"
+    migrator = ctx.run(
+        f"cf push {migrator_app} -f {manifest_filename}",
+        echo=True,
+        warn=True,
+    )
+    if not migrator.ok:
+        print("Failed to spin up migrator app.  Check logs.")
+        return False
+
+    # Run migrations
+    task = 'django-backend/manage.py migrate --no-input --traceback --verbosity 3'
+    migrations = ctx.run(
+        f"cf rt {migrator_app} --command '{task}' --name 'Run Migrations' --wait",
+        echo=True,
+        warn=True,
+    )
+    if not migrations.ok:
+        print("Failed to run migrations.  Check logs.")
+        return False
+
+    print("Migration process has finished successfully.")
+    return True
+
+
 @task
 def deploy(ctx, space=None, branch=None, login=False, help=False):
     """Deploy app to Cloud Foundry.
@@ -187,6 +241,17 @@ def deploy(ctx, space=None, branch=None, login=False, help=False):
     with open(".cfmeta", "w") as fp:
         json.dump({"user": os.getenv("USER"), "branch": branch}, fp)
 
+    # Runs migrations
+    # tasks.py does not continue until the migrations task has completed
+    migrator_app = 'fecfile-api-migrator'  # the app with this name will get deleted!
+    migrations_successful = _run_migrations(ctx, space, migrator_app)
+    migrator_app_deleted = _delete_migrator_app(ctx, space, migrator_app)
+
+    if not (migrations_successful and migrator_app_deleted):
+        print("Migrations failed and/or the migrator app was not deleted successfully.")
+        print("See the logs for more information.\nCanceling deploy...")
+        sys.exit(1)
+
     for app in [APP_NAME, WEB_SERVICES_NAME]:
         new_deploy = _do_deploy(ctx, space, app)