diff --git a/.circleci/config.yml b/.circleci/config.yml index ea972ad9f..1ace9d38e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -147,7 +147,7 @@ jobs: command: | mkdir -p $HOME/bin export PATH=$HOME/bin:$PATH - curl -L "https://cli.run.pivotal.io/stable?release=linux64-binary&version=7.1.0" | tar xzv -C $HOME/bin + curl -L "https://cli.run.pivotal.io/stable?release=linux64-binary&version=7.3.0" | tar xzv -C $HOME/bin - run: name: Deploy API diff --git a/bin/run-api.sh b/bin/run-api.sh index b958f015f..d229d065c 100755 --- a/bin/run-api.sh +++ b/bin/run-api.sh @@ -3,8 +3,6 @@ cd django-backend # Only Instance 0 runs migrations and creates views echo "------ Starting APP ------" if [ $CF_INSTANCE_INDEX = "0" ]; then - echo "----- Migrating Database -----" - python manage.py migrate --no-input --traceback --verbosity 3 echo "----- Creating committee views -----" python manage.py create_committee_views fi diff --git a/manifests/manifest-dev-migrator.yml b/manifests/manifest-dev-migrator.yml new file mode 100644 index 000000000..4f4b5c27f --- /dev/null +++ b/manifests/manifest-dev-migrator.yml @@ -0,0 +1,35 @@ +--- +applications: + - name: fecfile-api-migrator + instances: 1 + stack: cflinuxfs4 + buildpacks: + - python_buildpack + command: bin/run-api.sh + memory: 2G + services: + - fecfile-api-rds + - fecfile-api-s3 + - fecfile-api-redis + - fecfile-api-creds-dev + env: + DISABLE_COLLECTSTATIC: 1 + DJANGO_SETTINGS_MODULE: fecfiler.settings.production + FFAPI_COOKIE_DOMAIN: fecfile.fec.gov + LOGIN_REDIRECT_CLIENT_URL: https://dev.fecfile.fec.gov + LOGIN_REDIRECT_SERVER_URL: https://dev-api.fecfile.fec.gov/api/v1/oidc/login-redirect + LOGOUT_REDIRECT_URL: https://dev-api.fecfile.fec.gov/api/v1/oidc/logout-redirect + PRODUCTION_OPEN_FEC_API: https://api.open.fec.gov/v1/ + STAGE_OPEN_FEC_API: https://api-stage.open.fec.gov/v1/ + SESSION_COOKIE_AGE: 64800 + BP_PIP_VERSION: latest + LOG_FORMAT: KEY_VALUE + MOCK_EFO: True + SYSTEM_STATUS_CACHE_AGE: 60 + INITIAL_POLLING_INTERVAL: 10 + INITIAL_POLLING_DURATION: 900 + SECONDARY_POLLING_INTERVAL: 3600 + SECONDARY_POLLING_DURATION: 86400 + # ---- FEATURE FLAGS ---- + FLAG__COMMITTEE_DATA_SOURCE: TEST # Values are PRODUCTION, TEST, MOCKED + ENABLE_DEVELOPER_COMMANDS: True diff --git a/manifests/manifest-prod-migrator.yml b/manifests/manifest-prod-migrator.yml new file mode 100644 index 000000000..a1e17ae67 --- /dev/null +++ b/manifests/manifest-prod-migrator.yml @@ -0,0 +1,35 @@ +--- +applications: + - name: fecfile-api-migrator + instances: 1 + stack: cflinuxfs4 + buildpacks: + - python_buildpack + command: bin/run-api.sh + memory: 2G + services: + - fecfile-api-rds + - fecfile-api-s3 + - fecfile-api-redis + - fecfile-api-creds-prod + env: + DISABLE_COLLECTSTATIC: 1 + DJANGO_SETTINGS_MODULE: fecfiler.settings.production + FFAPI_COOKIE_DOMAIN: fecfile.fec.gov + LOGIN_REDIRECT_CLIENT_URL: https://fecfile.fec.gov + LOGIN_REDIRECT_SERVER_URL: https://api.fecfile.fec.gov/api/v1/oidc/login-redirect + LOGOUT_REDIRECT_URL: https://api.fecfile.fec.gov/api/v1/oidc/logout-redirect + PRODUCTION_OPEN_FEC_API: https://api.open.fec.gov/v1/ + STAGE_OPEN_FEC_API: https://api-stage.open.fec.gov/v1/ + FEC_FILING_API: https://efoservices.stage.efo.fec.gov + SESSION_COOKIE_AGE: 64800 + BP_PIP_VERSION: latest + LOG_FORMAT: KEY_VALUE + SYSTEM_STATUS_CACHE_AGE: 60 + INITIAL_POLLING_INTERVAL: 10 + INITIAL_POLLING_DURATION: 900 + SECONDARY_POLLING_INTERVAL: 3600 + SECONDARY_POLLING_DURATION: 86400 + # ---- FEATURE FLAGS ---- + FLAG__COMMITTEE_DATA_SOURCE: PRODUCTION # Values are PRODUCTION, TEST, and MOCKED + ENABLE_DEVELOPER_COMMANDS: False diff --git a/manifests/manifest-stage-migrator.yml b/manifests/manifest-stage-migrator.yml new file mode 100644 index 000000000..86f5a3551 --- /dev/null +++ b/manifests/manifest-stage-migrator.yml @@ -0,0 +1,35 @@ +--- +applications: + - name: fecfile-api-migrator + instances: 1 + stack: cflinuxfs4 + buildpacks: + - python_buildpack + command: bin/run-api.sh + memory: 2G + services: + - fecfile-api-rds + - fecfile-api-s3 + - fecfile-api-redis + - fecfile-api-creds-stage + env: + DISABLE_COLLECTSTATIC: 1 + DJANGO_SETTINGS_MODULE: fecfiler.settings.production + FFAPI_COOKIE_DOMAIN: fecfile.fec.gov + LOGIN_REDIRECT_CLIENT_URL: https://stage.fecfile.fec.gov + LOGIN_REDIRECT_SERVER_URL: https://stage-api.fecfile.fec.gov/api/v1/oidc/login-redirect + LOGOUT_REDIRECT_URL: https://stage-api.fecfile.fec.gov/api/v1/oidc/logout-redirect + PRODUCTION_OPEN_FEC_API: https://api.open.fec.gov/v1/ + STAGE_OPEN_FEC_API: https://api-stage.open.fec.gov/v1/ + FEC_FILING_API: https://efoservices.stage.efo.fec.gov + SESSION_COOKIE_AGE: 64800 + BP_PIP_VERSION: latest + LOG_FORMAT: KEY_VALUE + SYSTEM_STATUS_CACHE_AGE: 60 + INITIAL_POLLING_INTERVAL: 10 + INITIAL_POLLING_DURATION: 900 + SECONDARY_POLLING_INTERVAL: 3600 + SECONDARY_POLLING_DURATION: 86400 + # ---- FEATURE FLAGS ---- + FLAG__COMMITTEE_DATA_SOURCE: TEST # Values are TEST and PRODUCTION + ENABLE_DEVELOPER_COMMANDS: True diff --git a/manifests/manifest-test-migrator.yml b/manifests/manifest-test-migrator.yml new file mode 100644 index 000000000..220eb6668 --- /dev/null +++ b/manifests/manifest-test-migrator.yml @@ -0,0 +1,35 @@ +--- +applications: + - name: fecfile-api-migrator + instances: 1 + stack: cflinuxfs4 + buildpacks: + - python_buildpack + command: bin/run-api.sh + memory: 2G + services: + - fecfile-api-rds + - fecfile-api-s3 + - fecfile-api-redis + - fecfile-api-creds-test + env: + DISABLE_COLLECTSTATIC: 1 + DJANGO_SETTINGS_MODULE: fecfiler.settings.production + FFAPI_COOKIE_DOMAIN: fecfile.fec.gov + LOGIN_REDIRECT_CLIENT_URL: https://test.fecfile.fec.gov + LOGIN_REDIRECT_SERVER_URL: https://test-api.fecfile.fec.gov/api/v1/oidc/login-redirect + LOGOUT_REDIRECT_URL: https://test-api.fecfile.fec.gov/api/v1/oidc/logout-redirect + PRODUCTION_OPEN_FEC_API: https://api.open.fec.gov/v1/ + STAGE_OPEN_FEC_API: https://api-stage.open.fec.gov/v1/ + FEC_FILING_API: https://efoservices.stage.efo.fec.gov + SESSION_COOKIE_AGE: 64800 + BP_PIP_VERSION: latest + LOG_FORMAT: KEY_VALUE + SYSTEM_STATUS_CACHE_AGE: 60 + INITIAL_POLLING_INTERVAL: 10 + INITIAL_POLLING_DURATION: 900 + SECONDARY_POLLING_INTERVAL: 3600 + SECONDARY_POLLING_DURATION: 86400 + # ---- FEATURE FLAGS ---- + FLAG__COMMITTEE_DATA_SOURCE: TEST # Values are PRODUCTION, TEST, and MOCKED + ENABLE_DEVELOPER_COMMANDS: True diff --git a/tasks.py b/tasks.py index 4eea678eb..085fd7e7f 100644 --- a/tasks.py +++ b/tasks.py @@ -9,6 +9,7 @@ env = cfenv.AppEnv() APP_NAME = "fecfile-web-api" +MIGRATOR_APP_NAME = 'fecfile-api-migrator' # THE APP WITH THIS NAME WILL GET DELETED! WEB_SERVICES_NAME = "fecfile-web-services" PROXY_NAME = "fecfile-api-proxy" ORG_NAME = "fec-fecfileonline-prototyping" @@ -89,7 +90,6 @@ def _login_to_cf(ctx, space): def _do_deploy(ctx, space, app): - manifest_filename = f"manifests/manifest-{space}-{MANIFEST_LABEL.get(app)}.yml" existing_deploy = ctx.run(f"cf app {app}", echo=True, warn=True) print("\n") @@ -151,6 +151,61 @@ def _rollback(ctx, app): print("Unable to cancel deploy. Check logs.") +def _delete_migrator_app(ctx, space): + print("Deleting migrator app...") + + existing_migrator_app = ctx.run(f"cf app {MIGRATOR_APP_NAME}", echo=True, warn=True) + if not existing_migrator_app.ok: + print("No migrator app detected. There is nothing to delete.") + return True + + if MIGRATOR_APP_NAME == APP_NAME: + print(f"Possible error: could result in deleting main app - {APP_NAME}") + print("Canceling migrator app deletion attempt.") + return False + + delete_app = ctx.run( + f"cf delete {MIGRATOR_APP_NAME} -f", + echo=True, + warn=True + ) + if not delete_app.ok: + print('Failed to delete migrator app.') + print(f'Stray migrator app remains on {space}: "{MIGRATOR_APP_NAME}"') + return False + print("Migrator app deleted successfully.") + return True + + +def _run_migrations(ctx, space): + print("Running migrations...") + + # Start migrator app + manifest_filename = f"manifests/manifest-{space}-migrator.yml" + migrator = ctx.run( + f"cf push {MIGRATOR_APP_NAME} -f {manifest_filename}", + echo=True, + warn=True, + ) + if not migrator.ok: + print("Failed to spin up migrator app. Check logs.") + return False + + # Run migrations + task = 'django-backend/manage.py migrate --no-input --traceback --verbosity 3' + migrations = ctx.run( + f"cf rt {MIGRATOR_APP_NAME} --command '{task}' --name 'Run Migrations' --wait", + echo=True, + warn=True, + ) + if not migrations.ok: + print("Failed to run migrations. Check logs.") + return False + + print("Migration process has finished successfully.") + return True + + @task def deploy(ctx, space=None, branch=None, login=False, help=False): """Deploy app to Cloud Foundry. @@ -187,6 +242,16 @@ def deploy(ctx, space=None, branch=None, login=False, help=False): with open(".cfmeta", "w") as fp: json.dump({"user": os.getenv("USER"), "branch": branch}, fp) + # Runs migrations + # tasks.py does not continue until the migrations task has completed + migrations_successful = _run_migrations(ctx, space) + migrator_app_deleted = _delete_migrator_app(ctx, space) + + if not (migrations_successful and migrator_app_deleted): + print("Migrations failed and/or the migrator app was not deleted successfully.") + print("See the logs for more information.\nCanceling deploy...") + sys.exit(1) + for app in [APP_NAME, WEB_SERVICES_NAME]: new_deploy = _do_deploy(ctx, space, app)