Merge pull request #2026 from GSA/main

10/16/24 production deploy

.ds.baseline

@@ -133,7 +133,7 @@
         "filename": ".github/workflows/checks.yml",
         "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
         "is_verified": false,
-        "line_number": 66,
+        "line_number": 68,
         "is_secret": false
       }
     ],
@@ -684,5 +684,5 @@
       }
     ]
   },
-  "generated_at": "2024-09-03T17:36:57Z"
+  "generated_at": "2024-10-07T20:44:40Z"
 }

.github/actions/deploy-proxy/action.yml

@@ -1,6 +1,9 @@
 name: Deploy egress proxy
 description: Set egress space security groups and deploy proxy
 inputs:
+  cf_org:
+    description: The org the target app exists in.
+    required: true
   cf_space:
     description: The space the target app exists in.
     required: true
@@ -16,6 +19,19 @@ inputs:
 runs:
   using: composite
   steps:
+    - name: Install cf-cli
+      shell: bash
+      run: |
+        curl -A "cg-deploy-action" -v -L -o cf-cli_amd64.deb 'https://packages.cloudfoundry.org/stable?release=debian64&version=v8&source=github'
+        sudo dpkg -i cf-cli_amd64.deb
+    - name: Login to cf-cli
+      shell: bash
+      run: |
+        cf api api.fr.cloud.gov
+        cf auth
+    - name: Target org and space
+      shell: bash
+      run: cf target -o ${{ inputs.cf_org }} -s ${{ inputs.cf_space }}
     - name: Set restricted space egress
       shell: bash
       run: ./terraform/set_space_egress.sh -t -s ${{ inputs.cf_space }}

.github/workflows/checks.yml

@@ -53,11 +53,13 @@ jobs:
 
   end-to-end-tests:
     if: ${{ github.actor != 'dependabot[bot]' }}
+
     permissions:
       checks: write
       pull-requests: write
       contents: write
     runs-on: ubuntu-latest
+    environment: staging
     services:
       postgres:
         image: postgres
@@ -97,21 +99,25 @@ jobs:
         # insert this line:
         # tail -f admin-server.log &
         # above make e2e-test
+
+
         run: |
           make run-flask > admin-server.log 2>&1 &
+          tail -f admin-server.log &
           make e2e-test
+
         env:
           API_HOST_NAME: https://notify-api-staging.app.cloud.gov/
-          DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }}
           SECRET_KEY: ${{ secrets.SECRET_KEY }}
+          DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }}
           ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }}
           ADMIN_CLIENT_USERNAME: notify-admin
           NOTIFY_ENVIRONMENT: e2etest
           NOTIFY_E2E_AUTH_STATE_PATH: ${{ secrets.NOTIFY_E2E_AUTH_STATE_PATH }}
           NOTIFY_E2E_TEST_EMAIL: ${{ secrets.NOTIFY_E2E_TEST_EMAIL }}
           NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }}
           NOTIFY_E2E_TEST_URI: http://localhost:6012/
-
+          VCAP_SERVICES: ${{ secrets.VCAP_SERVICES }}
   validate-new-relic-config:
     runs-on: ubuntu-latest
     environment: staging

.github/workflows/deploy-demo.yml

@@ -47,7 +47,7 @@ jobs:
         run: poetry export --without-hashes --format=requirements.txt > requirements.txt
 
       - name: Deploy to cloud.gov
-        uses: 18f/cg-deploy-action@main
+        uses: cloud-gov/cg-cli-tools@main
         env:
           DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }}
           SECRET_KEY: ${{ secrets.SECRET_KEY }}
@@ -63,12 +63,14 @@ jobs:
           LOGIN_DOT_GOV_BASE_LOGOUT_URL: "https://secure.login.gov/openid_connect/logout?"
           LOGIN_DOT_GOV_SIGNOUT_REDIRECT: "https://notify-demo.app.cloud.gov/sign-out"
           LOGIN_DOT_GOV_INITIAL_SIGNIN_URL: "https://secure.login.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=NONCE&prompt=select_account&redirect_uri=https://notify-demo.app.cloud.gov/sign-in&response_type=code&scope=openid+email&state=STATE"
+          LOGIN_DOT_GOV_CERTS_URL: "https://secure.login.gov/api/openid_connect/certs"
         with:
           cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
           cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
           cf_org: gsa-tts-benefits-studio
           cf_space: notify-demo
           push_arguments: >-
+            push -f manifest.yml
             --vars-file deploy-config/demo.yml
             --var DANGEROUS_SALT="$DANGEROUS_SALT"
             --var SECRET_KEY="$SECRET_KEY"
@@ -85,6 +87,8 @@ jobs:
             --var LOGIN_DOT_GOV_BASE_LOGOUT_URL="$LOGIN_DOT_GOV_BASE_LOGOUT_URL"
             --var LOGIN_DOT_GOV_SIGNOUT_REDIRECT="$LOGIN_DOT_GOV_SIGNOUT_REDIRECT"
             --var LOGIN_DOT_GOV_INITIAL_SIGNIN_URL="$LOGIN_DOT_GOV_INITIAL_SIGNIN_URL"
+            --var LOGIN_DOT_GOV_CERTS_URL="$LOGIN_DOT_GOV_CERTS_URL"
+            --strategy rolling
 
       - name: Check for changes to egress config
         id: changed-egress-config

.github/workflows/deploy-prod.yml

@@ -47,7 +47,7 @@ jobs:
         run: poetry export --without-hashes --format=requirements.txt > requirements.txt
 
       - name: Deploy to cloud.gov
-        uses: 18f/cg-deploy-action@main
+        uses: cloud-gov/cg-cli-tools@main
         env:
           DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }}
           SECRET_KEY: ${{ secrets.SECRET_KEY }}
@@ -63,12 +63,14 @@ jobs:
           LOGIN_DOT_GOV_BASE_LOGOUT_URL: "https://secure.login.gov/openid_connect/logout?"
           LOGIN_DOT_GOV_SIGNOUT_REDIRECT: "https://beta.notify.gov/sign-out"
           LOGIN_DOT_GOV_INITIAL_SIGNIN_URL: "https://secure.login.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=NONCE&prompt=select_account&redirect_uri=https://beta.notify.gov/sign-in&response_type=code&scope=openid+email&state=STATE"
+          LOGIN_DOT_GOV_CERTS_URL: "https://secure.login.gov/api/openid_connect/certs"
         with:
           cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
           cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
           cf_org: gsa-tts-benefits-studio
           cf_space: notify-production
           push_arguments: >-
+            push -f manifest.yml
             --vars-file deploy-config/production.yml
             --var DANGEROUS_SALT="$DANGEROUS_SALT"
             --var SECRET_KEY="$SECRET_KEY"
@@ -85,6 +87,8 @@ jobs:
             --var LOGIN_DOT_GOV_BASE_LOGOUT_URL="$LOGIN_DOT_GOV_BASE_LOGOUT_URL"
             --var LOGIN_DOT_GOV_SIGNOUT_REDIRECT="$LOGIN_DOT_GOV_SIGNOUT_REDIRECT"
             --var LOGIN_DOT_GOV_INITIAL_SIGNIN_URL="$LOGIN_DOT_GOV_INITIAL_SIGNIN_URL"
+            --var LOGIN_DOT_GOV_CERTS_URL="$LOGIN_DOT_GOV_CERTS_URL"
+            --strategy rolling
 
       - name: Check for changes to egress config
         id: changed-egress-config

.github/workflows/deploy.yml

@@ -53,11 +53,12 @@ jobs:
 
 
     - name: Deploy to cloud.gov
-      uses: 18f/cg-deploy-action@main
+      uses: cloud-gov/cg-cli-tools@main
       env:
         DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }}
         SECRET_KEY: ${{ secrets.SECRET_KEY }}
         ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }}
+        ADMIN_CLIENT_USERNAME: "notify-admin"
         NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
         NR_BROWSER_KEY: ${{ secrets.NR_BROWSER_KEY }}
         COMMIT_HASH: ${{ github.sha }}
@@ -69,28 +70,32 @@ jobs:
         LOGIN_DOT_GOV_BASE_LOGOUT_URL: "https://secure.login.gov/openid_connect/logout?"
         LOGIN_DOT_GOV_SIGNOUT_REDIRECT: "https://notify-staging.app.cloud.gov/sign-out"
         LOGIN_DOT_GOV_INITIAL_SIGNIN_URL: "https://secure.login.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=NONCE&prompt=select_account&redirect_uri=https://notify-staging.app.cloud.gov/sign-in&response_type=code&scope=openid+email&state=STATEE"
+        LOGIN_DOT_GOV_CERTS_URL: "https://secure.login.gov/api/openid_connect/certs"
       with:
         cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
         cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
         cf_org: gsa-tts-benefits-studio
         cf_space: notify-staging
-        push_arguments: >-
+        cf_command: >-
+          push -f manifest.yml
           --vars-file deploy-config/staging.yml
           --var DANGEROUS_SALT="$DANGEROUS_SALT"
           --var SECRET_KEY="$SECRET_KEY"
-          --var ADMIN_CLIENT_USERNAME="notify-admin"
           --var ADMIN_CLIENT_SECRET="$ADMIN_CLIENT_SECRET"
+          --var ADMIN_CLIENT_USERNAME="$ADMIN_CLIENT_USERNAME"
           --var NEW_RELIC_LICENSE_KEY="$NEW_RELIC_LICENSE_KEY"
           --var NR_BROWSER_KEY="$NR_BROWSER_KEY"
           --var COMMIT_HASH="$COMMIT_HASH"
-          --var LOGIN_PEM="$LOGIN_PEM"
           --var LOGIN_DOT_GOV_CLIENT_ID="$LOGIN_DOT_GOV_CLIENT_ID"
           --var LOGIN_DOT_GOV_USER_INFO_URL="$LOGIN_DOT_GOV_USER_INFO_URL"
           --var LOGIN_DOT_GOV_ACCESS_TOKEN_URL="$LOGIN_DOT_GOV_ACCESS_TOKEN_URL"
           --var LOGIN_DOT_GOV_LOGOUT_URL="$LOGIN_DOT_GOV_LOGOUT_URL"
           --var LOGIN_DOT_GOV_BASE_LOGOUT_URL="$LOGIN_DOT_GOV_BASE_LOGOUT_URL"
           --var LOGIN_DOT_GOV_SIGNOUT_REDIRECT="$LOGIN_DOT_GOV_SIGNOUT_REDIRECT"
           --var LOGIN_DOT_GOV_INITIAL_SIGNIN_URL="$LOGIN_DOT_GOV_INITIAL_SIGNIN_URL"
+          --var LOGIN_DOT_GOV_CERTS_URL="$LOGIN_DOT_GOV_CERTS_URL"
+          --var LOGIN_PEM="$LOGIN_PEM"
+          --strategy rolling
 
 
     - name: Check for changes to egress config
@@ -102,9 +107,13 @@ jobs:
           .github/actions/deploy-proxy/action.yml
           .github/workflows/deploy.yml
     - name: Deploy egress proxy
-      if: steps.changed-egress-config.outputs.any_changed == 'true'
+      #if: steps.changed-egress-config.outputs.any_changed == 'true'
       uses: ./.github/actions/deploy-proxy
+      env:
+          CF_USERNAME: ${{ secrets.CLOUDGOV_USERNAME }}
+          CF_PASSWORD: ${{ secrets.CLOUDGOV_PASSWORD }}
       with:
+        cf_org: gsa-tts-benefits-studio
         cf_space: notify-staging
         app: notify-admin-staging
 

.gitignore

@@ -134,3 +134,8 @@ playwright/
 
 # Nodenv
 .node-version
+
+# BackstopJS
+backstop_data/bitmaps_test/
+backstop_data/html_report/
+backstop_data/engine_scripts/playwright/

.prettierrc

@@ -0,0 +1,3 @@
+{
+  "singleQuote": true
+}

README.md

@@ -507,3 +507,13 @@ insurance.
 For more information on what we're working on, the Notify tool, and how to get
 involved with our team,
 [see our flyer.](https://github.com/GSA/notifications-admin/blob/main/docs/notify-pilot-flyer.md)
+
+## Updating secrets for the E2E tests
+
+At some point, E2E tests will fail because the secrets held in VCAP_SERVICES have expired.  To refresh
+them, you will need to do the following:
+
+1. Log in the normal way to access cloudfoundry command line options
+2. In your terminal, run `chmod +x print_vcap.sh`
+3. In your terminal, run `./print_vcap.sh`
+4. Copy the value in your terminal and paste it into the VCAP_SERVICES secret in Github on the staging tier.

app/assets/error_pages/5xx.html

@@ -68,11 +68,8 @@ <h1 class="usa-heading-l">
           <p class="usa-body">
             Try again later.
           </p>
-          <p class="usa-body">
-            You can check our <a class="usa-link"
-              href="https://status.notifications.service.gov.uk/">system status</a> page to see if there are any known
-            issues.<br />To report a problem, email <a
-              href="mailto:[email protected]">[email protected]</a>
+          <p>To report a problem you can email us at <a class="usa-link" href="mailto:[email protected]">[email protected]</a>.</p>
+          <p>You can expect a response within one business day.</p>
         </div>
       </div>
     </main>

app/content/get-started.md

@@ -14,7 +14,7 @@ Learn how to [personalize messages](/using-notify/guidance) to increase response
 Learn about message _parts_ and [how limits are calculated](/using-notify/pricing).
 
 5. ## Start sending messages
-To remove the restrictions of Trial Mode and begin sending messages to clients complete the [Live Campaign Form](https://airtable.com/appe4n7jYOALPLcyU/shrIPWnLTw9U1fclL).
+To remove the restrictions of Trial Mode and begin sending messages to clients complete the <a class="usa-link usa-link--external" href="https://docs.google.com/forms/d/1fnaBtxuGf3q-OdGVyt2LqBKvp9_P21kmKJa0yIK8rWM/edit">Go-Live Form</a>.
 We’ll respond within one business day.
 
 ### Questions?

app/main/views/index.py

@@ -1,6 +1,15 @@
 import os
-
-from flask import abort, current_app, redirect, render_template, request, url_for
+import secrets
+
+from flask import (
+    abort,
+    current_app,
+    redirect,
+    render_template,
+    request,
+    session,
+    url_for,
+)
 from flask_login import current_user
 
 from app import status_api_client
@@ -23,8 +32,12 @@ def index():
     )
     url = os.getenv("LOGIN_DOT_GOV_INITIAL_SIGNIN_URL")
     # handle unit tests
+
+    nonce = secrets.token_urlsafe()
+    session["nonce"] = nonce
+
     if url is not None:
-        url = url.replace("NONCE", token)
+        url = url.replace("NONCE", nonce)
         url = url.replace("STATE", token)
     return render_template(
         "views/signedout.html",

app/main/views/send.py

@@ -802,9 +802,15 @@ def get_skip_link(step_index, template):
 )
 @user_has_permissions("send_messages", restrict_admin_usage=True)
 def send_one_off_to_myself(service_id, template_id):
-    db_template = current_service.get_template_with_user_permission_or_403(
-        template_id, current_user
-    )
+    current_app.logger.info("Send one off to myself")
+    try:
+        db_template = current_service.get_template_with_user_permission_or_403(
+            template_id, current_user
+        )
+    except Exception:
+        current_app.logger.exception("Couldnt get template for one off")
+        # Use 406 just because we're limited to certain codes here and it will point us back to a problem here
+        abort(406)
 
     if db_template["template_type"] not in ("sms", "email"):
         abort(404)
@@ -1009,9 +1015,16 @@ def _send_notification(service_id, template_id):
 
     keys = []
     values = []
+    # Guarantee that the real phone number comes last, because some
+    # users will have placeholders like "add your second phone number"
+    # or something like as custom placeholders.
     for k, v in session["placeholders"].items():
-        keys.append(k)
-        values.append(v)
+        if k != "phone number":
+            keys.append(k)
+            values.append(v)
+    if "phone number" in session["placeholders"].keys():
+        keys.append("phone number")
+        values.append(session["placeholders"]["phone number"])
 
     data = ",".join(keys)
     vals = ",".join(values)
@@ -1027,7 +1040,7 @@ def _send_notification(service_id, template_id):
     # on the API side to find out what happens to the message.
     current_app.logger.info(
         hilite(
-            f"One-off file: {filename} job_id: {upload_id} s3 location: service-{service_id}-notify/{upload_id}.csv"
+            f"One-off file: {filename} job_id: {upload_id} s3 location: {service_id}-service-notify/{upload_id}.csv"
         )
     )