Merge pull request #1765 from GSA/main

7/24/2024 Production Deploy

.ds.baseline

@@ -133,15 +133,15 @@
         "filename": ".github/workflows/checks.yml",
         "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
         "is_verified": false,
-        "line_number": 61,
+        "line_number": 67,
         "is_secret": false
       },
       {
         "type": "Basic Auth Credentials",
         "filename": ".github/workflows/checks.yml",
         "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
         "is_verified": false,
-        "line_number": 95,
+        "line_number": 101,
         "is_secret": false
       }
     ],

.github/workflows/checks.yml

@@ -23,6 +23,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "22.3.0"
+      - name: Install dependencies
+        run: npm install
       - uses: ./.github/actions/setup-project
       - uses: jwalton/gh-find-current-pr@v1
         id: findPr
@@ -86,10 +92,10 @@ jobs:
       - name: Clone API
         uses: actions/checkout@v4
         with:
-         repository: GSA/notifications-api
-         path: 'notifications-api'
+          repository: GSA/notifications-api
+          path: "notifications-api"
       - name: Install API dependencies
-        working-directory: 'notifications-api'
+        working-directory: "notifications-api"
         run: make bootstrap
         env:
           DATABASE_URL: postgresql://user:password@localhost:5432/test_notification_api
@@ -99,7 +105,7 @@ jobs:
           NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }}
           NOTIFY_ENVIRONMENT: development
       - name: Run API server
-        working-directory: 'notifications-api'
+        working-directory: "notifications-api"
         run: make run-procfile &
         env:
           DATABASE_URL: postgresql://user:password@localhost:5432/test_notification_api
@@ -185,12 +191,12 @@ jobs:
       - name: Run OWASP Baseline Scan
         uses: zaproxy/[email protected]
         with:
-          docker_name: 'ghcr.io/zaproxy/zaproxy:weekly'
-          target: 'http://localhost:6012'
+          docker_name: "ghcr.io/zaproxy/zaproxy:weekly"
+          target: "http://localhost:6012"
           fail_action: true
           allow_issue_writing: false
-          rules_file_name: 'zap.conf'
-          cmd_options: '-I'
+          rules_file_name: "zap.conf"
+          cmd_options: "-I"
 
   a11y-scan:
     runs-on: ubuntu-20.04

.gitignore

@@ -123,6 +123,8 @@ app/templates/vendor
 secrets.auto.tfvars
 terraform.tfstate
 terraform.tfstate.backup
+requirements.txt
+terraform/sandbox/requirements.txt
 
 # Playwright
 playwright/

app/__init__.py

@@ -368,18 +368,31 @@ def make_session_permanent():
 
 
 def create_beta_url(url):
-    url_created = urlparse(url)
-    url_list = list(url_created)
-    url_list[1] = "beta.notify.gov"
-    url_for_redirect = urlunparse(url_list)
-    return url_for_redirect
+    url_created = None
+    try:
+        url_created = urlparse(url)
+        url_list = list(url_created)
+        url_list[1] = "beta.notify.gov"
+        url_for_redirect = urlunparse(url_list)
+        return url_for_redirect
+    except ValueError:
+        # This might be happening due to IPv6, see issue # 1395.
+        # If we see "'RequestContext' object has no attribute 'service'" in the logs
+        # we can search around that timestamp and find this output, hopefully.
+        # It may be sufficient to just catch and log, and prevent the stack trace from being in the logs
+        # but we need to confirm the root cause first.
+        current_app.logger.error(
+            f"create_beta_url orig_url: {url} \
+                                 url_created = {str(url_created)} url_for_redirect {str(url_for_redirect)}"
+        )
 
 
 def redirect_notify_to_beta():
     if (
         current_app.config["NOTIFY_ENVIRONMENT"] == "production"
         and "beta.notify.gov" not in request.url
     ):
+        # TODO add debug here to trace what is going on with the URL for the 'RequestContext' error
         url_to_beta = create_beta_url(request.url)
         return redirect(url_to_beta, 302)
 

app/main/forms.py

@@ -50,6 +50,7 @@
     CommonlyUsedPassword,
     CsvFileValidator,
     DoesNotStartWithDoubleZero,
+    FieldCannotContainComma,
     LettersNumbersSingleQuotesFullStopsAndUnderscoresOnly,
     MustContainAlphanumericCharacters,
     NoCommasInPlaceHolders,
@@ -1650,7 +1651,11 @@ def get_placeholder_form_instance(
             )  # TODO: replace with us_mobile_number
     else:
         field = GovukTextInputField(
-            placeholder_name, validators=[DataRequired(message="Cannot be empty")]
+            placeholder_name,
+            validators=[
+                DataRequired(message="Cannot be empty"),
+                FieldCannotContainComma(),
+            ],
         )
 
     PlaceholderForm.placeholder_value = field

app/main/validators.py

@@ -161,6 +161,15 @@ def __call__(self, form, field):
             raise ValidationError(self.message)
 
 
+class FieldCannotContainComma:
+    def __init__(self, message="Cannot contain a comma"):
+        self.message = message
+
+    def __call__(self, form, field):
+        if field.data and "," in field.data:
+            raise ValidationError(self.message)
+
+
 class MustContainAlphanumericCharacters:
     regex = re.compile(r".*[a-zA-Z0-9].*[a-zA-Z0-9].*")
 

app/main/views/dashboard.py

@@ -1,5 +1,4 @@
 import calendar
-from collections import defaultdict
 from datetime import datetime
 from functools import partial
 from itertools import groupby
@@ -13,7 +12,6 @@
     billing_api_client,
     current_service,
     job_api_client,
-    notification_api_client,
     service_api_client,
     socketio,
     template_statistics_client,
@@ -83,18 +81,9 @@ def service_dashboard(service_id):
         return redirect(url_for("main.choose_template", service_id=service_id))
 
     job_response = job_api_client.get_jobs(service_id)["data"]
-    notifications_response = notification_api_client.get_notifications_for_service(
-        service_id
-    )["notifications"]
     service_data_retention_days = 7
 
-    aggregate_notifications_by_job = defaultdict(list)
-    for notification in notifications_response:
-        job_id = notification.get("job", {}).get("id", None)
-        if job_id:
-            aggregate_notifications_by_job[job_id].append(notification)
-
-    job_and_notifications = [
+    jobs = [
         {
             "job_id": job["id"],
             "time_left": get_time_left(job["created_at"]),
@@ -105,18 +94,20 @@ def service_dashboard(service_id):
                 ".view_job", service_id=current_service.id, job_id=job["id"]
             ),
             "created_at": job["created_at"],
+            "processing_finished": job.get("processing_finished"),
+            "processing_started": job.get("processing_started"),
             "notification_count": job["notification_count"],
             "created_by": job["created_by"],
-            "notifications": aggregate_notifications_by_job.get(job["id"], []),
+            "template_name": job["template_name"],
+            "original_file_name": job["original_file_name"],
         }
         for job in job_response
-        if aggregate_notifications_by_job.get(job["id"], [])
     ]
     return render_template(
         "views/dashboard/dashboard.html",
         updates_url=url_for(".service_dashboard_updates", service_id=service_id),
         partials=get_dashboard_partials(service_id),
-        job_and_notifications=job_and_notifications,
+        jobs=jobs,
         service_data_retention_days=service_data_retention_days,
     )
 

app/main/views/send.py

@@ -1053,8 +1053,13 @@ def get_email_reply_to_address_from_session():
 
 
 def get_sms_sender_from_session():
-    if session.get("sender_id"):
-        return current_service.get_sms_sender(session["sender_id"])["sms_sender"]
+    sender_id = session.get("sender_id")
+    if sender_id:
+        sms_sender = current_service.get_sms_sender(session["sender_id"])["sms_sender"]
+        current_app.logger.info(f"SMS Sender ({sender_id}) #: {sms_sender}")
+        return sms_sender
+    else:
+        current_app.logger.error("No SMS Sender!!!!!!")
 
 
 def get_spreadsheet_column_headings_from_template(template):

app/models/user.py

@@ -24,11 +24,15 @@
 
 
 def _get_service_id_from_view_args():
-    return str(request.view_args.get("service_id", "")) or None
+    if request and request.view_args:
+        return str(request.view_args.get("service_id", ""))
+    return None
 
 
 def _get_org_id_from_view_args():
-    return str(request.view_args.get("org_id", "")) or None
+    if request and request.view_args:
+        return str(request.view_args.get("org_id", ""))
+    return None
 
 
 class User(JSONModel, UserMixin):
@@ -227,7 +231,9 @@ def has_permissions(
         if not service_id and not org_id:
             # we shouldn't have any pages that require permissions, but don't specify a service or organization.
             # use @user_is_platform_admin for platform admin only pages
-            raise NotImplementedError
+            # raise NotImplementedError
+            current_app.logger.warn(f"VIEW ARGS ARE {request.view_args}")
+            pass
 
         log_msg = f"has_permissions user: {self.id} service: {service_id}"
         # platform admins should be able to do most things (except eg send messages, or create api keys)

app/notify_client/__init__.py

@@ -58,6 +58,10 @@ def check_inactive_service(self):
 
     def check_inactive_user(self, *args):
         still_signing_in = False
+
+        # TODO clean up and add testing etc.
+        # We really should be checking for exact matches
+        # and we only want to check the first arg
         for arg in args:
             arg = str(arg)
             if (
@@ -66,6 +70,7 @@ def check_inactive_user(self, *args):
                 or "/activate" in arg
                 or "/email-code" in arg
                 or "/verify/code" in arg
+                or "/user" in arg
             ):
                 still_signing_in = True
 

app/templates/new/components/footer.html

@@ -58,6 +58,20 @@ <h2 class="usa-sr-only">Support links</h2>
         </ul>
       </div>
     </nav>
+    <section class="usa-identifier__section usa-identifier__section--usagov"
+      aria-label="Github Repos">
+      <div class="usa-identifier__container">
+        <div class="usa-identifier__required-links-item">
+          Find us on Github:
+        </div>
+        <ul>
+          <li><a href="https://github.com/gsa/notifications-admin" class="usa-identifier__required-link">Notify.gov Admin repo</a></li>
+        <li><a href="https://github.com/gsa/notifications-api" class="usa-identifier__required-link">Notify.gov API repo</a></li>
+      </ul>
+
+      </div>
+    </section>
+
     <section class="usa-identifier__section usa-identifier__section--usagov"
       aria-label="Government information and services">
       <div class="usa-identifier__container">

app/templates/new/components/header.html

@@ -14,10 +14,17 @@
   {% endif %}
 
   {% if current_service %}
-    {% set secondaryNavigation = [
-        {"href": url_for('main.service_settings', service_id=current_service.id), "text": "Settings", "active": secondary_navigation.is_selected('settings')},
+    {% if current_user.has_permissions('manage_service') %}
+      {% set secondaryNavigation = [
+          {"href": url_for('main.service_settings', service_id=current_service.id), "text": "Settings", "active": secondary_navigation.is_selected('settings')},
+          {"href": url_for('main.sign_out'), "text": "Sign out"}
+      ] %}
+    {% else %}
+      {% set secondaryNavigation = [
         {"href": url_for('main.sign_out'), "text": "Sign out"}
-    ] %}
+      ] %}
+
+    {% endif %}
   {% else %}
     {% set secondaryNavigation = [{"href": url_for('main.sign_out'), "text": "Sign out"}] %}
   {% endif %}
@@ -31,7 +38,7 @@
         <div class="logo-img display-flex">
           <a href="/">
             <span class="usa-sr-only">Notify.gov logo</span>
-            <img src="{{ (asset_path | default('/static')) + 'images/notify-logo.png' }}" alt="Notify.gov logo" class="usa-flag-logo margin-right-1">
+            <img src="{{ (asset_path | default('/static')) + 'images/notify-logo.svg' }}" alt="Notify.gov logo" class="usa-flag-logo margin-right-1">
 
           </a>
         </div>

app/templates/partials/jobs/status.html

@@ -37,7 +37,9 @@ <h4 class="usa-alert__heading">Your text has been scheduled</h4>
           <div class="usa-alert__body">
             <h4 class="usa-alert__heading">Your text has been sent</h4>
             <p class="usa-alert__text">
-              {{ job.template_name }} - {{ current_service.name }} was sent on {{ job.created_at|format_datetime_normal }} by {{ job.created_by.name }}
+              {{ job.template_name }} - {{ current_service.name }} was sent on {% if job.processing_started %}
+              {{ job.processing_started|format_datetime_table }} {% else %}
+              {{ job.created_at|format_datetime_table }} {% endif %} by {{ job.created_by.name }}
             </p>
           </div>
         </div>

app/templates/views/dashboard/dashboard.html

@@ -41,7 +41,7 @@ <h2 class="margin-top-4 margin-bottom-1">Recent Batches</h2>
               <span>Template</span>
             </th>
             <th scope="col" class="table-field-heading">
-              <span>Time sent</span>
+              <span>Job status</span>
             </th>
             <th scope="col" class="table-field-heading">
               <span>Sender</span>
@@ -55,38 +55,37 @@ <h2 class="margin-top-4 margin-bottom-1">Recent Batches</h2>
           </tr>
         </thead>
         <tbody>
-          {% if job_and_notifications %}
-            {% for job in job_and_notifications[:5] %}
-              {% if job.job_id and job.notifications %}
+          {% if jobs %}
+            {% for job in jobs[:5] %}
                 {% set notification = job.notifications[0] %}
                 <tr class="table-row" id="{{ job.job_id }}">
                   <td class="table-field file-name">
-                    {{ notification.job.original_file_name[:12] if notification.job.original_file_name else 'Manually entered number'}}
+                    {{ job.original_file_name[:12] if job.original_file_name else 'Manually entered number'}}
                     <br>
                     <a class="usa-link file-list-filename" href="{{ job.view_job_link }}">View Batch</a>
                   </td>
                   <td class="table-field template">
-                    {{ notification.template.name }}
+                    {{ job.template_name }}
                   </td>
                   <td class="table-field time-sent">
-                    {{ job.created_at | format_datetime_table }}
+                    {{ (job.processing_finished if job.processing_finished else job.processing_started
+                      if job.processing_started else job.created_at)|format_datetime_table }}
                   </td>
                   <td class="table-field sender">
-                    {{ notification.created_by.name }}
+                    {{ job.created_by.name }}
                   </td>
                   <td class="table-field count-of-recipients">
                     {{ job.notification_count}}
                   </td>
                   <td class="table-field report">
-                    {% if notification and job.time_left != "Data no longer available" %}
+                    {% if job.time_left != "Data no longer available" %}
                       <a class="usa-link file-list-filename" href="{{ job.download_link }}">Download</a>
                       <span class="usa-hint">{{ job.time_left }}</span>
                     {% elif job %}
                       <span>{{ job.time_left }}</span>
                     {% endif %}
                   </td>
                 </tr>
-              {% endif %}
             {% endfor %}
           {% else %}
             <tr class="table-row">