[DSEC-936] more zap logging; use google cloud logging

.github/workflows/docs.yml

@@ -4,13 +4,14 @@
 name: Deploy docs to Github Pages
 
 on:
-  push:
-   branches:
-     - master
-  pull_request:
+  # push:
+  #  branches:
+  #    - master
+  # pull_request:
+  workflow_dispatch:
 
 jobs:
-  deploy:
+  deploy-docs:
     runs-on: ubuntu-20.04
     defaults:
       run:

.github/workflows/trivy.yml

@@ -1,8 +1,10 @@
 name: dsp-appsec-trivy
 on:
-  pull_request:
-  # schedule:
-  # - cron: '0 14 * * 1' # each Monday at 9am EST
+  # This workflow fails if run too frequently, due to rate limiting.
+  #pull_request:
+  schedule:
+  - cron: '0 14 * * 1' # each Monday at 9am EST
+  workflow_dispatch:
 
 jobs:
   appsec-trivy:

zap/requirements.txt

@@ -1,6 +1,7 @@
 aiohttp==3.10.2
 codedx-api @ git+https://github.com/broadinstitute/dsp-appsec-codedx-api-client-python.git@cf0de6fcdd0a4db98f2c2a35bd9d2ae7e8056d6b
 google-auth==1.30.0
+google-cloud-logging==2.7.2
 google-cloud-pubsub==2.5.0
 google-cloud-storage==1.38.0
 requests==2.32.0

zap/src/scan.py

@@ -14,14 +14,15 @@
 from typing import List
 from urllib.parse import urlparse, urlunparse
 
+import defectdojo_apiv2 as defectdojo
 import defusedxml.ElementTree as ET
+import drive_upload as drivehelper
+import google.cloud.logging
 from codedx_api.CodeDxAPI import CodeDx  # pylint: disable=import-error
 from google.cloud import storage
 from slack_sdk.web import WebClient as SlackClient
 
 from zap import ScanType, zap_compliance_scan, zap_connect
-import defectdojo_apiv2 as defectdojo
-import drive_upload as drivehelper
 
 
 def fetch_dojo_product_name(defect_dojo, defect_dojo_user, defect_dojo_key, product_id):
@@ -371,6 +372,8 @@ def main(): # pylint: disable=too-many-locals
     - Upload ZAP XML report to GCS, if needed
     - Send a Slack alert with Code Dx report, if needed.
     """
+    client = google.cloud.logging.Client()
+    client.setup_logging()
 
     max_retries = int(getenv("MAX_RETRIES", '1'))
     sleep_time = 10
@@ -537,7 +540,7 @@ def main(): # pylint: disable=too-many-locals
                 try:
                     error_slack_alert(error_message, slack_token, slack_channel)
                 except:
-                    logging(f"Slack could not post to {slack_channel}")
+                    logging.error(f"Slack could not post to {slack_channel}")
                 try:
                     zap = zap_connect()
                     zap.core.shutdown()

zap/src/trigger.py

@@ -6,6 +6,7 @@
 import argparse
 import concurrent
 import logging
+import google.cloud.logging
 import re
 import traceback
 from asyncio import Future
@@ -152,10 +153,9 @@ def main():
     - Fetch the list of endpoints from DefectDojo
     - Trigger the scans for all endpoints
     """
-    logging.basicConfig(
-        level=logging.INFO,
-        format=f"%(levelname)-8s [zap-trigger] %(message)s",
-    )
+    client = google.cloud.logging.Client()
+    client.setup_logging()
+
     defectdojo_url = getenv("DEFECT_DOJO_URL")
     defect_dojo_key = getenv("DEFECT_DOJO_KEY")
     zap_topic = getenv("ZAP_TOPIC_NAME")

zap/src/zap.py

@@ -5,18 +5,18 @@
 import logging
 import os
 import shutil
-from urllib.parse import urlparse
 from datetime import datetime
+from urllib.parse import urlparse
 
 import google.auth
 import requests
+import terra_auth
 from google.auth.transport.requests import Request as GoogleAuthRequest
 from google.oauth2 import id_token
-from zapv2 import ZAPv2
 from zap_common import (wait_for_zap_start, zap_access_target,
                         zap_wait_for_passive_scan)
 from zap_scan_type import ScanType
-import terra_auth
+from zapv2 import ZAPv2
 
 TIMEOUT_MINS = 5
 zap_port = int(os.getenv("ZAP_PORT", ""))
@@ -97,7 +97,7 @@ def zap_sa_auth(zap: ZAPv2, env):
         if tos:
             logging.info("SA has accepted the TOS.")
     else:
-        logging.info("ZAP Service Account failed to register with Terra.")
+        logging.error("ZAP Service Account failed to register with Terra.")
     return token
 
 
@@ -138,7 +138,7 @@ def leo_auth(host, path, token):
     if response.status_code == 204:
         logging.info("Set cookie was successful")
         return True
-    logging.info("Set cookie did not succeed")
+    logging.error("Set cookie did not succeed")
     return False
 
 
@@ -170,9 +170,10 @@ def zap_setup_cookie(zap, domain, context_id, cookie_name=None):
         zap.forcedUser.set_forced_user_mode_enabled(True)
         zap_access_target(zap, f"https://{domain}")
         return username, userid
-    except Exception:
-        logging.info("Cookie authenication setup failed.")
-        raise RuntimeError("Failed to set the provided cookie as the default session in Zap.")
+    except Exception as e:
+        logging.error("Cookie authentication setup failed.")
+        raise RuntimeError("Failed to set the provided cookie" +
+                           " as the default session in Zap.") from e
 
 
 def zap_api_import(zap: ZAPv2, target_url: str):
@@ -189,6 +190,7 @@ def get_gcp_token() -> str:
     """
     Generate a Google access token with custom scopes for the default identity.
     """
+    logging.info("get_gcp_token fetching access token for the default identity")
     credentials, _ = google.auth.default(
         scopes=[
             "profile",
@@ -209,6 +211,7 @@ def zap_set_iap_token(client_id):
     open_id_connect_token = id_token.fetch_id_token( GoogleAuthRequest(), 
                                                         client_id)
     bearer = f"Bearer {open_id_connect_token}"
+    logging.info("zap replacer auth id token (IAP)")
     zap.replacer.add_rule(
         description="auth",
         enabled=True,
@@ -217,6 +220,7 @@ def zap_set_iap_token(client_id):
         matchstring="Authorization",
         replacement=bearer
     )
+    logging.info("zap replacer beehive")
     zap.replacer.add_rule(
         description="beehive",
         enabled=True,
@@ -290,6 +294,7 @@ def zap_compliance_scan(
     """
     Run a ZAP compliance scan of a given type against the target URL.
     """
+    logging.info("=====  %s  %s  %s  =====", scan_type.name, project, target_url)
 
     host, _, path = parse_url(target_url)
 
@@ -329,12 +334,13 @@ def zap_compliance_scan(
                 # and forces all Zap requests to use that cookie
                 zap_setup_cookie(zap, host, context_id)
             else:
-                logging.info("Leo authentication was unsuccessful")
+                logging.error("Leo authentication was unsuccessful")
 
 
     if scan_type == ScanType.API:
         zap_api_import(zap, target_url)
 
+    logging.info("zap spider scan %s", target_url)
     zap.spider.scan(contextname=project, url=target_url)
 
     if scan_type in (ScanType.UI, ScanType.LEOAPP, ScanType.BEEHIVE):
@@ -343,8 +349,10 @@ def zap_compliance_scan(
     zap_wait_for_passive_scan(zap, timeout_in_secs=TIMEOUT_MINS * 60)
 
     if scan_type != ScanType.BASELINE:
+        logging.info("zap ajax spider scan %s", target_url)
         zap.ascan.scan(target_url, contextid=context_id, recurse=True)
 
+    logging.info("Scan complete. Cumulative alerts: %s", zap.alert.alerts_summary())
     filename = zap_report(zap, project, scan_type, f"https://{host}")
     session_file = zap_save_session(zap, project, scan_type)