Resolve merge conflict.

.github/workflows/cve-scan-pr.yml

@@ -0,0 +1,51 @@
+name: HMDA PR CVE Scan
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+permissions:
+  pull-requests: write
+
+jobs:
+  hmda-platform-cve-scan:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v4
+
+      - name: Build Docker image of HMDA platform
+        run: |
+          env JAVA_OPTS="-Xss256m -Xmx4096m" sbt "project hmda-platform" dockerPublishLocalSkipTests
+        continue-on-error: false
+
+      - name: Tag and name Docker image
+        run: docker tag hmda/hmda-platform:latest pr-cve-scan:latest
+
+      - name: Set up Homebrew
+        uses: Homebrew/actions/setup-homebrew@master
+
+      - name: Install Grype
+        run: |
+          brew install grype
+
+      - name: Run Grype scan
+        run: |
+          grype pr-cve-scan:latest > grype-report.txt
+          
+      - name: Upload Grype report to artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: cve-report
+          path: |
+            grype-report.txt
+
+      - name: Post comment with report link
+        uses: thollander/actions-comment-pull-request@v3
+        with:
+          message: CVE scan report generated by Grype are available. Check the Actions tab to download the reports.
+
+      - name: Remove Docker image
+        run: |
+          docker rmi pr-cve-scan:latest

.github/workflows/dockerhub-push.yml

@@ -31,8 +31,22 @@ jobs:
       - name: Push image to Docker Hub
         run: docker push ${{ secrets.DOCKERHUB_USERNAME }}/hmda:latest
 
-      - name: Check image for CVEs
+      - name: Run Docker Scout CVE scan
+        if: ${{ github.event_name != 'pull_request_target' }}
         uses: docker/scout-action@v1
         with:
           command: cves
-          image: ${{ secrets.DOCKERHUB_USERNAME }}/hmda:latest
+          image: ${{ secrets.DOCKERHUB_USERNAME }}/hmda:latest
+          sarif-file: sarif.output.json
+          summary: true
+
+      - name: Upload CVE scan to artifact
+        if: ${{ github.event_name != 'pull_request_target' }}
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: sarif.output.json
+
+      - name: Post comment with report link
+        uses: thollander/actions-comment-pull-request@v3
+        with:
+          message: CVE scan report generated by Docker Scout are available. Check the Actions tab to download the report.

.github/workflows/unit-test.yml → .github/workflows/unit-test.yaml

@@ -63,64 +63,64 @@ jobs:
           fi
 
   common_tests:
-      runs-on: ubuntu-latest
-
-      steps:
-        - name: Checkout
-          uses: actions/checkout@v4
-
-        - name: Setup JDK
-          uses: actions/setup-java@v4
-          with:
-            java-version: '11'
-            distribution: 'adopt'
-
-        - name: Run common Tests
-          run: |
-            sbt "project common" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file
-          continue-on-error: true
-
-        - name: Check Test Results
-          run: |
-            if [ $(grep 'All tests passed.' log-file | wc -l) -ne 1 ]; then
-              echo "One or more projects had failures. Please review the logs."
-              exit 1
-            else
-              echo "All tests passed."
-              exit 0
-            fi
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup JDK
+        uses: actions/setup-java@v4
+        with:
+          java-version: '11'
+          distribution: 'adopt'
+
+      - name: Run common Tests
+        run: |
+          sbt "project common" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file
+        continue-on-error: true
+
+      - name: Check Test Results
+        run: |
+          if [ $(grep 'All tests passed.' log-file | wc -l) -ne 1 ]; then
+            echo "One or more projects had failures. Please review the logs."
+            exit 1
+          else
+            echo "All tests passed."
+            exit 0
+          fi
 
   data_browser_tests:
-        runs-on: ubuntu-latest
-
-        steps:
-          - name: Checkout
-            uses: actions/checkout@v4
-
-          - name: Setup JDK
-            uses: actions/setup-java@v4
-            with:
-              java-version: '11'
-              distribution: 'adopt'
-
-          - name: Run data-browser Tests
-            run: |
-              sbt "project data-browser" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file
-            continue-on-error: true
-
-          - name: Check Test Results
-            run: |
-              if [ $(grep 'All tests passed.' log-file | wc -l) -ne 1 ]; then
-                echo "One or more projects had failures. Please review the logs."
-                exit 1
-              else
-                echo "All tests passed."
-                exit 0
-              fi
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup JDK
+        uses: actions/setup-java@v4
+        with:
+          java-version: '11'
+          distribution: 'adopt'
+
+      - name: Run data-browser Tests
+        run: |
+          sbt "project data-browser" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file
+        continue-on-error: true
+
+      - name: Check Test Results
+        run: |
+          if [ $(grep 'All tests passed.' log-file | wc -l) -ne 1 ]; then
+            echo "One or more projects had failures. Please review the logs."
+            exit 1
+          else
+            echo "All tests passed."
+            exit 0
+          fi
 
   hmda_analytics_tests:
     runs-on: ubuntu-latest
-    
+
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -367,4 +367,4 @@ jobs:
           else
             echo "All tests passed."
             exit 0
-          fi
+          fi

common/src/main/pyhmda/parse_census_file.py

@@ -24,15 +24,14 @@ def conv_scf(val: str) -> str:
     if val not in ["T", "S", "I"]: raise ValueError(f"invalid scf: \"{val}\"")
     return val    
 
-
 census_file_columns = {
-    0: ["CollectionYear", conv_num], 1: ["CBSACode", conv_dgstr], 2: ["FIPSStateCode", conv_dgstr],
-    3: ["FIPSCountyCode", conv_dgstr], 4: ["CensusTract", conv_dgstr],
-    13: ["FFIECMedianFamilyIncome", conv_num], 22: ["Population", conv_optnum],
-    28: ["MinorityPopulationPct", conv_optpct], 879: ["NumOwnerOccupiedUnits", conv_optnum],
-    899: ["Num1To4FamilyUnits", conv_optnum], 585: ["TractMFI", conv_optnum],
-    12: ["TractToMSAIncomePct", conv_optpct], 1057: ["MedianAge", conv_optnum],
-    6: ["SmallCounty", conv_scf]
+    0: ["Collection Year", conv_num], 1: ["MSA/MD", conv_dgstr], 2: ["State", conv_dgstr],
+    3: ["County", conv_dgstr], 4: ["Census Tract", conv_dgstr],
+    13: ["FFIEC Median Family Income", conv_num], 22: ["Population", conv_optnum],
+    28: ["Minority Population %", conv_optpct], 879: ["Number of Owner Occupied Units", conv_optnum],
+    899: ["Number of 1 to 4 Family Units", conv_optnum], 585: ["Tract MFI", conv_optnum],
+    12: ["Tract to MSA Income %", conv_optpct], 1057: ["Median Age", conv_optnum],
+    6: ["Small County", conv_scf]
 }
 cfkeys = census_file_columns.keys()
 cfcolnames = {k: v[0] for k, v in census_file_columns.items()}
@@ -50,7 +49,7 @@ def conv_scf(val: str) -> str:
 
 delineation_file_columns = {
     3: ["CBSATitle", conv_str], 5: ["MDTitle", conv_optstr],
-    9: ["FIPSStateCode", conv_dgstr], 10: ["FIPSCountyCode", conv_dgstr]
+    9: ["State", conv_dgstr], 10: ["County", conv_dgstr]
 }
 dfkeys = delineation_file_columns.keys()
 dfcolnames = {k: v[0] for k, v in delineation_file_columns.items()}
@@ -59,17 +58,17 @@ def conv_scf(val: str) -> str:
                               converters=dfconverters).rename(dfcolnames, axis=1)
 logging.info(f"Parsed {prepared_file}")
 
-parsed_delin_df["MSAOrMDTitle"] = parsed_delin_df.apply(lambda row:
+parsed_delin_df["MSA/MD Name"] = parsed_delin_df.apply(lambda row:
     row.MDTitle if pd.notna(row.MDTitle) else row.CBSATitle, axis=1)
 parsed_delin_df.drop(columns=["CBSATitle", "MDTitle"], inplace=True)
-logging.info("Calculated MSAOrMDTitles")
+logging.info("Calculated MSA/MD Names")
 
 output_file = args.output_file if args.output_file \
     else f"{os.path.splitext(args.censusfile)[0]}-parsed.txt"
 output_df = parsed_census_df.merge(parsed_delin_df,
-    how="left", on=["FIPSStateCode", "FIPSCountyCode"])
-output_df["MSAOrMDTitle"] = output_df.apply(lambda row:
-    "" if row.CBSACode == "99999" else row.MSAOrMDTitle, axis=1)
+    how="left", on=["State", "County"])
+output_df["MSA/MD Name"] = output_df.apply(lambda row:
+    "" if row["MSA/MD"] == "99999" else row["MSA/MD Name"], axis=1)
 output_df.to_csv(output_file, sep='|', index=False)
 logging.info(f"Wrote output file {output_file}")
 os.remove(prepared_file)

common/src/main/pyhmda/utils.py

@@ -87,7 +87,7 @@ def apply_authorized_modifications(modmap: dict, df: pd.DataFrame) -> pd.DataFra
 # Census Flat File Modifications
 
 def replace_MedianAge_2002_values(df: pd.DataFrame) -> pd.DataFrame:
-    df.loc[df["MedianAge"] == 2002, "MedianAge"] = 6
+    df.loc[df["Median Age"] == 2002, "Median Age"] = 6
     return df
 
 

hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala

@@ -156,9 +156,7 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo
 
   //submission_history table remains same regardless of the year. There is a sign_date column and submission_id column which would show which year the filing was for
   val histTableName    = config.getString("hmda.analytics.2018.historyTableName")
-
-
-
+
   val submissionHistoryRepository    = new SubmissionHistoryRepository(dbConfig, histTableName)
 
   val consumerSettings: ConsumerSettings[String, String] =