diff --git a/.github/workflows/new-relic-deployment.yml b/.github/workflows/new-relic-deployment.yml index 2a6e4049f6..01fc5a2e1e 100644 --- a/.github/workflows/new-relic-deployment.yml +++ b/.github/workflows/new-relic-deployment.yml @@ -17,7 +17,7 @@ jobs: run: echo "RELEASE_VERSION=${{ github.ref_name }}" >> $GITHUB_ENV # This step creates a new Change Tracking Marker - name: Add New Relic Application Deployment Marker - uses: newrelic/deployment-marker-action@v2.3.0 + uses: newrelic/deployment-marker-action@v2.5.0 with: apiKey: ${{ secrets.NEW_RELIC_API_KEY }} guid: ${{ secrets.NEW_RELIC_DEPLOYMENT_ENTITY_GUID }} diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index 834bc0c534..ab65b5be94 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -39,7 +39,7 @@ jobs: run: docker build -t ${{ env.DOCKER_NAME }}:${{ steps.date.outputs.date }} . - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.13.0 + uses: aquasecurity/trivy-action@0.13.1 with: image-ref: '${{ env.DOCKER_NAME }}:${{ steps.date.outputs.date }}' scan-type: 'image' @@ -74,7 +74,7 @@ jobs: run: docker pull ${{ matrix.image.name }} - name: Run Trivy vulnerability scanner on Third Party Images - uses: aquasecurity/trivy-action@0.13.0 + uses: aquasecurity/trivy-action@0.13.1 with: image-ref: '${{ matrix.image.name }}' scan-type: 'image' diff --git a/.github/workflows/zap-scan.yml b/.github/workflows/zap-scan.yml index 0ca80f9868..1367b7be33 100644 --- a/.github/workflows/zap-scan.yml +++ b/.github/workflows/zap-scan.yml @@ -18,7 +18,7 @@ jobs: uses: actions/checkout@v4 - name: ZAP Scan of ${{ env.url }} - uses: zaproxy/action-baseline@v0.9.0 + uses: zaproxy/action-baseline@v0.10.0 with: token: ${{ secrets.GITHUB_TOKEN }} docker_name: 'owasp/zap2docker-stable' diff --git a/backend/dissemination/api/api_v1_0_3/base.sql b/backend/dissemination/api/api_v1_0_3/base.sql new file mode 100644 index 0000000000..dedabe0cb7 --- /dev/null +++ b/backend/dissemination/api/api_v1_0_3/base.sql @@ -0,0 +1,29 @@ +DO +$do$ +BEGIN + IF EXISTS ( + SELECT FROM pg_catalog.pg_roles + WHERE rolname = 'authenticator') THEN + RAISE NOTICE 'Role "authenticator" already exists. Skipping.'; + ELSE + CREATE ROLE authenticator LOGIN NOINHERIT NOCREATEDB NOCREATEROLE NOSUPERUSER; + END IF; +END +$do$; + +DO +$do$ +BEGIN + IF EXISTS ( + SELECT FROM pg_catalog.pg_roles + WHERE rolname = 'api_fac_gov') THEN + RAISE NOTICE 'Role "api_fac_gov" already exists. Skipping.'; + ELSE + CREATE ROLE api_fac_gov NOLOGIN; + END IF; +END +$do$; + +GRANT api_fac_gov TO authenticator; + +NOTIFY pgrst, 'reload schema'; diff --git a/backend/dissemination/api/api_v1_0_3/create_functions.sql b/backend/dissemination/api/api_v1_0_3/create_functions.sql new file mode 100644 index 0000000000..a5c340ffab --- /dev/null +++ b/backend/dissemination/api/api_v1_0_3/create_functions.sql @@ -0,0 +1,60 @@ +-- WARNING +-- Under PostgreSQL 12, the functions below work. +-- Under PostgreSQL 14, these will break. +-- +-- Note the differences: +-- +-- raise info 'Works under PostgreSQL 12'; +-- raise info 'request.header.x-magic %', (SELECT current_setting('request.header.x-magic', true)); +-- raise info 'request.jwt.claim.expires %', (SELECT current_setting('request.jwt.claim.expires', true)); +-- raise info 'Works under PostgreSQL 14'; +-- raise info 'request.headers::json->>x-magic %', (SELECT current_setting('request.headers', true)::json->>'x-magic'); +-- raise info 'request.jwt.claims::json->expires %', (SELECT current_setting('request.jwt.claims', true)::json->>'expires'); +-- +-- To quote the work of Dav Pilkey, "remember this now." + +create or replace function getter(base text, item text) returns text +as $getter$ +begin + return current_setting(concat(base, '.', item), true); +end; +$getter$ language plpgsql; + +create or replace function get_jwt_claim(item text) returns text +as $get_jwt_claim$ +begin + return getter('request.jwt.claim', item); +end; +$get_jwt_claim$ language plpgsql; + +create or replace function get_header(item text) returns text +as $get_header$ +begin + raise info 'request.header % %', item, getter('request.header', item); + return getter('request.header', item); +end; +$get_header$ LANGUAGE plpgsql; + +-- https://api-umbrella.readthedocs.io/en/latest/admin/api-backends/http-headers.html +-- I'd like to go to a model where we provide the API keys. +-- However, for now, we're going to look for a role attached to an api.data.gov account. +-- These come in on `X-Api-Roles` as a comma-separated string. +create or replace function has_tribal_data_access() returns boolean +as $has_tribal_data_access$ +declare + roles text; +begin + select get_header('x-api-roles') into roles; + return (roles like '%fac_gov_tribal_access%'); +end; +$has_tribal_data_access$ LANGUAGE plpgsql; + +create or replace function has_public_data_access_only() returns boolean +as $has_public_data_access_only$ +begin + return not has_tribal_data_access(); +end; +$has_public_data_access_only$ LANGUAGE plpgsql; + + +NOTIFY pgrst, 'reload schema'; \ No newline at end of file diff --git a/backend/dissemination/api/api_v1_0_3/create_schema.sql b/backend/dissemination/api/api_v1_0_3/create_schema.sql new file mode 100644 index 0000000000..41372fffef --- /dev/null +++ b/backend/dissemination/api/api_v1_0_3/create_schema.sql @@ -0,0 +1,48 @@ +begin; + +do +$$ +begin + DROP SCHEMA IF EXISTS api_v1_0_3 CASCADE; + + if not exists (select schema_name from information_schema.schemata where schema_name = 'api_v1_0_3') then + create schema api_v1_0_3; + + -- Grant access to tables and views + alter default privileges + in schema api_v1_0_3 + grant select + -- this includes views + on tables + to api_fac_gov; + + -- Grant access to sequences, if we have them + grant usage on schema api_v1_0_3 to api_fac_gov; + grant select, usage on all sequences in schema api_v1_0_3 to api_fac_gov; + alter default privileges + in schema api_v1_0_3 + grant select, usage + on sequences + to api_fac_gov; + end if; +end +$$ +; + +-- This is the description +COMMENT ON SCHEMA api_v1_0_3 IS + 'The FAC dissemation API version 1.0.3.' +; + +-- https://postgrest.org/en/stable/references/api/openapi.html +-- This is the title +COMMENT ON SCHEMA api_v1_0_3 IS +$$v1.0.3 + +A RESTful API that serves data from the SF-SAC.$$; + +commit; + +notify pgrst, + 'reload schema'; + diff --git a/backend/dissemination/api/api_v1_0_3/create_views.sql b/backend/dissemination/api/api_v1_0_3/create_views.sql new file mode 100644 index 0000000000..183062693e --- /dev/null +++ b/backend/dissemination/api/api_v1_0_3/create_views.sql @@ -0,0 +1,329 @@ + +begin; + +--------------------------------------- +-- finding_text +--------------------------------------- +create view api_v1_0_3.findings_text as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + ft.finding_ref_number, + ft.contains_chart_or_table, + ft.finding_text + from + dissemination_findingtext ft, + dissemination_general gen + where + (ft.report_id = gen.report_id + and + gen.is_public = true) + or (gen.is_public = false and has_tribal_data_access()) + order by ft.id +; + +--------------------------------------- +-- additional_ueis +--------------------------------------- +create view api_v1_0_3.additional_ueis as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + uei.additional_uei + from + dissemination_general gen, + dissemination_additionaluei uei + where + (gen.report_id = uei.report_id + and + gen.is_public = true) + or (gen.is_public = false and has_tribal_data_access()) + order by uei.id +; + +--------------------------------------- +-- finding +--------------------------------------- +create view api_v1_0_3.findings as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + finding.award_reference, + finding.reference_number, + finding.is_material_weakness, + finding.is_modified_opinion, + finding.is_other_findings, + finding.is_other_matters, + finding.prior_finding_ref_numbers, + finding.is_questioned_costs, + finding.is_repeat_finding, + finding.is_significant_deficiency, + finding.type_requirement + from + dissemination_finding finding, + dissemination_general gen + where + (finding.report_id = gen.report_id + and + gen.is_public = true) + or (gen.is_public = false and has_tribal_data_access()) + order by finding.id +; + +--------------------------------------- +-- federal award +--------------------------------------- +create view api_v1_0_3.federal_awards as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + award.award_reference, + award.federal_agency_prefix, + award.federal_award_extension, + award.additional_award_identification, + award.federal_program_name, + award.amount_expended, + award.cluster_name, + award.other_cluster_name, + award.state_cluster_name, + award.cluster_total, + award.federal_program_total, + award.is_major, + award.is_loan, + award.loan_balance, + award.is_direct, + award.audit_report_type, + award.findings_count, + award.is_passthrough_award, + award.passthrough_amount + from + dissemination_federalaward award, + dissemination_general gen + where + (award.report_id = gen.report_id + and + gen.is_public = true) + or (gen.is_public = false and has_tribal_data_access()) + order by award.id +; + + +--------------------------------------- +-- corrective_action_plan +--------------------------------------- +create view api_v1_0_3.corrective_action_plans as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + ct.finding_ref_number, + ct.contains_chart_or_table, + ct.planned_action + from + dissemination_CAPText ct, + dissemination_General gen + where + (ct.report_id = gen.report_id + and + gen.is_public = true) + or (gen.is_public = false and has_tribal_data_access()) + order by ct.id +; + +--------------------------------------- +-- notes_to_sefa +--------------------------------------- +create view api_v1_0_3.notes_to_sefa as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + note.note_title as title, + note.accounting_policies, + note.is_minimis_rate_used, + note.rate_explained, + note.content, + note.contains_chart_or_table + from + dissemination_general gen, + dissemination_note note + where + (note.report_id = gen.report_id + and + gen.is_public = true) + or (gen.is_public = false and has_tribal_data_access()) + order by note.id +; + +--------------------------------------- +-- passthrough +--------------------------------------- +create view api_v1_0_3.passthrough as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + pass.award_reference, + pass.passthrough_id, + pass.passthrough_name + from + dissemination_general as gen, + dissemination_passthrough as pass + where + (gen.report_id = pass.report_id + and + gen.is_public = true) + or (gen.is_public = false and has_tribal_data_access()) + order by pass.id +; + + +--------------------------------------- +-- general +--------------------------------------- +create view api_v1_0_3.general as + select + -- every table starts with report_id, UEI, and year + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + gen.auditee_certify_name, + gen.auditee_certify_title, + gen.auditee_contact_name, + gen.auditee_email, + gen.auditee_name, + gen.auditee_phone, + gen.auditee_contact_title, + gen.auditee_address_line_1, + gen.auditee_city, + gen.auditee_state, + gen.auditee_ein, + gen.auditee_zip, + -- auditor + gen.auditor_phone, + gen.auditor_state, + gen.auditor_city, + gen.auditor_contact_title, + gen.auditor_address_line_1, + gen.auditor_zip, + gen.auditor_country, + gen.auditor_contact_name, + gen.auditor_email, + gen.auditor_firm_name, + gen.auditor_foreign_address, + gen.auditor_ein, + -- agency + gen.cognizant_agency, + gen.oversight_agency, + -- dates + gen.date_created, + gen.ready_for_certification_date, + gen.auditor_certified_date, + gen.auditee_certified_date, + gen.submitted_date, + gen.fac_accepted_date, + gen.fy_end_date, + gen.fy_start_date, + gen.audit_type, + gen.gaap_results, + gen.sp_framework_basis, + gen.is_sp_framework_required, + gen.sp_framework_opinions, + gen.is_going_concern_included, + gen.is_internal_control_deficiency_disclosed, + gen.is_internal_control_material_weakness_disclosed, + gen.is_material_noncompliance_disclosed, + gen.dollar_threshold, + gen.is_low_risk_auditee, + gen.agencies_with_prior_findings, + gen.entity_type, + gen.number_months, + gen.audit_period_covered, + gen.total_amount_expended, + gen.type_audit_code, + gen.is_public, + gen.data_source, + gen.is_aicpa_audit_guide_included, + gen.is_additional_ueis, + CASE + WHEN aud.general_information ->> 'multiple_eins_covered' = 'true' THEN 'Yes' + WHEN aud.general_information ->> 'multiple_eins_covered' = 'false' THEN 'No' + END AS is_multiple_eins, + CASE + WHEN aud.general_information ->> 'secondary_auditors_exist' = 'true' THEN 'Yes' + WHEN aud.general_information ->> 'secondary_auditors_exist' = 'false' THEN 'No' + END AS is_secondary_auditors + from + dissemination_General gen, + audit_singleauditchecklist aud + where + (aud.report_id = gen.report_id + and + gen.is_public = true) + or (gen.is_public = false and has_tribal_data_access()) + order by gen.id +; + +--------------------------------------- +-- auditor (secondary auditor) +--------------------------------------- +create view api_v1_0_3.secondary_auditors as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + sa.auditor_ein, + sa.auditor_name, + sa.contact_name, + sa.contact_title, + sa.contact_email, + sa.contact_phone, + sa.address_street, + sa.address_city, + sa.address_state, + sa.address_zipcode + from + dissemination_General gen, + dissemination_SecondaryAuditor sa + where + (sa.report_id = gen.report_id + and + gen.is_public=True) + or (gen.is_public=false and has_tribal_data_access()) + order by sa.id +; + +create view api_v1_0_3.additional_eins as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + ein.additional_ein + from + dissemination_general gen, + dissemination_additionalein ein + where + (gen.report_id = ein.report_id + and + gen.is_public = true) + or (gen.is_public = false and has_tribal_data_access()) + order by ein.id +; + + +commit; + +notify pgrst, + 'reload schema'; diff --git a/backend/dissemination/api/api_v1_0_3/drop.sql b/backend/dissemination/api/api_v1_0_3/drop.sql new file mode 100644 index 0000000000..cf1aca6d91 --- /dev/null +++ b/backend/dissemination/api/api_v1_0_3/drop.sql @@ -0,0 +1,11 @@ + +begin; + +DROP SCHEMA IF EXISTS api_v1_0_3 CASCADE; +-- DROP ROLE IF EXISTS authenticator; +-- DROP ROLE IF EXISTS api_fac_gov; + +commit; + +notify pgrst, + 'reload schema'; diff --git a/backend/dissemination/api/api_v1_0_3/drop_schema.sql b/backend/dissemination/api/api_v1_0_3/drop_schema.sql new file mode 100644 index 0000000000..cf1aca6d91 --- /dev/null +++ b/backend/dissemination/api/api_v1_0_3/drop_schema.sql @@ -0,0 +1,11 @@ + +begin; + +DROP SCHEMA IF EXISTS api_v1_0_3 CASCADE; +-- DROP ROLE IF EXISTS authenticator; +-- DROP ROLE IF EXISTS api_fac_gov; + +commit; + +notify pgrst, + 'reload schema'; diff --git a/backend/dissemination/api/api_v1_0_3/drop_views.sql b/backend/dissemination/api/api_v1_0_3/drop_views.sql new file mode 100644 index 0000000000..beae413e69 --- /dev/null +++ b/backend/dissemination/api/api_v1_0_3/drop_views.sql @@ -0,0 +1,15 @@ +begin; + + drop table if exists api_v1_0_3.metadata; + drop view if exists api_v1_0_3.general; + drop view if exists api_v1_0_3.auditor; + drop view if exists api_v1_0_3.federal_award; + drop view if exists api_v1_0_3.finding; + drop view if exists api_v1_0_3.finding_text; + drop view if exists api_v1_0_3.cap_text; + drop view if exists api_v1_0_3.note; + +commit; + +notify pgrst, + 'reload schema'; diff --git a/backend/dissemination/api_versions.py b/backend/dissemination/api_versions.py index c9263cf3ec..88f89ea3a7 100644 --- a/backend/dissemination/api_versions.py +++ b/backend/dissemination/api_versions.py @@ -4,13 +4,13 @@ # These are API versions we want live. live = ( # These are API versions we have in flight. - "api_v1_0_2", + "api_v1_0_3", ) # These are API versions we have deprecated. # They will be removed. It should be safe to leave them # here for repeated runs. -deprecated = ("api", "api_v1_0_0", "api_v1_0_1") +deprecated = ("api", "api_v1_0_0", "api_v1_0_1", "api_v1_0_2") def get_conn_string(): diff --git a/backend/dissemination/search.py b/backend/dissemination/search.py index 2d264fe268..98a5ae7380 100644 --- a/backend/dissemination/search.py +++ b/backend/dissemination/search.py @@ -1,9 +1,10 @@ from django.db.models import Q -from dissemination.models import General +from dissemination.models import General, FederalAward def search_general( + alns=None, names=None, uei_or_eins=None, start_date=None, @@ -14,14 +15,11 @@ def search_general( ): query = Q(is_public=True) - # TODO: use something like auditee_name__contains - # SELECT * WHERE auditee_name LIKE '%SomeString%' + if alns: + query.add(_get_aln_match_query(alns), Q.AND) + if names: - names_match = Q() - for name in names: - names_match.add(Q(auditee_name__search=name), Q.OR) - names_match.add(Q(auditor_firm_name__search=name), Q.OR) - query.add(names_match, Q.AND) + query.add(_get_names_match_query(names), Q.AND) if uei_or_eins: uei_or_ein_match = Q( @@ -52,3 +50,76 @@ def search_general( results = General.objects.filter(query).order_by("-fac_accepted_date") return results + + +def _get_aln_match_query(alns): + """ + Create the match query for ALNs. + Takes: A list of (potential) ALNs. + Returns: A query object matching on relevant report_ids found in the FederalAward table. + + # ALNs are a little weird, because they are stored per-award in the FederalAward table. To search on ALNs, we: + # 1. Split the given ALNs into a set with their prefix and extention. + # 2. Search the FederalAward table for awards with matching federal_agency_prefix and federal_award_extension. + # If there's just a prefix, search on all prefixes. + # If there's a prefix and extention, search on both. + # 3. Add the report_ids from the identified awards to the search params. + """ + # Split each ALN into (prefix, extention) + split_alns = set() + agency_numbers = set() + for aln in alns: + if len(aln) == 2: + # If we don't wrap the `aln` with [], the string + # goes in as individual characters. A weirdness of Python sets. + agency_numbers.update([aln]) + else: + split_aln = aln.split(".") + if len(split_aln) == 2: + # The [wrapping] is so the tuple goes into the set as a tuple. + # Otherwise, the individual elements go in unpaired. + split_alns.update([tuple(split_aln)]) + + # Search for relevant awards + report_ids = _get_aln_report_ids(split_alns) + + for agency_number in agency_numbers: + matching_awards = FederalAward.objects.filter( + federal_agency_prefix=agency_number + ).values() + if matching_awards: + for matching_award in matching_awards: + report_ids.update([matching_award.get("report_id")]) + # Add the report_id's from the award search to the full search params + alns_match = Q() + for report_id in report_ids: + alns_match.add(Q(report_id=report_id), Q.OR) + return alns_match + + +def _get_aln_report_ids(split_alns): + """ + Given a set of split ALNs, find the relevant awards and return their report_ids. + """ + report_ids = set() + for aln_list in split_alns: + matching_awards = FederalAward.objects.filter( + federal_agency_prefix=aln_list[0], federal_award_extension=aln_list[1] + ).values() + if matching_awards: + for matching_award in matching_awards: + # Again, adding in a string requires [] so the individual + # characters of the report ID don't go in... we want the whole string. + report_ids.update([matching_award.get("report_id")]) + return report_ids + + +def _get_names_match_query(names): + """ + Given a list of (potential) names, return the query object that searches auditee and firm names. + """ + names_match = Q() + for name in names: + names_match.add(Q(auditee_name__search=name), Q.OR) + names_match.add(Q(auditor_firm_name__search=name), Q.OR) + return names_match diff --git a/backend/dissemination/templates/search.html b/backend/dissemination/templates/search.html index 0017b70ddf..d37f8e3bf6 100644 --- a/backend/dissemination/templates/search.html +++ b/backend/dissemination/templates/search.html @@ -7,6 +7,7 @@