diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e6e142905110..eddc9a9f5627 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -238,6 +238,7 @@ jobs: include: ${{ fromJSON(needs.artifacts.outputs.testable-packages) }} with: build-artifact-name: ${{ matrix.artifact }} + runs-on: ${{ github.repository == 'hashicorp/vault' && '"ubuntu-latest"' || '["self-hosted","linux","small"]' }} sample-max: 1 sample-name: ${{ matrix.sample }} ssh-key-name: ${{ github.event.repository.name }}-ci-ssh-key diff --git a/.github/workflows/test-run-enos-scenario-matrix.yml b/.github/workflows/test-run-enos-scenario-matrix.yml index 535e971da9c4..ca5f71e0e71b 100644 --- a/.github/workflows/test-run-enos-scenario-matrix.yml +++ b/.github/workflows/test-run-enos-scenario-matrix.yml @@ -44,20 +44,38 @@ on: jobs: metadata: runs-on: ${{ fromJSON(inputs.runs-on) }} + permissions: + id-token: write # vault-auth + contents: read outputs: build-date: ${{ steps.metadata.outputs.build-date }} + is-enterprise: ${{ steps.metadata.outputs.is-enterprise }} sample: ${{ steps.metadata.outputs.sample }} vault-version: ${{ steps.metadata.outputs.vault-version }} steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.vault-revision }} + - if: inputs.vault-edition != 'ce' + id: vault-auth + name: Vault Authenticate + run: vault-auth + - if: inputs.vault-edition != 'ce' + id: vault-secrets + name: Fetch Vault Secrets + uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3.0.0 + with: + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/github-token token | ELEVATED_GITHUB_TOKEN; - uses: hashicorp/action-setup-enos@v1 with: - github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + github-token: ${{ inputs.vault-edition == 'ce' && secrets.ELEVATED_GITHUB_TOKEN || steps.vault-secrets.outputs.ELEVATED_GITHUB_TOKEN }} - uses: ./.github/actions/create-dynamic-config with: - github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + github-token: ${{ inputs.vault-edition == 'ce' && secrets.ELEVATED_GITHUB_TOKEN || steps.vault-secrets.outputs.ELEVATED_GITHUB_TOKEN }} vault-version: ${{ inputs.vault-version }} vault-edition: ${{ inputs.vault-edition }} - id: metadata @@ -78,9 +96,10 @@ jobs: sample=$(enos scenario sample observe "${{ inputs.sample-name }}" --chdir ./enos --min 1 --max "${{ inputs.sample-max }}" --seed "${sample_seed}" --format json | jq -c ".observation.elements") { echo "build-date=${build_date}" - echo "vault-version=${vault_version}" + echo "is-enterprise=${{ inputs.vault-edition != 'ce' }}" echo "sample=${sample}" echo "sample-seed=${sample_seed}" # This isn't used outside of here but is nice to know for duplicating observations + echo "vault-version=${vault_version}" } | tee -a "$GITHUB_OUTPUT" # Run the Enos test scenario(s) @@ -92,33 +111,99 @@ jobs: matrix: include: ${{ fromJSON(needs.metadata.outputs.sample) }} runs-on: ${{ fromJSON(inputs.runs-on) }} - env: - GITHUB_TOKEN: ${{ secrets.ELEVATED_GITHUB_TOKEN }} - # Pass in enos variables - ENOS_VAR_aws_region: ${{ matrix.attributes.aws_region }} - ENOS_VAR_aws_ssh_keypair_name: ${{ inputs.ssh-key-name }} - ENOS_VAR_aws_ssh_private_key_path: ./support/private_key.pem - ENOS_VAR_tfc_api_token: ${{ secrets.TF_API_TOKEN }} - ENOS_VAR_artifactory_username: ${{ secrets.ARTIFACTORY_USER }} - ENOS_VAR_artifactory_token: ${{ secrets.ARTIFACTORY_TOKEN }} - ENOS_VAR_terraform_plugin_cache_dir: ./support/terraform-plugin-cache - ENOS_VAR_vault_artifact_path: ./support/downloads/${{ inputs.build-artifact-name }} - ENOS_VAR_vault_build_date: ${{ needs.metadata.outputs.build-date }} - ENOS_VAR_vault_product_version: ${{ needs.metadata.outputs.vault-version }} - ENOS_VAR_vault_revision: ${{ inputs.vault-revision }} - ENOS_VAR_vault_upgrade_initial_version: ${{ matrix.attributes.upgrade_initial_version }} - ENOS_VAR_consul_license_path: ./support/consul.hclic - ENOS_VAR_vault_license_path: ./support/vault.hclic - ENOS_VAR_distro_version_amzz: ${{ matrix.attributes.distro_version_amzn }} - ENOS_VAR_distro_version_leap: ${{ matrix.attributes.distro_version_leap }} - ENOS_VAR_distro_version_rhel: ${{ matrix.attributes.distro_version_rhel }} - ENOS_VAR_distro_version_sles: ${{ matrix.attributes.distro_version_sles }} - ENOS_VAR_distro_version_ubuntu: ${{ matrix.attributes.distro_version_ubuntu }} - ENOS_DEBUG_DATA_ROOT_DIR: /tmp/enos-debug-data + permissions: + id-token: write # vault-auth + contents: read steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.vault-revision }} + - if: needs.metadata.outputs.is-enterprise == 'true' + id: vault-auth + name: Vault Authenticate + run: vault-auth + - if: needs.metadata.outputs.is-enterprise == 'true' + id: vault-secrets + name: Fetch Vault Secrets + uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3.0.0 + with: + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/artifactory token | ARTIFACTORY_TOKEN; + kv/data/github/${{ github.repository }}/artifactory username | ARTIFACTORY_USER; + kv/data/github/${{ github.repository }}/aws access-key-id | AWS_ACCESS_KEY_ID_CI; + kv/data/github/${{ github.repository }}/aws secret-access-key | AWS_SECRET_ACCESS_KEY_CI; + kv/data/github/${{ github.repository }}/aws role-arn | AWS_ROLE_ARN_CI; + kv/data/github/${{ github.repository }}/consul license | CONSUL_LICENSE; + kv/data/github/${{ github.repository }}/vault-radar license | RADAR_LICENSE; + kv/data/github/${{ github.repository }}/enos slack-webhook-url | SLACK_WEBHOOK_URL; + kv/data/github/${{ github.repository }}/enos ssh-key | SSH_KEY_PRIVATE_CI; + kv/data/github/${{ github.repository }}/license license_1 | VAULT_LICENSE; + kv/data/github/${{ github.repository }}/github-token token | ELEVATED_GITHUB_TOKEN; + - id: secrets + run: | + if [[ "${{ needs.metadata.outputs.is-enterprise }}" != 'true' ]]; then + { + echo "artifactory-user=${{ secrets.ARTIFACTORY_USER }}" + echo "artifactory-token=${{ secrets.ARTIFACTORY_TOKEN }}" + echo "aws-access-key-id=${{ secrets.AWS_ACCESS_KEY_ID_CI }}" + echo "aws-secret-access-key=${{ secrets.AWS_SECRET_ACCESS_KEY_CI }}" + echo "aws-role-arn=${{ secrets.AWS_ROLE_ARN_CI }}" + echo "consul-license=${{ secrets.CONSUL_LICENSE }}" + echo "github-token=${{ secrets.ELEVATED_GITHUB_TOKEN }}" + echo "radar-license=${{ secrets.RADAR_LICENSE }}" + echo "slack-webhook-url=${{ secrets.SLACK_WEBHOOK_URL }}" + echo 'ssh-key< "./enos/support/private_key.pem" + echo "${{ steps.secrets.outputs.ssh-key }}" > "./enos/support/private_key.pem" chmod 600 "./enos/support/private_key.pem" + sha256sum "./enos/support/private_key.pem" + du -h "./enos/support/private_key.pem" echo "debug_data_artifact_name=enos-debug-data_$(echo "${{ matrix.scenario }}" | sed -e 's/ /_/g' | sed -e 's/:/=/g')" >> "$GITHUB_OUTPUT" - if: contains(inputs.sample-name, 'build') uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 @@ -154,24 +241,26 @@ jobs: path: ./enos/support/downloads - if: contains(inputs.sample-name, 'ent') name: Configure Vault license - run: echo "${{ secrets.VAULT_LICENSE }}" > ./enos/support/vault.hclic || true + run: echo "${{ steps.secrets.outputs.vault-license }}" > ./enos/support/vault.hclic || true - if: contains(matrix.scenario.id.filter, 'consul_edition:ent') name: Configure Consul license run: | - echo "matrix.scenario.id.filter: ${{ matrix.scenario.id.filter }}" - echo "${{ secrets.CONSUL_LICENSE }}" > ./enos/support/consul.hclic || true + echo "${{ steps.secrets.outputs.consul-license }}" > ./enos/support/consul.hclic || true + - name: Configure Vault Radar license + run: | + echo "${{ steps.secrets.outputs.radar-license }}" > ./enos/support/vault-radar.hclic || true - id: launch name: enos scenario launch ${{ matrix.scenario.id.filter }} # Continue once and retry to handle occasional blips when creating infrastructure. continue-on-error: true - run: enos scenario launch --timeout 60m0s --chdir ./enos ${{ matrix.scenario.id.filter }} + run: enos scenario launch --timeout 45m0s --chdir ./enos ${{ matrix.scenario.id.filter }} - if: steps.launch.outcome == 'failure' id: launch_retry name: Retry enos scenario launch ${{ matrix.scenario.id.filter }} - run: enos scenario launch --timeout 60m0s --chdir ./enos ${{ matrix.scenario.id.filter }} + run: enos scenario launch --timeout 45m0s --chdir ./enos ${{ matrix.scenario.id.filter }} - name: Upload Debug Data if: failure() - uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: # The name of the artifact is the same as the matrix scenario name with the spaces replaced with underscores and colons replaced by equals. name: ${{ steps.prepare_scenario.outputs.debug_data_artifact_name }} @@ -182,12 +271,12 @@ jobs: id: destroy name: enos scenario destroy ${{ matrix.scenario.id.filter }} continue-on-error: true - run: enos scenario destroy --timeout 60m0s --chdir ./enos ${{ matrix.scenario.id.filter }} + run: enos scenario destroy --timeout 10m0s --chdir ./enos ${{ matrix.scenario.id.filter }} - if: steps.destroy.outcome == 'failure' id: destroy_retry name: Retry enos scenario destroy ${{ matrix.scenario.id.filter }} continue-on-error: true - run: enos scenario destroy --timeout 60m0s --chdir ./enos ${{ matrix.scenario.id.filter }} + run: enos scenario destroy --timeout 10m0s --chdir ./enos ${{ matrix.scenario.id.filter }} - name: Clean up Enos runtime directories id: cleanup if: ${{ always() }} @@ -205,25 +294,25 @@ jobs: with: failure-message: "enos scenario launch ${{ matrix.scenario.id.filter}} failed. \nTriggering event: `${{ github.event_name }}` \nActor: `${{ github.actor }}`" status: ${{ steps.launch.outcome }} - slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }} + slack-webhook-url: ${{ steps.secrets.outputs.slack-webhook-url }} - if: ${{ always() && ! cancelled() }} name: Notify retry launch failed uses: hashicorp/actions-slack-status@v2.0.1 with: failure-message: "retry enos scenario launch ${{ matrix.scenario.id.filter}} failed. \nTriggering event: `${{ github.event_name }}` \nActor: `${{ github.actor }}`" status: ${{ steps.launch_retry.outcome }} - slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }} + slack-webhook-url: ${{ steps.secrets.outputs.slack-webhook-url }} - if: ${{ always() && ! cancelled() }} name: Notify destroy failed uses: hashicorp/actions-slack-status@v2.0.1 with: failure-message: "enos scenario destroy ${{ matrix.scenario.id.filter}} failed. \nTriggering event: `${{ github.event_name }}` \nActor: `${{ github.actor }}`" status: ${{ steps.destroy.outcome }} - slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }} + slack-webhook-url: ${{ steps.secrets.outputs.slack-webhook-url }} - if: ${{ always() && ! cancelled() }} name: Notify retry destroy failed uses: hashicorp/actions-slack-status@v2.0.1 with: failure-message: "retry enos scenario destroy ${{ matrix.scenario.id.filter}} failed. \nTriggering event: `${{ github.event_name }}` \nActor: `${{ github.actor }}`" status: ${{ steps.destroy_retry.outcome }} - slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }} + slack-webhook-url: ${{ steps.secrets.outputs.slack-webhook-url }} diff --git a/enos/enos-descriptions.hcl b/enos/enos-descriptions.hcl index 6ec60150ecc8..5d381edd2a5e 100644 --- a/enos/enos-descriptions.hcl +++ b/enos/enos-descriptions.hcl @@ -115,6 +115,10 @@ globals { Vault running in Agent mode uses templates to create log output. EOF + verify_log_secrets = <<-EOF + Verify that the vault audit log and systemd journal do not leak secret values. + EOF + verify_raft_cluster_all_nodes_are_voters = <<-EOF When configured with a 'backend:raft' variant, verify that all nodes in the cluster are healthy and are voters. @@ -198,7 +202,7 @@ globals { EOF verify_billing_start_date = <<-EOF - Verify that the billing start date has successfully rolled over to the latest billing year if needed. + Verify that the billing start date has successfully rolled over to the latest billing year if needed. EOF } diff --git a/enos/enos-modules.hcl b/enos/enos-modules.hcl index 9a11042d5f7b..64f29cab22e1 100644 --- a/enos/enos-modules.hcl +++ b/enos/enos-modules.hcl @@ -350,6 +350,12 @@ module "vault_wait_for_seal_rewrap" { vault_install_dir = var.vault_install_dir } +module "verify_log_secrets" { + source = "./modules/verify_log_secrets" + + radar_license_path = var.vault_radar_license_path != null ? abspath(var.vault_radar_license_path) : null +} + module "verify_seal_type" { source = "./modules/verify_seal_type" @@ -363,4 +369,3 @@ module "vault_verify_billing_start_date" { vault_instance_count = var.vault_instance_count vault_cluster_addr_port = global.ports["vault_cluster"]["port"] } - diff --git a/enos/enos-qualities.hcl b/enos/enos-qualities.hcl index 698ef6a57bc0..36dcd185d382 100644 --- a/enos/enos-qualities.hcl +++ b/enos/enos-qualities.hcl @@ -405,6 +405,10 @@ quality "vault_audit_log" { description = "The Vault audit sub-system is enabled with the log and writes to a log" } +quality "vault_audit_log_secrets" { + description = "The Vault audit sub-system does not output secret values" +} + quality "vault_audit_socket" { description = "The Vault audit sub-system is enabled with the socket and writes to a socket" } @@ -490,6 +494,10 @@ quality "vault_init" { description = "Vault initializes the cluster with the given seal parameters" } +quality "vault_journal_secrets" { + description = "The Vault systemd journal does not output secret values" +} + quality "vault_license_required_ent" { description = "Vault Enterprise requires a license in order to start" } @@ -532,6 +540,14 @@ quality "vault_proxy_cli_access" { EOF } +quality "vault_radar_index_create" { + description = "Vault radar is able to create an index from KVv2 mounts" +} + +quality "vault_radar_scan_file" { + description = "Vault radar is able to scan a file for secrets" +} + quality "vault_raft_voters" { description = global.description.verify_raft_cluster_all_nodes_are_voters } diff --git a/enos/enos-scenario-agent.hcl b/enos/enos-scenario-agent.hcl index 0f362a085f08..034cd1b8df22 100644 --- a/enos/enos-scenario-agent.hcl +++ b/enos/enos-scenario-agent.hcl @@ -564,6 +564,34 @@ scenario "agent" { } } + step "verify_log_secrets" { + skip_step = !var.vault_enable_audit_devices || !var.verify_log_secrets + + description = global.description.verify_log_secrets + module = module.verify_log_secrets + depends_on = [ + step.verify_secrets_engines_read, + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_audit_log_secrets, + quality.vault_journal_secrets, + quality.vault_radar_index_create, + quality.vault_radar_scan_file, + ] + + variables { + audit_log_file_path = step.create_vault_cluster.audit_device_file_path + leader_host = step.get_vault_cluster_ips.leader_host + vault_addr = step.create_vault_cluster.api_addr_localhost + vault_root_token = step.create_vault_cluster.root_token + } + } + step "verify_ui" { description = global.description.verify_ui module = module.vault_verify_ui diff --git a/enos/enos-scenario-autopilot.hcl b/enos/enos-scenario-autopilot.hcl index bac9d9b7aca9..816b0c803603 100644 --- a/enos/enos-scenario-autopilot.hcl +++ b/enos/enos-scenario-autopilot.hcl @@ -572,6 +572,34 @@ scenario "autopilot" { } } + step "verify_log_secrets" { + skip_step = !var.vault_enable_audit_devices || !var.verify_log_secrets + + description = global.description.verify_log_secrets + module = module.verify_log_secrets + depends_on = [ + step.verify_secrets_engines_read, + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_audit_log_secrets, + quality.vault_journal_secrets, + quality.vault_radar_index_create, + quality.vault_radar_scan_file, + ] + + variables { + audit_log_file_path = step.create_vault_cluster.audit_device_file_path + leader_host = step.get_updated_vault_cluster_ips.leader_host + vault_addr = step.upgrade_vault_cluster_with_autopilot.api_addr_localhost + vault_root_token = step.create_vault_cluster.root_token + } + } + step "raft_remove_peers" { description = <<-EOF Remove the nodes that were running the prior version of Vault from the raft cluster diff --git a/enos/enos-scenario-proxy.hcl b/enos/enos-scenario-proxy.hcl index 6865d1b62146..30556da2d19c 100644 --- a/enos/enos-scenario-proxy.hcl +++ b/enos/enos-scenario-proxy.hcl @@ -541,6 +541,34 @@ scenario "proxy" { } } + step "verify_log_secrets" { + skip_step = !var.vault_enable_audit_devices || !var.verify_log_secrets + + description = global.description.verify_log_secrets + module = module.verify_log_secrets + depends_on = [ + step.verify_secrets_engines_read, + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_audit_log_secrets, + quality.vault_journal_secrets, + quality.vault_radar_index_create, + quality.vault_radar_scan_file, + ] + + variables { + audit_log_file_path = step.create_vault_cluster.audit_device_file_path + leader_host = step.get_vault_cluster_ips.leader_host + vault_addr = step.create_vault_cluster.api_addr_localhost + vault_root_token = step.create_vault_cluster.root_token + } + } + step "verify_ui" { description = global.description.verify_ui module = module.vault_verify_ui diff --git a/enos/enos-scenario-seal-ha.hcl b/enos/enos-scenario-seal-ha.hcl index 5478a6a99842..555569e880a8 100644 --- a/enos/enos-scenario-seal-ha.hcl +++ b/enos/enos-scenario-seal-ha.hcl @@ -794,6 +794,34 @@ scenario "seal_ha" { } } + step "verify_log_secrets" { + skip_step = !var.vault_enable_audit_devices || !var.verify_log_secrets + + description = global.description.verify_log_secrets + module = module.verify_log_secrets + depends_on = [ + step.verify_secrets_engines_read, + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_audit_log_secrets, + quality.vault_journal_secrets, + quality.vault_radar_index_create, + quality.vault_radar_scan_file, + ] + + variables { + audit_log_file_path = step.create_vault_cluster.audit_device_file_path + leader_host = step.get_updated_cluster_ips.leader_host + vault_addr = step.create_vault_cluster.api_addr_localhost + vault_root_token = step.create_vault_cluster.root_token + } + } + step "verify_ui" { description = global.description.verify_ui module = module.vault_verify_ui diff --git a/enos/enos-scenario-smoke.hcl b/enos/enos-scenario-smoke.hcl index cfbfe3a07ca2..d883a3ff9572 100644 --- a/enos/enos-scenario-smoke.hcl +++ b/enos/enos-scenario-smoke.hcl @@ -583,6 +583,34 @@ scenario "smoke" { } } + step "verify_log_secrets" { + skip_step = !var.vault_enable_audit_devices || !var.verify_log_secrets + + description = global.description.verify_log_secrets + module = module.verify_log_secrets + depends_on = [ + step.verify_secrets_engines_read, + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_audit_log_secrets, + quality.vault_journal_secrets, + quality.vault_radar_index_create, + quality.vault_radar_scan_file, + ] + + variables { + audit_log_file_path = step.create_vault_cluster.audit_device_file_path + leader_host = step.get_vault_cluster_ips.leader_host + vault_addr = step.create_vault_cluster.api_addr_localhost + vault_root_token = step.create_vault_cluster.root_token + } + } + step "verify_ui" { description = global.description.verify_ui module = module.vault_verify_ui diff --git a/enos/enos-scenario-upgrade.hcl b/enos/enos-scenario-upgrade.hcl index 405cfd47c17e..00eb2204e42e 100644 --- a/enos/enos-scenario-upgrade.hcl +++ b/enos/enos-scenario-upgrade.hcl @@ -649,6 +649,37 @@ scenario "upgrade" { } } + step "verify_log_secrets" { + // Only verify log secrets if the audit devices are turned on and we've enabled the check (as + // it requires a radar license). Some older versions have known issues so we'll skip this step + // in the event that we're upgrading from them, see VAULT-30557 for more information. + skip_step = !var.vault_enable_audit_devices || !var.verify_log_secrets || semverconstraint(var.vault_upgrade_initial_version, "=1.17.3 || =1.17.4 || =1.16.7 || =1.16.8") + + description = global.description.verify_log_secrets + module = module.verify_log_secrets + depends_on = [ + step.verify_secrets_engines_read, + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_audit_log_secrets, + quality.vault_journal_secrets, + quality.vault_radar_index_create, + quality.vault_radar_scan_file, + ] + + variables { + audit_log_file_path = step.create_vault_cluster.audit_device_file_path + leader_host = step.get_updated_vault_cluster_ips.leader_host + vault_addr = step.create_vault_cluster.api_addr_localhost + vault_root_token = step.create_vault_cluster.root_token + } + } + step "verify_raft_auto_join_voter" { description = global.description.verify_raft_cluster_all_nodes_are_voters skip_step = matrix.backend != "raft" diff --git a/enos/enos-variables.hcl b/enos/enos-variables.hcl index 298f295cd378..91402071a979 100644 --- a/enos/enos-variables.hcl +++ b/enos/enos-variables.hcl @@ -188,6 +188,12 @@ variable "vault_product_version" { default = null } +variable "vault_radar_license_path" { + description = "The license for vault-radar which is used to verify the audit log" + type = string + default = null +} + variable "vault_revision" { description = "The git sha of Vault artifact we are testing" type = string @@ -199,3 +205,9 @@ variable "vault_upgrade_initial_version" { type = string default = "1.13.13" } + +variable "verify_log_secrets" { + description = "If true and var.vault_enable_audit_devices is true we'll verify that the audit log does not contain unencrypted secrets. Requires var.vault_radar_license_path to be set to a valid license file." + type = bool + default = false +} diff --git a/enos/modules/build_local/scripts/build.sh b/enos/modules/build_local/scripts/build.sh index 06fc03f39462..b7b095003f2a 100755 --- a/enos/modules/build_local/scripts/build.sh +++ b/enos/modules/build_local/scripts/build.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: BUSL-1.1 diff --git a/enos/modules/generate_failover_secondary_token/scripts/generate-failover-secondary-token.sh b/enos/modules/generate_failover_secondary_token/scripts/generate-failover-secondary-token.sh index e8e0e3094b12..05da4a440749 100644 --- a/enos/modules/generate_failover_secondary_token/scripts/generate-failover-secondary-token.sh +++ b/enos/modules/generate_failover_secondary_token/scripts/generate-failover-secondary-token.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash ## Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: BUSL-1.1 diff --git a/enos/modules/install_packages/scripts/install-packages.sh b/enos/modules/install_packages/scripts/install-packages.sh index 6c6e5dd7dc0e..0b9bfdecf78c 100644 --- a/enos/modules/install_packages/scripts/install-packages.sh +++ b/enos/modules/install_packages/scripts/install-packages.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: BUSL-1.1 diff --git a/enos/modules/softhsm_create_vault_keys/scripts/create-keys.sh b/enos/modules/softhsm_create_vault_keys/scripts/create-keys.sh index aa271cde1dd4..6518779f4087 100644 --- a/enos/modules/softhsm_create_vault_keys/scripts/create-keys.sh +++ b/enos/modules/softhsm_create_vault_keys/scripts/create-keys.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: BUSL-1.1 diff --git a/enos/modules/softhsm_create_vault_keys/scripts/get-keys.sh b/enos/modules/softhsm_create_vault_keys/scripts/get-keys.sh index 6409943f51fd..953880f666fc 100644 --- a/enos/modules/softhsm_create_vault_keys/scripts/get-keys.sh +++ b/enos/modules/softhsm_create_vault_keys/scripts/get-keys.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: BUSL-1.1 diff --git a/enos/modules/softhsm_distribute_vault_keys/scripts/distribute-token.sh b/enos/modules/softhsm_distribute_vault_keys/scripts/distribute-token.sh index 95f896c756d1..34279915c9ff 100644 --- a/enos/modules/softhsm_distribute_vault_keys/scripts/distribute-token.sh +++ b/enos/modules/softhsm_distribute_vault_keys/scripts/distribute-token.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: BUSL-1.1 diff --git a/enos/modules/softhsm_init/scripts/init-softhsm.sh b/enos/modules/softhsm_init/scripts/init-softhsm.sh index c36db5304306..3181d9eb2f20 100644 --- a/enos/modules/softhsm_init/scripts/init-softhsm.sh +++ b/enos/modules/softhsm_init/scripts/init-softhsm.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: BUSL-1.1 diff --git a/enos/modules/softhsm_install/scripts/find-shared-object.sh b/enos/modules/softhsm_install/scripts/find-shared-object.sh index 4afaee8b16b2..52b720d86512 100644 --- a/enos/modules/softhsm_install/scripts/find-shared-object.sh +++ b/enos/modules/softhsm_install/scripts/find-shared-object.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash ## Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: BUSL-1.1 diff --git a/enos/modules/vault_upgrade/scripts/maybe-remove-old-unit-file.sh b/enos/modules/vault_upgrade/scripts/maybe-remove-old-unit-file.sh index 1d584d76d7aa..e5c673a94816 100644 --- a/enos/modules/vault_upgrade/scripts/maybe-remove-old-unit-file.sh +++ b/enos/modules/vault_upgrade/scripts/maybe-remove-old-unit-file.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: BUSL-1.1 diff --git a/enos/modules/vault_upgrade/scripts/restart-vault.sh b/enos/modules/vault_upgrade/scripts/restart-vault.sh index ba067fc88ce1..352199479e92 100644 --- a/enos/modules/vault_upgrade/scripts/restart-vault.sh +++ b/enos/modules/vault_upgrade/scripts/restart-vault.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: BUSL-1.1 diff --git a/enos/modules/vault_verify_autopilot/scripts/smoke-verify-autopilot.sh b/enos/modules/vault_verify_autopilot/scripts/smoke-verify-autopilot.sh index eb0a1a1baf65..6408c760f8ff 100755 --- a/enos/modules/vault_verify_autopilot/scripts/smoke-verify-autopilot.sh +++ b/enos/modules/vault_verify_autopilot/scripts/smoke-verify-autopilot.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: BUSL-1.1 diff --git a/enos/modules/vault_verify_default_lcq/scripts/smoke-verify-default-lcq.sh b/enos/modules/vault_verify_default_lcq/scripts/smoke-verify-default-lcq.sh index 57a943654157..64e8e0f1e7e8 100755 --- a/enos/modules/vault_verify_default_lcq/scripts/smoke-verify-default-lcq.sh +++ b/enos/modules/vault_verify_default_lcq/scripts/smoke-verify-default-lcq.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: BUSL-1.1 diff --git a/enos/modules/vault_verify_undo_logs/scripts/smoke-verify-undo-logs.sh b/enos/modules/vault_verify_undo_logs/scripts/smoke-verify-undo-logs.sh index 99bc7018c7d7..77363317434a 100644 --- a/enos/modules/vault_verify_undo_logs/scripts/smoke-verify-undo-logs.sh +++ b/enos/modules/vault_verify_undo_logs/scripts/smoke-verify-undo-logs.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: BUSL-1.1 diff --git a/enos/modules/verify_log_secrets/main.tf b/enos/modules/verify_log_secrets/main.tf new file mode 100644 index 000000000000..1f40874b69b8 --- /dev/null +++ b/enos/modules/verify_log_secrets/main.tf @@ -0,0 +1,96 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +terraform { + required_providers { + enos = { + source = "registry.terraform.io/hashicorp-forge/enos" + } + } +} + +variable "audit_log_file_path" { + type = string +} + +variable "leader_host" { + type = object({ + ipv6 = string + private_ip = string + public_ip = string + }) + description = "The cluster leader host. Only the leader write to the audit log" +} + +variable "radar_install_dir" { + type = string + description = "The directory where the Vault binary will be installed" + default = "/opt/vault-radar/bin" +} + +variable "radar_license_path" { + description = "The path to a vault-radar license file" +} + +variable "radar_version" { + description = "The version of Vault Radar to install" + default = "0.17.0" # must be >= 0.17.0 + // NOTE: A `semverconstraint` validation condition would be very useful here + // when we get around to exporting our custom enos funcs in the provider. +} + +variable "vault_addr" { + type = string + description = "The local vault API listen address" +} + +variable "vault_root_token" { + type = string + description = "The vault root token" +} + +variable "vault_unit_name" { + type = string + description = "The vault unit name" + default = "vault" +} + +resource "enos_bundle_install" "radar" { + destination = var.radar_install_dir + + release = { + product = "vault-radar" + version = var.radar_version + // Radar doesn't have CE/Ent editions. CE is equivalent to no edition metadata. + edition = "ce" + } + + transport = { + ssh = { + host = var.leader_host.public_ip + } + } +} + +resource "enos_remote_exec" "scan_logs_for_secrets" { + depends_on = [ + enos_bundle_install.radar, + ] + + environment = { + AUDIT_LOG_FILE_PATH = var.audit_log_file_path + VAULT_ADDR = var.vault_addr + VAULT_RADAR_INSTALL_DIR = var.radar_install_dir + VAULT_RADAR_LICENSE = file(var.radar_license_path) + VAULT_TOKEN = var.vault_root_token + VAULT_UNIT_NAME = var.vault_unit_name + } + + scripts = [abspath("${path.module}/scripts/scan_logs_for_secrets.sh")] + + transport = { + ssh = { + host = var.leader_host.public_ip + } + } +} diff --git a/enos/modules/verify_log_secrets/scripts/scan_logs_for_secrets.sh b/enos/modules/verify_log_secrets/scripts/scan_logs_for_secrets.sh new file mode 100644 index 000000000000..7e924d6f292c --- /dev/null +++ b/enos/modules/verify_log_secrets/scripts/scan_logs_for_secrets.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +fail() { + echo "$1" 1>&2 + exit 1 +} + +verify_radar_scan_output_file() { + # Given a file with a radar scan output, filter out tagged false positives and verify that no + # other secrets remain. + if ! jq -eMcn '[inputs] | [.[] | select((.tags == null) or (.tags | contains(["ignore_rule"]) | not ))] | length == 0' < "$2"; then + found=$(jq -eMn '[inputs] | [.[] | select((.tags == null) or (.tags | contains(["ignore_rule"]) | not ))]' < "$2") + fail "failed to radar secrets output: vault radar detected secrets in $1!: $found" + fi +} + +set -e + +[[ -z "$AUDIT_LOG_FILE_PATH" ]] && fail "AUDIT_LOG_FILE_PATH env variable has not been set" +[[ -z "$VAULT_RADAR_INSTALL_DIR" ]] && fail "VAULT_RADAR_INSTALL_DIR env variable has not been set" +# Radar implicitly requires the following for creating the index and running radar itself +[[ -z "$VAULT_RADAR_LICENSE" ]] && fail "VAULT_RADAR_LICENSE env variable has not been set" +[[ -z "$VAULT_ADDR" ]] && fail "VAULT_ADDR env variable has not been set" +[[ -z "$VAULT_TOKEN" ]] && fail "VAULT_TOKEN env variable has not been set" +[[ -z "$VAULT_UNIT_NAME" ]] && fail "VAULT_UNIT_NAME env variable has not been set" + +radar_bin_path=${VAULT_RADAR_INSTALL_DIR}/vault-radar +test -x "$radar_bin_path" || fail "failed to scan vault audit log: unable to locate radar binary at $radar_bin_path" + +# Make sure our audit log file exists. +if [ ! -f "$AUDIT_LOG_FILE_PATH" ]; then + fail "failed to scan vault audit log: no audit logifile found at $AUDIT_LOG_FILE_PATH" +fi + +# Create a readable copy of the audit log. +if ! sudo cp "$AUDIT_LOG_FILE_PATH" audit.log; then + fail "failed to scan vault audit log: could not copy audit log for scanning" +fi + +if ! sudo chmod +r audit.log; then + fail "failed to scan vault audit log: could not make audit log copy readable" +fi + +# Create a radar index file of our KVv2 secret values. +if ! out=$($radar_bin_path index vault --offline --disable-ui --outfile index.jsonl 2>&1); then + fail "failed to generate vault-radar index of vault cluster: $out" +fi + +# Write our ignore rules to avoid known false positives. +mkdir -p "$HOME/.hashicorp/vault-radar" +cat >> "$HOME/.hashicorp/vault-radar/ignore.yaml" << EOF +- secret_values: + - "hmac-sha256:*" +EOF + +# Scan the audit log for known secrets via the audit log and other secrets using radars built-in +# secret types. +if ! out=$("$radar_bin_path" scan file --offline --disable-ui -p audit.log --index-file index.jsonl -f json -o audit-secrets.json 2>&1); then + fail "failed to scan vault audit log: vault-radar scan file failed: $out" +fi + +verify_radar_scan_output_file vault-audit-log audit-secrets.json + +# Scan the vault journal for known secrets via the audit log and other secrets using radars built-in +# secret types. +if ! out=$(sudo journalctl --no-pager -u "$VAULT_UNIT_NAME" -a | "$radar_bin_path" scan file --offline --disable-ui --index-file index.jsonl -f json -o journal-secrets.json 2>&1); then + fail "failed to scan vault journal: vault-radar scan file failed: $out" +fi + +verify_radar_scan_output_file vault-journal journal-secrets.json diff --git a/enos/modules/verify_seal_type/scripts/verify-seal-type.sh b/enos/modules/verify_seal_type/scripts/verify-seal-type.sh index 73ce06fd9e03..82a79856e1bf 100644 --- a/enos/modules/verify_seal_type/scripts/verify-seal-type.sh +++ b/enos/modules/verify_seal_type/scripts/verify-seal-type.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: BUSL-1.1 diff --git a/enos/modules/verify_secrets_engines/modules/create/kv.tf b/enos/modules/verify_secrets_engines/modules/create/kv.tf index 269f64b73eec..c721718590d6 100644 --- a/enos/modules/verify_secrets_engines/modules/create/kv.tf +++ b/enos/modules/verify_secrets_engines/modules/create/kv.tf @@ -8,6 +8,7 @@ locals { kv_write_policy_name = "kv_writer" # sys/policy/kv_writer kv_test_data_path_prefix = "smoke" kv_test_data_value_prefix = "fire" + kv_version = 2 // Response data identity_group_kv_writers_data = jsondecode(enos_remote_exec.identity_group_kv_writers.stdout).data @@ -17,6 +18,7 @@ locals { reader_group_name = local.group_name_kv_writers writer_policy_name = local.kv_write_policy_name mount = local.kv_mount + version = local.kv_version test = { path_prefix = local.kv_test_data_path_prefix value_prefix = local.kv_test_data_value_prefix @@ -36,6 +38,7 @@ resource "enos_remote_exec" "secrets_enable_kv_secret" { environment = { ENGINE = "kv" MOUNT = local.kv_mount + SECRETS_META = "-version=${local.kv_version}" VAULT_ADDR = var.vault_addr VAULT_TOKEN = var.vault_root_token VAULT_INSTALL_DIR = var.vault_install_dir diff --git a/enos/modules/verify_secrets_engines/modules/read/kv.tf b/enos/modules/verify_secrets_engines/modules/read/kv.tf index cfa4b7829e13..6983e749b3e4 100644 --- a/enos/modules/verify_secrets_engines/modules/read/kv.tf +++ b/enos/modules/verify_secrets_engines/modules/read/kv.tf @@ -8,6 +8,7 @@ resource "enos_remote_exec" "kv_get_verify_test_data" { MOUNT = var.create_state.kv.mount SECRET_PATH = "${var.create_state.kv.test.path_prefix}-${each.key}" KEY = "${var.create_state.kv.test.path_prefix}-${each.key}" + KV_VERSION = var.create_state.kv.version VALUE = "${var.create_state.kv.test.value_prefix}-${each.key}" VAULT_ADDR = var.vault_addr VAULT_TOKEN = local.user_login_data.auth.client_token diff --git a/enos/modules/verify_secrets_engines/scripts/kv-verify-value.sh b/enos/modules/verify_secrets_engines/scripts/kv-verify-value.sh index 72427d869642..64d6f29d8b7e 100644 --- a/enos/modules/verify_secrets_engines/scripts/kv-verify-value.sh +++ b/enos/modules/verify_secrets_engines/scripts/kv-verify-value.sh @@ -21,8 +21,9 @@ binpath=${VAULT_INSTALL_DIR}/vault test -x "$binpath" || fail "unable to locate vault binary at $binpath" export VAULT_FORMAT=json -if res=$("$binpath" kv get "$MOUNT/$SECRET_PATH"); then - if jq -Merc --arg VALUE "$VALUE" --arg KEY "$KEY" '.data[$KEY] == $VALUE' <<< "$res"; then +if res=$("$binpath" kv get -mount="$MOUNT" "$SECRET_PATH"); then + # Note that this expects KVv2 response payloads. KVv1 does not include doubly nested .data + if jq -Merc --arg VALUE "$VALUE" --arg KEY "$KEY" '.data.data[$KEY] == $VALUE' <<< "$res"; then printf "kv %s/%s %s=%s is valid\n" "$MOUNT" "$SECRET_PATH" "$KEY" "$VALUE" exit 0 fi diff --git a/enos/modules/verify_secrets_engines/scripts/secrets-enable.sh b/enos/modules/verify_secrets_engines/scripts/secrets-enable.sh index 7cc957a290bf..0e8174a80ee0 100644 --- a/enos/modules/verify_secrets_engines/scripts/secrets-enable.sh +++ b/enos/modules/verify_secrets_engines/scripts/secrets-enable.sh @@ -19,4 +19,4 @@ binpath=${VAULT_INSTALL_DIR}/vault test -x "$binpath" || fail "unable to locate vault binary at $binpath" export VAULT_FORMAT=json -"$binpath" secrets enable -path="$MOUNT" "$ENGINE" +eval "$binpath" secrets enable -path="$MOUNT" "$SECRETS_META" "$ENGINE"