VAULT-32188: Enos test for PKI certificates

enos/enos-scenario-agent.hcl

@@ -582,6 +582,7 @@ scenario "agent" {
       hosts             = step.get_vault_cluster_ips.follower_hosts
       vault_addr        = step.create_vault_cluster.api_addr_localhost
       vault_install_dir = global.vault_install_dir[matrix.artifact_type]
+      vault_root_token  = step.create_vault_cluster.root_token
     }
   }
 

enos/enos-scenario-autopilot.hcl

@@ -592,6 +592,7 @@ scenario "autopilot" {
       hosts             = step.get_updated_vault_cluster_ips.follower_hosts
       vault_addr        = step.upgrade_vault_cluster_with_autopilot.api_addr_localhost
       vault_install_dir = local.vault_install_dir
+      vault_root_token  = step.create_vault_cluster.root_token
     }
   }
 

enos/enos-scenario-dr-replication.hcl

@@ -1111,6 +1111,8 @@ scenario "dr_replication" {
       hosts             = step.get_secondary_cluster_ips.follower_hosts
       vault_addr        = step.create_secondary_cluster.api_addr_localhost
       vault_install_dir = global.vault_install_dir[matrix.artifact_type]
+      vault_root_token  = step.create_secondary_cluster.root_token
+      verify_pki_certs  = false
     }
   }
 
@@ -1247,6 +1249,8 @@ scenario "dr_replication" {
       hosts             = step.get_secondary_cluster_ips.follower_hosts
       vault_addr        = step.create_secondary_cluster.api_addr_localhost
       vault_install_dir = global.vault_install_dir[matrix.artifact_type]
+      vault_root_token  = step.create_secondary_cluster.root_token
+      verify_pki_certs  = false
     }
   }
 

enos/enos-scenario-pr-replication.hcl

@@ -943,6 +943,8 @@ scenario "pr_replication" {
       hosts             = step.get_secondary_cluster_ips.follower_hosts
       vault_addr        = step.create_secondary_cluster.api_addr_localhost
       vault_install_dir = global.vault_install_dir[matrix.artifact_type]
+      vault_root_token  = step.create_secondary_cluster.root_token
+      verify_pki_certs  = false
     }
   }
 

enos/enos-scenario-proxy.hcl

@@ -559,6 +559,7 @@ scenario "proxy" {
       hosts             = step.get_vault_cluster_ips.follower_hosts
       vault_addr        = step.create_vault_cluster.api_addr_localhost
       vault_install_dir = global.vault_install_dir[matrix.artifact_type]
+      vault_root_token  = step.create_vault_cluster.root_token
     }
   }
 

enos/enos-scenario-seal-ha.hcl

@@ -812,6 +812,7 @@ scenario "seal_ha" {
       hosts             = step.get_updated_cluster_ips.follower_hosts
       vault_addr        = step.create_vault_cluster.api_addr_localhost
       vault_install_dir = global.vault_install_dir[matrix.artifact_type]
+      vault_root_token  = step.create_vault_cluster.root_token
     }
   }
 
@@ -1040,6 +1041,7 @@ scenario "seal_ha" {
       hosts             = step.get_cluster_ips_after_migration.follower_hosts
       vault_addr        = step.create_vault_cluster.api_addr_localhost
       vault_install_dir = global.vault_install_dir[matrix.artifact_type]
+      vault_root_token  = step.create_vault_cluster.root_token
     }
   }
 

enos/enos-scenario-smoke.hcl

@@ -601,6 +601,7 @@ scenario "smoke" {
       hosts             = step.get_vault_cluster_ips.follower_hosts
       vault_addr        = step.create_vault_cluster.api_addr_localhost
       vault_install_dir = global.vault_install_dir[matrix.artifact_type]
+      vault_root_token  = step.create_vault_cluster.root_token
     }
   }
 

enos/enos-scenario-upgrade.hcl

@@ -668,6 +668,7 @@ scenario "upgrade" {
       hosts             = step.get_updated_vault_cluster_ips.follower_hosts
       vault_addr        = step.create_vault_cluster.api_addr_localhost
       vault_install_dir = global.vault_install_dir[matrix.artifact_type]
+      vault_root_token  = step.create_vault_cluster.root_token
     }
   }
 

enos/modules/verify_secrets_engines/modules/create/main.tf

@@ -49,5 +49,6 @@ output "state" {
     auth     = local.auth_output
     identity = local.identity_output
     kv       = local.kv_output
+    pki      = local.pki_output
   }
 }

enos/modules/verify_secrets_engines/modules/create/pki.tf

@@ -0,0 +1,69 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+locals {
+  // Variables
+  pki_mount       = "pki" # secret
+  pki_issuer_name = "issuer"
+  pki_common_name = "common"
+  pki_default_ttl = "72h"
+  pki_test_dir    = "tmp-test-results"
+
+  // Output
+  pki_output = {
+    common_name = local.pki_common_name
+    issuer_name = local.pki_issuer_name
+    mount       = local.pki_mount
+    ttl         = local.pki_default_ttl
+    test_dir    = local.pki_test_dir
+  }
+
+}
+
+output "pki" {
+  value = local.pki_output
+}
+
+# Enable pki secrets engine
+resource "enos_remote_exec" "secrets_enable_pki_secret" {
+  environment = {
+    ENGINE            = local.pki_mount
+    MOUNT             = local.pki_mount
+    VAULT_ADDR        = var.vault_addr
+    VAULT_TOKEN       = var.vault_root_token
+    VAULT_INSTALL_DIR = var.vault_install_dir
+  }
+
+  scripts = [abspath("${path.module}/../../scripts/secrets-enable.sh")]
+
+  transport = {
+    ssh = {
+      host = var.leader_host.public_ip
+    }
+  }
+}
+
+# Issue RSA Certificate
+resource "enos_remote_exec" "pki_issue_certificates" {
+  depends_on = [enos_remote_exec.secrets_enable_pki_secret]
+  for_each   = var.hosts
+
+  environment = {
+    MOUNT             = local.pki_mount
+    VAULT_ADDR        = var.vault_addr
+    VAULT_INSTALL_DIR = var.vault_install_dir
+    VAULT_TOKEN       = var.vault_root_token
+    COMMON_NAME       = local.pki_common_name
+    ISSUER_NAME       = local.pki_issuer_name
+    TTL               = local.pki_default_ttl
+    TEST_DIR          = local.pki_test_dir
+  }
+
+  scripts = [abspath("${path.module}/../../scripts/pki-issue-certificates.sh")]
+
+  transport = {
+    ssh = {
+      host = each.value.public_ip
+    }
+  }
+}

enos/modules/verify_secrets_engines/modules/read/main.tf

@@ -38,6 +38,12 @@ variable "vault_root_token" {
   default     = null
 }
 
+variable "verify_pki_certs" {
+  type        = bool
+  description = "Flag to verify pki certificates"
+  default     = true
+}
+
 locals {
   vault_bin_path = "${var.vault_install_dir}/vault"
 }

enos/modules/verify_secrets_engines/modules/read/pki.tf

@@ -0,0 +1,31 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+# Verify PKI Certificate
+resource "enos_remote_exec" "pki_verify_certificates" {
+  for_each = var.hosts
+
+  environment = {
+    MOUNT             = var.create_state.pki.mount
+    AUTH_PATH         = "${var.create_state.auth.userpass.path}"
+    USERNAME          = "${var.create_state.auth.userpass.user.name}"
+    PASSWORD          = "${var.create_state.auth.userpass.user.password}"
+    VAULT_ADDR        = var.vault_addr
+    VAULT_INSTALL_DIR = var.vault_install_dir
+    VAULT_TOKEN       = var.vault_root_token
+    COMMON_NAME       = var.create_state.pki.common_name
+    ISSUER_NAME       = var.create_state.pki.issuer_name
+    TTL               = var.create_state.pki.ttl
+    TEST_DIR          = var.create_state.pki.test_dir
+    VERIFY_PKI_CERTS  = var.verify_pki_certs
+  }
+
+  scripts = [abspath("${path.module}/../../scripts/pki-verify-certificates.sh")]
+
+  transport = {
+    ssh = {
+      host = each.value.public_ip
+    }
+  }
+}
+

enos/modules/verify_secrets_engines/scripts/pki-issue-certificates.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+set -e
+
+fail() {
+  echo "$1" 1>&2
+  exit 1
+}
+
+[[ -z "$MOUNT" ]] && fail "MOUNT env variable has not been set"
+[[ -z "$VAULT_ADDR" ]] && fail "VAULT_ADDR env variable has not been set"
+[[ -z "$VAULT_INSTALL_DIR" ]] && fail "VAULT_INSTALL_DIR env variable has not been set"
+[[ -z "$VAULT_TOKEN" ]] && fail "VAULT_TOKEN env variable has not been set"
+[[ -z "$COMMON_NAME" ]] && fail "COMMON_NAME env variable has not been set"
+[[ -z "$ISSUER_NAME" ]] && fail "ISSUER_NAME env variable has not been set"
+[[ -z "$TTL" ]] && fail "TTL env variable has not been set"
+[[ -z "$TEST_DIR" ]] && fail "TEST_DIR env variable has not been set"
+
+binpath=${VAULT_INSTALL_DIR}/vault
+test -x "$binpath" || fail "unable to locate vault binary at $binpath"
+export VAULT_FORMAT=json
+
+# ------ Generate and sign certificate ------
+CA_NAME="${MOUNT}-ca.pem"
+ISSUED_CERT_NAME="${MOUNT}-issued.pem"
+ROLE_NAME="${COMMON_NAME}-role"
+SUBJECT="test.${COMMON_NAME}"
+TMP_TTL="1h"
+rm -rf "${TEST_DIR}"
+mkdir "${TEST_DIR}"
+
+## Setting AIA fields for Certificate
+"$binpath" write "${MOUNT}/config/urls" issuing_certificates="${VAULT_ADDR}/v1/pki/ca" crl_distribution_points="${VAULT_ADDR}/v1/pki/crl"
+
+# Generating CA Certificate
+"$binpath" write "${MOUNT}/root/generate/internal" common_name="${COMMON_NAME}.com" issuer_name="${ISSUER_NAME}" ttl="${TTL}" | jq -r '.data.issuing_ca' > "${TEST_DIR}/${CA_NAME}"
+# Creating a role
+"$binpath" write "${MOUNT}/roles/${ROLE_NAME}" allowed_domains="${COMMON_NAME}.com" allow_subdomains=true max_ttl="${TMP_TTL}"
+# Issuing Signed Certificate
+"$binpath" write "${MOUNT}/issue/${ROLE_NAME}" common_name="${SUBJECT}.com" ttl="${TMP_TTL}" | jq -r '.data.certificate' > "${TEST_DIR}/${ISSUED_CERT_NAME}"
+
+# ------ Generate and sign intermediate ------
+INTERMEDIATE_COMMON_NAME="intermediate-${COMMON_NAME}"
+INTERMEDIATE_ISSUER_NAME="intermediate-${ISSUER_NAME}"
+INTERMEDIATE_ROLE_NAME="intermediate-${COMMON_NAME}-role"
+INTERMEDIATE_CA_NAME="${MOUNT}-${INTERMEDIATE_COMMON_NAME}.pem"
+INTERMEDIATE_SIGNED_NAME="${MOUNT}-${INTERMEDIATE_COMMON_NAME}-ca.pem"
+INTERMEDIATE_ISSUED_NAME="${MOUNT}-${INTERMEDIATE_COMMON_NAME}-issued.pem"
+
+# Generate Intermediate CSR
+"$binpath" write "${MOUNT}/intermediate/generate/internal" common_name="${INTERMEDIATE_COMMON_NAME}.com" issuer_name="${INTERMEDIATE_ISSUER_NAME}" ttl="${TTL}" | jq -r '.data.csr' > "${TEST_DIR}/${INTERMEDIATE_CA_NAME}"
+# Creating a intermediate role
+"$binpath" write "${MOUNT}/roles/${INTERMEDIATE_ROLE_NAME}" allowed_domains="${INTERMEDIATE_COMMON_NAME}.com" allow_subdomains=true max_ttl="${TMP_TTL}"
+# Sign Intermediate Certificate
+"$binpath" write "${MOUNT}/root/sign-intermediate" csr="@${TEST_DIR}/${INTERMEDIATE_CA_NAME}" format=pem_bundle ttl="${TMP_TTL}" | jq -r '.data.certificate' > "${TEST_DIR}/${INTERMEDIATE_SIGNED_NAME}"
+# Import Signed Intermediate Certificate into Vault
+"$binpath" write "${MOUNT}/intermediate/set-signed" certificate="@${TEST_DIR}/${INTERMEDIATE_SIGNED_NAME}"
+# Issuing Signed Certificate with the intermediate role
+"$binpath" write "${MOUNT}/issue/${INTERMEDIATE_ROLE_NAME}" common_name="www.${INTERMEDIATE_COMMON_NAME}.com" ttl="${TMP_TTL}" | jq -r '.data.certificate' > "${TEST_DIR}/${INTERMEDIATE_ISSUED_NAME}"

enos/modules/verify_secrets_engines/scripts/pki-verify-certificates.sh

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+set -e
+
+fail() {
+  echo "$1" 1>&2
+  exit 1
+}
+
+[[ -z "$AUTH_PATH" ]] && fail "AUTH_PATH env variable has not been set"
+[[ -z "$USERNAME" ]] && fail "USERNAME env variable has not been set"
+[[ -z "$PASSWORD" ]] && fail "PASSWORD env variable has not been set"
+[[ -z "$VERIFY_PKI_CERTS" ]] && fail "VERIFY_CERT_DETAILS env variable has not been set"
+[[ -z "$MOUNT" ]] && fail "MOUNT env variable has not been set"
+[[ -z "$VAULT_ADDR" ]] && fail "VAULT_ADDR env variable has not been set"
+[[ -z "$VAULT_INSTALL_DIR" ]] && fail "VAULT_INSTALL_DIR env variable has not been set"
+[[ -z "$VAULT_TOKEN" ]] && fail "VAULT_TOKEN env variable has not been set"
+[[ -z "$COMMON_NAME" ]] && fail "COMMON_NAME env variable has not been set"
+[[ -z "$ISSUER_NAME" ]] && fail "ISSUER_NAME env variable has not been set"
+[[ -z "$TTL" ]] && fail "TTL env variable has not been set"
+[[ -z "$TEST_DIR" ]] && fail "TEST_DIR env variable has not been set"
+
+binpath=${VAULT_INSTALL_DIR}/vault
+test -x "$binpath" || fail "unable to locate vault binary at $binpath" || fail "The certificate appears to be improperly configured or contains errors"
+export VAULT_FORMAT=json
+
+# Log in so this vault instance have access to the primary pki roles, issuers, and etc
+if [ "${VERIFY_PKI_CERTS}" = false ]; then
+  echo "Logging in Vault with username and password: ${USERNAME}"
+  VAULT_TOKEN=$("$binpath" write "auth/$AUTH_PATH/login/$USERNAME" password="$PASSWORD" | jq -r '.auth.client_token')
+fi
+
+# Verifying List Roles
+ROLE=$("$binpath" list "${MOUNT}/roles" | jq -r '.[]')
+[[ -z "$ROLE" ]] && fail "No roles created!"
+
+# Verifying List Issuer
+ISSUER=$("$binpath" list "${MOUNT}/issuers" | jq -r '.[]')
+[[ -z "$ISSUER" ]] && fail "No issuers created!"
+
+# Verifying Root CA Certificate
+ROOT_CA_CERT=$("$binpath" read pki/cert/ca | jq -r '.data.certificate')
+[[ -z "$ROOT_CA_CERT" ]] && fail "No root ca certificate generated"
+
+# Verifying Certificates
+if [ "${VERIFY_PKI_CERTS}" = true ]; then
+  if [ ! -d "${TEST_DIR}" ]; then
+      echo "Directory does not exist. Creating it now."
+      mkdir -p "${TEST_DIR}" # Need to create this directory for Enterprise test
+  fi
+  TMP_FILE="tmp-vault-cert.pem"
+
+  # Verify List Certificate
+  VAULT_CERTS=$("$binpath" list "${MOUNT}/certs" | jq -r '.[]')
+  [[ -z "$VAULT_CERTS" ]] && fail "VAULT_CERTS should include vault certificates"
+  for CERT in $VAULT_CERTS; do
+    echo "Getting certificate from Vault PKI: ${CERT}"
+    "$binpath" read "${MOUNT}/cert/${CERT}" | jq -r '.data.certificate' > "${TEST_DIR}/${TMP_FILE}"
+    echo "Verifying certificate contents..."
+    openssl x509 -in "${TEST_DIR}/${TMP_FILE}" -text -noout || fail "The certificate appears to be improperly configured or contains errors"
+    CURR_CERT_SERIAL=$(echo "${CERT}" | tr -d ':' | tr '[:lower:]' '[:upper:]')
+    TMP_CERT_SUBJECT=$(openssl x509 -in "${TEST_DIR}/${TMP_FILE}" -noout -subject | awk -F'= ' '{print $2}')
+    TMP_CERT_ISSUER=$(openssl x509 -in "${TEST_DIR}/${TMP_FILE}" -noout -issuer | awk -F'= ' '{print $2}')
+    TMP_CERT_SERIAL=$(openssl x509 -in "${TEST_DIR}/${TMP_FILE}" -noout -serial | awk -F'=' '{print $2}')
+    [[ "${TMP_CERT_SUBJECT}" == *"${COMMON_NAME}.com"* ]] || fail "Subject is incorrect. Actual Subject: ${TMP_CERT_SUBJECT}"
+    [[ "${TMP_CERT_ISSUER}" == *"${COMMON_NAME}.com"* ]] || fail "Issuer is incorrect. Actual Issuer: ${TMP_CERT_ISSUER}"
+    [[ "${TMP_CERT_SERIAL}" == *"${CURR_CERT_SERIAL}"* ]] || fail "Certificate Serial is incorrect. Actual certificate Serial: ${CURR_CERT_SERIAL},${TMP_CERT_SERIAL}"
+    echo "Successfully verified certificate contents."
+
+    # Setting up variables for types of certificates
+    IS_CA=$(openssl x509 -in "${TEST_DIR}/${TMP_FILE}" -text -noout | grep -q "CA:TRUE" && echo "TRUE" || echo "FALSE")
+    if [[ "${IS_CA}" == "TRUE" ]]; then
+      if [[ "${COMMON_NAME}.com" == "${TMP_CERT_SUBJECT}" ]]; then
+        CA_CERT=${CERT}
+      elif [[ "intermediate-${COMMON_NAME}.com" == "${TMP_CERT_SUBJECT}" ]]; then
+        INTERMEDIATE_CA_CERT=${CERT}
+      fi
+    elif [[ "${IS_CA}" == "FALSE" ]]; then
+      INTERMEDIATE_ISSUED_CERT=${CERT}
+    fi
+
+  done
+
+  echo "Verifying that Vault PKI has successfully generated valid certificates for the CA, Intermediate CA, and issued certificates..."
+  if [[ -n "${CA_CERT}" ]] && [[ -n "${INTERMEDIATE_CA_CERT}" ]] && [[ -n "${INTERMEDIATE_ISSUED_CERT}" ]]; then
+    CA_NAME="ca.pem"
+    INTERMEDIATE_CA_NAME="intermediate-ca.pem"
+    ISSUED_NAME="issued.pem"
+    "$binpath" read "${MOUNT}/cert/${CA_CERT}" | jq -r '.data.certificate' > "${TEST_DIR}/${CA_NAME}"
+    "$binpath" read "${MOUNT}/cert/${INTERMEDIATE_CA_CERT}" | jq -r '.data.certificate' > "${TEST_DIR}/${INTERMEDIATE_CA_NAME}"
+    "$binpath" read "${MOUNT}/cert/${INTERMEDIATE_ISSUED_CERT}" | jq -r '.data.certificate' > "${TEST_DIR}/${ISSUED_NAME}"
+    openssl verify --CAfile "${TEST_DIR}/${CA_NAME}" -untrusted "${TEST_DIR}/${INTERMEDIATE_CA_NAME}" "${TEST_DIR}/${ISSUED_NAME}" || fail "One or more Certificate is not valid."
+  else
+    echo "CA Cert: ${CA_CERT}, Intermedidate Cert: ${INTERMEDIATE_CA_CERT}, Issued Cert: ${INTERMEDIATE_ISSUED_CERT}"
+  fi
+
+  echo "Revoking certificate: ${INTERMEDIATE_ISSUED_CERT}"
+  "$binpath" write "${MOUNT}/revoke" serial_number="${INTERMEDIATE_ISSUED_CERT}" || fail "Could not revoke certificate ${INTERMEDIATE_ISSUED_CERT}"
+  echo "Verifying Revoked Certificate"
+  REVOKED_CERT_FROM_LIST=$("$binpath" list "${MOUNT}/certs/revoked" | jq -r '.[0]')
+  [[ "${INTERMEDIATE_ISSUED_CERT}" == "${REVOKED_CERT_FROM_LIST}" ]] || fail "Expected: ${INTERMEDIATE_ISSUED_CERT}, actual: ${REVOKED_CERT_FROM_LIST}"
+  echo "Successfully verified revoked certificate"
+else
+  echo "Skipping verify certificates!"
+fi