Skip to content

Commit

Permalink
CI: test against n-2 versions of Vault (#604)
Browse files Browse the repository at this point in the history
  • Loading branch information
benashz authored Apr 16, 2024
1 parent 8e63e94 commit 0e10b04
Show file tree
Hide file tree
Showing 6 changed files with 227 additions and 78 deletions.
116 changes: 116 additions & 0 deletions .github/actions/integration-test/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: MPL-2.0

name: Integration test
description: Run the integration tests against a single version of k8s and Vault
inputs:
k8s-version:
description: 'Kubernetes version to use for the kind cluster'
required: true
version:
description: 'Vault K8s version to use for the tests'
required: true
vault-version:
description: 'Vault version to use for the tests'
required: true
tarball-file:
description: 'Name of the tarball file artifact to download'
required: true
kind-cluster-name:
description: 'Name of the kind cluster to create and test against'
default: 'vault-k8s'
bats-version:
description: 'Version of bats to run tests with'
default: '1.11.0'
runs:
using: "composite"
steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
- run: sudo npm install -g bats@${{ inputs.bats-version }}
shell: bash
- run: bats -v
shell: bash

- uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
with:
python-version: '3.10'
- run: pip install yq
shell: bash

# Checkout this repo.
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
path: "vault-k8s"

# Checkout vault-helm for acceptance test code.
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
repository: "hashicorp/vault-helm"
ref: "v0.27.0"
path: "vault-helm"

- name: Create Kind Cluster
uses: helm/kind-action@99576bfa6ddf9a8e612d83b513da5a75875caced # v1.9.0
with:
cluster_name: ${{ inputs.kind-cluster-name }}
config: vault-helm/test/kind/config.yaml
node_image: kindest/node:v${{ inputs.k8s-version }}
version: "v0.22.0"

- name: Create kind export log root
id: create_kind_export_log_root
shell: bash
run: |
vault_flavor=community
log_artifact_name="kind-${{ inputs.kind-cluster-name }}-$(git rev-parse --short ${{ github.sha }})-${{ inputs.k8s-version }}-${{ inputs.vault-version }}-${vault_flavor}-helm-logs"
log_root="/tmp/${log_artifact_name}"
mkdir -p "${log_root}"
echo "log_root=${log_root}" >> $GITHUB_OUTPUT
echo "log_artifact_name=${log_artifact_name}" >> $GITHUB_OUTPUT
- uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
with:
name: ${{ inputs.tarball-file }}

- name: Load vault-k8s ${{ inputs.version }} image
shell: bash
run: |
[ -n "${{ inputs.tarball-file }}" ] || (echo "inputs.tarball-file not set" >&2 ; exit 1)
docker image load --input "${{ inputs.tarball-file }}"
kind load docker-image --name ${{ inputs.kind-cluster-name }} hashicorp/vault-k8s:"${{ inputs.version }}"
- name: Makefile tests
shell: bash
working-directory: vault-k8s
run: make deploy exercise teardown VERSION="${{ inputs.version }}" VAULT_VERSION="${{ inputs.vault-version }}"

- name: bats tests
shell: bash
working-directory: vault-helm
run: |
# assumes Python based yq :(
yq --in-place --yaml-roundtrip '.injector.image.tag |= "${{ inputs.version }}"' ./values.yaml
yq --in-place --yaml-roundtrip '.server.image.tag |= "${{ inputs.vault-version }}"' ./values.yaml
yq --in-place --yaml-roundtrip '.injector.agentImage.tag |= "${{ inputs.vault-version }}"' ./values.yaml
bats ./test/acceptance -t --filter injector
- name: export kind cluster logs
if: always()
shell: bash
run: |
kind export logs --name ${{ inputs.kind-cluster-name }} ${{ steps.create_kind_export_log_root.outputs.log_root }}
- name: Store kind cluster logs
if: success()
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
with:
name: ${{ steps.create_kind_export_log_root.outputs.log_artifact_name }}
path: ${{ steps.create_kind_export_log_root.outputs.log_root }}

- name: Store kind cluster logs failure
if: failure()
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
with:
name: ${{ steps.create_kind_export_log_root.outputs.log_artifact_name }}-failed
path: ${{ steps.create_kind_export_log_root.outputs.log_root }}
105 changes: 53 additions & 52 deletions .github/workflows/tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,12 @@ on: [push, workflow_dispatch]
jobs:
test:
env:
# vault-k8s version
VERSION: "0.0.0-dev"
TARBALL_FILE: vault-k8s-image.docker.tar
outputs:
version: ${{ env.VERSION }}
tarball_file: ${{ env.TARBALL_FILE }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
Expand All @@ -18,74 +23,70 @@ jobs:
- name: Build
run: |
REGISTRY=hashicorp make build image
docker save --output "${TARBALL_FILE}" hashicorp/vault-k8s:0.0.0-dev
docker save --output "${TARBALL_FILE}" hashicorp/vault-k8s:${{ env.VERSION }}
- name: Test
run: make test

- uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
with:
name: vault-k8s-image
name: ${{ env.TARBALL_FILE }}
path: ${{ env.TARBALL_FILE }}

acceptance:
versions:
runs-on: ubuntu-latest
steps:
- run: echo "setting versions"
outputs:
# vault-k8s version
K8S_VERSIONS: '["1.29.2", "1.28.7", "1.27.11", "1.26.14", "1.25.16"]'
VAULT_N: "1.16.1"
VAULT_N_1: "1.15.6"
VAULT_N_2: "1.14.10"

latest-vault:
name: vault:${{ matrix.vault-version }} kind:${{ matrix.k8s-version }}
runs-on: ubuntu-latest
needs:
- test
- versions
strategy:
fail-fast: false
matrix:
kind-k8s-version: [1.25.16, 1.26.14, 1.27.11, 1.28.7, 1.29.2]
runs-on: ubuntu-latest
steps:
# Setup test tools from https://github.com/hashicorp/vault-helm/blob/main/.github/workflows/setup-test-tools/action.yaml
- uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
- run: sudo npm install -g bats@${BATS_VERSION}
shell: bash
env:
BATS_VERSION: '1.11.0'
- run: bats -v
shell: bash
- uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
with:
python-version: '3.10'
- run: pip install yq
shell: bash
vault-version:
- ${{ needs.versions.outputs.VAULT_N }}
k8s-version: ${{ fromJson(needs.versions.outputs.K8S_VERSIONS) }}

# Checkout this repo.
steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- uses: ./.github/actions/integration-test
name: vault:${{ matrix.vault-version }} kind:${{ matrix.k8s-version }}
with:
path: "vault-k8s"
k8s-version: ${{ matrix.k8s-version }}
vault-version: ${{ matrix.vault-version }}
tarball-file: ${{ needs.test.outputs.tarball_file }}
version: ${{ needs.test.outputs.version }}

# Checkout vault-helm for acceptance test code.
latest-k8s:
name: vault:${{ matrix.vault-version }} kind:${{ matrix.k8s-version }}
needs:
- versions
- test
strategy:
fail-fast: false
matrix:
k8s-version:
- ${{ fromJson(needs.versions.outputs.K8S_VERSIONS)[0] }}
vault-version:
- ${{ needs.versions.outputs.VAULT_N_1 }}
- ${{ needs.versions.outputs.VAULT_N_2 }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- uses: ./.github/actions/integration-test
name: vault:${{ matrix.vault-version }} kind:${{ matrix.k8s-version }}
with:
repository: "hashicorp/vault-helm"
ref: "v0.27.0"
path: "vault-helm"

- name: Create K8s Kind Cluster
uses: helm/kind-action@99576bfa6ddf9a8e612d83b513da5a75875caced # v1.9.0
with:
cluster_name: kind
config: vault-helm/test/kind/config.yaml
node_image: kindest/node:v${{ matrix.kind-k8s-version }}
version: v0.22.0

- uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
with:
name: vault-k8s-image

- name: Load vault-k8s dev image
run: |
docker image load --input vault-k8s-image.docker.tar
kind load docker-image hashicorp/vault-k8s:0.0.0-dev
- name: Makefile tests
working-directory: vault-k8s
run: make deploy exercise teardown

- name: bats tests
working-directory: vault-helm
run: |
yq --in-place --yaml-roundtrip '.injector.image.tag |= "0.0.0-dev"' ./values.yaml
bats ./test/acceptance -t --filter injector
k8s-version: ${{ matrix.k8s-version }}
vault-version: ${{ matrix.vault-version }}
tarball-file: ${{ needs.test.outputs.tarball_file }}
version: ${{ needs.test.outputs.version }}
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,7 @@
/pkg
/dist/

# directory for misc build files etc.
scratch/
build/
vault-helm/
64 changes: 42 additions & 22 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,23 +1,41 @@
REGISTRY_NAME?=docker.io/hashicorp
IMAGE_NAME=vault-k8s
VERSION?=0.0.0-dev
IMAGE_TAG?=$(REGISTRY_NAME)/$(IMAGE_NAME):$(VERSION)
PUBLISH_LOCATION?=https://releases.hashicorp.com
DOCKER_DIR=./build/docker
BUILD_DIR=dist
GOOS?=linux
GOARCH?=amd64
BIN_NAME=$(IMAGE_NAME)
GOFMT_FILES?=$$(find . -name '*.go' | grep -v vendor)
XC_PUBLISH?=
PKG=github.com/hashicorp/vault-k8s/version
LDFLAGS?="-X '$(PKG).Version=v$(VERSION)'"
REGISTRY_NAME ?= docker.io/hashicorp
IMAGE_NAME = vault-k8s
VERSION ?= 0.0.0-dev
VAULT_VERSION ?= 1.16.1
IMAGE_TAG ?= $(REGISTRY_NAME)/$(IMAGE_NAME):$(VERSION)
PUBLISH_LOCATION ?= https://releases.hashicorp.com
DOCKER_DIR = ./build/docker
BUILD_DIR = dist
GOOS ?= linux
GOARCH ?= amd64
BIN_NAME = $(IMAGE_NAME)
GOFMT_FILES ?= $$(find . -name '*.go' | grep -v vendor)
XC_PUBLISH ?=
PKG = github.com/hashicorp/vault-k8s/version
LDFLAGS ?= "-X '$(PKG).Version=v$(VERSION)'"
TESTARGS ?= '-test.v'

VAULT_HELM_CHART_VERSION ?= 0.27.0
VAULT_HELM_FLAGS?=--repo https://helm.releases.hashicorp.com --version=$(VAULT_HELM_CHART_VERSION) \
# TODO: add support for testing against enterprise

TEST_WITHOUT_VAULT_TLS ?=
ifndef TEST_WITHOUT_VAULT_TLS
VAULT_VERSION_PARTS := $(subst ., , $(VAULT_VERSION))
VAULT_MAJOR_VERSION := $(word 1, $(VAULT_VERSION_PARTS))
VAULT_MINOR_VERSION := $(word 2, $(VAULT_VERSION_PARTS))
TEST_WITHOUT_VAULT_TLS := $(shell test $(VAULT_MAJOR_VERSION) -le 1 -a $(VAULT_MINOR_VERSION) -lt 15 && echo 1)
endif

HELM_VALUES_FILE ?= test/vault/dev.values.yaml
ifdef TEST_WITHOUT_VAULT_TLS
HELM_VALUES_FILE = test/vault/dev-no-tls.values.yaml
endif

VAULT_HELM_DEFAULT_ARGS ?= --repo https://helm.releases.hashicorp.com --version=$(VAULT_HELM_CHART_VERSION) \
--wait --timeout=5m \
--values=test/vault/dev.values.yaml \
--values=$(HELM_VALUES_FILE) \
--set server.image.tag=$(VAULT_VERSION) \
--set injector.agentImage.tag=$(VAULT_VERSION) \
--set 'injector.image.tag=$(VERSION)'

.PHONY: all test build image clean version deploy exercise teardown
Expand All @@ -37,20 +55,22 @@ image: build

# Deploys Vault dev server and a locally built Agent Injector.
# Run multiple times to deploy new builds of the injector.
VAULT_HELM_POST_INSTALL_ARGS ?=
ifndef TEST_WITHOUT_VAULT_TLS
VAULT_HELM_POST_INSTALL_ARGS = "--set=injector.extraEnvironmentVars.AGENT_INJECT_VAULT_CACERT_BYTES=$$(kubectl exec vault-0 -- sh -c 'cat /tmp/vault-ca.pem | base64 -w0')"
endif
deploy:
kind load docker-image hashicorp/vault-k8s:$(VERSION)
helm upgrade --install vault vault $(VAULT_HELM_FLAGS) \
helm upgrade --install vault vault $(VAULT_HELM_DEFAULT_ARGS) \
--set "injector.enabled=false"
kubectl delete pod -l "app.kubernetes.io/instance=vault"
kubectl wait --for=condition=Ready --timeout=5m pod -l "app.kubernetes.io/instance=vault"
helm upgrade --install vault vault $(VAULT_HELM_FLAGS) \
--set "injector.extraEnvironmentVars.AGENT_INJECT_VAULT_CACERT_BYTES=$$(kubectl exec vault-0 -- cat /tmp/vault-ca.pem | base64)"
helm upgrade --install vault vault $(VAULT_HELM_DEFAULT_ARGS) $(VAULT_HELM_POST_INSTALL_ARGS)

# Populates the Vault dev server with a secret, configures kubernetes auth, and
# deploys an nginx pod with annotations to have the secret injected.
exercise:
kubectl exec vault-0 -- vault kv put secret/test-app hello=world
kubectl exec vault-0 -- vault auth enable kubernetes || true
kubectl exec vault-0 -- vault auth list -format json | jq -e '."kubernetes/"' || kubectl exec vault-0 -- vault auth enable kubernetes
kubectl exec vault-0 -- sh -c 'vault write auth/kubernetes/config kubernetes_host="https://$$KUBERNETES_PORT_443_TCP_ADDR:443"'
echo 'path "secret/data/*" { capabilities = ["read"] }' | kubectl exec -i vault-0 -- vault policy write test-app -
kubectl exec vault-0 -- vault write auth/kubernetes/role/test-app \
Expand All @@ -70,7 +90,7 @@ exercise:

# Teardown any resources created in deploy and exercise targets.
teardown:
helm uninstall vault || true
helm uninstall --namespace default vault --wait 2> /dev/null || true
kubectl delete --ignore-not-found serviceaccount test-app-sa
kubectl delete --ignore-not-found pod nginx

Expand Down
13 changes: 13 additions & 0 deletions test/vault/dev-no-tls.values.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: MPL-2.0

global:
tlsDisable: true
injector:
image:
pullPolicy: Never
affinity: null
server:
dev:
enabled: true
logLevel: debug
4 changes: 0 additions & 4 deletions test/vault/dev.values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,7 @@ injector:
image:
pullPolicy: Never
affinity: null
agentImage:
tag: 1.16.1
server:
image:
tag: 1.16.1
dev:
enabled: true
logLevel: debug
Expand Down

0 comments on commit 0e10b04

Please sign in to comment.