Skip to content

Commit

Permalink
feat: Use shared workflow for ci and nightly
Browse files Browse the repository at this point in the history
  • Loading branch information
@mailo-nr
mailo-nr committed Feb 5, 2025
1 parent 4929c48 commit 674c24c
Show file tree
Hide file tree
Showing 9 changed files with 262 additions and 208 deletions.
181 changes: 181 additions & 0 deletions .github/workflows/base-build.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,181 @@
name: 🧩 Base Build

on:
workflow_call:
inputs:
distribution:
required: true
type: string
nightly:
required: false
type: boolean
default: false
test_cluster_name:
required: false
type: string
default: 'ci-e2etest'
secrets:
registry:
required: false
docker_hub_username:
required: true
docker_hub_password:
required: true
gpg_private_key:
required: true
gpg_passphrase:
required: true
nr_backend_url:
required: false
nr_ingest_key:
required: false
nr_account_id:
required: false
nr_api_base_url:
required: false

jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0 # required for tag metadata

- name: Setup Go
uses: actions/setup-go@v5
with:
go-version: '1.23'
check-latest: true

- name: Tidy go.mod files
run: go mod tidy

- name: Verify build
run: make ci DISTRIBUTIONS=${{ inputs.distribution }}

- name: Login to Docker
uses: docker/login-action@v3
if: ${{ env.ACT }}
with:
registry: docker.io
username: ${{ secrets.docker_hub_username }}
password: ${{ secrets.docker_hub_password }}

- uses: docker/setup-qemu-action@v2

- uses: docker/setup-buildx-action@v2

- name: Import GPG key
id: import_gpg
uses: crazy-max/ghaction-import-gpg@v6
with:
gpg_private_key: ${{ secrets.gpg_private_key }}
passphrase: ${{ secrets.gpg_passphrase }}

- name: Write GPG to path in memory for signing rpm/deb
id: write_gpg_to_path
run: |
GPG_KEY_PATH="$(mktemp /dev/shm/gpg.XXXXXX)"
echo "$GPG_PRIVATE_KEY" | base64 -d >> "$GPG_KEY_PATH"
echo "gpg_key_path=$GPG_KEY_PATH" >> $GITHUB_OUTPUT
env:
GPG_PRIVATE_KEY: ${{ secrets.gpg_private_key }}

- name: Build Args
run: |
if [ ${{ inputs.nightly }} = "true" ]; then
echo "goreleaser_args=--snapshot --clean --skip=publish,validate --timeout 2h --config .goreleaser-nightly.yaml" >> $GITHUB_ENV
else
echo "goreleaser_args=--snapshot --clean --skip=publish,validate --timeout 2h" >> $GITHUB_ENV
fi
- name: Build binaries & packages with GoReleaser
id: goreleaser
uses: goreleaser/goreleaser-action@v6
env:
NFPM_PASSPHRASE: ${{ secrets.gpg_passphrase }}
GPG_FINGERPRINT: ${{ steps.import_gpg.outputs.fingerprint }}
GPG_KEY_PATH: ${{ steps.write_gpg_to_path.outputs.gpg_key_path }}
REGISTRY: "${{ secrets.registry }}"
with:
distribution: goreleaser
version: '~> v2'
args: ${{ env.goreleaser_args }}
workdir: distributions/${{ inputs.distribution}}

- name: Extract image version and arch
run: |
VERSION=$(echo '${{ steps.goreleaser.outputs.metadata }}' | jq -r '.version')
ARCH=$(echo '${{ runner.arch }}' | sed 's/X/amd/g')
echo "version=$VERSION" >> $GITHUB_ENV
echo "arch=${ARCH@L}" >> $GITHUB_ENV
echo "image_ref=$IMAGE_REF" >> $GITHUB_ENV
if [ ${{ inputs.nightly }} = "true" ]; then
echo "image_tag=$VERSION-nightly-$ARCH" >> $GITHUB_ENV
else
echo "image_tag=$VERSION-$ARCH" >> $GITHUB_ENV
fi
- name: Show local images
run: |
docker image ls
- name: Setup local kind cluster
uses: helm/kind-action@v1
with:
version: v0.21.0
cluster_name: ${{ inputs.test_cluster_name }}
wait: 60s

- uses: azure/[email protected]

- name: Run local e2e tests
if: ${{ !inputs.nightly }}
env:
KIND_CLUSTER_NAME: ${{ inputs.test_cluster_name }}
IMAGE_TAG: ${{ env.image_tag }}
IMAGE_REPO: ${{ secrets.registry }}/${{ inputs.distribution}}
DISTRO: ${{ inputs.distribution }}
run: |
make -f ./test/e2e/Makefile ci_test-fast
- name: Run slow local tests
if: ${{ inputs.nightly }}
env:
KIND_CLUSTER_NAME: ${{ inputs.test_cluster_name }}
IMAGE_TAG: ${{ env.image_tag }}
IMAGE_REPO: ${{ secrets.registry }}/${{ inputs.distribution}}
DISTRO: ${{ inputs.distribution }}
NR_BACKEND_URL: ${{ secrets.nr_backend_url }}
NR_INGEST_KEY: ${{ secrets.nr_ingest_key }}
NR_API_KEY: ${{ secrets.nr_api_key }}
NR_ACCOUNT_ID: ${{ secrets.nr_account_id }}
NR_API_BASE_URL: ${{ secrets.nr_api_base_url }}
run: |
make -f ./test/e2e/Makefile ci_test-slow
- name: Trivy security check
uses: aquasecurity/[email protected]
with:
image-ref: "${{ secrets.registry }}/${{ inputs.distribution}}:${{ env.image_tag }}"
format: 'table'
exit-code: '1'
ignore-unfixed: true
vuln-type: 'os,library'
severity: "HIGH,CRITICAL"
env:
# dbs are downloaded async in trivy-cache.yml
TRIVY_SKIP_DB_UPDATE: ${{ !env.ACT }}
TRIVY_SKIP_JAVA_DB_UPDATE: true

- name: Upload Artifacts
uses: actions/[email protected]
if: ${{ inputs.nightly }}
with:
name: artifacts-${{ inputs.distribution }}-${{ env.version }}
path: distributions/${{ inputs.distribution }}/dist/**/*
if-no-files-found: error

173 changes: 68 additions & 105 deletions .github/workflows/ci-nightly.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,112 +16,75 @@ env:
REGISTRY: ${{ secrets.OTELCOMM_AWS_TEST_ACC_ACCOUNT_ID }}.dkr.ecr.us-east-1.amazonaws.com

jobs:
slow-tests:
name: Run slow tests
build-nightly:
strategy:
matrix:
distribution:
- nr-otel-collector
uses: ./.github/workflows/base-build.yaml
with:
nightly: true
distribution: ${{ matrix.distribution }}
test_cluster_name: 'ci-e2etest-nightly'
secrets:
docker_hub_username: ${{ secrets.OTELCOMM_DOCKER_HUB_USERNAME }}
docker_hub_password: ${{ secrets.OTELCOMM_DOCKER_HUB_PASSWORD }}
gpg_private_key: ${{ secrets.OHAI_GPG_PRIVATE_KEY_BASE64 }}
gpg_passphrase: ${{ secrets.OHAI_GPG_PASSPHRASE }}
registry: '${{ secrets.OTELCOMM_AWS_TEST_ACC_ACCOUNT_ID }}.dkr.ecr.us-east-1.amazonaws.com'
nr_backend_url: ${{ secrets.NR_STAGING_BACKEND_URL }}
nr_ingest_key: ${{ secrets.OTELCOMM_NR_INGEST_KEY }}
nr_account_id: ${{ secrets.OTELCOMM_NR_TEST_ACCOUNT_ID }}
nr_api_base_url: ${{ secrets.NR_STAGING_API_BASE_URL }}

publish-nightly:
runs-on: ubuntu-latest
needs: build-nightly
strategy:
matrix:
distribution:
- nr-otel-collector
steps:
- name: Checkout repository
uses: actions/checkout@v4
- uses: actions/[email protected]
with:
ref: ${{ inputs.branch || 'main' }}
fetch-depth: 0 # required for tag metadata
pattern: artifacts-${{ inputs.distribution }}-*
path: distributions/${{ inputs.distribution }}/dist
merge-multiple: true

- name: Display structure of downloaded files
run: ls -R distributions/${{ inputs.distribution }}/dist

# - name: Configure AWS Credentials
# uses: aws-actions/configure-aws-credentials@v4
# with:
# aws-access-key-id: ${{ secrets.OTELCOMM_AWS_TEST_ACC_ACCESS_KEY_ID }}
# aws-secret-access-key: ${{ secrets.OTELCOMM_AWS_TEST_ACC_SECRET_ACCESS_KEY }}
# aws-region: us-east-1
# role-to-assume: arn:aws:iam::${{ secrets.OTELCOMM_AWS_TEST_ACC_ACCOUNT_ID }}:role/resource-provisioner
# role-skip-session-tagging: true
#
# - name: Login to ECR
# uses: docker/login-action@v3
# with:
# registry: ${{ secrets.OTELCOMM_AWS_TEST_ACC_ACCOUNT_ID }}.dkr.ecr.us-east-1.amazonaws.com/${{ matrix.distribution }}
#
# - name: Build and publish nightly binaries & packages with GoReleaser
# uses: goreleaser/goreleaser-action@v6
# env:
# NFPM_PASSPHRASE: ${{ secrets.OHAI_GPG_PASSPHRASE }}
# GPG_FINGERPRINT: ${{ steps.import_gpg.outputs.fingerprint }}
# GPG_KEY_PATH: ${{ steps.write_gpg_to_path.outputs.gpg_key_path }}
# with:
# distribution: goreleaser
# version: '~> v2'
# args: --skip=announce,validate --clean --timeout 2h --config .goreleaser-nightly.yaml
# workdir: distributions/${{ matrix.distribution }}
#
# - name: Run nightly tests
# run: |
# NR_API_KEY=${{ secrets.OTELCOMM_NR_API_KEY }} \
# NR_ACCOUNT_ID=${{ secrets.OTELCOMM_NR_TEST_ACCOUNT_ID }} \
# NR_API_BASE_URL=${{ secrets.NR_STAGING_API_BASE_URL }} \
# DISTRO=${{ matrix.distribution }} \
# make -f ./test/e2e/Makefile ci_test-nightly

- name: Setup Go
uses: actions/setup-go@v5
with:
go-version: '1.23'
check-latest: true

- name: Verify build
run: make ci

- uses: docker/setup-qemu-action@v2

- uses: docker/setup-buildx-action@v2

- name: Import GPG key
id: import_gpg
uses: crazy-max/ghaction-import-gpg@v6
with:
gpg_private_key: ${{ secrets.OHAI_GPG_PRIVATE_KEY_BASE64 }}
passphrase: ${{ secrets.OHAI_GPG_PASSPHRASE }}

- name: Write GPG to path in memory for signing rpm/deb
id: write_gpg_to_path
run: |
GPG_KEY_PATH="$(mktemp /dev/shm/gpg.XXXXXX)"
echo "$GPG_PRIVATE_KEY" | base64 -d >> "$GPG_KEY_PATH"
echo "gpg_key_path=$GPG_KEY_PATH" >> $GITHUB_OUTPUT
env:
GPG_PRIVATE_KEY: ${{ secrets.OHAI_GPG_PRIVATE_KEY_BASE64 }}

- name: Build binaries & packages with GoReleaser
id: goreleaser_snapshot
uses: goreleaser/goreleaser-action@v6
env:
NFPM_PASSPHRASE: ${{ secrets.OHAI_GPG_PASSPHRASE }}
GPG_FINGERPRINT: ${{ steps.import_gpg.outputs.fingerprint }}
GPG_KEY_PATH: ${{ steps.write_gpg_to_path.outputs.gpg_key_path }}
REGISTRY: "newrelic/nr-otel-collector"
with:
distribution: goreleaser
version: '~> v2'
args: --snapshot --clean --skip=publish,validate --timeout 2h --config .goreleaser-nightly.yaml
workdir: distributions/nr-otel-collector

- name: List contents of dist folder
run: ls -la distributions/nr-otel-collector/dist

- name: Extract image version
run: echo "version=$(echo '${{ steps.goreleaser_snapshot.outputs.metadata }}' | jq -r '.version')" >> $GITHUB_ENV

- name: Setup local kind cluster
uses: helm/kind-action@v1
with:
version: v0.21.0
cluster_name: ${{ env.TEST_CLUSTER_NAME }}
wait: 60s

- name: Run slow local tests
run: |
IMAGE_TAG=${{ env.version }}-nightly-amd64 \
KIND_CLUSTER_NAME=${{ env.TEST_CLUSTER_NAME }} \
NR_BACKEND_URL=${{ secrets.NR_STAGING_BACKEND_URL }} \
NR_INGEST_KEY=${{ secrets.OTELCOMM_NR_INGEST_KEY }} \
NR_API_KEY=${{ secrets.OTELCOMM_NR_API_KEY }} \
NR_ACCOUNT_ID=${{ secrets.OTELCOMM_NR_TEST_ACCOUNT_ID }} \
NR_API_BASE_URL=${{ secrets.NR_STAGING_API_BASE_URL }} \
make -f ./test/e2e/Makefile ci_test-slow
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.OTELCOMM_AWS_TEST_ACC_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.OTELCOMM_AWS_TEST_ACC_SECRET_ACCESS_KEY }}
aws-region: us-east-1
role-to-assume: arn:aws:iam::${{ secrets.OTELCOMM_AWS_TEST_ACC_ACCOUNT_ID }}:role/resource-provisioner
role-skip-session-tagging: true

- name: Login to ECR
uses: docker/login-action@v3
with:
registry: ${{ secrets.OTELCOMM_AWS_TEST_ACC_ACCOUNT_ID }}.dkr.ecr.us-east-1.amazonaws.com/nr-otel-collector

- name: Build and publish nightly binaries & packages with GoReleaser
uses: goreleaser/goreleaser-action@v6
env:
NFPM_PASSPHRASE: ${{ secrets.OHAI_GPG_PASSPHRASE }}
GPG_FINGERPRINT: ${{ steps.import_gpg.outputs.fingerprint }}
GPG_KEY_PATH: ${{ steps.write_gpg_to_path.outputs.gpg_key_path }}
with:
distribution: goreleaser
version: '~> v2'
args: --skip=announce,validate --clean --timeout 2h --config .goreleaser-nightly.yaml
workdir: distributions/nr-otel-collector

- name: Run nightly tests
run: |
NR_API_KEY=${{ secrets.OTELCOMM_NR_API_KEY }} \
NR_ACCOUNT_ID=${{ secrets.OTELCOMM_NR_TEST_ACCOUNT_ID }} \
NR_API_BASE_URL=${{ secrets.NR_STAGING_API_BASE_URL }} \
make -f ./test/e2e/Makefile ci_test-nightly
Loading

0 comments on commit 674c24c

Please sign in to comment.