Skip to content

Commit

Permalink
feat: Use shared workflow for ci and nightly (#208)
Browse files Browse the repository at this point in the history
* feat: Use shared workflow for ci and nightly

* chore: Expose image repo

* chore: Add publish step

* chore: PR feedback and cleanup
  • Loading branch information
mailo-nr authored Feb 6, 2025
1 parent 3158482 commit 50b9f03
Show file tree
Hide file tree
Showing 17 changed files with 264 additions and 194 deletions.
170 changes: 170 additions & 0 deletions .github/workflows/ci-base.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,170 @@
name: 🧩 CI Base

on:
workflow_call:
inputs:
distribution:
required: true
type: string
nightly:
required: false
type: boolean
default: false
test_cluster_name:
required: false
type: string
default: 'ci-e2etest'
secrets:
registry:
required: false
docker_hub_username:
required: true
docker_hub_password:
required: true
gpg_private_key:
required: true
gpg_passphrase:
required: true
nr_backend_url:
required: false
nr_ingest_key:
required: false
nr_account_id:
required: false
nr_api_base_url:
required: false
nr_api_key:
required: false

jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0 # required for tag metadata

- name: Setup Go
uses: actions/setup-go@v5
with:
go-version: '1.23'
check-latest: true

- name: Tidy go.mod files
run: go mod tidy

- name: Verify build
run: make ci DISTRIBUTIONS=${{ inputs.distribution }}

- name: Login to Docker
uses: docker/login-action@v3
if: ${{ env.ACT }}
with:
registry: docker.io
username: ${{ secrets.docker_hub_username }}
password: ${{ secrets.docker_hub_password }}

- uses: docker/setup-qemu-action@v2

- uses: docker/setup-buildx-action@v2

- name: Import GPG key
id: import_gpg
uses: crazy-max/ghaction-import-gpg@v6
with:
gpg_private_key: ${{ secrets.gpg_private_key }}
passphrase: ${{ secrets.gpg_passphrase }}

- name: Write GPG to path in memory for signing rpm/deb
id: write_gpg_to_path
run: |
GPG_KEY_PATH="$(mktemp /dev/shm/gpg.XXXXXX)"
echo "$GPG_PRIVATE_KEY" | base64 -d >> "$GPG_KEY_PATH"
echo "gpg_key_path=$GPG_KEY_PATH" >> $GITHUB_OUTPUT
env:
GPG_PRIVATE_KEY: ${{ secrets.gpg_private_key }}

- name: Build Args
run: |
if [ ${{ inputs.nightly }} = "true" ]; then
echo "goreleaser_args=--snapshot --clean --skip=publish,validate --timeout 2h --config .goreleaser-nightly.yaml" >> $GITHUB_ENV
else
echo "goreleaser_args=--snapshot --clean --skip=publish,validate --timeout 2h" >> $GITHUB_ENV
fi
- name: Build binaries & packages with GoReleaser
id: goreleaser
uses: goreleaser/goreleaser-action@v6
env:
NFPM_PASSPHRASE: ${{ secrets.gpg_passphrase }}
GPG_FINGERPRINT: ${{ steps.import_gpg.outputs.fingerprint }}
GPG_KEY_PATH: ${{ steps.write_gpg_to_path.outputs.gpg_key_path }}
REGISTRY: "${{ secrets.registry }}"
with:
distribution: goreleaser
version: '~> v2'
args: ${{ env.goreleaser_args }}
workdir: distributions/${{ inputs.distribution }}

- name: Extract image version and arch
run: |
VERSION=$(echo '${{ steps.goreleaser.outputs.metadata }}' | jq -r '.version')
ARCH=$(echo '${{ runner.arch }}' | sed 's/X/amd/g')
ARCH=${ARCH@L}
echo "version=$VERSION" >> $GITHUB_ENV
echo "arch=$ARCH" >> $GITHUB_ENV
if [ ${{ inputs.nightly }} = "true" ]; then
echo "image_tag=$VERSION-nightly-$ARCH" >> $GITHUB_ENV
else
echo "image_tag=$VERSION-$ARCH" >> $GITHUB_ENV
fi
- name: Setup local kind cluster
uses: helm/kind-action@v1
with:
version: v0.21.0
cluster_name: ${{ inputs.test_cluster_name }}
wait: 60s

- uses: azure/[email protected]

- name: Run local e2e tests
if: ${{ !inputs.nightly }}
env:
KIND_CLUSTER_NAME: ${{ inputs.test_cluster_name }}
IMAGE_TAG: ${{ env.image_tag }}
IMAGE_REPO: ${{ secrets.registry }}/${{ inputs.distribution}}
DISTRO: ${{ inputs.distribution }}
run: |
make -f ./test/e2e/Makefile ci_test-fast
- name: Run slow local tests
if: ${{ inputs.nightly }}
env:
KIND_CLUSTER_NAME: ${{ inputs.test_cluster_name }}
IMAGE_TAG: ${{ env.image_tag }}
IMAGE_REPO: ${{ secrets.registry }}/${{ inputs.distribution}}
DISTRO: ${{ inputs.distribution }}
NR_BACKEND_URL: ${{ secrets.nr_backend_url }}
NR_INGEST_KEY: ${{ secrets.nr_ingest_key }}
NR_API_KEY: ${{ secrets.nr_api_key }}
NR_ACCOUNT_ID: ${{ secrets.nr_account_id }}
NR_API_BASE_URL: ${{ secrets.nr_api_base_url }}
run: |
make -f ./test/e2e/Makefile ci_test-slow
- name: Trivy security check
if: ${{ !inputs.nightly }}
uses: aquasecurity/[email protected]
with:
image-ref: "${{ secrets.registry }}/${{ inputs.distribution}}:${{ env.image_tag }}"
format: 'table'
exit-code: '1'
ignore-unfixed: true
vuln-type: 'os,library'
severity: "HIGH,CRITICAL"
env:
# dbs are downloaded async in trivy-cache.yml
TRIVY_SKIP_DB_UPDATE: ${{ !env.ACT }}
TRIVY_SKIP_JAVA_DB_UPDATE: true
102 changes: 53 additions & 49 deletions .github/workflows/ci-nightly.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,46 @@ on:
- cron: '0 15 * * 1-5'

env:
TEST_CLUSTER_NAME: ci-e2etest-nightly
REGISTRY: ${{ secrets.OTELCOMM_AWS_TEST_ACC_ACCOUNT_ID }}.dkr.ecr.us-east-1.amazonaws.com
REGISTRY: '${{ secrets.OTELCOMM_AWS_TEST_ACC_ACCOUNT_ID }}.dkr.ecr.us-east-1.amazonaws.com'

jobs:
slow-tests:
name: Run slow tests
build:
name: Build & Validate
strategy:
matrix:
distribution:
- nr-otel-collector
- nrdot-collector-host
uses: ./.github/workflows/ci-base.yaml
with:
nightly: true
distribution: ${{ matrix.distribution }}
test_cluster_name: 'ci-e2etest-nightly'
secrets:
docker_hub_username: ${{ secrets.OTELCOMM_DOCKER_HUB_USERNAME }}
docker_hub_password: ${{ secrets.OTELCOMM_DOCKER_HUB_PASSWORD }}
gpg_private_key: ${{ secrets.OHAI_GPG_PRIVATE_KEY_BASE64 }}
gpg_passphrase: ${{ secrets.OHAI_GPG_PASSPHRASE }}
registry: '${{ secrets.OTELCOMM_AWS_TEST_ACC_ACCOUNT_ID }}.dkr.ecr.us-east-1.amazonaws.com'
nr_backend_url: ${{ secrets.NR_STAGING_BACKEND_URL }}
nr_ingest_key: ${{ secrets.OTELCOMM_NR_INGEST_KEY }}
nr_account_id: ${{ secrets.OTELCOMM_NR_TEST_ACCOUNT_ID }}
nr_api_base_url: ${{ secrets.NR_STAGING_API_BASE_URL }}
nr_api_key: ${{ secrets.OTELCOMM_NR_API_KEY }}

publish:
name: Build & Publish Nightly
runs-on: ubuntu-latest
needs: build
strategy:
matrix:
distribution:
- nr-otel-collector
- nrdot-collector-host
steps:
- name: Checkout repository
- name: Checkout
uses: actions/checkout@v4
with:
ref: ${{ inputs.branch || 'main' }}
fetch-depth: 0 # required for tag metadata

- name: Setup Go
Expand All @@ -32,8 +60,19 @@ jobs:
go-version: '1.23'
check-latest: true

- name: Tidy go.mod files
run: go mod tidy

- name: Verify build
run: make ci
run: make ci DISTRIBUTIONS=${{ matrix.distribution }}

- name: Login to Docker
uses: docker/login-action@v3
if: ${{ env.ACT }}
with:
registry: docker.io
username: ${{ secrets.OTELCOMM_DOCKER_HUB_USERNAME }}
password: ${{ secrets.OTELCOMM_DOCKER_HUB_PASSWORD }}

- uses: docker/setup-qemu-action@v2

Expand All @@ -55,44 +94,6 @@ jobs:
env:
GPG_PRIVATE_KEY: ${{ secrets.OHAI_GPG_PRIVATE_KEY_BASE64 }}

- name: Build binaries & packages with GoReleaser
id: goreleaser_snapshot
uses: goreleaser/goreleaser-action@v6
env:
NFPM_PASSPHRASE: ${{ secrets.OHAI_GPG_PASSPHRASE }}
GPG_FINGERPRINT: ${{ steps.import_gpg.outputs.fingerprint }}
GPG_KEY_PATH: ${{ steps.write_gpg_to_path.outputs.gpg_key_path }}
REGISTRY: "newrelic/nr-otel-collector"
with:
distribution: goreleaser
version: '~> v2'
args: --snapshot --clean --skip=publish,validate --timeout 2h --config .goreleaser-nightly.yaml
workdir: distributions/nr-otel-collector

- name: List contents of dist folder
run: ls -la distributions/nr-otel-collector/dist

- name: Extract image version
run: echo "version=$(echo '${{ steps.goreleaser_snapshot.outputs.metadata }}' | jq -r '.version')" >> $GITHUB_ENV

- name: Setup local kind cluster
uses: helm/kind-action@v1
with:
version: v0.21.0
cluster_name: ${{ env.TEST_CLUSTER_NAME }}
wait: 60s

- name: Run slow local tests
run: |
IMAGE_TAG=${{ env.version }}-nightly-amd64 \
KIND_CLUSTER_NAME=${{ env.TEST_CLUSTER_NAME }} \
NR_BACKEND_URL=${{ secrets.NR_STAGING_BACKEND_URL }} \
NR_INGEST_KEY=${{ secrets.OTELCOMM_NR_INGEST_KEY }} \
NR_API_KEY=${{ secrets.OTELCOMM_NR_API_KEY }} \
NR_ACCOUNT_ID=${{ secrets.OTELCOMM_NR_TEST_ACCOUNT_ID }} \
NR_API_BASE_URL=${{ secrets.NR_STAGING_API_BASE_URL }} \
make -f ./test/e2e/Makefile ci_test-slow
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
Expand All @@ -105,23 +106,26 @@ jobs:
- name: Login to ECR
uses: docker/login-action@v3
with:
registry: ${{ secrets.OTELCOMM_AWS_TEST_ACC_ACCOUNT_ID }}.dkr.ecr.us-east-1.amazonaws.com/nr-otel-collector
registry: ${{ env.REGISTRY}}/${{ matrix.distribution }}

- name: Build and publish nightly binaries & packages with GoReleaser
- name: Build and publish ${{ matrix.distribution }} nightly binaries & packages with GoReleaser
if: ${{ !env.ACT }}
uses: goreleaser/goreleaser-action@v6
env:
NFPM_PASSPHRASE: ${{ secrets.OHAI_GPG_PASSPHRASE }}
GPG_FINGERPRINT: ${{ steps.import_gpg.outputs.fingerprint }}
GPG_KEY_PATH: ${{ steps.write_gpg_to_path.outputs.gpg_key_path }}
REGISTRY: ${{ env.REGISTRY }}
with:
distribution: goreleaser
version: '~> v2'
args: --skip=announce,validate --clean --timeout 2h --config .goreleaser-nightly.yaml
workdir: distributions/nr-otel-collector
workdir: distributions/${{ matrix.distribution }}

- name: Run nightly tests
- name: Run nightly tests for ${{ matrix.distribution }}
run: |
NR_API_KEY=${{ secrets.OTELCOMM_NR_API_KEY }} \
NR_ACCOUNT_ID=${{ secrets.OTELCOMM_NR_TEST_ACCOUNT_ID }} \
NR_API_BASE_URL=${{ secrets.NR_STAGING_API_BASE_URL }} \
DISTRO=${{ matrix.distribution }} \
make -f ./test/e2e/Makefile ci_test-nightly
Loading

0 comments on commit 50b9f03

Please sign in to comment.