Merge branch 'main' into tbs-memory-usage

.buildkite/hooks/pre-command

@@ -84,6 +84,14 @@ if command -v docker &>/dev/null; then
     unset DOCKER_USERNAME_SECRET DOCKER_PASSWORD_SECRET
 fi
 
+echo "--- Setting up the :github: environment..."
+# release.mk uses gh cli to know the current release tag. release.mk is used by the root Makefile
+# regardless is explicitly used, so it's loaded with the include directive.
+# this will avoid printing warnings about the missing gh cli configurations
+# VAULT_GITHUB_TOKEN is a primitive env variable created by the Buildkite automation.
+GH_TOKEN="${VAULT_GITHUB_TOKEN}"
+export GH_TOKEN
+
 echo "--- Setting up the :golang: environment..."
 GO_VERSION=$(cat .go-version)
 WORKSPACE=$(git rev-parse --show-toplevel)

.buildkite/package.yml

@@ -7,7 +7,7 @@ notify:
 env:
   IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2004"
   IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
-  GENERATE_CHAINGUARD_IMAGES: true
+  GENERATE_WOLFI_IMAGES: true
 
 steps:
   - group: "Package"

.buildkite/scripts/dra.sh

@@ -82,7 +82,19 @@ dra() {
       --commit $BUILDKITE_COMMIT \
       --workflow $workflow \
       --artifact-set main \
-      --version $VERSION
+      --version $VERSION | tee rm-output.txt
+
+  # Create Buildkite annotation similarly done in Beats:
+  # https://github.com/elastic/beats/blob/90f9e8f6e48e76a83331f64f6c8c633ae6b31661/.buildkite/scripts/dra.sh#L74-L81
+  if [[ "$command" == "collect" ]]; then
+    # extract the summary URL from a release manager output line like:
+    # Report summary-18.22.0.html can be found at https://artifacts-staging.elastic.co/apm-server/18.22.0-ABCDEFGH/summary-18.22.0.html
+    SUMMARY_URL=$(grep -E '^Report summary-.* can be found at ' rm-output.txt | grep -oP 'https://\S+' | awk '{print $1}')
+    rm rm-output.txt
+
+    # and make it easily clickable as a Builkite annotation
+    printf "**${workflow} summary link:** [${SUMMARY_URL}](${SUMMARY_URL})\n" | buildkite-agent annotate --style=success --append
+  fi
 }
 
 dra "snapshot" "$dra_command"

.github/CODEOWNERS

@@ -1 +1 @@
-* @elastic/apm-server
+* @elastic/obs-ds-intake-services

.github/dependabot.yml

@@ -11,8 +11,6 @@ updates:
       otel:
         patterns:
         - "go.opentelemetry.io/*"
-      jaeger:
-        patterns:
         - "github.com/jaegertracing/jaeger"
         - "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger"
       spf13:
@@ -29,6 +27,24 @@ updates:
     directory: "tools/"
     schedule:
       interval: "weekly"
+    groups:
+      dependencies:
+        patterns:
+          - "*"
+  # go systemtest
+  - package-ecosystem: "gomod"
+    directory: "systemtest/"
+    schedule:
+      interval: "weekly"
+    groups:
+      dependencies:
+        patterns:
+          - "*"
+  # docker
+  - package-ecosystem: "docker"
+    directory: "/packaging/docker/"
+    schedule:
+      interval: "weekly"
 
   # GitHub actions
   - package-ecosystem: "github-actions"

.github/workflows/benchmarks.yml

@@ -46,6 +46,13 @@ jobs:
       TF_VAR_ENVIRONMENT: ci
       TF_VAR_REPO: ${{ github.repository }}
       GOBENCH_TAGS: branch=${{ github.head_ref || github.ref }},commit=${{ github.sha }},target_branch=${{ github.base_ref }}
+      GOBENCH_PASSWORD: ${{ secrets.GOBENCH_PASSWORD }}
+      GOBENCH_USERNAME: ${{ secrets.GOBENCH_USERNAME }}
+      GOBENCH_HOST: ${{ secrets.GOBENCH_HOST }}
+      # TODO: use keyless
+      EC_API_KEY: ${{ secrets.OBSERVABILITY_EC_API_KEY }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.OBSERVABILITY_AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.OBSERVABILITY_AWS_SECRET_ACCESS_KEY }}
     steps:
       - uses: actions/checkout@v4
 
@@ -70,36 +77,13 @@ jobs:
             echo "BENCHMARK_RUN=${{ inputs.benchmarkRun }}" >> "$GITHUB_ENV"
           fi
 
-      - uses: hashicorp/[email protected]
-        env:
-          AWS_CREDENTIALS_PATH: secret/observability-team/ci/elastic-observability-aws-account-auth
-          BENCHMARK_CLOUD_CREDENTIALS_PATH: secret/observability-team/ci/benchmark-cloud
-          EC_CREDENTIALS_PATH: secret/observability-team/ci/elastic-cloud/observability-team-pro
-          KIBANA_CREDENTIALS_PATH: secret/observability-team/ci/apm-benchmark-kibana
-        with:
-          url: ${{ secrets.VAULT_ADDR }}
-          roleId: ${{ secrets.VAULT_ROLE_ID }}
-          secretId: ${{ secrets.VAULT_SECRET_ID }}
-          method: approle
-          secrets: |
-            ${{ env.AWS_CREDENTIALS_PATH }} access_key | AWS_ACCESS_KEY_ID ;
-            ${{ env.AWS_CREDENTIALS_PATH }} secret_key | AWS_SECRET_ACCESS_KEY ;
-            ${{ env.BENCHMARK_CLOUD_CREDENTIALS_PATH }} user | GOBENCH_USERNAME ;
-            ${{ env.BENCHMARK_CLOUD_CREDENTIALS_PATH }} password | GOBENCH_PASSWORD ;
-            ${{ env.BENCHMARK_CLOUD_CREDENTIALS_PATH }} url | GOBENCH_HOST ;
-            ${{ env.EC_CREDENTIALS_PATH }} apiKey | EC_API_KEY ;
-            ${{ env.KIBANA_CREDENTIALS_PATH }} user | KIBANA_USERNAME ;
-            ${{ env.KIBANA_CREDENTIALS_PATH }} password | KIBANA_PASSWORD ;
-            ${{ env.KIBANA_CREDENTIALS_PATH }} kibana_url | KIBANA_ENDPOINT ;
-            ${{ env.KIBANA_CREDENTIALS_PATH }} kibana_dashboard_url | KIBANA_DASHBOARD_URL ;
-
-      - uses: elastic/apm-pipeline-library/.github/actions/docker-login@main
+      - name: Log in to the Elastic Container registry
+        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
         with:
-          registry: docker.elastic.co
-          secret: secret/observability-team/ci/docker-registry/prod
-          url: ${{ secrets.VAULT_ADDR }}
-          roleId: ${{ secrets.VAULT_ROLE_ID }}
-          secretId: ${{ secrets.VAULT_SECRET_ID }}
+          registry: ${{ secrets.ELASTIC_DOCKER_REGISTRY }}
+          username: ${{ secrets.ELASTIC_DOCKER_USERNAME }}
+          password: ${{ secrets.ELASTIC_DOCKER_PASSWORD }}
+
       - uses: hashicorp/setup-terraform@v3
         with:
           terraform_version: 1.3.7
@@ -131,9 +115,9 @@ jobs:
       - name: Download PNG
         run: >-
           ${{ github.workspace }}/.ci/scripts/download-png-from-kibana.sh
-          $KIBANA_ENDPOINT
-          $KIBANA_USERNAME
-          $KIBANA_PASSWORD
+          ${{ secrets.KIBANA_BENCH_ENDPOINT }}
+          ${{ secrets.KIBANA_BENCH_USERNAME }}
+          ${{ secrets.KIBANA_BENCH_PASSWORD }}
           $PNG_REPORT_FILE
 
       - name: Upload PNG
@@ -166,23 +150,20 @@ jobs:
 
       # Notify failure to Slack only on schedule (nightly run)
       - if: failure() && github.event_name == 'schedule'
-        uses: elastic/apm-pipeline-library/.github/actions/notify-build-status@current
+        uses: elastic/oblt-actions/slack/notify-[email protected]
         with:
+          bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
+          channel-id: "#apm-server"
           message: |
             Nightly APM Server benchmarks failed! SDH Duty assignee, please have a look and follow this <https://github.com/elastic/observability-dev/blob/main/docs/apm/apm-server/runbooks/benchmarks.md|Runbook>!
-          vaultUrl: ${{ secrets.VAULT_ADDR }}
-          vaultRoleId: ${{ secrets.VAULT_ROLE_ID }}
-          vaultSecretId: ${{ secrets.VAULT_SECRET_ID }}
-          slackChannel: '#apm-server'
 
       # Notify result to Slack only on schedule (nightly run)
       - if: github.event_name == 'schedule'
-        uses: elastic/apm-pipeline-library/.github/actions/slack-message@current
+        uses: slackapi/slack-github-action@70cd7be8e40a46e8b0eced40b0de447bdb42f68e # v1.26.0
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
         with:
-          url: ${{ secrets.VAULT_ADDR }}
-          roleId: ${{ secrets.VAULT_ROLE_ID }}
-          secretId: ${{ secrets.VAULT_SECRET_ID }}
-          channel: '#apm-server'
+          channel-id: "#apm-server"
           payload: |
             {
                 "blocks": [
@@ -218,7 +199,7 @@ jobs:
                                     "type": "plain_text",
                                     "text": "Benchmarks dashboard"
                                 },
-                                "url": "${{ env.KIBANA_DASHBOARD_URL }}",
+                                "url": "${{ secrets.KIBANA_BENCH_DASHBOARD }}",
                                 "action_id": "kibana-dashboard-button"
                             },
                             {

.github/workflows/bump-elastic-stack.yml

@@ -30,15 +30,15 @@ jobs:
         with:
           ref: ${{ matrix.branch }}
 
-      - uses: elastic/oblt-actions/updatecli/run@v1.2.0
+      - uses: elastic/oblt-actions/updatecli/run@v1.9.1
         with:
           command: --experimental apply --config .ci/bump-elastic-stack-snapshot.yml
         env:
           BRANCH: ${{ matrix.branch }}
           GITHUB_TOKEN: ${{ secrets.UPDATECLI_GH_TOKEN }}
 
       - if: ${{ failure()  }}
-        uses: elastic/oblt-actions/slack/send@v1.2.0
+        uses: elastic/oblt-actions/slack/send@v1.9.1
         with:
           channel-id: '#apm-server'
           message: ":traffic_cone: updatecli failed for `${{ github.repository }}@${{ github.ref_name }}`, @robots-ci please look what's going on <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|here>"

.github/workflows/bump-golang.yml

@@ -16,14 +16,14 @@ jobs:
 
       - uses: actions/checkout@v4
 
-      - uses: elastic/oblt-actions/updatecli/run@v1.2.0
+      - uses: elastic/oblt-actions/updatecli/run@v1.9.1
         with:
           command: --experimental apply --config .ci/bump-golang.yml
         env:
           GITHUB_TOKEN: ${{ secrets.UPDATECLI_GH_TOKEN }}
 
       - if: ${{ failure() }}
-        uses: elastic/oblt-actions/slack/send@v1.2.0
+        uses: elastic/oblt-actions/slack/send@v1.9.1
         with:
           bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
           channel-id: "#apm-server"

.github/workflows/check-docker-compose.yml

@@ -52,10 +52,8 @@ jobs:
           needs: ${{ toJSON(needs) }}
       - run: ${{ steps.check.outputs.isSuccess }}
       - if: failure()
-        uses: elastic/apm-pipeline-library/.github/actions/notify-build-status@current
+        uses: elastic/oblt-actions/slack/notify-result@v1
         with:
+          bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
+          channel-id: "#apm-server"
           status: ${{ steps.check.outputs.status }}
-          vaultUrl: ${{ secrets.VAULT_ADDR }}
-          vaultRoleId: ${{ secrets.VAULT_ROLE_ID }}
-          vaultSecretId: ${{ secrets.VAULT_SECRET_ID }}
-          slackChannel: "#apm-server"

.github/workflows/ci.yml

@@ -68,6 +68,7 @@ jobs:
           GOTESTFLAGS: "-v -json"
           OUTPUT_JSON_FILE: "build/TEST-go-system_tests.out.json"
           OUTPUT_JUNIT_FILE: "build/TEST-go-system_tests.xml"
+          GH_TOKEN: ${{ github.token }}
         run: |
           go run -modfile=tools/go.mod gotest.tools/gotestsum \
           --no-color -f standard-quiet --jsonfile "$OUTPUT_JSON_FILE" --junitfile "$OUTPUT_JUNIT_FILE" \
@@ -92,22 +93,22 @@ jobs:
 
   test-package-and-push:
     runs-on: ubuntu-latest
-    if: github.event_name != 'pull_request' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false)
+    # If no PR event or if a PR event that's caused by a non-fork and non dependabot actor
+    if: github.event_name != 'pull_request' || ( github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false && github.actor != 'dependabot[bot]' )
     env:
-      GENERATE_CHAINGUARD_IMAGES: true
+      GENERATE_WOLFI_IMAGES: true
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
           go-version-file: go.mod
           cache: false
-      - uses: elastic/apm-pipeline-library/.github/actions/docker-login@current
+      - name: Log in to the Elastic Container registry
+        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
         with:
-          registry: docker.elastic.co
-          secret: secret/observability-team/ci/docker-registry/prod
-          url: ${{ secrets.VAULT_ADDR }}
-          roleId: ${{ secrets.VAULT_ROLE_ID }}
-          secretId: ${{ secrets.VAULT_SECRET_ID }}
+          registry: ${{ secrets.ELASTIC_DOCKER_REGISTRY }}
+          username: ${{ secrets.ELASTIC_DOCKER_USERNAME }}
+          password: ${{ secrets.ELASTIC_DOCKER_PASSWORD }}
       - run: make package-snapshot
         env:
           GH_TOKEN: ${{ github.token }}

.github/workflows/generate-smoke-tests-list/action.yml

@@ -4,15 +4,12 @@ name: generate-smoke-tests-list
 description: Generate smoke tests list
 
 inputs:
-  vault-url:
-    description: 'Vault URL'
-    required: false
-  vault-role-id:
-    description: 'Vault role ID'
-    required: false
-  vault-secret-id:
-    description: 'Vault secret ID'
-    required: false
+  aws-csv-file:
+    description: 'Content with the AWS csv file'
+    required: true
+  aws-profile:
+    description: 'aws profile'
+    required: true
 outputs:
   tests:
     description: "List of smoke tests"
@@ -27,9 +24,8 @@ runs:
     - name: Setup cluster env
       uses: ./.github/workflows/setup-cluster-env
       with:
-        vault-url: ${{ inputs.vault-url }}
-        vault-role-id: ${{ inputs.vault-role-id }}
-        vault-secret-id: ${{ inputs.vault-secret-id }}
+        aws-csv-file: ${{ inputs.aws-csv-file }}
+        aws-profile: ${{ inputs.aws-profile }}
     - id: generate
       name: Generate matrix and date
       run: |

.github/workflows/microbenchmark.yml

@@ -22,7 +22,7 @@ jobs:
     timeout-minutes: 5
     steps:
       - name: Run buildkite pipeline
-        uses: elastic/oblt-actions/buildkite/run@v1.4.0
+        uses: elastic/oblt-actions/buildkite/run@v1.9.1
         with:
           token: ${{ secrets.BUILDKITE_TOKEN }}
           pipeline: apm-agent-microbenchmark