Merge branch 'guance-1340' into guance

.circleci/config.continue.yml.j2

@@ -36,7 +36,7 @@ instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation
 debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core"
 profiling_modules: &profiling_modules "dd-java-agent/agent-profiling"
 
-default_system_tests_commit: &default_system_tests_commit a0064c4012f19fbec5b9f0583efc80ab5f37c35e
+default_system_tests_commit: &default_system_tests_commit 832e4d8514a06cbe9484ace1fedb3b9344a9ef44
 
 parameters:
   nightly:
@@ -113,7 +113,7 @@ commands:
       version of Testcontainers.
     steps:
       - setup_remote_docker:
-          version: 20.10.18
+          version: docker24
           # DLC shares Docker layers across jobs (at an extra cost).
           # But its time to setup (~1min) exceeds the time required to prefetch all images we use.
           docker_layer_caching: false
@@ -242,7 +242,7 @@ commands:
           paths:
             # Gradle version specific cache for incremental builds. Needs to match version in
             # gradle/wrapper/gradle-wrapper.properties
-            - ~/.gradle/caches/8.3
+            - ~/.gradle/caches/8.4
             # Workspace
             - ~/dd-trace-java/.gradle
             - ~/dd-trace-java/workspace
@@ -287,10 +287,16 @@ jobs:
       collectLibs:
         type: boolean
         default: false
+      triggeredBy:
+        type: string
+        default: ".*"
 
     steps:
       - setup_code
 
+      - skip_unless_matching_files_changed:
+          pattern: << parameters.triggeredBy >>
+
       - restore_dependency_cache:
           cacheType: << parameters.cacheType >>
 
@@ -374,13 +380,20 @@ jobs:
         type: string
       cacheType:
         type: string
+      triggeredBy:
+        type: string
+        default: ".*"
 
     resource_class: medium+
 
     parallelism: << parameters.parallelism >>
 
     steps:
       - setup_code
+
+      - skip_unless_matching_files_changed:
+          pattern: << parameters.triggeredBy >>
+
       - restore_dependency_cache:
           cacheType: << parameters.cacheType >>
       - restore_build_cache:
@@ -947,6 +960,7 @@ build_test_jobs: &build_test_jobs
       name: build_inst
       gradleTarget: :instrumentationTest
       cacheType: inst
+      triggeredBy: *instrumentation_modules
   - build:
       name: build_latestdep
       gradleTarget: :instrumentationLatestDepTest
@@ -986,6 +1000,7 @@ build_test_jobs: &build_test_jobs
       parallelism: 4
       gradleTarget: ":instrumentationCheck"
       cacheType: inst
+      triggeredBy: *instrumentation_modules
 
   - check:
       requires:
@@ -1319,6 +1334,7 @@ build_test_jobs: &build_test_jobs
 {% endfor %}
         - profiling
         - debugger
+        - system-tests
       name: required
       stage: required
 

.circleci/render_config.py

@@ -28,7 +28,7 @@
 }
 # Version to use for all the base Docker images, see
 # https://github.com/DataDog/dd-trace-java-docker-build/pkgs/container/dd-trace-java-docker-build
-DOCKER_IMAGE_VERSION="v24.01"
+DOCKER_IMAGE_VERSION="v24.04"
 
 # Get labels from pull requests to override some defaults for jobs to run.
 # `run-tests: all` will run all tests.

.github/CODEOWNERS

@@ -21,6 +21,7 @@ dd-java-agent/instrumentation/junit-4.10/  @DataDog/ci-app-libraries-java
 dd-java-agent/instrumentation/junit-5.3/   @DataDog/ci-app-libraries-java
 dd-java-agent/instrumentation/karate/      @DataDog/ci-app-libraries-java
 dd-java-agent/instrumentation/scalatest/   @DataDog/ci-app-libraries-java
+dd-java-agent/instrumentation/selenium/    @DataDog/ci-app-libraries-java
 dd-java-agent/instrumentation/testng/      @DataDog/ci-app-libraries-java
 dd-java-agent/instrumentation/gradle/      @DataDog/ci-app-libraries-java
 dd-java-agent/instrumentation/maven-3.2.1/ @DataDog/ci-app-libraries-java
@@ -47,3 +48,7 @@ dd-java-agent/instrumentation/spring-security-5/ @DataDog/asm-java
 **/iast/                               @DataDog/asm-java
 **/Iast*.java                          @DataDog/asm-java
 **/Iast*.groovy                        @DataDog/asm-java
+
+# @DataDog/data-jobs-monitoring
+dd-java-agent/instrumentation/spark/            @DataDog/data-jobs-monitoring
+dd-java-agent/instrumentation/spark-executor/   @DataDog/data-jobs-monitoring

.github/workflows/README.md

@@ -4,16 +4,6 @@ This lists and describes the repository GitHub actions.
 
 ## Release Management
 
-### add-assets-to-release [🔗](add-assets-to-release.yaml)
-
-_Trigger:_ When a release is published.
-
-_Actions:_
-* Ensure the release name is properly formatted (using `x.y.z` format),
-* Download `dd-java-agent`, `dd-trace-api` and `dd-trace-ot` artifacts from Sonatype (aka _Maven Central_ and upload them to the release (`dd-java-agent` will also be uploaded without version number).
-
-_Recovery:_ Download artifacts and upload them manually to the release.
-
 ### add-milestone-to-pull-requests [🔗](add-milestone-to-pull-requests.yaml)
 
 _Trigger:_ When a PR to `master` is closed.

.github/workflows/codeql-analysis.yml

@@ -33,7 +33,7 @@ jobs:
           # queries: ./path/to/local/query, your-org/your-repo/queries@main
 
       - name: Build dd-trace-java for creating the CodeQL database
-        run: JAVA_HOME=$JAVA_HOME_8_X64 JAVA_8_HOME=$JAVA_HOME_8_X64 JAVA_11_HOME=$JAVA_HOME_11_X64 JAVA_17_HOME=$JAVA_HOME_17_X64 ./gradlew clean :dd-java-agent:shadowJar --build-cache --parallel --stacktrace --no-daemon --max-workers=8
+        run: JAVA_HOME=$JAVA_HOME_8_X64 JAVA_8_HOME=$JAVA_HOME_8_X64 JAVA_11_HOME=$JAVA_HOME_11_X64 JAVA_17_HOME=$JAVA_HOME_17_X64 JAVA_21_HOME=$JAVA_HOME_21_X64 ./gradlew clean :dd-java-agent:shadowJar --build-cache --parallel --stacktrace --no-daemon --max-workers=8
 
       - name: Perform CodeQL Analysis
         uses: github/codeql-action/analyze@1a927e9307bc11970b2c679922ebc4d03a5bd980 # 1.0.31

.github/workflows/draft-release-notes-on-tag.yaml

@@ -16,12 +16,13 @@ jobs:
           result-encoding: string
           script: |
             // Get the milestone title ("X.Y.Z") from tag name ("vX.Y.Z(-rc)")
-            const match = '${{github.event.ref}}'.match(/v(\d+\.\d+\.\d+)(?:-rc\d+)?/i)
+            const match = '${{github.event.ref}}'.match(/v(\d+\.\d+\.\d+)(-rc\d+)?/i)
             if (!match) {
               core.setFailed('Failed to parse tag name into milestone name: ${{github.event.ref}}')
               return
             }
             const milestoneTitle = match[1]
+            const isReleaseCandidate = match[2] !== undefined
 
             // Look for the milestone
             const milestone = (await github.paginate('GET /repos/{owner}/{repo}/milestones', {
@@ -135,8 +136,10 @@ jobs:
             }
 
             var changelog = ''
-            if ('${{github.event.ref}}'.match(/rc/i)) {
-              changelog += ':warning:  This is a RELEASE CANDIDATE and is NOT intended for use in production. Please contact Datadog support regarding any problems in this RC.\n'
+            if (isReleaseCandidate) {
+              changelog += '> [!WARNING]\n' +
+                '> This is a **release candidate** and is **not** intended for use in production.  \n' +
+                'Please [open an issue](https://github.com/DataDog/dd-trace-java/issues/new) regarding any problems in this release candidate.\n\n'
             }
             if (prByComponents.size > 0) {
               changelog += '# Components\n\n';

.github/workflows/lib-injection.yaml

@@ -65,26 +65,32 @@ jobs:
     permissions:
       contents: read
       packages: write
-    strategy:
-      matrix:
-        lib-injection-connection: [ 'network','uds']
-        lib-injection-use-admission-controller: ['', 'use-admission-controller']
-        weblog-variant: [ 'dd-lib-java-init-test-app']
-      fail-fast: false
     env:
       TEST_LIBRARY: java
-      WEBLOG_VARIANT: ${{ matrix.weblog-variant }}
-      LIBRARY_INJECTION_CONNECTION: ${{ matrix.lib-injection-connection }}
-      LIBRARY_INJECTION_ADMISSION_CONTROLLER: ${{ matrix.lib-injection-use-admission-controller }}
+      WEBLOG_VARIANT: dd-lib-java-init-test-app
       DOCKER_REGISTRY_IMAGES_PATH: ghcr.io/datadog
       DOCKER_IMAGE_TAG: ${{ github.sha }}
-      MODE: manual
-    steps:     
-      - name: lib-injection test runner
-        id: lib-injection-test-runner
-        uses: DataDog/system-tests/lib-injection/runner@89d754340046eafcc72dc5c5fd6ea651ca7b920e # main
-        with:
-          docker-registry: ghcr.io
-          docker-registry-username: ${{ github.repository_owner }}
-          docker-registry-password: ${{ secrets.GITHUB_TOKEN }}
-          test-script: ./lib-injection/run-manual-lib-injection.sh
+      BUILDX_PLATFORMS: linux/amd64
+    steps:
+    - name: Checkout system tests
+      uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f # 2.3.4
+      with:
+          repository: 'DataDog/system-tests'
+
+    - name: Install runner
+      uses: ./.github/actions/install_runner 
+
+    - name: Run K8s Lib Injection Tests
+      run: ./run.sh K8S_LIB_INJECTION_BASIC
+
+    - name: Compress logs
+      id: compress_logs
+      if: always()
+      run: tar -czvf artifact.tar.gz $(ls | grep logs)
+
+    - name: Upload artifact
+      if: always()
+      uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2 # 2.3.1
+      with:
+        name: logs_k8s_lib_injection
+        path: artifact.tar.gz

.github/workflows/trivy-analysis.yml

@@ -31,7 +31,7 @@ jobs:
 
       - name: Build and publish artifacts locally
         run: |
-          GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G'" JAVA_HOME=$JAVA_HOME_8_X64 JAVA_8_HOME=$JAVA_HOME_8_X64 JAVA_11_HOME=$JAVA_HOME_11_X64 JAVA_17_HOME=$JAVA_HOME_17_X64 ./gradlew clean publishToMavenLocal --build-cache --parallel --stacktrace --no-daemon --max-workers=4
+          GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G'" JAVA_HOME=$JAVA_HOME_8_X64 JAVA_8_HOME=$JAVA_HOME_8_X64 JAVA_11_HOME=$JAVA_HOME_11_X64 JAVA_17_HOME=$JAVA_HOME_17_X64 JAVA_21_HOME=$JAVA_HOME_21_X64 ./gradlew clean publishToMavenLocal --build-cache --parallel --stacktrace --no-daemon --max-workers=4
 
       - name: Copy published artifacts
         run: |

.gitignore

@@ -73,3 +73,4 @@ upstream.env
 # Benchmarks #
 benchmark/reports
 benchmark/tracer
+benchmark/dacapo/scratch

.gitlab-ci.yml

@@ -4,6 +4,7 @@ include:
   - local: ".gitlab/benchmarks.yml"
   - local: ".gitlab/exploration-tests.yml"
   - local: ".gitlab/ci-visibility-tests.yml"
+  - local: ".gitlab/single-step-instrumentation-tests.yml"
 
 stages:
   - build
@@ -12,6 +13,7 @@ stages:
   - benchmarks
   - exploration-tests
   - ci-visibility-tests
+  - single-step-instrumentation-tests
   - generate-signing-key
 
 variables:
@@ -46,6 +48,14 @@ cache: &default_cache
   image: ghcr.io/datadog/dd-trace-java-docker-build:v23.10-base
   before_script:
     - export GRADLE_USER_HOME=`pwd`/.gradle
+    # for weird reasons, gradle will always "chmod 700" the .gradle folder
+    # with Gitlab caching, .gradle is always owned by root and thus gradle's chmod invocation fails
+    # This dance is a hack to have .gradle owned by the Gitlab runner user
+    - mkdir -p .gradle
+    - cp -r .gradle .gradle-copy
+    - rm -rf .gradle
+    - mv .gradle-copy .gradle
+    - ls -la
 
 build: &build
   <<: *gradle_build
@@ -84,12 +94,21 @@ package:
   when: on_success # this can't use 'needs: [build]', since build is not available in the scheduled pipeline
   script:
     - ../.gitlab/build_java_package.sh
+    - find . -iregex '.*\.\(deb\|rpm\)' -printf '%f\0' | xargs -0 dd-pkg lint
 
 package-arm:
   extends: .package-arm
   when: on_success # this can't use 'needs: [build]', since build is not available in the scheduled pipeline
   script:
     - ../.gitlab/build_java_package.sh
+    - find . -iregex '.*\.\(deb\|rpm\)' -printf '%f\0' | xargs -0 dd-pkg lint
+
+package-oci:
+  stage: package
+  extends: .package-oci
+  when: on_success # this can't use 'needs: [build]', since build is not available in the scheduled pipeline
+  script:
+    - ../.gitlab/build_java_package_oci.sh
 
 .release-package:
   stage: deploy
@@ -139,27 +158,6 @@ deploy_to_profiling_backend:
     UPSTREAM_TAG: $CI_COMMIT_TAG
     FORCE_TRIGGER: $FORCE_TRIGGER
 
-deploy_to_di_backend:automatic:
-  stage: deploy
-  rules:
-    - if: '$POPULATE_CACHE'
-      when: never
-    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
-      when: on_success
-    - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+$/'
-      when: on_success
-  trigger:
-    project: DataDog/debugger-demos
-    branch: main
-  variables:
-    UPSTREAM_PACKAGE_JOB: build
-    UPSTREAM_PROJECT_ID: $CI_PROJECT_ID
-    UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME
-    UPSTREAM_PIPELINE_ID: $CI_PIPELINE_ID
-    UPSTREAM_BRANCH: $CI_COMMIT_BRANCH
-    UPSTREAM_TAG: $CI_COMMIT_TAG
-    UPSTREAM_COMMIT_SHORT_SHA: $CI_COMMIT_SHORT_SHA
-
 deploy_to_di_backend:manual:
   stage: deploy
   rules:

.gitlab/benchmarks.yml

@@ -1,7 +1,7 @@
 variables:
   BASE_CI_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:dd-trace-java-benchmarks
 
-benchmarks:
+.benchmarks:
   stage: benchmarks
   when: on_success
   interruptible: true
@@ -12,11 +12,6 @@ benchmarks:
     - export ARTIFACTS_DIR="$(pwd)/reports" && mkdir -p "${ARTIFACTS_DIR}"
     - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/"
     - git clone --branch dd-trace-java/tracer-benchmarks https://github.com/DataDog/benchmarking-platform.git /platform && cd /platform
-    - ./steps/capture-hardware-software-info.sh
-    - ./steps/run-benchmarks.sh
-    - ./steps/analyze-results.sh
-    - ./steps/upload-results-to-s3.sh
-    - ./steps/post-pr-comment.sh
   artifacts:
     name: "reports"
     paths:
@@ -31,6 +26,44 @@ benchmarks:
     KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: dd-trace-java
     FF_USE_LEGACY_KUBERNETES_EXECUTION_STRATEGY: "true"
 
+benchmarks-startup:
+  extends: .benchmarks
+  script:
+    - !reference [ .benchmarks, script ]
+    - ./steps/capture-hardware-software-info.sh
+    - ./steps/run-benchmarks.sh startup
+    - ./steps/analyze-results.sh startup
+
+benchmarks-load:
+  extends: .benchmarks
+  script:
+    - !reference [ .benchmarks, script ]
+    - ./steps/capture-hardware-software-info.sh
+    - ./steps/run-benchmarks.sh load
+    - ./steps/analyze-results.sh load
+
+benchmarks-dacapo:
+  extends: .benchmarks
+  script:
+    - !reference [ .benchmarks, script ]
+    - ./steps/capture-hardware-software-info.sh
+    - ./steps/run-benchmarks.sh dacapo
+    - ./steps/analyze-results.sh dacapo
+
+benchmarks-post-results:
+  extends: .benchmarks
+  script:
+    - !reference [ .benchmarks, script ]
+    - ./steps/upload-results-to-s3.sh
+    - ./steps/post-pr-comment.sh
+  needs:
+    - job: benchmarks-startup
+      artifacts: true
+    - job: benchmarks-load
+      artifacts: true
+    - job: benchmarks-dacapo
+      artifacts: true
+
 .dsm-kafka-benchmarks:
   stage: benchmarks
   rules:
@@ -41,6 +74,7 @@ benchmarks:
         compare_to: "master"
       when: on_success
     - when: manual
+      allow_failure: true
   tags: ["runner:apm-k8s-tweaked-metal"]
   interruptible: true
   timeout: 1h