Enable packaging tests in exhaustive pipeline (#15807) (#15781)

This commit adds the packaging tests (that were refactored in #15754
to not rely on Vagrant) in a new "acceptance" group.

One difference from #15781 is that we use ubuntu 2004 for building packages
(as done in DRA/7.17) due to jruby/jruby#7611 (comment)

Relates: elastic/ingest-dev#1722

(cherry picked from commit 16c4d88)

.buildkite/scripts/common/vm-agent-multi-jdk.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+# **************************************************************
+# This file contains prerequisite bootstrap invocations
+# required for Logstash CI when using custom multi-jdk VM images
+# It is primarily used by the exhaustive BK pipeline.
+# **************************************************************
+
+set -euo pipefail
+
+source .ci/java-versions.properties
+export BUILD_JAVA_HOME=/opt/buildkite-agent/.java/$LS_BUILD_JAVA
+
+export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$BUILD_JAVA_HOME/bin:$PATH"
+
+eval "$(rbenv init -)"

.buildkite/scripts/exhaustive-tests/generate-steps.py

@@ -9,6 +9,9 @@
 
 VM_IMAGES_FILE = ".buildkite/scripts/common/vm-images.json"
 VM_IMAGE_PREFIX = "platform-ingest-logstash-multi-jdk-"
+
+ACCEPTANCE_LINUX_OSES = ["ubuntu-2204", "ubuntu-2004", "debian-11", "debian-10", "rhel-8", "oraclelinux-7", "rocky-linux-8", "opensuse-leap-15", "amazonlinux-2023"]
+
 CUR_PATH = os.path.dirname(os.path.abspath(__file__))
 
 def slugify_bk_key(key: str) -> str:
@@ -99,6 +102,62 @@ def gcp_agent(vm_name: str, instance_type: str = "n2-standard-4", image_prefix:
         "diskType": "pd-ssd",
     }
 
+def acceptance_linux_vms() -> typing.List[str]:
+    acceptance_linux_vms = os.getenv("ACCEPTANCE_LINUX_OSES")
+    if acceptance_linux_vms:
+        acceptance_linux_vms = acceptance_linux_vms.split(",")
+    else:
+        acceptance_linux_vms = ACCEPTANCE_LINUX_OSES
+
+    return acceptance_linux_vms
+
+def acceptance_linux_steps() -> list[typing.Any]:
+    steps = []
+
+    build_artifacts_step = {
+        "label": "Build artifacts",
+        "key": "acceptance-build-artifacts",
+        # use the same agent as the one we use for building DRA artifacts
+        # NOTE! specifically on 7.17 we must use ubuntu 20.04 or less due to https://github.com/jruby/jruby/pull/7611#issuecomment-1750387837
+        "agents": gcp_agent("ubuntu-2004", instance_type="n2-standard-16", image_prefix="family/platform-ingest-logstash"),
+        "command": LiteralScalarString("""#!/usr/bin/env bash
+set -eo pipefail
+source .buildkite/scripts/common/vm-agent.sh
+echo "--- Building all artifacts"
+./gradlew clean bootstrap
+rake artifact:deb artifact:rpm
+"""),
+        "artifact_paths": [
+            "build/*rpm",
+            "build/*deb",
+            "build/*tar.gz",
+        ],
+    }
+
+    steps.append(build_artifacts_step)
+
+    for vm in acceptance_linux_vms():
+        step = {
+            "label": vm,
+            "key": slugify_bk_key(vm),
+            "agents": aws_agent(vm,instance_type="m5.4xlarge") if "amazonlinux" in vm else gcp_agent(vm),
+            "depends_on": "acceptance-build-artifacts",
+            "retry": {"automatic": [{"limit": 3}]},
+            "command": LiteralScalarString("""#!/usr/bin/env bash
+set -eo pipefail
+source .buildkite/scripts/common/vm-agent-multi-jdk.sh
+source /etc/os-release
+if [[ "$$(echo $$ID_LIKE | tr '[:upper:]' '[:lower:]')" =~ (rhel|fedora) && "$${VERSION_ID%.*}" -le 7 ]]; then
+  # jruby-9.3.10.0 unavailable on centos-7 / oel-7, see https://github.com/jruby/jruby/issues/7579#issuecomment-1425885324 / https://github.com/jruby/jruby/issues/7695
+  # we only need a working jruby to run the acceptance test framework -- the packages have been prebuilt in a previous stage
+  rbenv local jruby-9.4.5.0
+fi
+ci/acceptance_tests.sh"""),
+        }
+        steps.append(step)
+
+    return steps
+
 def acceptance_docker_steps()-> list[typing.Any]:
     steps = []
     for flavor in ["full", "oss", "ubi8"]:
@@ -146,6 +205,13 @@ def acceptance_docker_steps()-> list[typing.Any]:
             "steps": [compat_windows_step(imagesuffix=windows_test_os)],
     })
 
+    structure["steps"].append({
+            "group": "Acceptance / Packaging",
+            "key": "acceptance-packaging",
+            "depends_on": ["testing-phase"],
+            "steps": acceptance_linux_steps(),
+    })
+
     structure["steps"].append({
             "group": "Acceptance / Docker",
             "key": "acceptance-docker",

.ci/java-versions.properties

@@ -1,2 +1,2 @@
-LS_BUILD_JAVA=adoptopenjdk11
-LS_RUNTIME_JAVA=adoptopenjdk11
+LS_BUILD_JAVA=adoptopenjdk_11
+LS_RUNTIME_JAVA=adoptopenjdk_11