diff --git a/.buildkite/scripts/common/vm-agent-multi-jdk.sh b/.buildkite/scripts/common/vm-agent-multi-jdk.sh new file mode 100644 index 00000000000..364b0ba2484 --- /dev/null +++ b/.buildkite/scripts/common/vm-agent-multi-jdk.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +# ************************************************************** +# This file contains prerequisite bootstrap invocations +# required for Logstash CI when using custom multi-jdk VM images +# It is primarily used by the exhaustive BK pipeline. +# ************************************************************** + +set -euo pipefail + +source .ci/java-versions.properties +export BUILD_JAVA_HOME=/opt/buildkite-agent/.java/$LS_BUILD_JAVA + +export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$BUILD_JAVA_HOME/bin:$PATH" + +eval "$(rbenv init -)" diff --git a/.buildkite/scripts/exhaustive-tests/generate-steps.py b/.buildkite/scripts/exhaustive-tests/generate-steps.py index 46a54a3cc58..ed8c129c27a 100644 --- a/.buildkite/scripts/exhaustive-tests/generate-steps.py +++ b/.buildkite/scripts/exhaustive-tests/generate-steps.py @@ -9,6 +9,9 @@ VM_IMAGES_FILE = ".buildkite/scripts/common/vm-images.json" VM_IMAGE_PREFIX = "platform-ingest-logstash-multi-jdk-" + +ACCEPTANCE_LINUX_OSES = ["ubuntu-2204", "ubuntu-2004", "debian-11", "debian-10", "rhel-8", "oraclelinux-7", "rocky-linux-8", "opensuse-leap-15", "amazonlinux-2023"] + CUR_PATH = os.path.dirname(os.path.abspath(__file__)) def slugify_bk_key(key: str) -> str: @@ -99,6 +102,62 @@ def gcp_agent(vm_name: str, instance_type: str = "n2-standard-4", image_prefix: "diskType": "pd-ssd", } +def acceptance_linux_vms() -> typing.List[str]: + acceptance_linux_vms = os.getenv("ACCEPTANCE_LINUX_OSES") + if acceptance_linux_vms: + acceptance_linux_vms = acceptance_linux_vms.split(",") + else: + acceptance_linux_vms = ACCEPTANCE_LINUX_OSES + + return acceptance_linux_vms + +def acceptance_linux_steps() -> list[typing.Any]: + steps = [] + + build_artifacts_step = { + "label": "Build artifacts", + "key": "acceptance-build-artifacts", + # use the same agent as the one we use for building DRA artifacts + # NOTE! specifically on 7.17 we must use ubuntu 20.04 or less due to https://github.com/jruby/jruby/pull/7611#issuecomment-1750387837 + "agents": gcp_agent("ubuntu-2004", instance_type="n2-standard-16", image_prefix="family/platform-ingest-logstash"), + "command": LiteralScalarString("""#!/usr/bin/env bash +set -eo pipefail +source .buildkite/scripts/common/vm-agent.sh +echo "--- Building all artifacts" +./gradlew clean bootstrap +rake artifact:deb artifact:rpm +"""), + "artifact_paths": [ + "build/*rpm", + "build/*deb", + "build/*tar.gz", + ], + } + + steps.append(build_artifacts_step) + + for vm in acceptance_linux_vms(): + step = { + "label": vm, + "key": slugify_bk_key(vm), + "agents": aws_agent(vm,instance_type="m5.4xlarge") if "amazonlinux" in vm else gcp_agent(vm), + "depends_on": "acceptance-build-artifacts", + "retry": {"automatic": [{"limit": 3}]}, + "command": LiteralScalarString("""#!/usr/bin/env bash +set -eo pipefail +source .buildkite/scripts/common/vm-agent-multi-jdk.sh +source /etc/os-release +if [[ "$$(echo $$ID_LIKE | tr '[:upper:]' '[:lower:]')" =~ (rhel|fedora) && "$${VERSION_ID%.*}" -le 7 ]]; then + # jruby-9.3.10.0 unavailable on centos-7 / oel-7, see https://github.com/jruby/jruby/issues/7579#issuecomment-1425885324 / https://github.com/jruby/jruby/issues/7695 + # we only need a working jruby to run the acceptance test framework -- the packages have been prebuilt in a previous stage + rbenv local jruby-9.4.5.0 +fi +ci/acceptance_tests.sh"""), + } + steps.append(step) + + return steps + def acceptance_docker_steps()-> list[typing.Any]: steps = [] for flavor in ["full", "oss", "ubi8"]: @@ -146,6 +205,13 @@ def acceptance_docker_steps()-> list[typing.Any]: "steps": [compat_windows_step(imagesuffix=windows_test_os)], }) + structure["steps"].append({ + "group": "Acceptance / Packaging", + "key": "acceptance-packaging", + "depends_on": ["testing-phase"], + "steps": acceptance_linux_steps(), + }) + structure["steps"].append({ "group": "Acceptance / Docker", "key": "acceptance-docker", diff --git a/.ci/java-versions.properties b/.ci/java-versions.properties index ee75e09ce87..0b4302be0f5 100644 --- a/.ci/java-versions.properties +++ b/.ci/java-versions.properties @@ -1,2 +1,2 @@ -LS_BUILD_JAVA=adoptopenjdk11 -LS_RUNTIME_JAVA=adoptopenjdk11 +LS_BUILD_JAVA=adoptopenjdk_11 +LS_RUNTIME_JAVA=adoptopenjdk_11