diff --git a/build-support/common-build-env.sh b/build-support/common-build-env.sh index c7c46833bb1c..2cf5b820e1e8 100644 --- a/build-support/common-build-env.sh +++ b/build-support/common-build-env.sh @@ -2603,7 +2603,7 @@ set_prebuilt_thirdparty_url() { # Transform "thin-lto" or "full-lto" into "thin" or "full" respectively. thirdparty_tool_cmd_line+=( "--lto=${YB_LINKING_TYPE%%-lto}" ) fi - if [[ ! ${build_type} =~ ^(asan|tsan)$ && ${YB_COMPILER_TYPE} == clang* ]]; then + if [[ is_alma && ! ${build_type} =~ ^(asan|tsan)$ && ${YB_COMPILER_TYPE} == clang* ]]; then thirdparty_tool_cmd_line+=( "--allow-older-os" ) fi "${thirdparty_tool_cmd_line[@]}" diff --git a/build-support/thirdparty_archives.yml b/build-support/thirdparty_archives.yml index 181c4f2ca84d..0e3b62aebb4c 100644 --- a/build-support/thirdparty_archives.yml +++ b/build-support/thirdparty_archives.yml @@ -1,163 +1,198 @@ -sha_for_local_checkout: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419 +sha_for_local_checkout: 5b9d0c65af588696e47baf8e24b6a64a177a4157 archives: - os_type: almalinux8 architecture: x86_64 - compiler_type: clang15 + compiler_type: clang16 is_linuxbrew: false - sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419 + sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157 lto_type: - tag: v2.20-20231114005155-44dd1e39bc-almalinux8-x86_64-clang15 + tag: v2.20-20240509185516-5b9d0c65af-almalinux8-x86_64-clang16 - os_type: almalinux8 architecture: x86_64 - compiler_type: clang15 + compiler_type: clang16 is_linuxbrew: true - sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419 + sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157 lto_type: - tag: v2.20-20231114005150-44dd1e39bc-almalinux8-x86_64-clang15-linuxbrew + tag: v2.20-20240509185512-5b9d0c65af-almalinux8-x86_64-clang16-linuxbrew - os_type: almalinux8 architecture: x86_64 - compiler_type: clang15 + compiler_type: clang16 is_linuxbrew: true - sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419 + sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157 lto_type: full - tag: v2.20-20231114005229-44dd1e39bc-almalinux8-x86_64-clang15-linuxbrew-full-lto + tag: v2.20-20240509185512-5b9d0c65af-almalinux8-x86_64-clang16-linuxbrew-full-lto - os_type: almalinux8 architecture: x86_64 - compiler_type: clang16 + compiler_type: clang17 is_linuxbrew: false - sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419 + sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157 lto_type: - tag: v2.20-20231114005304-44dd1e39bc-almalinux8-x86_64-clang16 + tag: v2.20-20240509185516-5b9d0c65af-almalinux8-x86_64-clang17 - os_type: almalinux8 architecture: x86_64 - compiler_type: clang16 + compiler_type: clang17 is_linuxbrew: true - sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419 + sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157 lto_type: - tag: v2.20-20231114005319-44dd1e39bc-almalinux8-x86_64-clang16-linuxbrew + tag: v2.20-20240509185520-5b9d0c65af-almalinux8-x86_64-clang17-linuxbrew - os_type: almalinux8 architecture: x86_64 - compiler_type: clang16 + compiler_type: clang17 is_linuxbrew: true - sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419 + sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157 lto_type: full - tag: v2.20-20231114005149-44dd1e39bc-almalinux8-x86_64-clang16-linuxbrew-full-lto + tag: v2.20-20240509185505-5b9d0c65af-almalinux8-x86_64-clang17-linuxbrew-full-lto - os_type: almalinux8 architecture: x86_64 compiler_type: gcc11 is_linuxbrew: false - sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419 + sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157 lto_type: - tag: v2.20-20231114005150-44dd1e39bc-almalinux8-x86_64-gcc11 + tag: v2.20-20240509185529-5b9d0c65af-almalinux8-x86_64-gcc11 - os_type: almalinux9 architecture: x86_64 compiler_type: clang16 is_linuxbrew: false - sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419 + sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157 lto_type: - tag: v2.20-20231114005254-44dd1e39bc-almalinux9-x86_64-clang16 + tag: v2.20-20240509185522-5b9d0c65af-almalinux9-x86_64-clang16 + - os_type: almalinux9 + architecture: x86_64 + compiler_type: clang17 + is_linuxbrew: false + sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157 + lto_type: + tag: v2.20-20240509185514-5b9d0c65af-almalinux9-x86_64-clang17 - os_type: almalinux9 architecture: x86_64 compiler_type: gcc12 is_linuxbrew: false - sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419 + sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157 lto_type: - tag: v2.20-20231114005251-44dd1e39bc-almalinux9-x86_64-gcc12 - - os_type: centos7 + tag: v2.20-20240625152657-ffb781854e-almalinux9-x86_64-gcc12 + - os_type: amzn2 architecture: aarch64 - compiler_type: clang15 + compiler_type: clang17 is_linuxbrew: false - sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419 + sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157 lto_type: - tag: v2.20-20231114005317-44dd1e39bc-centos7-aarch64-clang15 - - os_type: centos7 + tag: v2.20-20240625152843-ffb781854e-amazonlinux2-aarch64-clang17 + - os_type: amzn2 architecture: aarch64 - compiler_type: clang15 + compiler_type: clang17 is_linuxbrew: false - sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419 + sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157 lto_type: full - tag: v2.20-20231114005340-44dd1e39bc-centos7-aarch64-clang15-full-lto + tag: v2.20-20240625152849-ffb781854e-amazonlinux2-aarch64-clang17-full-lto + - os_type: amzn2 + architecture: x86_64 + compiler_type: clang17 + is_linuxbrew: false + sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157 + lto_type: + tag: v2.20-20240509185556-5b9d0c65af-amazonlinux2-x86_64-clang17 - os_type: centos7 architecture: aarch64 compiler_type: clang16 is_linuxbrew: false - sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419 + sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157 lto_type: - tag: v2.20-20231114005341-44dd1e39bc-centos7-aarch64-clang16 + tag: v2.20-20240509185638-5b9d0c65af-centos7-aarch64-clang16 - os_type: centos7 architecture: aarch64 compiler_type: clang16 is_linuxbrew: false - sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419 + sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157 lto_type: full - tag: v2.20-20231114005319-44dd1e39bc-centos7-aarch64-clang16-full-lto + tag: v2.20-20240509185634-5b9d0c65af-centos7-aarch64-clang16-full-lto - os_type: centos7 - architecture: x86_64 - compiler_type: clang15 + architecture: aarch64 + compiler_type: clang17 is_linuxbrew: false - sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419 + sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157 lto_type: - tag: v2.20-20231114005245-44dd1e39bc-centos7-x86_64-clang15 + tag: v2.20-20240509185634-5b9d0c65af-centos7-aarch64-clang17 - os_type: centos7 - architecture: x86_64 - compiler_type: clang15 + architecture: aarch64 + compiler_type: clang17 is_linuxbrew: false - sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419 + sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157 lto_type: full - tag: v2.20-20231114005141-44dd1e39bc-centos7-x86_64-clang15-full-lto + tag: v2.20-20240509185632-5b9d0c65af-centos7-aarch64-clang17-full-lto - os_type: centos7 architecture: x86_64 compiler_type: clang16 is_linuxbrew: false - sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419 + sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157 lto_type: - tag: v2.20-20231114005157-44dd1e39bc-centos7-x86_64-clang16 + tag: v2.20-20240509185520-5b9d0c65af-centos7-x86_64-clang16 - os_type: centos7 architecture: x86_64 compiler_type: clang16 is_linuxbrew: false - sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419 + sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157 + lto_type: full + tag: v2.20-20240509185505-5b9d0c65af-centos7-x86_64-clang16-full-lto + - os_type: centos7 + architecture: x86_64 + compiler_type: clang17 + is_linuxbrew: false + sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157 + lto_type: + tag: v2.20-20240509185518-5b9d0c65af-centos7-x86_64-clang17 + - os_type: centos7 + architecture: x86_64 + compiler_type: clang17 + is_linuxbrew: false + sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157 lto_type: full - tag: v2.20-20231114005147-44dd1e39bc-centos7-x86_64-clang16-full-lto + tag: v2.20-20240509185509-5b9d0c65af-centos7-x86_64-clang17-full-lto - os_type: centos7 architecture: x86_64 compiler_type: gcc11 is_linuxbrew: false - sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419 + sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157 lto_type: - tag: v2.20-20231114005152-44dd1e39bc-centos7-x86_64-gcc11 + tag: v2.20-20240509185520-5b9d0c65af-centos7-x86_64-gcc11 - os_type: macos architecture: arm64 compiler_type: clang is_linuxbrew: false - sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419 + sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157 lto_type: - tag: v2.20-20231114170325-44dd1e39bc-macos-arm64 + tag: v2.20-20240517153912-5b9d0c65af-macos-arm64 - os_type: macos architecture: x86_64 compiler_type: clang is_linuxbrew: false - sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419 + sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157 lto_type: - tag: v2.20-20231114023841-44dd1e39bc-macos-x86_64 + tag: v2.20-20240509185549-5b9d0c65af-macos-x86_64 - os_type: ubuntu20.04 architecture: x86_64 - compiler_type: clang15 + compiler_type: clang16 + is_linuxbrew: false + sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157 + lto_type: + tag: v2.20-20240509185512-5b9d0c65af-ubuntu2004-x86_64-clang16 + - os_type: ubuntu22.04 + architecture: x86_64 + compiler_type: clang16 is_linuxbrew: false - sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419 + sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157 lto_type: - tag: v2.20-20231114005138-44dd1e39bc-ubuntu2004-x86_64-clang15 + tag: v2.20-20240509185514-5b9d0c65af-ubuntu2204-x86_64-clang16 - os_type: ubuntu22.04 architecture: x86_64 - compiler_type: clang15 + compiler_type: clang17 is_linuxbrew: false - sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419 + sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157 lto_type: - tag: v2.20-20231114005141-44dd1e39bc-ubuntu2204-x86_64-clang15 + tag: v2.20-20240509185507-5b9d0c65af-ubuntu2204-x86_64-clang17 - os_type: ubuntu22.04 architecture: x86_64 compiler_type: gcc11 is_linuxbrew: false - sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419 + sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157 lto_type: - tag: v2.20-20231114005141-44dd1e39bc-ubuntu2204-x86_64-gcc11 + tag: v2.20-20240509185513-5b9d0c65af-ubuntu2204-x86_64-gcc11 diff --git a/build-support/yugabyte-bash-common-sha1.txt b/build-support/yugabyte-bash-common-sha1.txt index c2fcc2e0e5f6..63a04c15b7f3 100644 --- a/build-support/yugabyte-bash-common-sha1.txt +++ b/build-support/yugabyte-bash-common-sha1.txt @@ -1 +1 @@ -06fdecffba1970934a139b3f2dfdf684789c212c +bcceabfde9f45afc0ad24bd99aab925191707da6 diff --git a/jenkins_jobs.yml b/jenkins_jobs.yml index e936554e769f..6fb134ed7034 100644 --- a/jenkins_jobs.yml +++ b/jenkins_jobs.yml @@ -6,13 +6,13 @@ jobs: build_type: asan release_artifact: false - - os: centos7 + - os: alma8 compiler: clang16 build_type: tsan release_artifact: false - - os: centos7 - compiler: clang16 + - os: alma8 + compiler: gcc11 build_type: debug release_artifact: false @@ -21,8 +21,8 @@ jobs: build_type: release release_artifact: true - - os: centos7 - compiler: clang16 + - os: amzn2 + compiler: clang17 build_type: release architecture: aarch64 release_artifact: true @@ -33,24 +33,22 @@ jobs: architecture: aarch64 release_artifact: false - - os: alma8 - compiler: gcc11 - build_type: fastdebug - release_artifact: false - - os: mac12 compiler: clang build_type: release architecture: arm64 release_artifact: true + test_opts: YB_COMPILE_ONLY=1 - os: mac12 compiler: clang build_type: release architecture: x86_64 release_artifact: true + test_opts: YB_COMPILE_ONLY=1 - os: ubuntu20.04 compiler: clang16 build_type: debug release_artifact: false + test_opts: YB_COMPILE_ONLY=1 diff --git a/python/yugabyte/os_versions.py b/python/yugabyte/os_versions.py index 2c6746242e71..752d13197dcc 100644 --- a/python/yugabyte/os_versions.py +++ b/python/yugabyte/os_versions.py @@ -14,6 +14,7 @@ UBUNTU_OS_TYPE_RE = re.compile(r'^(ubuntu)([0-9]{2})([0-9]{2})$') +AMZN_OS_TYPE_RE = re.compile(r'^(amzn|amazonlinux)([0-9]+)$') RHEL_FAMILY_RE = re.compile(r'^(almalinux|centos|rhel)([0-9]+)$') @@ -22,6 +23,10 @@ def adjust_os_type(os_type: str) -> str: if match: # Convert e.g. ubuntu2004 -> ubuntu20.04 for clarity. return f'{match.group(1)}{match.group(2)}.{match.group(3)}' + match = AMZN_OS_TYPE_RE.match(os_type) + if match: + # Convert e.g. amazonlinux2 -> amzn2 to match OS ID name + return f'amzn{match.group(2)}' return os_type diff --git a/python/yugabyte/release_util.py b/python/yugabyte/release_util.py index 36c560085e44..4c2a7629d74b 100644 --- a/python/yugabyte/release_util.py +++ b/python/yugabyte/release_util.py @@ -261,8 +261,10 @@ def get_release_file(self) -> str: # in our downsstream release code. So here we munge the name to 'centos' to keep things # working while we fix downstream code. # TODO(jharveymsith): Remove the almalinux to centos mapping once downstream is fixed. + # DEVOPS-3154 if distro.id() == "centos" and distro.major_version() == "7" \ - or distro.id() == "almalinux" and platform.machine().lower() == "x86_64": + or distro.id() == "almalinux" and platform.machine().lower() == "x86_64" \ + or distro.id() == "amzn" and platform.machine().lower() == "aarch64": system = "centos" elif distro.id == "ubuntu": system = distro.id() + distro.version() diff --git a/python/yugabyte/thirdparty_tool.py b/python/yugabyte/thirdparty_tool.py index 2575b5cfcd87..92eb78c3bc08 100755 --- a/python/yugabyte/thirdparty_tool.py +++ b/python/yugabyte/thirdparty_tool.py @@ -258,19 +258,22 @@ def is_consistent_with_yb_version(self, yb_version: str) -> bool: def should_skip_as_too_os_specific(self) -> bool: """ + Centos7 is EOL June 2024, this condition no longer applies. + Certain build types of specific OSes could be skipped because we can use the CentOS 7 build instead. We can do that in cases we know we don't need to run ASAN/TSAN. We know that we don't use ASAN/TSAN on aarch64 or for LTO builds as of 11/07/2022. Also we don't skip Linuxbrew builds or GCC builds. """ - return ( - self.os_type != 'centos7' and - self.compiler_type.startswith('clang') and - # We handle Linuxbrew builds in a special way, e.g. they could be built on AlmaLinux 8. - not self.is_linuxbrew and - # We don't run ASAN/TSAN on aarch64 or with LTO yet. - (self.architecture == 'aarch64' or self.lto_type is not None) - ) + # return ( + # self.os_type != 'centos7' and + # self.compiler_type.startswith('clang') and + # # We handle Linuxbrew builds in a special way, e.g. they could be built on AlmaLinux 8. + # not self.is_linuxbrew and + # # We don't run ASAN/TSAN on aarch64 or with LTO yet. + # (self.architecture == 'aarch64' or self.lto_type is not None) + # ) + return False @ruamel_yaml_object.register_class diff --git a/requirements_frozen.txt b/requirements_frozen.txt index 7d44654629d7..e93b2d793364 100644 --- a/requirements_frozen.txt +++ b/requirements_frozen.txt @@ -20,7 +20,7 @@ exceptiongroup==1.1.3 idna==3.4 iniconfig==2.0.0 jmespath==1.0.1 -llvm-installer==1.3.11 +llvm-installer==1.4.3 mypy-extensions==1.0.0 mypy==1.5.1 overrides==7.4.0 @@ -37,7 +37,7 @@ ruamel.yaml==0.17.32 semantic-version==2.10.0 shutilwhich==1.1.0 six==1.16.0 -sys-detection==1.3.0 +sys-detection==1.3.4 tomli==2.0.1 types-psutil==5.9.5.16 typing_extensions==4.8.0