From 898e27fffb10e04349bd99d6d8990ca6bfbe83d6 Mon Sep 17 00:00:00 2001 From: James Bartlett Date: Thu, 22 Jun 2023 09:20:47 -0700 Subject: [PATCH] [dev artifacts] Create releases for development artifacts. (#1553) Summary: We need to store our development artifacts somewhere other than gcs. This PR creates releases on the `pixie-io/dev-artifacts` repo, storing the artifacts in these releases. This prevents having random releases on the main repo, but still gives us a stable location for the dev artifacts. Type of change: /kind cleanup Test Plan: Ran `bazel clean --expunge && bazel build //...`. Also relying existing tests. Signed-off-by: James Bartlett --- bazel/cc_toolchains/sysroots/sysroots.bzl | 40 +++++++++--- bazel/linux_headers.bzl | 10 ++- bazel/pl_qemu_kernels.bzl | 19 +++--- bazel/repositories.bzl | 1 + bazel/repository_locations.bzl | 65 +++++++++++++++---- scripts/create_release_for_dev_artifact.sh | 42 ++++++++++++ .../px_dev_extras/attributes/linux_debs.rb | 2 +- tools/docker/Makefile | 24 ++++--- 8 files changed, 161 insertions(+), 42 deletions(-) create mode 100755 scripts/create_release_for_dev_artifact.sh diff --git a/bazel/cc_toolchains/sysroots/sysroots.bzl b/bazel/cc_toolchains/sysroots/sysroots.bzl index bb37154a908..07a8d80b349 100644 --- a/bazel/cc_toolchains/sysroots/sysroots.bzl +++ b/bazel/cc_toolchains/sysroots/sysroots.bzl @@ -21,42 +21,66 @@ SYSROOT_LOCATIONS = dict( sysroot_x86_64_glibc2_36_runtime = dict( sha256 = "0f6c8147394c41a5c10715d0859ca2ef6e03d65a52b5959f6e11f1de7513f3a4", strip_prefix = "", - urls = ["https://storage.googleapis.com/pixie-dev-public/sysroots/pl8/sysroot-amd64-runtime.tar.gz"], + urls = [ + "https://github.com/pixie-io/dev-artifacts/releases/download/sysroots%2Fpl8/sysroot-amd64-runtime.tar.gz", + "https://storage.googleapis.com/pixie-dev-public/sysroots/pl8/sysroot-amd64-runtime.tar.gz", + ], ), sysroot_x86_64_glibc2_36_build = dict( sha256 = "ccf1c8426baa180c8b1b178b58acf9c06c7e7b16383cf92f42719a2a05585058", strip_prefix = "", - urls = ["https://storage.googleapis.com/pixie-dev-public/sysroots/pl8/sysroot-amd64-build.tar.gz"], + urls = [ + "https://github.com/pixie-io/dev-artifacts/releases/download/sysroots%2Fpl8/sysroot-amd64-build.tar.gz", + "https://storage.googleapis.com/pixie-dev-public/sysroots/pl8/sysroot-amd64-build.tar.gz", + ], ), sysroot_x86_64_glibc2_36_test = dict( sha256 = "226b41c76e55899e9cc3bcbfed4646a610df5102dd9358add9450c80c5a14336", strip_prefix = "", - urls = ["https://storage.googleapis.com/pixie-dev-public/sysroots/pl8/sysroot-amd64-test.tar.gz"], + urls = [ + "https://github.com/pixie-io/dev-artifacts/releases/download/sysroots%2Fpl8/sysroot-amd64-test.tar.gz", + "https://storage.googleapis.com/pixie-dev-public/sysroots/pl8/sysroot-amd64-test.tar.gz", + ], ), sysroot_x86_64_glibc2_36_debug = dict( sha256 = "331d6c258a4abf5b877d93ae1d672fdd8758f5dafd7e614e19c5c20916ca1585", strip_prefix = "", - urls = ["https://storage.googleapis.com/pixie-dev-public/sysroots/pl8/sysroot-amd64-debug.tar.gz"], + urls = [ + "https://github.com/pixie-io/dev-artifacts/releases/download/sysroots%2Fpl8/sysroot-amd64-debug.tar.gz", + "https://storage.googleapis.com/pixie-dev-public/sysroots/pl8/sysroot-amd64-debug.tar.gz", + ], ), sysroot_aarch64_glibc2_36_runtime = dict( sha256 = "2974c29dd6baef1416d5cc2781306823885e64acaf0ce8bd3b447929b6103f4e", strip_prefix = "", - urls = ["https://storage.googleapis.com/pixie-dev-public/sysroots/pl8/sysroot-arm64-runtime.tar.gz"], + urls = [ + "https://github.com/pixie-io/dev-artifacts/releases/download/sysroots%2Fpl8/sysroot-arm64-runtime.tar.gz", + "https://storage.googleapis.com/pixie-dev-public/sysroots/pl8/sysroot-arm64-runtime.tar.gz", + ], ), sysroot_aarch64_glibc2_36_build = dict( sha256 = "04b957f6be840372e54075b966f76ea4238294028620ac8f8257c9e0fdb15bef", strip_prefix = "", - urls = ["https://storage.googleapis.com/pixie-dev-public/sysroots/pl8/sysroot-arm64-build.tar.gz"], + urls = [ + "https://github.com/pixie-io/dev-artifacts/releases/download/sysroots%2Fpl8/sysroot-arm64-build.tar.gz", + "https://storage.googleapis.com/pixie-dev-public/sysroots/pl8/sysroot-arm64-build.tar.gz", + ], ), sysroot_aarch64_glibc2_36_test = dict( sha256 = "59934822ae0ffe1e990fba589a94651eb92d0bddf2e40419f81299cf600eb62f", strip_prefix = "", - urls = ["https://storage.googleapis.com/pixie-dev-public/sysroots/pl8/sysroot-arm64-test.tar.gz"], + urls = [ + "https://github.com/pixie-io/dev-artifacts/releases/download/sysroots%2Fpl8/sysroot-arm64-test.tar.gz", + "https://storage.googleapis.com/pixie-dev-public/sysroots/pl8/sysroot-arm64-test.tar.gz", + ], ), sysroot_aarch64_glibc2_36_debug = dict( sha256 = "910d4ba853dedb6b9fcd05c862ed3e4a933cea941334064b7b62f8ae7fe4b87d", strip_prefix = "", - urls = ["https://storage.googleapis.com/pixie-dev-public/sysroots/pl8/sysroot-arm64-debug.tar.gz"], + urls = [ + "https://github.com/pixie-io/dev-artifacts/releases/download/sysroots%2Fpl8/sysroot-arm64-debug.tar.gz", + "https://storage.googleapis.com/pixie-dev-public/sysroots/pl8/sysroot-arm64-debug.tar.gz", + ], ), ) diff --git a/bazel/linux_headers.bzl b/bazel/linux_headers.bzl index 24d826dc994..a73243073cc 100644 --- a/bazel/linux_headers.bzl +++ b/bazel/linux_headers.bzl @@ -27,13 +27,19 @@ def linux_headers(): http_file( name = "linux_headers_merged_x86_64_tar_gz", - urls = ["https://storage.googleapis.com/pixie-dev-public/linux-headers/pl6/linux-headers-merged-x86_64-pl6.tar.gz"], + urls = [ + "https://github.com/pixie-io/dev-artifacts/releases/download/linux-headers%2Fpl6/linux-headers-merged-x86_64-pl6.tar.gz", + "https://storage.googleapis.com/pixie-dev-public/linux-headers/pl6/linux-headers-merged-x86_64-pl6.tar.gz", + ], sha256 = "b99ac9def26c0c1ef03f34583a1e62b96b3b5f8b038428fa582ed28aacbe0ac3", downloaded_file_path = "linux-headers-merged-x86_64.tar.gz", ) http_file( name = "linux_headers_merged_arm64_tar_gz", - urls = ["https://storage.googleapis.com/pixie-dev-public/linux-headers/pl6/linux-headers-merged-arm64-pl6.tar.gz"], + urls = [ + "https://github.com/pixie-io/dev-artifacts/releases/download/linux-headers%2Fpl6/linux-headers-merged-arm64-pl6.tar.gz", + "https://storage.googleapis.com/pixie-dev-public/linux-headers/pl6/linux-headers-merged-arm64-pl6.tar.gz", + ], sha256 = "3daa69cc0e0516c2e9190b6c90ed42cbd8cdabc377c30571fc7037354dc5c57f", downloaded_file_path = "linux-headers-merged-arm64.tar.gz", ) diff --git a/bazel/pl_qemu_kernels.bzl b/bazel/pl_qemu_kernels.bzl index 01e88e45c0b..fa759565a7b 100644 --- a/bazel/pl_qemu_kernels.bzl +++ b/bazel/pl_qemu_kernels.bzl @@ -16,14 +16,14 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") -kernel_build_date = 20230516231333 +kernel_build_date = 20230620214456 kernel_catalog = { - "4.14.254": "17e9ec8e081f98f38d23e5deb4ed0d29cc9785e19e842289d2f8ecd4f0263605", - "4.19.254": "2246f6f7a7949b721e228b94affd93fdb3a6e4262a589f870df232624daf6123", - "5.10.173": "d9033d193d44a3abcd1e14ab7757052e1ba104753e9a45f371356aac222f7381", - "5.15.101": "50d6150186856211ec0a3fc752568daf89f569c0c7b911c53cba402dd330041a", - "5.4.235": "29ec911b99a55deebdc1538903b350c93dc33fe2e4d3a92dbd4020f6992d8aba", - "6.1.18": "29f85b949a57b4b7f3c88c4c5459420b61247a71c05a7c1aa9a8d038fdc7a80b", + "4.14.254": "857030b52d0a48bc95ab544439cf52da07bce8cc2b9bef2e0382f02c182fab78", + "4.19.254": "5203a52b48a966ddfc8564ed2d0fd1e84f20ef417cbb5d41eb0b1c8413e4853d", + "5.10.173": "8c746c5ee94964323ca69533950370f595426efd671bfed85473e5a4b570ec26", + "5.15.101": "e18b8c5e91e3856dd6705a324ec7ebb2f1be5ff503c1322f5ddc6c44255272e4", + "5.4.235": "828f69979065800a4f214a1be369ab9756fdc2c586bd5bbfcc6d2a0f501601d5", + "6.1.18": "31f125451a70249e206a1eba555e432d5e08d173a4ab9d675f0b5d57f879f81b", } def kernel_version_to_name(version): @@ -33,7 +33,10 @@ def qemu_kernel_deps(): for version, sha in kernel_catalog.items(): http_file( name = kernel_version_to_name(version), - url = "https://storage.googleapis.com/pixie-dev-public/kernel-build/{}/linux-build-{}.tar.gz".format(kernel_build_date, version), + urls = [ + "https://github.com/pixie-io/dev-artifacts/releases/download/kernel-build%2F{}/linux-build-{}.tar.gz".format(kernel_build_date, version), + "https://storage.googleapis.com/pixie-dev-public/kernel-build/{}/linux-build-{}.tar.gz".format(kernel_build_date, version), + ], sha256 = sha, downloaded_file_path = "linux-build.tar.gz", ) diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index cd05bca3a9b..c5c3b3af2a8 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -196,6 +196,7 @@ def _java_deps(): ], sha256 = "102db28b450ff5eb8c497aacaececc5263a4e50e64b7cdc5c7baa8b216e73531", urls = [ + "https://github.com/pixie-io/dev-artifacts/releases/download/graalvm%2Fpl1/graalvm-native-image-22.3.0-pl1.tar.gz", "https://storage.googleapis.com/pixie-dev-public/graalvm-native-image-22.3.0-pl1.tar.gz", ], ) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index e11b40b9ac6..b604c4521c2 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -283,79 +283,118 @@ REPOSITORY_LOCATIONS = dict( ), com_llvm_clang_15 = dict( sha256 = "a275cde426a790cdf6eaecc2a33bac320ee61ceb4f5e76d2c73d3c555fdcedfc", - urls = ["https://storage.googleapis.com/pixie-dev-public/clang/15.0-pl11/clang-min-15.0-pl11.tar.gz"], + urls = [ + "https://github.com/pixie-io/dev-artifacts/releases/download/clang%2F15.0-pl11/clang-min-15.0-pl11.tar.gz", + "https://storage.googleapis.com/pixie-dev-public/clang/15.0-pl11/clang-min-15.0-pl11.tar.gz", + ], manual_license_name = "llvm/llvm-project", ), com_llvm_lib_x86_64_glibc_host = dict( sha256 = "f832e8cf5285aa7bc66d9e958cd10e14587a0d6d2376e293e6644651fe064b05", strip_prefix = "", - urls = ["https://storage.googleapis.com/pixie-dev-public/clang/15.0-pl11/llvm-15.0-pl11-libstdc%2B%2B.tar.gz"], + urls = [ + "https://github.com/pixie-io/dev-artifacts/releases/download/clang%2F15.0-pl11/llvm-15.0-pl11-libstdc%2B%2B.tar.gz", + "https://storage.googleapis.com/pixie-dev-public/clang/15.0-pl11/llvm-15.0-pl11-libstdc%2B%2B.tar.gz", + ], manual_license_name = "llvm/llvm-project", ), com_llvm_lib_libcpp_x86_64_glibc_host = dict( sha256 = "83d6c8f39688646e733b9341d3d8a67978ac33dc6874d730338492ccec61e40b", strip_prefix = "", - urls = ["https://storage.googleapis.com/pixie-dev-public/clang/15.0-pl11/llvm-15.0-pl11-libcxx.tar.gz"], + urls = [ + "https://github.com/pixie-io/dev-artifacts/releases/download/clang%2F15.0-pl11/llvm-15.0-pl11-libcxx.tar.gz", + "https://storage.googleapis.com/pixie-dev-public/clang/15.0-pl11/llvm-15.0-pl11-libcxx.tar.gz", + ], manual_license_name = "llvm/llvm-project", ), com_llvm_lib_libcpp_x86_64_glibc_host_asan = dict( sha256 = "5b0e518bc36b308d8f64a074bdce45f709f4f661f45a0c1e8c4a92ead6ef7b7d", strip_prefix = "", - urls = ["https://storage.googleapis.com/pixie-dev-public/clang/15.0-pl11/llvm-15.0-pl11-libcxx-asan.tar.gz"], + urls = [ + "https://github.com/pixie-io/dev-artifacts/releases/download/clang%2F15.0-pl11/llvm-15.0-pl11-libcxx-asan.tar.gz", + "https://storage.googleapis.com/pixie-dev-public/clang/15.0-pl11/llvm-15.0-pl11-libcxx-asan.tar.gz", + ], manual_license_name = "llvm/llvm-project", ), com_llvm_lib_libcpp_x86_64_glibc_host_msan = dict( sha256 = "39e959d02a2ae753af64d61a3ae264fd73650e63c258738905f61f944e8a2c18", strip_prefix = "", - urls = ["https://storage.googleapis.com/pixie-dev-public/clang/15.0-pl11/llvm-15.0-pl11-libcxx-msan.tar.gz"], + urls = [ + "https://github.com/pixie-io/dev-artifacts/releases/download/clang%2F15.0-pl11/llvm-15.0-pl11-libcxx-msan.tar.gz", + "https://storage.googleapis.com/pixie-dev-public/clang/15.0-pl11/llvm-15.0-pl11-libcxx-msan.tar.gz", + ], manual_license_name = "llvm/llvm-project", ), com_llvm_lib_libcpp_x86_64_glibc_host_tsan = dict( sha256 = "4e39d01e4719d0491b4d970bb963a8c4e90c75a1002c90e7cee5fc08fd003f83", strip_prefix = "", - urls = ["https://storage.googleapis.com/pixie-dev-public/clang/15.0-pl11/llvm-15.0-pl11-libcxx-tsan.tar.gz"], + urls = [ + "https://github.com/pixie-io/dev-artifacts/releases/download/clang%2F15.0-pl11/llvm-15.0-pl11-libcxx-tsan.tar.gz", + "https://storage.googleapis.com/pixie-dev-public/clang/15.0-pl11/llvm-15.0-pl11-libcxx-tsan.tar.gz", + ], manual_license_name = "llvm/llvm-project", ), com_llvm_lib_x86_64_glibc2_36 = dict( sha256 = "13a37fdba0e9bf386873509ded490177200265839a4abdbdccacb8bf2b9795ab", strip_prefix = "", - urls = ["https://storage.googleapis.com/pixie-dev-public/clang/15.0-pl11/llvm-15.0-pl11-libstdc++-x86_64-sysroot.tar.gz"], + urls = [ + "https://github.com/pixie-io/dev-artifacts/releases/download/clang%2F15.0-pl11/llvm-15.0-pl11-libstdc++-x86_64-sysroot.tar.gz", + "https://storage.googleapis.com/pixie-dev-public/clang/15.0-pl11/llvm-15.0-pl11-libstdc++-x86_64-sysroot.tar.gz", + ], manual_license_name = "llvm/llvm-project", ), com_llvm_lib_libcpp_x86_64_glibc2_36 = dict( sha256 = "6909bf0f95c2b0d798365651a8fb8e93c9b01994c5e68e76e71fb7e5100a3965", strip_prefix = "", - urls = ["https://storage.googleapis.com/pixie-dev-public/clang/15.0-pl11/llvm-15.0-pl11-libcxx-x86_64-sysroot.tar.gz"], + urls = [ + "https://github.com/pixie-io/dev-artifacts/releases/download/clang%2F15.0-pl11/llvm-15.0-pl11-libcxx-x86_64-sysroot.tar.gz", + "https://storage.googleapis.com/pixie-dev-public/clang/15.0-pl11/llvm-15.0-pl11-libcxx-x86_64-sysroot.tar.gz", + ], manual_license_name = "llvm/llvm-project", ), com_llvm_lib_aarch64_glibc2_36 = dict( sha256 = "e1d0b628a7915113e79ac0fc1dfd4bc730177d0652fb684026ecfb87694416b0", strip_prefix = "", - urls = ["https://storage.googleapis.com/pixie-dev-public/clang/15.0-pl11/llvm-15.0-pl11-libstdc++-aarch64-sysroot.tar.gz"], + urls = [ + "https://github.com/pixie-io/dev-artifacts/releases/download/clang%2F15.0-pl11/llvm-15.0-pl11-libstdc++-aarch64-sysroot.tar.gz", + "https://storage.googleapis.com/pixie-dev-public/clang/15.0-pl11/llvm-15.0-pl11-libstdc++-aarch64-sysroot.tar.gz", + ], manual_license_name = "llvm/llvm-project", ), com_llvm_lib_libcpp_aarch64_glibc2_36 = dict( sha256 = "a0199c3a4247ea0e82a5b6914efac0cf339ad75ef64f927e353f84e2d0f8b60b", strip_prefix = "", - urls = ["https://storage.googleapis.com/pixie-dev-public/clang/15.0-pl11/llvm-15.0-pl11-libcxx-aarch64-sysroot.tar.gz"], + urls = [ + "https://github.com/pixie-io/dev-artifacts/releases/download/clang%2F15.0-pl11/llvm-15.0-pl11-libcxx-aarch64-sysroot.tar.gz", + "https://storage.googleapis.com/pixie-dev-public/clang/15.0-pl11/llvm-15.0-pl11-libcxx-aarch64-sysroot.tar.gz", + ], manual_license_name = "llvm/llvm-project", ), com_llvm_libcxx_x86_64_glibc_host = dict( sha256 = "2e123059d395daec37153fc413fe377a6a551b2919cd658fd9a302b11fea5201", strip_prefix = "", - urls = ["https://storage.googleapis.com/pixie-dev-public/clang/15.0-pl11/libcxx-15.0-pl11.tar.gz"], + urls = [ + "https://github.com/pixie-io/dev-artifacts/releases/download/clang%2F15.0-pl11/libcxx-15.0-pl11.tar.gz", + "https://storage.googleapis.com/pixie-dev-public/clang/15.0-pl11/libcxx-15.0-pl11.tar.gz", + ], manual_license_name = "llvm/llvm-project", ), com_llvm_libcxx_x86_64_glibc2_36 = dict( sha256 = "51135b52804d3a899e58e1a7162ed065e95eb5d3822cab39edcfe9aed466eede", strip_prefix = "", - urls = ["https://storage.googleapis.com/pixie-dev-public/clang/15.0-pl11/libcxx-15.0-pl11-x86_64-sysroot.tar.gz"], + urls = [ + "https://github.com/pixie-io/dev-artifacts/releases/download/clang%2F15.0-pl11/libcxx-15.0-pl11-x86_64-sysroot.tar.gz", + "https://storage.googleapis.com/pixie-dev-public/clang/15.0-pl11/libcxx-15.0-pl11-x86_64-sysroot.tar.gz", + ], manual_license_name = "llvm/llvm-project", ), com_llvm_libcxx_aarch64_glibc2_36 = dict( sha256 = "a90304bd3e6530e6228beb8fede84fc06c893cefdab4b12859d5e93661aac7ce", strip_prefix = "", - urls = ["https://storage.googleapis.com/pixie-dev-public/clang/15.0-pl11/libcxx-15.0-pl11-aarch64-sysroot.tar.gz"], + urls = [ + "https://github.com/pixie-io/dev-artifacts/releases/download/clang%2F15.0-pl11/libcxx-15.0-pl11-aarch64-sysroot.tar.gz", + "https://storage.googleapis.com/pixie-dev-public/clang/15.0-pl11/libcxx-15.0-pl11-aarch64-sysroot.tar.gz", + ], manual_license_name = "llvm/llvm-project", ), com_oracle_openjdk_18 = dict( diff --git a/scripts/create_release_for_dev_artifact.sh b/scripts/create_release_for_dev_artifact.sh new file mode 100755 index 00000000000..669d53b9e48 --- /dev/null +++ b/scripts/create_release_for_dev_artifact.sh @@ -0,0 +1,42 @@ +#!/bin/bash -e + +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +repo="pixie-io/dev-artifacts" + +if [[ $# -lt 3 ]]; then + echo "Usage: $0 [...]" + exit 1 +fi + +release_name="$1" +version="$2" +artifacts=( "${@:3}" ) + +tag_name="${release_name}/${version}" + +repo_dir="$(mktemp -d)" +git clone git@github.com:pixie-io/dev-artifacts.git "${repo_dir}" + +pushd "${repo_dir}" &> /dev/null +git tag "${tag_name}" +git push origin "${tag_name}" +popd &> /dev/null +rm -rf "${repo_dir}" + +gh release create --repo="${repo}" "${tag_name}" --title "${release_name} ${version}" --notes "" +gh release upload --repo="${repo}" "${tag_name}" "${artifacts[@]}" diff --git a/tools/chef/cookbooks/px_dev_extras/attributes/linux_debs.rb b/tools/chef/cookbooks/px_dev_extras/attributes/linux_debs.rb index b9aa06f2e75..0704d044c17 100644 --- a/tools/chef/cookbooks/px_dev_extras/attributes/linux_debs.rb +++ b/tools/chef/cookbooks/px_dev_extras/attributes/linux_debs.rb @@ -26,6 +26,6 @@ default['gperftools']['version'] = '2.10-pl1' default['gperftools']['deb'] = - "https://storage.googleapis.com/pixie-dev-public/gperftools-pixie-#{default['gperftools']['version']}.deb" + "https://github.com/pixie-io/dev-artifacts/releases/download/gperftools%2F#{default['gperftools']['version']}/gperftools-pixie-#{default['gperftools']['version']}.deb" default['gperftools']['deb_sha256'] = '0920a93a8a8716b714b9b316c8d7e8f2ecc242a85147f7bec5e1543d88c203dc' diff --git a/tools/docker/Makefile b/tools/docker/Makefile index ab64cad4d8d..b6efbf3bf26 100644 --- a/tools/docker/Makefile +++ b/tools/docker/Makefile @@ -22,6 +22,8 @@ BUILD_DIR ?= .build TOT := $$(git rev-parse --show-toplevel) DOCKER_PROPERTIES_FILE := $(TOT)/docker.properties +GH_RELEASE_UPLOAD := $(TOT)/scripts/create_release_for_dev_artifact.sh + DEV_EXTRAS_TAG = $$(grep DOCKER_IMAGE_TAG $(DOCKER_PROPERTIES_FILE) | cut -d'=' -f2) DEV_EXTRAS_DIGEST = $$(grep DEV_IMAGE_WITH_EXTRAS_DIGEST $(DOCKER_PROPERTIES_FILE) | cut -d'=' -f2) DEV_EXTRAS_IMAGE = "gcr.io/pixie-oss/pixie-dev-public/dev_image_with_extras:$(DEV_EXTRAS_TAG)" @@ -36,6 +38,7 @@ clang_deb_fname := "clang-$(CLANG_TAG).deb" clang_linters_deb_fname := "clang-linters-$(CLANG_TAG).deb" clang_gs_path := gs://pixie-dev-public/clang/$(CLANG_TAG) clang_deb_image_tag := "gcr.io/pixie-oss/pixie-dev-public/clang_deb_builder_image:$(CLANG_VERSION)" +CLANG_BUILD_DIR := "$(BUILD_DIR)/clang-$(CLANG_TAG)" ## gperftools parameters GPERFTOOLS_DEB_IMAGE_VERSION := 2.10-pl1 @@ -100,10 +103,6 @@ LINUX_HEADERS_X86_64_MERGED_FILE := $(LINUX_HEADER_BUILD_DIR)/linux-headers-merg LINUX_HEADERS_ARM64_MERGED_FILE := $(LINUX_HEADER_BUILD_DIR)/linux-headers-merged-arm64-$(LINUX_HEADERS_REV).tar.gz LINUX_HEADERS_GS_PATH := gs://pixie-dev-public/linux-headers/$(LINUX_HEADERS_REV) -## Curl image parameters. -CURL_IMAGE_VERSION := 1.0 -curl_image_tag := "gcr.io/pixie-oss/pixie-dev-public/curl:$(CURL_IMAGE_VERSION)" - ## NATS image parameters. NATS_IMAGE_VERSION := 2.9.19 nats_image_tag := "gcr.io/pixie-oss/pixie-prod/vizier-deps/nats:$(NATS_IMAGE_VERSION)-scratch" @@ -143,13 +142,14 @@ build_clang_deb_image: .PHONY: upload_clang_deb upload_clang_deb: build_clang_deb_image ## Target to build and upload clang deb image - mkdir -p $(BUILD_DIR) - $(DOCKER) run --rm -e CLANG_SUFFIX=$(CLANG_SUFFIX) -e CLANG_VERSION=$(CLANG_VERSION) -v $(PWD)/$(BUILD_DIR):/image $(clang_deb_image_tag) + @mkdir -p $(CLANG_BUILD_DIR) + $(DOCKER) run --rm -e CLANG_SUFFIX=$(CLANG_SUFFIX) -e CLANG_VERSION=$(CLANG_VERSION) -v $(PWD)/$(CLANG_BUILD_DIR):/image $(clang_deb_image_tag) - sha256sum $(BUILD_DIR)/* > $(BUILD_DIR)/sha256sums - gsutil cp $(BUILD_DIR)/* $(clang_gs_path)/ + sha256sum $(CLANG_BUILD_DIR)/* > $(CLANG_BUILD_DIR)/sha256sums + gsutil cp $(CLANG_BUILD_DIR)/* $(clang_gs_path)/ + $(GH_RELEASE_UPLOAD) clang $(CLANG_TAG) $(CLANG_BUILD_DIR)/* - cat $(BUILD_DIR)/sha256sums + cat $(CLANG_BUILD_DIR)/sha256sums ############################################## @@ -164,6 +164,7 @@ upload_gperftools_deb: build_gperftools_deb_image ## Target to build and upload mkdir -p $(BUILD_DIR) $(DOCKER) run --rm -e DEB_NAME=$(gperftools_deb_fname) -e DEB_VERSION=$(GPERFTOOLS_DEB_IMAGE_VERSION) -v $(PWD)/$(BUILD_DIR):/image $(gperftools_deb_image_tag) gsutil cp $(PWD)/$(BUILD_DIR)/$(gperftools_deb_fname) $(gperftools_deb_gs_path) + $(GH_RELEASE_UPLOAD) gperftools $(GPERFTOOLS_DEB_IMAGE_VERSION) $(PWD)/$(BUILD_DIR)/$(gperftools_deb_fname) @echo "SHA: " sha256sum $(PWD)/$(BUILD_DIR)/$(gperftools_deb_fname) @@ -180,6 +181,7 @@ upload_graalvm_archive: build_graalvm_archive_image mkdir -p $(BUILD_DIR) $(DOCKER) run --rm -v $(PWD)/$(BUILD_DIR):/archive $(GRAALVM_ARCHIVE_IMAGE_TAG) gsutil cp $(PWD)/$(BUILD_DIR)/$(GRAALVM_ARCHIVE_FNAME) $(GRAALVM_ARCHIVE_GS_PATH) + $(GH_RELEASE_UPLOAD) graalvm $(GRAALVM_IMAGE_VERSION) $(PWD)/$(BUILD_DIR)/$(GRAALVM_ARCHIVE_FNAME) @echo "SHA: " sha256sum $(PWD)/$(BUILD_DIR)/$(GRAALVM_ARCHIVE_FNAME) @@ -217,6 +219,7 @@ $(LINUX_HEADERS_ARM64_MERGED_FILE): $(LINUX_HEADER_ARM64_TARGETS) .PHONY: upload_linux_headers upload_linux_headers: $(LINUX_HEADERS_X86_64_MERGED_FILE) $(LINUX_HEADERS_ARM64_MERGED_FILE) ## Target to build and upload linux headers image gsutil cp $^ $(LINUX_HEADERS_GS_PATH) + $(GH_RELEASE_UPLOAD) linux-headers $(LINUX_HEADERS_REV) $^ sha256sum $^ ############################################## @@ -231,9 +234,9 @@ build_all_sysroots: sysroot_creator/Dockerfile sysroot_creator/build_all_sysroot .PHONY: upload_all_sysroots upload_all_sysroots: build_all_sysroots gsutil cp $(SYSROOT_BUILD_DIR)/* $(SYSROOT_GS_PATH) + $(GH_RELEASE_UPLOAD) sysroots $(SYSROOT_REV) $(SYSROOT_BUILD_DIR)/* sha256sum $(SYSROOT_BUILD_DIR)/* - ############################################## # Custom docker images. ############################################## @@ -268,6 +271,7 @@ $(KERNEL_BUILD_DIR)/linux-build-%.tar.gz: kernel_builder/Dockerfile kernel_build .PHONY: upload_kernel_build upload_kernel_build: $(KERNEL_BUILD_TARGETS) gsutil cp $^ $(KERNEL_BUILD_GS_PATH) + $(GH_RELEASE_UPLOAD) kernel-build $(KERNEL_BUILD_TS) $^ @echo "Add this to the qemu bzl file:" @echo "=============================" @echo "kernel_build_date = ${KERNEL_BUILD_TS}"