From a994e7fbe83e6e21da36bc929ddf9cd50ffc792a Mon Sep 17 00:00:00 2001 From: Vihang Mehta Date: Sat, 21 Jan 2023 22:58:13 +0000 Subject: [PATCH] Support templated replacements in kustomize_build Summary: This changes kustomize_build's handling of replacement and uses the string format syntax combined with ctx.vars to enable Make var like substitution. This allows us to reduce the dupe in vizier images and just use the var instead. Test Plan: Built the vizier images with diffrent values for build_type. Checked to ensure that the prefix and version were as expected. Reviewers: jamesbartlett, zasgar, michelle Reviewed By: jamesbartlett Signed-off-by: Vihang Mehta Differential Revision: https://phab.corp.pixielabs.ai/D12887 GitOrigin-RevId: 261a91f30361c7e8a27dbdf8c8dbf7d76d2236c4 --- bazel/images.bzl | 2 +- bazel/kustomize.bzl | 12 +- k8s/operator/BUILD.bazel | 3 - k8s/vizier/BUILD.bazel | 278 +++++++++------------------------------ 4 files changed, 71 insertions(+), 224 deletions(-) diff --git a/bazel/images.bzl b/bazel/images.bzl index 5b5ff9d5f51..8244f6f6cab 100644 --- a/bazel/images.bzl +++ b/bazel/images.bzl @@ -25,7 +25,7 @@ def image_replacements(image_map, existing_prefix, new_prefix): for image in image_map.keys(): image = image.removeprefix("$(IMAGE_PREFIX)").removesuffix(":$(BUNDLE_VERSION)") - replacements[existing_prefix + image] = new_prefix + image + replacements[existing_prefix + image] = new_prefix + image + ":{BUNDLE_VERSION}" return replacements diff --git a/bazel/kustomize.bzl b/bazel/kustomize.bzl index f4ce517d48e..a87311dc616 100644 --- a/bazel/kustomize.bzl +++ b/bazel/kustomize.bzl @@ -32,7 +32,9 @@ def _kustomize_build_impl(ctx): cmds.append('pushd "$TMP/$(dirname "{}")" &> /dev/null'.format(ctx.file.kustomization.path)) for old, new in ctx.attr.replacements.items(): - cmds.append('"$KUSTOMIZE_BIN" edit set image {}={}:{}'.format(old, new, ctx.var["BUNDLE_VERSION"])) + old_expanded = old.format(**ctx.var) + new_expanded = new.format(**ctx.var) + cmds.append('"$KUSTOMIZE_BIN" edit set image {}={}'.format(old_expanded, new_expanded)) cmds.append("popd &> /dev/null") @@ -60,7 +62,13 @@ kustomize_build = rule( mandatory = True, allow_single_file = True, ), - "replacements": attr.string_dict(), + "replacements": attr.string_dict( + doc = """ + Will be passed as args to `kustomize edit set image =` + Supports make vars as templates. i.e. {MY_VAR} will be replaced with + values ctx.vars["MY_VAR"] if available. + """, + ), "srcs": attr.label_list( mandatory = True, allow_files = True, diff --git a/k8s/operator/BUILD.bazel b/k8s/operator/BUILD.bazel index 2d0f48fa9f2..a0ace1e658d 100644 --- a/k8s/operator/BUILD.bazel +++ b/k8s/operator/BUILD.bazel @@ -48,9 +48,6 @@ kustomize_build( exclude = ["crd/base/kustomization.yaml"], ), kustomization = "crd/base/kustomization.yaml", - toolchains = [ - "//k8s:bundle_version", - ], ) pkg_tar( diff --git a/k8s/vizier/BUILD.bazel b/k8s/vizier/BUILD.bazel index a07deb8efdf..219b0ce9ea4 100644 --- a/k8s/vizier/BUILD.bazel +++ b/k8s/vizier/BUILD.bazel @@ -17,7 +17,7 @@ load("@io_bazel_rules_docker//container:container.bzl", "container_bundle") load("@io_bazel_rules_docker//contrib:push-all.bzl", "container_push") load("@rules_pkg//pkg:tar.bzl", "pkg_tar") -load("//bazel:images.bzl", "DEV_PREFIX", "PROPRIETARY_PREFIX", "PUBLIC_PREFIX", "image_replacements") +load("//bazel:images.bzl", "DEV_PREFIX", "image_replacements") load("//bazel:kustomize.bzl", "kustomize_build") package(default_visibility = ["//visibility:public"]) @@ -32,8 +32,8 @@ VIZIER_IMAGE_TO_LABEL = { "$(IMAGE_PREFIX)vizier/vizier_updater_image:$(BUNDLE_VERSION)": "//src/utils/pixie_updater:vizier_updater_image", } -filegroup( - name = "etcd_base", +kustomize_build( + name = "etcd_metadata", srcs = glob( [ "base/**/*.yaml", @@ -47,10 +47,20 @@ filegroup( "persistent_metadata/**", ], ), + kustomization = "etcd_metadata/kustomization.yaml", + replacements = image_replacements( + existing_prefix = DEV_PREFIX, + image_map = VIZIER_IMAGE_TO_LABEL, + new_prefix = "{IMAGE_PREFIX}", + ), + toolchains = [ + "//k8s:bundle_version", + "//k8s:image_prefix", + ], ) -filegroup( - name = "etcd_autopilot", +kustomize_build( + name = "etcd_metadata_autopilot", srcs = glob( [ "base/**/*.yaml", @@ -63,10 +73,20 @@ filegroup( "persistent_metadata/**", ], ), + kustomization = "etcd_metadata/autopilot/kustomization.yaml", + replacements = image_replacements( + existing_prefix = DEV_PREFIX, + image_map = VIZIER_IMAGE_TO_LABEL, + new_prefix = "{IMAGE_PREFIX}", + ), + toolchains = [ + "//k8s:bundle_version", + "//k8s:image_prefix", + ], ) -filegroup( - name = "persistent_base", +kustomize_build( + name = "persistent_metadata", srcs = glob( [ "base/**/*.yaml", @@ -80,10 +100,20 @@ filegroup( "etcd_metadata/**", ], ), + kustomization = "persistent_metadata/kustomization.yaml", + replacements = image_replacements( + existing_prefix = DEV_PREFIX, + image_map = VIZIER_IMAGE_TO_LABEL, + new_prefix = "{IMAGE_PREFIX}", + ), + toolchains = [ + "//k8s:bundle_version", + "//k8s:image_prefix", + ], ) -filegroup( - name = "persistent_autopilot", +kustomize_build( + name = "persistent_metadata_autopilot", srcs = glob( [ "base/**/*.yaml", @@ -96,173 +126,15 @@ filegroup( "etcd_metadata/**", ], ), -) - -kustomize_build( - name = "public_etcd", - srcs = [":etcd_base"], - kustomization = "etcd_metadata/kustomization.yaml", - replacements = image_replacements( - existing_prefix = DEV_PREFIX, - image_map = VIZIER_IMAGE_TO_LABEL, - new_prefix = PUBLIC_PREFIX, - ), - toolchains = [ - "//k8s:bundle_version", - ], -) - -kustomize_build( - name = "private_etcd", - srcs = [":etcd_base"], - kustomization = "etcd_metadata/kustomization.yaml", - replacements = image_replacements( - existing_prefix = DEV_PREFIX, - image_map = VIZIER_IMAGE_TO_LABEL, - new_prefix = PROPRIETARY_PREFIX, - ), - toolchains = [ - "//k8s:bundle_version", - ], -) - -kustomize_build( - name = "dev_etcd", - srcs = [":etcd_base"], - kustomization = "etcd_metadata/kustomization.yaml", - replacements = image_replacements( - existing_prefix = DEV_PREFIX, - image_map = VIZIER_IMAGE_TO_LABEL, - new_prefix = DEV_PREFIX, - ), - toolchains = [ - "//k8s:bundle_version", - ], -) - -kustomize_build( - name = "public_etcd_autopilot", - srcs = [":etcd_autopilot"], - kustomization = "etcd_metadata/autopilot/kustomization.yaml", - replacements = image_replacements( - existing_prefix = DEV_PREFIX, - image_map = VIZIER_IMAGE_TO_LABEL, - new_prefix = PUBLIC_PREFIX, - ), - toolchains = [ - "//k8s:bundle_version", - ], -) - -kustomize_build( - name = "private_etcd_autopilot", - srcs = [":etcd_autopilot"], - kustomization = "etcd_metadata/autopilot/kustomization.yaml", - replacements = image_replacements( - existing_prefix = DEV_PREFIX, - image_map = VIZIER_IMAGE_TO_LABEL, - new_prefix = PROPRIETARY_PREFIX, - ), - toolchains = [ - "//k8s:bundle_version", - ], -) - -kustomize_build( - name = "dev_etcd_autopilot", - srcs = [":etcd_autopilot"], - kustomization = "etcd_metadata/autopilot/kustomization.yaml", - replacements = image_replacements( - existing_prefix = DEV_PREFIX, - image_map = VIZIER_IMAGE_TO_LABEL, - new_prefix = DEV_PREFIX, - ), - toolchains = [ - "//k8s:bundle_version", - ], -) - -kustomize_build( - name = "public_persist", - srcs = [":persistent_base"], - kustomization = "persistent_metadata/kustomization.yaml", - replacements = image_replacements( - existing_prefix = DEV_PREFIX, - image_map = VIZIER_IMAGE_TO_LABEL, - new_prefix = PUBLIC_PREFIX, - ), - toolchains = [ - "//k8s:bundle_version", - ], -) - -kustomize_build( - name = "private_persist", - srcs = [":persistent_base"], - kustomization = "persistent_metadata/kustomization.yaml", - replacements = image_replacements( - existing_prefix = DEV_PREFIX, - image_map = VIZIER_IMAGE_TO_LABEL, - new_prefix = PROPRIETARY_PREFIX, - ), - toolchains = [ - "//k8s:bundle_version", - ], -) - -kustomize_build( - name = "dev_persist", - srcs = [":persistent_base"], - kustomization = "persistent_metadata/kustomization.yaml", - replacements = image_replacements( - existing_prefix = DEV_PREFIX, - image_map = VIZIER_IMAGE_TO_LABEL, - new_prefix = DEV_PREFIX, - ), - toolchains = [ - "//k8s:bundle_version", - ], -) - -kustomize_build( - name = "public_persist_autopilot", - srcs = [":persistent_autopilot"], - kustomization = "persistent_metadata/autopilot/kustomization.yaml", - replacements = image_replacements( - existing_prefix = DEV_PREFIX, - image_map = VIZIER_IMAGE_TO_LABEL, - new_prefix = PUBLIC_PREFIX, - ), - toolchains = [ - "//k8s:bundle_version", - ], -) - -kustomize_build( - name = "private_persist_autopilot", - srcs = [":persistent_autopilot"], - kustomization = "persistent_metadata/autopilot/kustomization.yaml", - replacements = image_replacements( - existing_prefix = DEV_PREFIX, - image_map = VIZIER_IMAGE_TO_LABEL, - new_prefix = PROPRIETARY_PREFIX, - ), - toolchains = [ - "//k8s:bundle_version", - ], -) - -kustomize_build( - name = "dev_persist_autopilot", - srcs = [":persistent_autopilot"], kustomization = "persistent_metadata/autopilot/kustomization.yaml", replacements = image_replacements( existing_prefix = DEV_PREFIX, image_map = VIZIER_IMAGE_TO_LABEL, - new_prefix = DEV_PREFIX, + new_prefix = "{IMAGE_PREFIX}", ), toolchains = [ "//k8s:bundle_version", + "//k8s:image_prefix", ], ) @@ -283,11 +155,8 @@ container_push( genrule( name = "vizier_etcd_image_list", - srcs = select({ - "//k8s:dev": [":dev_etcd"], - "//k8s:proprietary": [":private_etcd"], - "//k8s:public": [":public_etcd"], - }) + [ + srcs = [ + ":etcd_metadata", "//k8s/vizier_deps:nats_prod", "//k8s/vizier_deps:etcd_prod", ], @@ -299,11 +168,8 @@ genrule( genrule( name = "vizier_image_list", - srcs = select({ - "//k8s:dev": [":dev_persist"], - "//k8s:proprietary": [":private_persist"], - "//k8s:public": [":public_persist"], - }) + [ + srcs = [ + ":persistent_metadata", "//k8s/vizier_deps:nats_prod", ], outs = ["vizier_image_list.txt"], @@ -314,48 +180,24 @@ genrule( pkg_tar( name = "vizier_yamls", - srcs = - select({ - "//k8s:dev": [ - ":dev_etcd", - ":dev_persist", - ":dev_etcd_autopilot", - ":dev_persist_autopilot", - ], - "//k8s:proprietary": [ - ":private_etcd", - ":private_persist", - ":private_etcd_autopilot", - ":private_persist_autopilot", - ], - "//k8s:public": [ - ":public_etcd", - ":public_persist", - ":public_etcd_autopilot", - ":public_persist_autopilot", - ], - }) + [ - "//k8s/vizier_deps:nats_prod", - "//k8s/vizier_deps:etcd_prod", - ":vizier_etcd_image_list", - ":vizier_image_list", - "public/secrets.yaml", - ], + srcs = [ + "public/secrets.yaml", + ":etcd_metadata", + ":etcd_metadata_autopilot", + ":persistent_metadata", + ":persistent_metadata_autopilot", + ":vizier_etcd_image_list", + ":vizier_image_list", + "//k8s/vizier_deps:etcd_prod", + "//k8s/vizier_deps:nats_prod", + ], package_dir = "/yamls", remap_paths = { - "/vizier/dev_etcd.yaml": "vizier/vizier_etcd_metadata_prod.yaml", - "/vizier/dev_etcd_autopilot.yaml": "vizier/vizier_etcd_metadata_autopilot_prod.yaml", - "/vizier/dev_persist.yaml": "vizier/vizier_metadata_persist_prod.yaml", - "/vizier/dev_persist_autopilot.yaml": "vizier/vizier_metadata_persist_autopilot_prod.yaml", - "/vizier/private_etcd.yaml": "vizier/vizier_etcd_metadata_prod.yaml", - "/vizier/private_etcd_autopilot.yaml": "vizier/vizier_etcd_metadata_autopilot_prod.yaml", - "/vizier/private_persist.yaml": "vizier/vizier_metadata_persist_prod.yaml", - "/vizier/private_persist_autopilot.yaml": "vizier/vizier_metadata_persist_autopilot_prod.yaml", + "/vizier/etcd.yaml": "vizier/vizier_etcd_metadata_prod.yaml", + "/vizier/etcd_autopilot.yaml": "vizier/vizier_etcd_metadata_autopilot_prod.yaml", + "/vizier/persist.yaml": "vizier/vizier_metadata_persist_prod.yaml", + "/vizier/persist_autopilot.yaml": "vizier/vizier_metadata_persist_autopilot_prod.yaml", "/vizier/public/secrets.yaml": "vizier/secrets.yaml", - "/vizier/public_etcd.yaml": "vizier/vizier_etcd_metadata_prod.yaml", - "/vizier/public_etcd_autopilot.yaml": "vizier/vizier_etcd_metadata_autopilot_prod.yaml", - "/vizier/public_persist.yaml": "vizier/vizier_metadata_persist_prod.yaml", - "/vizier/public_persist_autopilot.yaml": "vizier/vizier_metadata_persist_autopilot_prod.yaml", "/vizier/vizier_etcd_image_list.txt": "images/vizier_etcd_image_list.txt", "/vizier/vizier_image_list.txt": "images/vizier_image_list.txt", },