From 33d8ee563945eeca6cc0b5c84768f7390d3e7093 Mon Sep 17 00:00:00 2001 From: Christian Schlotter Date: Fri, 29 Dec 2023 16:05:29 +0100 Subject: [PATCH] hack: implement prowjob-generator prowjob-generator allows to generate the periodic and presubmit configuration files in test-infra from a configuration file which simplifies and automates introducing and chaning tests for branches. --- hack/tools/go.mod | 2 + hack/tools/go.sum | 4 + hack/tools/prowjob-generator/config.go | 45 ++ hack/tools/prowjob-generator/main.go | 175 ++++++++ hack/tools/prowjob-generator/main_test.go | 30 ++ hack/tools/prowjob-generator/template.go | 26 ++ hack/tools/prowjob-generator/template_test.go | 29 ++ .../templates/periodics-upgrades.yaml | 57 +++ .../templates/periodics.yaml | 218 +++++++++ .../templates/presubmits.yaml | 418 ++++++++++++++++++ 10 files changed, 1004 insertions(+) create mode 100644 hack/tools/prowjob-generator/config.go create mode 100644 hack/tools/prowjob-generator/main.go create mode 100644 hack/tools/prowjob-generator/main_test.go create mode 100644 hack/tools/prowjob-generator/template.go create mode 100644 hack/tools/prowjob-generator/template_test.go create mode 100644 hack/tools/prowjob-generator/templates/periodics-upgrades.yaml create mode 100644 hack/tools/prowjob-generator/templates/periodics.yaml create mode 100644 hack/tools/prowjob-generator/templates/presubmits.yaml diff --git a/hack/tools/go.mod b/hack/tools/go.mod index e9a6686949e9..313393c53268 100644 --- a/hack/tools/go.mod +++ b/hack/tools/go.mod @@ -8,6 +8,7 @@ replace sigs.k8s.io/cluster-api/test => ../../test require ( cloud.google.com/go/storage v1.36.0 + github.com/Masterminds/sprig v2.22.0+incompatible github.com/blang/semver/v4 v4.0.0 github.com/onsi/gomega v1.30.0 github.com/pkg/errors v0.9.1 @@ -37,6 +38,7 @@ require ( cloud.google.com/go/iam v1.1.5 // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect + github.com/Masterminds/semver v1.5.0 // indirect github.com/Masterminds/semver/v3 v3.2.0 // indirect github.com/Masterminds/sprig/v3 v3.2.3 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect diff --git a/hack/tools/go.sum b/hack/tools/go.sum index 07198c3293d1..4fdd72b5fcc8 100644 --- a/hack/tools/go.sum +++ b/hack/tools/go.sum @@ -16,8 +16,12 @@ github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZCSYp4Z0m2dk6cEM60= +github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= diff --git a/hack/tools/prowjob-generator/config.go b/hack/tools/prowjob-generator/config.go new file mode 100644 index 000000000000..96846160d4ab --- /dev/null +++ b/hack/tools/prowjob-generator/config.go @@ -0,0 +1,45 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +type ProwIgnoredConfig struct { + ProwIgnored Config `json:"prow_ignored"` +} + +// Config is the configuration file struct. +type Config struct { + Branches map[string]BranchConfig `json:"branches"` + Versions map[string]VersionMapper `json:"versions"` +} + +// BranchConfig is the branch-based configuration struct. +type BranchConfig struct { + Interval string `json:"interval"` + KubekinsImage string `json:"kubekinsImage"` + KubernetesVersionManagement string `json:"kubernetesVersionManagement"` + KubebuilderEnvtestKubernetesVersion string `json:"kubebuilderEnvtestKubernetesVersion"` + Upgrades []Upgrade `json:"upgrades"` +} + +// Upgrade describes a kubernetes upgrade. +type Upgrade struct { + From string `json:"from"` + To string `json:"to"` +} + +// VersionMapper is a key value map strings to versions. +type VersionMapper map[string]string diff --git a/hack/tools/prowjob-generator/main.go b/hack/tools/prowjob-generator/main.go new file mode 100644 index 000000000000..cb99c0f93788 --- /dev/null +++ b/hack/tools/prowjob-generator/main.go @@ -0,0 +1,175 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// main is the main package for prowjob-generator. +package main + +import ( + "bytes" + "flag" + "fmt" + "os" + "path" + "strings" + "text/template" + + "github.com/Masterminds/sprig" + "github.com/pkg/errors" + "k8s.io/klog/v2" + "sigs.k8s.io/yaml" +) + +var ( + configFile = flag.String("config", "", "Path to the config file") + outputDir = flag.String("output-dir", "", "Path to the directory to create the files in") + fileNamePrefix = flag.String("file-name-prefix", "cluster-api-", "File name prefix for the created prowjob files") +) + +func main() { + flag.Parse() + + if *outputDir == "" { + klog.Fatal("Expected flag \"output-dir\" to be set") + } + + if *configFile == "" { + klog.Fatal("Expected flag \"config\" to be set") + } + + rawConfig, err := os.ReadFile(*configFile) + if err != nil { + klog.Fatalf("Failed to read config file %q: %v", *configFile, err) + } + + prowIgnoredConfig := ProwIgnoredConfig{} + if err := yaml.Unmarshal(rawConfig, &prowIgnoredConfig); err != nil { + klog.Fatalf("Failed to parse config file %q: %v", *configFile, err) + } + + g, err := newGenerator(prowIgnoredConfig.ProwIgnored, *outputDir) + if err != nil { + klog.Fatalf("Failed to initialize generator: %v", err) + } + + if err := g.generate(); err != nil { + klog.Fatalf("Failed to generate prowjobs: %v", err) + } +} + +func k8sShortVersionString(s string) string { + if strings.HasPrefix(s, "ci/latest-") { + return "latest" + } + return strings.ReplaceAll(strings.TrimPrefix(s, "stable-"), ".", "-") +} + +func (g *generator) k8sVersionLookup(version, component string) string { + v, ok := g.config.Versions[version] + if !ok { + klog.Fatalf("Failed to lookup version (%q) in config", version) + } + c, ok := v[component] + if !ok { + klog.Fatalf("Failed to lookup component version (%q) for version %q in config", component, version) + } + return c +} + +func (g *generator) lastUpgradeVersionFrom(branch string) string { + upgrades := g.config.Branches[branch].Upgrades + return upgrades[len(upgrades)-1].From +} +func (g *generator) lastUpgradeVersionTo(branch string) string { + upgrades := g.config.Branches[branch].Upgrades + return upgrades[len(upgrades)-1].To +} + +func newGenerator(config Config, outputDir string) (*generator, error) { + g := &generator{ + config: config, + outputDir: outputDir, + } + funcs := sprig.HermeticTxtFuncMap() + funcs["k8sShortVersionString"] = k8sShortVersionString + funcs["k8sVersionLookup"] = g.k8sVersionLookup + funcs["lastUpgradeVersionFrom"] = g.lastUpgradeVersionFrom + funcs["lastUpgradeVersionTo"] = g.lastUpgradeVersionTo + + var err error + g.templates, err = template.New("").Funcs(funcs).ParseFS(templatesFS, "templates/*.yaml") + if err != nil { + return nil, err + } + + return g, err +} + +type generator struct { + templates *template.Template + config Config + outputDir string +} + +var jobTypes = [][]string{ + {"periodics"}, + {"periodics", "upgrades"}, + {"presubmits"}, +} + +func (g *generator) generate() error { + for _, jobType := range jobTypes { + for branch := range g.config.Branches { + if err := g.generateProwjobs(branch, jobType...); err != nil { + return errors.Wrapf(err, "Generating prowjobs for type %s", strings.Join(jobType, "-")) + } + } + } + return nil +} + +func (g *generator) generateProwjobs(branch string, jobType ...string) error { + fileName := g.generateFilename(branch, jobType...) + templateName := strings.Join(jobType, "-") + ".yaml" + + klog.Infof("Generating prowjobs to %q using template %q", fileName, templateName) + + data := map[string]interface{}{ + "branch": branch, + "config": g.config.Branches[branch], + } + + var out bytes.Buffer + if err := g.templates.ExecuteTemplate(&out, templateName, data); err != nil { + return errors.Wrapf(err, "Executing template %q for branch %q", templateName, branch) + } + + filePath := path.Join(g.outputDir, fileName) + if err := os.WriteFile(filePath, out.Bytes(), 0644); err != nil { //nolint:gosec + return errors.Wrapf(err, "Writing prowjob to %q", filePath) + } + + return nil +} + +func (g *generator) generateFilename(branch string, jobType ...string) string { + fileFriendlyBranchName := strings.ReplaceAll(branch, ".", "-") + + n := fmt.Sprintf("%s%s-%s", *fileNamePrefix, jobType[0], fileFriendlyBranchName) + if len(jobType) == 2 { + n = fmt.Sprintf("%s-%s", n, jobType[1]) + } + return n + ".yaml" +} diff --git a/hack/tools/prowjob-generator/main_test.go b/hack/tools/prowjob-generator/main_test.go new file mode 100644 index 000000000000..66f7aaee9829 --- /dev/null +++ b/hack/tools/prowjob-generator/main_test.go @@ -0,0 +1,30 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// main is the main package for prowjob-generator. +package main + +import ( + "testing" +) + +func Test_newGenerator(t *testing.T) { + _, err := newGenerator(Config{}, "") + if err != nil { + t.Errorf("newGenerator() error = %v", err) + return + } +} diff --git a/hack/tools/prowjob-generator/template.go b/hack/tools/prowjob-generator/template.go new file mode 100644 index 000000000000..0edf70ac72f8 --- /dev/null +++ b/hack/tools/prowjob-generator/template.go @@ -0,0 +1,26 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "embed" +) + +var ( + //go:embed templates/*.yaml + templatesFS embed.FS +) diff --git a/hack/tools/prowjob-generator/template_test.go b/hack/tools/prowjob-generator/template_test.go new file mode 100644 index 000000000000..8f8e1168b7a8 --- /dev/null +++ b/hack/tools/prowjob-generator/template_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + _ "embed" + "testing" + "text/template" +) + +func Test_Templates(t *testing.T) { + if _, err := template.ParseFS(templatesFS, "templates/*.yaml"); err != nil { + t.Errorf("Error parsing embedded templates: %v", err) + } +} diff --git a/hack/tools/prowjob-generator/templates/periodics-upgrades.yaml b/hack/tools/prowjob-generator/templates/periodics-upgrades.yaml new file mode 100644 index 000000000000..dc5c19d942be --- /dev/null +++ b/hack/tools/prowjob-generator/templates/periodics-upgrades.yaml @@ -0,0 +1,57 @@ +periodics: +{{- range $_, $upgrade := .config.Upgrades }} +- name: periodic-cluster-api-e2e-workload-upgrade-{{ $upgrade.From | k8sShortVersionString }}-{{ $upgrade.To | k8sShortVersionString }}-{{ $.branch | replace "." "-" }} + cluster: eks-prow-build-cluster + interval: 24h + decorate: true + rerun_auth_config: + github_team_slugs: + - org: kubernetes-sigs + slug: cluster-api-maintainers + labels: + preset-dind-enabled: "true" + preset-kind-volume-mounts: "true" + extra_refs: + - org: kubernetes-sigs + repo: cluster-api + base_ref: {{ $.branch }} + path_alias: sigs.k8s.io/cluster-api + - org: kubernetes + repo: kubernetes + base_ref: master + path_alias: k8s.io/kubernetes + spec: + containers: + - image: {{ $.config.KubekinsImage }} + args: + - runner.sh + - "./scripts/ci-e2e.sh" + env: + - name: ALWAYS_BUILD_KIND_IMAGES + value: "true" + - name: KUBERNETES_VERSION_UPGRADE_FROM + value: "{{ k8sVersionLookup $upgrade.From "k8sRelease" }}" + - name: KUBERNETES_VERSION_UPGRADE_TO + value: "{{ k8sVersionLookup $upgrade.To "k8sRelease" }}" + - name: ETCD_VERSION_UPGRADE_TO + value: "{{ k8sVersionLookup $upgrade.From "etcd" }}" + - name: COREDNS_VERSION_UPGRADE_TO + value: "{{ k8sVersionLookup $upgrade.From "coreDNS" }}" + - name: GINKGO_FOCUS + value: "\\[K8s-Upgrade\\]" + # we need privileged mode in order to do docker in docker + securityContext: + privileged: true + resources: + requests: + cpu: 7300m + memory: 32Gi + limits: + cpu: 7300m + memory: 32Gi + annotations: + testgrid-dashboards: sig-cluster-lifecycle-cluster-api{{ if eq $.branch "main" | not -}}{{ $.branch | trimPrefix "release" }}{{- end }} + testgrid-tab-name: capi-e2e-{{ $.branch | replace "." "-" }}-{{ $upgrade.From | k8sShortVersionString }}-{{ $upgrade.To | k8sShortVersionString }} + testgrid-alert-email: sig-cluster-lifecycle-cluster-api-alerts@kubernetes.io + testgrid-num-failures-to-alert: "4" +{{ end -}} diff --git a/hack/tools/prowjob-generator/templates/periodics.yaml b/hack/tools/prowjob-generator/templates/periodics.yaml new file mode 100644 index 000000000000..4233aee009aa --- /dev/null +++ b/hack/tools/prowjob-generator/templates/periodics.yaml @@ -0,0 +1,218 @@ +periodics: +- name: periodic-cluster-api-test-{{ $.branch | replace "." "-" }} + cluster: eks-prow-build-cluster + interval: {{ .config.Interval }} + decorate: true + rerun_auth_config: + github_team_slugs: + - org: kubernetes-sigs + slug: cluster-api-maintainers + extra_refs: + - org: kubernetes-sigs + repo: cluster-api + base_ref: {{ $.branch }} + path_alias: sigs.k8s.io/cluster-api + spec: + containers: + - image: {{ .config.KubekinsImage }} + command: + - runner.sh + - ./scripts/ci-test.sh + resources: + requests: + cpu: 7300m + memory: 9Gi + limits: + cpu: 7300m + memory: 9Gi + annotations: + testgrid-dashboards: sig-cluster-lifecycle-cluster-api{{ if eq $.branch "main" | not -}}{{ $.branch | trimPrefix "release" }}{{- end }} + testgrid-tab-name: capi-test-{{ $.branch | replace "." "-" }} + testgrid-alert-email: sig-cluster-lifecycle-cluster-api-alerts@kubernetes.io + testgrid-num-failures-to-alert: "4" +- name: periodic-cluster-api-test-mink8s-{{ $.branch | replace "." "-" }} + cluster: eks-prow-build-cluster + interval: {{ .config.Interval }} + decorate: true + rerun_auth_config: + github_team_slugs: + - org: kubernetes-sigs + slug: cluster-api-maintainers + extra_refs: + - org: kubernetes-sigs + repo: cluster-api + base_ref: {{ $.branch }} + path_alias: sigs.k8s.io/cluster-api + spec: + containers: + - image: {{ .config.KubekinsImage }} + command: + - runner.sh + - ./scripts/ci-test.sh + env: + # This value determines the minimum Kubernetes + # supported version for Cluster API management cluster + # and can be found by referring to [Supported Kubernetes Version](https://cluster-api.sigs.k8s.io/reference/versions.html#supported-kubernetes-versions) + # docs (look for minimum supported k8s version for management cluster, i.e N-3). + # + # To check the latest available envtest in Kubebuilder for the minor version we determined above, please + # refer to https://github.com/kubernetes-sigs/kubebuilder/tree/tools-releases. + - name: KUBEBUILDER_ENVTEST_KUBERNETES_VERSION + value: "{{ .config.KubebuilderEnvtestKubernetesVersion }}" + resources: + requests: + cpu: 7300m + memory: 9Gi + limits: + cpu: 7300m + memory: 9Gi + annotations: + testgrid-dashboards: sig-cluster-lifecycle-cluster-api{{ if eq $.branch "main" | not -}}{{ $.branch | trimPrefix "release" }}{{- end }} + testgrid-tab-name: capi-test-mink8s-{{ $.branch | replace "." "-" }} + testgrid-alert-email: sig-cluster-lifecycle-cluster-api-alerts@kubernetes.io + testgrid-num-failures-to-alert: "4" +- name: periodic-cluster-api-e2e-{{ $.branch | replace "." "-" }} + cluster: eks-prow-build-cluster + interval: {{ .config.Interval }} + decorate: true + rerun_auth_config: + github_team_slugs: + - org: kubernetes-sigs + slug: cluster-api-maintainers + labels: + preset-dind-enabled: "true" + preset-kind-volume-mounts: "true" + extra_refs: + - org: kubernetes-sigs + repo: cluster-api + base_ref: {{ $.branch }} + path_alias: sigs.k8s.io/cluster-api + - org: kubernetes + repo: kubernetes + base_ref: master + path_alias: k8s.io/kubernetes + spec: + containers: + - image: {{ .config.KubekinsImage }} + args: + - runner.sh + - "./scripts/ci-e2e.sh" + env: + - name: GINKGO_SKIP + value: "\\[Conformance\\] \\[K8s-Upgrade\\]|\\[IPv6\\]" + # we need privileged mode in order to do docker in docker + securityContext: + privileged: true + resources: + requests: + cpu: 7300m + memory: 32Gi + limits: + cpu: 7300m + memory: 32Gi + annotations: + testgrid-dashboards: sig-cluster-lifecycle-cluster-api{{ if eq $.branch "main" | not -}}{{ $.branch | trimPrefix "release" }}{{- end }} + testgrid-tab-name: capi-e2e-{{ $.branch | replace "." "-" }} + testgrid-alert-email: sig-cluster-lifecycle-cluster-api-alerts@kubernetes.io + testgrid-num-failures-to-alert: "4" +{{ if eq $.branch "release-1.4" | not -}} +- name: periodic-cluster-api-e2e-dualstack-and-ipv6-{{ $.branch | replace "." "-" }} + cluster: eks-prow-build-cluster + interval: {{ .config.Interval }} + decorate: true + rerun_auth_config: + github_team_slugs: + - org: kubernetes-sigs + slug: cluster-api-maintainers + labels: + preset-dind-enabled: "true" + preset-kind-volume-mounts: "true" + extra_refs: + - org: kubernetes-sigs + repo: cluster-api + base_ref: {{ $.branch }} + path_alias: sigs.k8s.io/cluster-api + - org: kubernetes + repo: kubernetes + base_ref: master + path_alias: k8s.io/kubernetes + spec: + containers: + - image: {{ .config.KubekinsImage }} + args: + - runner.sh + - "./scripts/ci-e2e.sh" + env: + # enable IPV6 in bootstrap image + - name: "DOCKER_IN_DOCKER_IPV6_ENABLED" + value: "true" + - name: GINKGO_SKIP + value: "\\[Conformance\\] \\[K8s-Upgrade\\]" + # we need privileged mode in order to do docker in docker + securityContext: + privileged: true + resources: + requests: + cpu: 7300m + memory: 32Gi + limits: + cpu: 7300m + memory: 32Gi + annotations: + testgrid-dashboards: sig-cluster-lifecycle-cluster-api{{ if eq $.branch "main" | not -}}{{ $.branch | trimPrefix "release" }}{{- end }} + testgrid-tab-name: capi-e2e-dualstack-and-ipv6-{{ $.branch | replace "." "-" }} + testgrid-alert-email: sig-cluster-lifecycle-cluster-api-alerts@kubernetes.io + testgrid-num-failures-to-alert: "4" +{{ end -}} +- name: periodic-cluster-api-e2e-mink8s-{{ $.branch | replace "." "-" }} + cluster: eks-prow-build-cluster + interval: {{ .config.Interval }} + decorate: true + rerun_auth_config: + github_team_slugs: + - org: kubernetes-sigs + slug: cluster-api-maintainers + labels: + preset-dind-enabled: "true" + preset-kind-volume-mounts: "true" + extra_refs: + - org: kubernetes-sigs + repo: cluster-api + base_ref: {{ $.branch }} + path_alias: sigs.k8s.io/cluster-api + - org: kubernetes + repo: kubernetes + base_ref: master + path_alias: k8s.io/kubernetes + spec: + containers: + - image: {{ .config.KubekinsImage }} + args: + - runner.sh + - "./scripts/ci-e2e.sh" + env: + - name: GINKGO_SKIP + value: "\\[Conformance\\] \\[K8s-Upgrade\\]|\\[IPv6\\]" + # This value determines the minimum Kubernetes + # supported version for Cluster API management cluster + # and can be found by referring to [Supported Kubernetes Version](https://cluster-api.sigs.k8s.io/reference/versions.html#supported-kubernetes-versions) + # docs (look for minimum supported k8s version for management cluster, i.e N-3). + # Please also make sure to refer a version where a kindest/node image exists + # for (see https://github.com/kubernetes-sigs/kind/releases/) + - name: KUBERNETES_VERSION_MANAGEMENT + value: "{{ .config.KubernetesVersionManagement }}" + # we need privileged mode in order to do docker in docker + securityContext: + privileged: true + resources: + requests: + cpu: 7300m + memory: 32Gi + limits: + cpu: 7300m + memory: 32Gi + annotations: + testgrid-dashboards: sig-cluster-lifecycle-cluster-api{{ if eq $.branch "main" | not -}}{{ $.branch | trimPrefix "release" }}{{- end }} + testgrid-tab-name: capi-e2e-mink8s-{{ $.branch | replace "." "-" }} + testgrid-alert-email: sig-cluster-lifecycle-cluster-api-alerts@kubernetes.io + testgrid-num-failures-to-alert: "4" diff --git a/hack/tools/prowjob-generator/templates/presubmits.yaml b/hack/tools/prowjob-generator/templates/presubmits.yaml new file mode 100644 index 000000000000..50412852cbd1 --- /dev/null +++ b/hack/tools/prowjob-generator/templates/presubmits.yaml @@ -0,0 +1,418 @@ +presubmits: + kubernetes-sigs/cluster-api: + - name: pull-cluster-api-build-{{ $.branch | replace "." "-" }} + cluster: eks-prow-build-cluster + decorate: true + path_alias: sigs.k8s.io/cluster-api + always_run: true + branches: + # The script this job runs is not in all branches. + - ^{{ $.branch }}$ + spec: + containers: + - image: {{ .config.KubekinsImage }} + command: + - runner.sh + - ./scripts/ci-build.sh + resources: + requests: + cpu: 7300m + memory: 9Gi + limits: + cpu: 7300m + memory: 9Gi + annotations: + testgrid-dashboards: sig-cluster-lifecycle-cluster-api{{ if eq $.branch "main" | not -}}{{ $.branch | trimPrefix "release" }}{{- end }} + testgrid-tab-name: capi-pr-build-{{ $.branch | replace "." "-" }} + - name: pull-cluster-api-apidiff-{{ $.branch | replace "." "-" }} + cluster: eks-prow-build-cluster + decorate: true + path_alias: sigs.k8s.io/cluster-api + optional: true + branches: + # The script this job runs is not in all branches. + - ^{{ $.branch }}$ + run_if_changed: '^((api|bootstrap|cmd|config|controllers|controlplane|errors|exp|feature|hack|internal|scripts|test|util|webhooks|version)/|main\.go|go\.mod|go\.sum|Dockerfile|Makefile)' + spec: + containers: + - command: + - runner.sh + - ./scripts/ci-apidiff.sh + image: {{ .config.KubekinsImage }} + resources: + requests: + cpu: 7300m + memory: 9Gi + limits: + cpu: 7300m + memory: 9Gi + annotations: + testgrid-dashboards: sig-cluster-lifecycle-cluster-api{{ if eq $.branch "main" | not -}}{{ $.branch | trimPrefix "release" }}{{- end }} + testgrid-tab-name: capi-pr-apidiff-{{ $.branch | replace "." "-" }} + - name: pull-cluster-api-verify-{{ $.branch | replace "." "-" }} + cluster: eks-prow-build-cluster + decorate: true + path_alias: sigs.k8s.io/cluster-api + always_run: true + labels: + preset-dind-enabled: "true" + branches: + # The script this job runs is not in all branches. + - ^{{ $.branch }}$ + spec: + containers: + - image: {{ .config.KubekinsImage }} + command: + - "runner.sh" + - ./scripts/ci-verify.sh + resources: + requests: + cpu: 7300m + memory: 9Gi + limits: + cpu: 7300m + memory: 9Gi + securityContext: + privileged: true + annotations: + testgrid-dashboards: sig-cluster-lifecycle-cluster-api{{ if eq $.branch "main" | not -}}{{ $.branch | trimPrefix "release" }}{{- end }} + testgrid-tab-name: capi-pr-verify-{{ $.branch | replace "." "-" }} + - name: pull-cluster-api-test-{{ $.branch | replace "." "-" }} + cluster: eks-prow-build-cluster + decorate: true + path_alias: sigs.k8s.io/cluster-api + branches: + # The script this job runs is not in all branches. + - ^{{ $.branch }}$ + run_if_changed: '^((api|bootstrap|cmd|config|controllers|controlplane|errors|exp|feature|hack|internal|scripts|test|util|webhooks|version)/|main\.go|go\.mod|go\.sum|Dockerfile|Makefile)' + spec: + containers: + - image: {{ .config.KubekinsImage }} + args: + - runner.sh + - ./scripts/ci-test.sh + resources: + requests: + cpu: 7300m + memory: 9Gi + limits: + cpu: 7300m + memory: 9Gi + annotations: + testgrid-dashboards: sig-cluster-lifecycle-cluster-api{{ if eq $.branch "main" | not -}}{{ $.branch | trimPrefix "release" }}{{- end }} + testgrid-tab-name: capi-pr-test-{{ $.branch | replace "." "-" }} + - name: pull-cluster-api-test-mink8s-{{ $.branch | replace "." "-" }} + cluster: eks-prow-build-cluster + decorate: true + path_alias: sigs.k8s.io/cluster-api + always_run: false + branches: + # The script this job runs is not in all branches. + - ^{{ $.branch }}$ + spec: + containers: + - image: {{ .config.KubekinsImage }} + args: + - runner.sh + - ./scripts/ci-test.sh + env: + # This value determines the minimum Kubernetes + # supported version for Cluster API management cluster + # and can be found by referring to [Supported Kubernetes Version](https://cluster-api.sigs.k8s.io/reference/versions.html#supported-kubernetes-versions) + # docs (look for minimum supported k8s version for management cluster, i.e N-3). + # + # To check the latest available envtest in Kubebuilder for the minor version we determined above, please + # refer to https://github.com/kubernetes-sigs/kubebuilder/tree/tools-releases. + - name: KUBEBUILDER_ENVTEST_KUBERNETES_VERSION + value: "{{ .config.KubebuilderEnvtestKubernetesVersion }}" + resources: + requests: + cpu: 7300m + memory: 9Gi + limits: + cpu: 7300m + memory: 9Gi + annotations: + testgrid-dashboards: sig-cluster-lifecycle-cluster-api{{ if eq $.branch "main" | not -}}{{ $.branch | trimPrefix "release" }}{{- end }} + testgrid-tab-name: capi-pr-test-mink8s-{{ $.branch | replace "." "-" }} +{{- if eq $.branch "release-1.4" | not }} + - name: pull-cluster-api-e2e-mink8s-{{ $.branch | replace "." "-" }} + cluster: eks-prow-build-cluster + labels: + preset-dind-enabled: "true" + preset-kind-volume-mounts: "true" + decorate: true + path_alias: sigs.k8s.io/cluster-api + always_run: false + branches: + # The script this job runs is not in all branches. + - ^{{ $.branch }}$ + spec: + containers: + - image: {{ .config.KubekinsImage }} + args: + - runner.sh + - ./scripts/ci-e2e.sh + env: + - name: GINKGO_SKIP + value: "\\[Conformance\\] \\[K8s-Upgrade\\]|\\[IPv6\\]" + # This value determines the minimum Kubernetes + # supported version for Cluster API management cluster + # and can be found by referring to [Supported Kubernetes Version](https://cluster-api.sigs.k8s.io/reference/versions.html#supported-kubernetes-versions) + # docs (look for minimum supported k8s version for management cluster, i.e N-3). + # Please also make sure to refer a version where a kindest/node image exists + # for (see https://github.com/kubernetes-sigs/kind/releases/) + - name: KUBERNETES_VERSION_MANAGEMENT + value: "{{ .config.KubernetesVersionManagement }}" + # we need privileged mode in order to do docker in docker + securityContext: + privileged: true + resources: + requests: + cpu: 7300m + memory: 32Gi + limits: + cpu: 7300m + memory: 32Gi + annotations: + testgrid-dashboards: sig-cluster-lifecycle-cluster-api{{ if eq $.branch "main" | not -}}{{ $.branch | trimPrefix "release" }}{{- end }} + testgrid-tab-name: capi-pr-e2e-mink8s-{{ $.branch | replace "." "-" }} +{{- end }} + - name: pull-cluster-api-e2e-{{ $.branch | replace "." "-" }} + cluster: eks-prow-build-cluster + labels: + preset-dind-enabled: "true" + preset-kind-volume-mounts: "true" + decorate: true + branches: + # The script this job runs is not in all branches. + - ^{{ $.branch }}$ + path_alias: sigs.k8s.io/cluster-api + run_if_changed: '^((api|bootstrap|cmd|config|controllers|controlplane|errors|exp|feature|hack|internal|scripts|test|util|webhooks|version)/|main\.go|go\.mod|go\.sum|Dockerfile|Makefile)' + spec: + containers: + - image: {{ .config.KubekinsImage }} + args: + - runner.sh + - "./scripts/ci-e2e.sh" + env: + - name: GINKGO_FOCUS + value: "\\[PR-Blocking\\]" + # we need privileged mode in order to do docker in docker + securityContext: + privileged: true + resources: + requests: + cpu: 7300m + memory: 32Gi + limits: + cpu: 7300m + memory: 32Gi + annotations: + testgrid-dashboards: sig-cluster-lifecycle-cluster-api{{ if eq $.branch "main" | not -}}{{ $.branch | trimPrefix "release" }}{{- end }} + testgrid-tab-name: capi-pr-e2e-{{ $.branch | replace "." "-" }} +{{- if has $.branch (list "release-1.4" "release-1.5") }} + - name: pull-cluster-api-e2e-informing-{{ $.branch | replace "." "-" }} + cluster: eks-prow-build-cluster + labels: + preset-dind-enabled: "true" + preset-kind-volume-mounts: "true" + decorate: true + optional: true + branches: + # The script this job runs is not in all branches. + - ^{{ $.branch }}$ + path_alias: sigs.k8s.io/cluster-api + run_if_changed: '^((api|bootstrap|cmd|config|controllers|controlplane|errors|exp|feature|hack|internal|scripts|test|util|webhooks|version)/|main\.go|go\.mod|go\.sum|Dockerfile|Makefile)' + spec: + containers: + - image: {{ .config.KubekinsImage }} + args: + - runner.sh + - "./scripts/ci-e2e.sh" + env: + - name: GINKGO_FOCUS + value: "\\[PR-Informing\\]" +{{- if eq $.branch "release-1.4" }} + - name: GINKGO_SKIP + value: "\\[IPv6\\]" +{{- end }} + # we need privileged mode in order to do docker in docker + securityContext: + privileged: true + resources: + requests: + cpu: 7300m + memory: 32Gi + limits: + cpu: 7300m + memory: 32Gi + annotations: + testgrid-dashboards: sig-cluster-lifecycle-cluster-api{{ if eq $.branch "main" | not -}}{{ $.branch | trimPrefix "release" }}{{- end }} + testgrid-tab-name: capi-pr-e2e-informing-{{ $.branch | replace "." "-" }} +{{- end }} +{{- if eq $.branch "release-1.4" | not }} + - name: pull-cluster-api-e2e-full-dualstack-and-ipv6-{{ $.branch | replace "." "-" }} + cluster: eks-prow-build-cluster + labels: + preset-dind-enabled: "true" + preset-kind-volume-mounts: "true" + decorate: true + always_run: false + branches: + # The script this job runs is not in all branches. + - ^{{ $.branch }}$ + path_alias: sigs.k8s.io/cluster-api + spec: + containers: + - image: {{ .config.KubekinsImage }} + args: + - runner.sh + - "./scripts/ci-e2e.sh" + env: + # enable IPV6 in bootstrap image + - name: "DOCKER_IN_DOCKER_IPV6_ENABLED" + value: "true" + # Since the PR-Blocking tests are run as part of the cluster-api-e2e job + # and the upgrade tests are being run as part of the periodic upgrade jobs. + # This jobs ends up running all the other tests in the E2E suite + - name: GINKGO_SKIP + value: "\\[PR-Blocking\\] \\[Conformance\\] \\[K8s-Upgrade\\]" + # we need privileged mode in order to do docker in docker + securityContext: + privileged: true + resources: + requests: + cpu: 7300m + memory: 32Gi + limits: + cpu: 7300m + memory: 32Gi + annotations: + testgrid-dashboards: sig-cluster-lifecycle-cluster-api{{ if eq $.branch "main" | not -}}{{ $.branch | trimPrefix "release" }}{{- end }} + testgrid-tab-name: capi-pr-e2e-full-dualstack-and-ipv6-{{ $.branch | replace "." "-" }} +{{ else }} + - name: pull-cluster-api-e2e-informing-ipv6-{{ $.branch | replace "." "-" }} + cluster: eks-prow-build-cluster + labels: + preset-dind-enabled: "true" + preset-kind-volume-mounts: "true" + decorate: true + optional: true + branches: + # The script this job runs is not in all branches. + - ^{{ $.branch }}$ + path_alias: sigs.k8s.io/cluster-api + run_if_changed: '^((api|bootstrap|cmd|config|controllers|controlplane|errors|exp|feature|hack|internal|scripts|test|util|webhooks|version)/|main\.go|go\.mod|go\.sum|Dockerfile|Makefile)' + spec: + containers: + - image: {{ .config.KubekinsImage }} + args: + - runner.sh + - "./scripts/ci-e2e.sh" + env: + # enable IPV6 in bootstrap image + - name: "DOCKER_IN_DOCKER_IPV6_ENABLED" + value: "true" + - name: GINKGO_FOCUS + value: "\\[IPv6\\] \\[PR-Informing\\]" + - name: IP_FAMILY + value: "IPv6" + # we need privileged mode in order to do docker in docker + securityContext: + privileged: true + resources: + requests: + cpu: 7300m + memory: 32Gi + limits: + cpu: 7300m + memory: 32Gi + annotations: + testgrid-dashboards: sig-cluster-lifecycle-cluster-api{{ if eq $.branch "main" | not -}}{{ $.branch | trimPrefix "release" }}{{- end }} + testgrid-tab-name: capi-pr-e2e-informing-ipv6-{{ $.branch | replace "." "-" }} +{{- end }} + - name: pull-cluster-api-e2e-full-{{ $.branch | replace "." "-" }} + cluster: eks-prow-build-cluster + labels: + preset-dind-enabled: "true" + preset-kind-volume-mounts: "true" + decorate: true + always_run: false + branches: + # The script this job runs is not in all branches. + - ^{{ $.branch }}$ + path_alias: sigs.k8s.io/cluster-api + spec: + containers: + - image: {{ .config.KubekinsImage }} + args: + - runner.sh + - "./scripts/ci-e2e.sh" + env: + # Since the PR-Blocking tests are run as part of the cluster-api-e2e job + # and the upgrade tests are being run as part of the periodic upgrade jobs. + # This jobs ends up running all the other tests in the E2E suite + - name: GINKGO_SKIP + value: "\\[PR-Blocking\\] \\[Conformance\\] \\[K8s-Upgrade\\]|\\[IPv6\\]" + # we need privileged mode in order to do docker in docker + securityContext: + privileged: true + resources: + requests: + cpu: 7300m + memory: 32Gi + limits: + cpu: 7300m + memory: 32Gi + annotations: + testgrid-dashboards: sig-cluster-lifecycle-cluster-api{{ if eq $.branch "main" | not -}}{{ $.branch | trimPrefix "release" }}{{- end }} + testgrid-tab-name: capi-pr-e2e-full-{{ $.branch | replace "." "-" }} + - name: pull-cluster-api-e2e-workload-upgrade-{{ lastUpgradeVersionFrom $.branch | replace "." "-" }}-{{ lastUpgradeVersionTo $.branch | replace "." "-" }}-{{ $.branch | replace "." "-" }} + cluster: eks-prow-build-cluster + labels: + preset-dind-enabled: "true" + preset-kind-volume-mounts: "true" + decorate: true + always_run: false + branches: + # The script this job runs is not in all branches. + - ^{{ $.branch }}$ + path_alias: sigs.k8s.io/cluster-api + extra_refs: + - org: kubernetes + repo: kubernetes + base_ref: master + path_alias: k8s.io/kubernetes + spec: + containers: + - image: {{ .config.KubekinsImage }} + args: + - runner.sh + - "./scripts/ci-e2e.sh" + env: + - name: ALWAYS_BUILD_KIND_IMAGES + value: "true" + - name: KUBERNETES_VERSION_UPGRADE_FROM + value: "{{ k8sVersionLookup (lastUpgradeVersionFrom $.branch) "k8sRelease" }}" + - name: KUBERNETES_VERSION_UPGRADE_TO + value: "{{ k8sVersionLookup (lastUpgradeVersionTo $.branch) "k8sRelease" }}" + - name: ETCD_VERSION_UPGRADE_TO + value: "{{ k8sVersionLookup (lastUpgradeVersionFrom $.branch) "etcd" }}" + - name: COREDNS_VERSION_UPGRADE_TO + value: "{{ k8sVersionLookup (lastUpgradeVersionFrom $.branch) "coreDNS" }}" + - name: GINKGO_FOCUS + value: "\\[K8s-Upgrade\\]" + # we need privileged mode in order to do docker in docker + securityContext: + privileged: true + resources: + requests: + cpu: 7300m + memory: 32Gi + limits: + cpu: 7300m + memory: 32Gi + annotations: + testgrid-dashboards: sig-cluster-lifecycle-cluster-api{{ if eq $.branch "main" | not -}}{{ $.branch | trimPrefix "release" }}{{- end }} + testgrid-tab-name: capi-pr-e2e-{{ $.branch | replace "." "-" }}-{{ lastUpgradeVersionFrom $.branch | replace "." "-" }}-{{ lastUpgradeVersionTo $.branch | replace "." "-" }}