fix: set config_path in Containerd config

pkg/handlers/generic/mutation/mirrors/inject.go

@@ -99,7 +99,7 @@ func (h *globalMirrorPatchHandler) Mutate(
 			if err != nil {
 				return err
 			}
-			files, generateErr := generateFiles(
+			files, commands, generateErr := generateFilesAndCommands(
 				mirrorConfig,
 				globalMirror)
 			if generateErr != nil {
@@ -115,6 +115,15 @@ func (h *globalMirrorPatchHandler) Mutate(
 				files...,
 			)
 
+			log.WithValues(
+				"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
+				"patchedObjectName", ctrlclient.ObjectKeyFromObject(obj),
+			).Info("adding PreKubeadmCommands to control plane kubeadm config spec")
+			obj.Spec.Template.Spec.KubeadmConfigSpec.PreKubeadmCommands = append(
+				obj.Spec.Template.Spec.KubeadmConfigSpec.PreKubeadmCommands,
+				commands...,
+			)
+
 			return nil
 		}); err != nil {
 		return err
@@ -132,7 +141,7 @@ func (h *globalMirrorPatchHandler) Mutate(
 			if err != nil {
 				return err
 			}
-			files, generateErr := generateFiles(
+			files, commands, generateErr := generateFilesAndCommands(
 				mirrorConfig,
 				globalMirror)
 			if generateErr != nil {
@@ -144,6 +153,13 @@ func (h *globalMirrorPatchHandler) Mutate(
 				"patchedObjectName", ctrlclient.ObjectKeyFromObject(obj),
 			).Info("adding global registry mirror files to worker node kubeadm config template")
 			obj.Spec.Template.Spec.Files = append(obj.Spec.Template.Spec.Files, files...)
+
+			log.WithValues(
+				"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
+				"patchedObjectName", ctrlclient.ObjectKeyFromObject(obj),
+			).Info("adding PreKubeadmCommands to worker node kubeadm config template")
+			obj.Spec.Template.Spec.PreKubeadmCommands = append(obj.Spec.Template.Spec.PreKubeadmCommands, commands...)
+
 			return nil
 		}); err != nil {
 		return err
@@ -152,18 +168,29 @@ func (h *globalMirrorPatchHandler) Mutate(
 	return nil
 }
 
-func generateFiles(
+func generateFilesAndCommands(
 	mirrorConfig *mirrorConfig,
 	globalMirror v1alpha1.GlobalImageRegistryMirror,
-) ([]bootstrapv1.File, error) {
-	// Generate default registry mirror file
+) ([]bootstrapv1.File, []string, error) {
+	// generate default registry mirror file
 	files, err := generateGlobalRegistryMirrorFile(mirrorConfig)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	// generate CA certificate file for registry mirror
 	mirrorCAFile := generateMirrorCACertFile(mirrorConfig, globalMirror)
 	files = append(files, mirrorCAFile...)
+	// generate Containerd registry config drop-in file
+	registryConfigDropIn := generateContainerdRegistryConfigDropInFile()
+	files = append(files, registryConfigDropIn...)
+	// generate Containerd apply patches script and command
+	var commands []string
+	applyPatchesFile, applyPatchesCommand, err := generateContainerdApplyPatchesScript()
+	if err != nil {
+		return nil, nil, err
+	}
+	files = append(files, applyPatchesFile...)
+	commands = append(commands, applyPatchesCommand)
 
-	return files, err
+	return files, commands, err
 }

pkg/handlers/generic/mutation/mirrors/mirror.go

@@ -23,6 +23,11 @@ const (
 	mirrorCACertPathOnRemote                = "/etc/certs/mirror.pem"
 	defaultRegistryMirrorConfigPathOnRemote = "/etc/containerd/certs.d/_default/hosts.toml"
 	secretKeyForMirrorCACert                = "ca.crt"
+
+	tomlMergeImage                              = "ghcr.io/mesosphere/toml-merge:v0.2.0"
+	containerdPatchesDirOnRemote                = "/etc/containerd/cre.d"
+	containerdApplyPatchesScriptOnRemote        = "/etc/containerd/apply-patches.sh"
+	containerdApplyPatchesScriptOnRemoteCommand = "/bin/bash " + containerdApplyPatchesScriptOnRemote
 )
 
 var (
@@ -32,6 +37,16 @@ var (
 	defaultRegistryMirrorPatchTemplate = template.Must(
 		template.New("").Parse(string(defaultRegistryMirrorPatch)),
 	)
+
+	//go:embed templates/containerd-registry-config-drop-in.toml
+	containerdRegistryConfigDropIn             []byte
+	containerdRegistryConfigDropInFileOnRemote = path.Join(
+		containerdPatchesDirOnRemote,
+		"registry-config.toml",
+	)
+
+	//go:embed templates/containerd-apply-patches.sh.gotmpl
+	containerdApplyConfigPatchesScript []byte
 )
 
 type mirrorConfig struct {
@@ -130,10 +145,10 @@ func generateGlobalRegistryMirrorFile(mirror *mirrorConfig) ([]cabpkv1.File, err
 }
 
 func generateMirrorCACertFile(
-	config *mirrorConfig,
+	mirror *mirrorConfig,
 	globalMirror v1alpha1.GlobalImageRegistryMirror,
 ) []cabpkv1.File {
-	if config == nil || config.CACert == "" {
+	if mirror == nil || mirror.CACert == "" {
 		return nil
 	}
 	return []cabpkv1.File{
@@ -166,3 +181,42 @@ func formatURLForContainerd(uri string) (string, error) {
 	// using path.Join on all elements incorrectly drops a "/" from "https://"
 	return fmt.Sprintf("%s/%s", mirror, mirrorPath), nil
 }
+
+func generateContainerdRegistryConfigDropInFile() []cabpkv1.File {
+	return []cabpkv1.File{
+		{
+			Path:        containerdRegistryConfigDropInFileOnRemote,
+			Content:     string(containerdRegistryConfigDropIn),
+			Permissions: "0600",
+		},
+	}
+}
+
+func generateContainerdApplyPatchesScript() ([]cabpkv1.File, string, error) {
+	t, err := template.New("").Parse(string(containerdApplyConfigPatchesScript))
+	if err != nil {
+		return nil, "", fmt.Errorf("failed to parse go template: %w", err)
+	}
+
+	templateInput := struct {
+		TOMLMergeImage string
+		PatchDir       string
+	}{
+		TOMLMergeImage: tomlMergeImage,
+		PatchDir:       containerdPatchesDirOnRemote,
+	}
+
+	var b bytes.Buffer
+	err = t.Execute(&b, templateInput)
+	if err != nil {
+		return nil, "", fmt.Errorf("failed executing template: %w", err)
+	}
+
+	return []cabpkv1.File{
+		{
+			Path:        containerdApplyPatchesScriptOnRemote,
+			Content:     b.String(),
+			Permissions: "0700",
+		},
+	}, containerdApplyPatchesScriptOnRemoteCommand, nil
+}

pkg/handlers/generic/mutation/mirrors/mirror_test.go

@@ -156,3 +156,58 @@ func Test_generateMirrorCACertFile(t *testing.T) {
 		})
 	}
 }
+
+func Test_generateContainerdRegistryConfigDropInFile(t *testing.T) {
+	want := []cabpkv1.File{
+		{
+			Path:        "/etc/containerd/cre.d/registry-config.toml",
+			Owner:       "",
+			Permissions: "0600",
+			Encoding:    "",
+			Append:      false,
+			Content: `[plugins."io.containerd.grpc.v1.cri".registry]
+  config_path = "/etc/containerd/certs.d"
+`,
+		},
+	}
+	file := generateContainerdRegistryConfigDropInFile()
+	assert.Equal(t, want, file)
+}
+
+func Test_generateContainerdApplyPatchesScript(t *testing.T) {
+	wantFile := []cabpkv1.File{
+		{
+			Path:        "/etc/containerd/apply-patches.sh",
+			Owner:       "",
+			Permissions: "0700",
+			Encoding:    "",
+			Append:      false,
+			//nolint:lll // just a long string
+			Content: `#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+declare -r TOML_MERGE_IMAGE="ghcr.io/mesosphere/toml-merge:v0.2.0"
+
+if ! ctr --namespace k8s.io images check "name==${TOML_MERGE_IMAGE}" | grep "${TOML_MERGE_IMAGE}" >/dev/null; then
+  ctr --namespace k8s.io images pull "${TOML_MERGE_IMAGE}"
+fi
+
+cleanup() {
+  ctr images unmount "${tmp_ctr_mount_dir}" || true
+}
+
+trap 'cleanup' EXIT
+
+readonly tmp_ctr_mount_dir="$(mktemp -d)"
+
+ctr --namespace k8s.io images mount "${TOML_MERGE_IMAGE}" "${tmp_ctr_mount_dir}"
+"${tmp_ctr_mount_dir}/usr/local/bin/toml-merge" -i --patch-file "/etc/containerd/cre.d/*.toml" /etc/containerd/config.toml
+`,
+		},
+	}
+	wantCmd := "/bin/bash /etc/containerd/apply-patches.sh"
+	file, cmd, _ := generateContainerdApplyPatchesScript()
+	assert.Equal(t, wantFile, file)
+	assert.Equal(t, wantCmd, cmd)
+}

pkg/handlers/generic/mutation/mirrors/templates/containerd-apply-patches.sh.gotmpl

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+declare -r TOML_MERGE_IMAGE="{{ .TOMLMergeImage }}"
+
+if ! ctr --namespace k8s.io images check "name==${TOML_MERGE_IMAGE}" | grep "${TOML_MERGE_IMAGE}" >/dev/null; then
+  ctr --namespace k8s.io images pull "${TOML_MERGE_IMAGE}"
+fi
+
+cleanup() {
+  ctr images unmount "${tmp_ctr_mount_dir}" || true
+}
+
+trap 'cleanup' EXIT
+
+readonly tmp_ctr_mount_dir="$(mktemp -d)"
+
+ctr --namespace k8s.io images mount "${TOML_MERGE_IMAGE}" "${tmp_ctr_mount_dir}"
+"${tmp_ctr_mount_dir}/usr/local/bin/toml-merge" -i --patch-file "{{ .PatchDir }}/*.toml" /etc/containerd/config.toml

pkg/handlers/generic/mutation/mirrors/templates/containerd-registry-config-drop-in.toml

@@ -0,0 +1,2 @@
+[plugins."io.containerd.grpc.v1.cri".registry]
+  config_path = "/etc/containerd/certs.d"

pkg/handlers/generic/mutation/mirrors/tests/generate_patches.go

@@ -69,6 +69,19 @@ func TestGeneratePatches(
 						gomega.HaveKeyWithValue(
 							"path", "/etc/containerd/certs.d/_default/hosts.toml",
 						),
+						gomega.HaveKeyWithValue(
+							"path", "/etc/containerd/cre.d/registry-config.toml",
+						),
+						gomega.HaveKeyWithValue(
+							"path", "/etc/containerd/apply-patches.sh",
+						),
+					),
+				},
+				{
+					Operation: "add",
+					Path:      "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands",
+					ValueMatcher: gomega.ContainElement(
+						"/bin/bash /etc/containerd/apply-patches.sh",
 					),
 				},
 			},
@@ -101,6 +114,19 @@ func TestGeneratePatches(
 						gomega.HaveKeyWithValue(
 							"path", "/etc/certs/mirror.pem",
 						),
+						gomega.HaveKeyWithValue(
+							"path", "/etc/containerd/cre.d/registry-config.toml",
+						),
+						gomega.HaveKeyWithValue(
+							"path", "/etc/containerd/apply-patches.sh",
+						),
+					),
+				},
+				{
+					Operation: "add",
+					Path:      "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands",
+					ValueMatcher: gomega.ContainElement(
+						"/bin/bash /etc/containerd/apply-patches.sh",
 					),
 				},
 			},
@@ -133,6 +159,19 @@ func TestGeneratePatches(
 						gomega.HaveKeyWithValue(
 							"path", "/etc/containerd/certs.d/_default/hosts.toml",
 						),
+						gomega.HaveKeyWithValue(
+							"path", "/etc/containerd/cre.d/registry-config.toml",
+						),
+						gomega.HaveKeyWithValue(
+							"path", "/etc/containerd/apply-patches.sh",
+						),
+					),
+				},
+				{
+					Operation: "add",
+					Path:      "/spec/template/spec/preKubeadmCommands",
+					ValueMatcher: gomega.ContainElement(
+						"/bin/bash /etc/containerd/apply-patches.sh",
 					),
 				},
 			},
@@ -173,6 +212,19 @@ func TestGeneratePatches(
 						gomega.HaveKeyWithValue(
 							"path", "/etc/certs/mirror.pem",
 						),
+						gomega.HaveKeyWithValue(
+							"path", "/etc/containerd/cre.d/registry-config.toml",
+						),
+						gomega.HaveKeyWithValue(
+							"path", "/etc/containerd/apply-patches.sh",
+						),
+					),
+				},
+				{
+					Operation: "add",
+					Path:      "/spec/template/spec/preKubeadmCommands",
+					ValueMatcher: gomega.ContainElement(
+						"/bin/bash /etc/containerd/apply-patches.sh",
 					),
 				},
 			},