From 711533034b58178923ab0a17f79b80d93f89269e Mon Sep 17 00:00:00 2001 From: Marc Barry <4965634+marc-barry@users.noreply.github.com> Date: Mon, 8 Jan 2024 13:26:28 -0500 Subject: [PATCH 1/4] Adjust the operator to support a common convention and the new Qtap functionality. --- api/v1/egress.go | 62 +++++++++++++++++++++-------------- config/webhook/configmap.yaml | 30 +++++++++++------ 2 files changed, 58 insertions(+), 34 deletions(-) diff --git a/api/v1/egress.go b/api/v1/egress.go index 1d84028..e937309 100644 --- a/api/v1/egress.go +++ b/api/v1/egress.go @@ -3,6 +3,7 @@ package v1 import ( "fmt" "math" + "net" "strconv" corev1 "k8s.io/api/core/v1" @@ -21,7 +22,7 @@ var ( func MutateEgress(pod *corev1.Pod, config *Config) error { // fetch the init image tag - tag := config.GetAnnotation("egress-init-tag") + tag := config.GetAnnotation("qtap-init-tag") // create an init container initContainer := corev1.Container{ @@ -41,7 +42,7 @@ func MutateEgress(pod *corev1.Pod, config *Config) error { } // TO_ADDR - if toAddr := config.GetAnnotation("egress-to-addr"); toAddr != "" { + if toAddr := config.GetAnnotation("qtap-init-egress-to-addr"); toAddr != "" { initContainer.Env = append(initContainer.Env, corev1.EnvVar{ Name: "TO_ADDR", Value: toAddr, @@ -49,7 +50,7 @@ func MutateEgress(pod *corev1.Pod, config *Config) error { } // TO_DOMAIN - if toDomain := config.GetAnnotation("egress-to-domain"); toDomain != "" { + if toDomain := config.GetAnnotation("qtap-init-egress-to-domain"); toDomain != "" { initContainer.Env = append(initContainer.Env, corev1.EnvVar{ Name: "TO_DOMAIN", Value: toDomain, @@ -57,7 +58,7 @@ func MutateEgress(pod *corev1.Pod, config *Config) error { } // PORT_MAPPING - if portMapping := config.GetAnnotation("egress-port-mapping"); portMapping != "" { + if portMapping := config.GetAnnotation("qtap-init-egress-port-mapping"); portMapping != "" { initContainer.Env = append(initContainer.Env, corev1.EnvVar{ Name: "PORT_MAPPING", Value: portMapping, @@ -65,7 +66,7 @@ func MutateEgress(pod *corev1.Pod, config *Config) error { } // ACCEPT_UIDS - if acceptUids := config.GetAnnotation("egress-accept-uids"); acceptUids != "" { + if acceptUids := config.GetAnnotation("qtap-init-egress-accept-uids"); acceptUids != "" { initContainer.Env = append(initContainer.Env, corev1.EnvVar{ Name: "ACCEPT_UIDS", Value: acceptUids, @@ -73,7 +74,7 @@ func MutateEgress(pod *corev1.Pod, config *Config) error { } // ACCEPT_GIDS - if acceptGids := config.GetAnnotation("egress-accept-gids"); acceptGids != "" { + if acceptGids := config.GetAnnotation("qtap-init-egress-accept-gids"); acceptGids != "" { initContainer.Env = append(initContainer.Env, corev1.EnvVar{ Name: "ACCEPT_GIDS", Value: acceptGids, @@ -146,6 +147,18 @@ func MutateInjection(pod *corev1.Pod, config *Config) error { } } + statusListen := config.GetAnnotation("qtap-status-listen") + var statusPort int32 = 10001 + if statusListen != "" { + if _, port, err := net.SplitHostPort(statusListen); err == nil { + portInt, err := strconv.ParseInt(port, 0, 16) + if err != nil { + return fmt.Errorf("invalid port: %w", err) + } + statusPort = int32(portInt) + } + } + // create an init container qtapContainer := corev1.Container{ Name: "qtap", @@ -163,7 +176,7 @@ func MutateInjection(pod *corev1.Pod, config *Config) error { HTTPGet: &corev1.HTTPGetAction{ Path: "/readyz", Port: intstr.IntOrString{ - IntVal: 8080, + IntVal: statusPort, }, }, }, @@ -178,7 +191,7 @@ func MutateInjection(pod *corev1.Pod, config *Config) error { HTTPGet: &corev1.HTTPGetAction{ Path: "/readyz", Port: intstr.IntOrString{ - IntVal: 8080, + IntVal: statusPort, }, }, }, @@ -193,7 +206,7 @@ func MutateInjection(pod *corev1.Pod, config *Config) error { HTTPGet: &corev1.HTTPGetAction{ Path: "/healthz", Port: intstr.IntOrString{ - IntVal: 8080, + IntVal: statusPort, }, }, }, @@ -206,7 +219,7 @@ func MutateInjection(pod *corev1.Pod, config *Config) error { } // LOG_LEVEL - if logLevel := config.GetAnnotation("log-level"); logLevel != "" { + if logLevel := config.GetAnnotation("qtap-log-level"); logLevel != "" { qtapContainer.Env = append(qtapContainer.Env, corev1.EnvVar{ Name: "LOG_LEVEL", Value: logLevel, @@ -214,7 +227,7 @@ func MutateInjection(pod *corev1.Pod, config *Config) error { } // LOG_ENCODING - if logEncoding := config.GetAnnotation("log-encoding"); logEncoding != "" { + if logEncoding := config.GetAnnotation("qtap-log-encoding"); logEncoding != "" { qtapContainer.Env = append(qtapContainer.Env, corev1.EnvVar{ Name: "LOG_ENCODING", Value: logEncoding, @@ -222,7 +235,7 @@ func MutateInjection(pod *corev1.Pod, config *Config) error { } // LOG_CALLER - if logCaller := config.GetAnnotation("log-caller"); logCaller != "" { + if logCaller := config.GetAnnotation("qtap-log-caller"); logCaller != "" { qtapContainer.Env = append(qtapContainer.Env, corev1.EnvVar{ Name: "LOG_CALLER", Value: logCaller, @@ -230,31 +243,32 @@ func MutateInjection(pod *corev1.Pod, config *Config) error { } // HTTP_LISTEN - if httpListen := config.GetAnnotation("http-listen"); httpListen != "" { + if httpListen := config.GetAnnotation("qtap-egress-http-listen"); httpListen != "" { qtapContainer.Env = append(qtapContainer.Env, corev1.EnvVar{ - Name: "HTTP_LISTEN", + Name: "EGRESS_HTTP_LISTEN", Value: httpListen, }) } // HTTPS_LISTEN - if httpsListen := config.GetAnnotation("https-listen"); httpsListen != "" { + if httpsListen := config.GetAnnotation("qtap-egress-https-listen"); httpsListen != "" { qtapContainer.Env = append(qtapContainer.Env, corev1.EnvVar{ - Name: "HTTPS_LISTEN", + Name: "EGRESS_HTTPS_LISTEN", Value: httpsListen, }) } - // TCP_LISTEN - if tcpListen := config.GetAnnotation("tcp-listen"); tcpListen != "" { + // STATUS_LISTEN + // The annotation was already read above as it is needed to determine the Kubernetes probe port + if statusListen != "" { qtapContainer.Env = append(qtapContainer.Env, corev1.EnvVar{ - Name: "TCP_LISTEN", - Value: tcpListen, + Name: "STATUS_LISTEN", + Value: statusListen, }) } // BLOCK_UNKNOWN - if blockUnknown := config.GetAnnotation("block-unknown"); blockUnknown != "" { + if blockUnknown := config.GetAnnotation("qtap-block-unknown"); blockUnknown != "" { qtapContainer.Env = append(qtapContainer.Env, corev1.EnvVar{ Name: "BLOCK_UNKNOWN", Value: blockUnknown, @@ -262,7 +276,7 @@ func MutateInjection(pod *corev1.Pod, config *Config) error { } // ENVOY_LOG_LEVEL - if envoyLogLevel := config.GetAnnotation("envoy-log-level"); envoyLogLevel != "" { + if envoyLogLevel := config.GetAnnotation("qtap-envoy-log-level"); envoyLogLevel != "" { qtapContainer.Env = append(qtapContainer.Env, corev1.EnvVar{ Name: "ENVOY_LOG_LEVEL", Value: envoyLogLevel, @@ -270,7 +284,7 @@ func MutateInjection(pod *corev1.Pod, config *Config) error { } // DNS_LOOKUP_FAMILY - if dnsLookupFamily := config.GetAnnotation("dns-lookup-family"); dnsLookupFamily != "" { + if dnsLookupFamily := config.GetAnnotation("qtap-dns-lookup-family"); dnsLookupFamily != "" { qtapContainer.Env = append(qtapContainer.Env, corev1.EnvVar{ Name: "DNS_LOOKUP_FAMILY", Value: dnsLookupFamily, @@ -278,7 +292,7 @@ func MutateInjection(pod *corev1.Pod, config *Config) error { } // API_ENDPOINT - if apiEndpoint := config.GetAnnotation("api-endpoint"); apiEndpoint != "" { + if apiEndpoint := config.GetAnnotation("qtap-api-endpoint"); apiEndpoint != "" { qtapContainer.Env = append(qtapContainer.Env, corev1.EnvVar{ Name: "ENDPOINT", Value: apiEndpoint, diff --git a/config/webhook/configmap.yaml b/config/webhook/configmap.yaml index 68f8bcd..d4d3623 100644 --- a/config/webhook/configmap.yaml +++ b/config/webhook/configmap.yaml @@ -6,9 +6,12 @@ metadata: data: annotations.yaml: | qpoint.io/inject-ca: "true" - qpoint.io/egress-init-tag: "v0.0.7" - qpoint.io/egress-to-domain: "qtap-gateway.qpoint.svc.cluster.local" - qpoint.io/egress-port-mapping: "10080:80,10443:443" + qpoint.io/qtap-init-tag: "v0.0.8" + qpoint.io/qtap-init-egress-to-addr: "" + qpoint.io/qtap-init-egress-to-domain: "qtap-gateway.qpoint.svc.cluster.local" + qpoint.io/qtap-init-egress-port-mapping: "10080:80,10443:443" + qpoint.io/qtap-init-egress-accept-uids: "1010" + qpoint.io/qtap-init-egress-accept-gids: "1010" --- apiVersion: v1 kind: ConfigMap @@ -18,13 +21,20 @@ metadata: data: annotations.yaml: | qpoint.io/inject-ca: "true" - qpoint.io/egress-init-tag: "v0.0.7" + qpoint.io/qtap-init-tag: "v0.0.8" qpoint.io/qtap-tag: "v0.0.10" - qpoint.io/egress-port-mapping: "10080:80,10443:443,10000:" - qpoint.io/egress-accept-uids: "1010" - qpoint.io/egress-accept-gids: "1010" - qpoint.io/log-level: "info" - qpoint.io/block-unknown: "false" - qpoint.io/dns-lookup-family: "V4_ONLY" + qpoint.io/qtap-init-egress-port-mapping: "10080:80,10443:443" + qpoint.io/qtap-init-egress-accept-uids: "1010" + qpoint.io/qtap-init-egress-accept-gids: "1010" qpoint.io/qtap-uid: "1010" qpoint.io/qtap-gid: "1010" + qpoint.io/qtap-log-level: "info" + qpoint.io/qtap-log-encoding: "json" + qpoint.io/qtap-log-caller: "false" + qpoint.io/qtap-egress-http-listen: "0.0.0.0:10080" + qpoint.io/qtap-egress-https-listen: "0.0.0.0:10443" + qpoint.io/qtap-status-listen: "0.0.0.0:10001" + qpoint.io/qtap-block-unknown: "false" + qpoint.io/qtap-envoy-log-level: "error" + qpoint.io/qtap-dns-lookup-family: "V4_ONLY" + qpoint.io/qtap-api-endpoint: "https://api.qpoint.io" From db68d1437d6ebb5ff6fbfee0b20a1445d8768fe0 Mon Sep 17 00:00:00 2001 From: Marc Barry <4965634+marc-barry@users.noreply.github.com> Date: Mon, 8 Jan 2024 14:10:42 -0500 Subject: [PATCH 2/4] Fix incorrect endpoint. --- config/webhook/configmap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/webhook/configmap.yaml b/config/webhook/configmap.yaml index d4d3623..1aabc5b 100644 --- a/config/webhook/configmap.yaml +++ b/config/webhook/configmap.yaml @@ -37,4 +37,4 @@ data: qpoint.io/qtap-block-unknown: "false" qpoint.io/qtap-envoy-log-level: "error" qpoint.io/qtap-dns-lookup-family: "V4_ONLY" - qpoint.io/qtap-api-endpoint: "https://api.qpoint.io" + qpoint.io/qtap-api-endpoint: "https://app.qpoint.io" From a657a0d524487de105d8c2f53136c828091be319 Mon Sep 17 00:00:00 2001 From: Marc Barry <4965634+marc-barry@users.noreply.github.com> Date: Mon, 8 Jan 2024 14:38:06 -0500 Subject: [PATCH 3/4] Fix incorrect endpoint. --- config/webhook/configmap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/webhook/configmap.yaml b/config/webhook/configmap.yaml index 1aabc5b..d4d3623 100644 --- a/config/webhook/configmap.yaml +++ b/config/webhook/configmap.yaml @@ -37,4 +37,4 @@ data: qpoint.io/qtap-block-unknown: "false" qpoint.io/qtap-envoy-log-level: "error" qpoint.io/qtap-dns-lookup-family: "V4_ONLY" - qpoint.io/qtap-api-endpoint: "https://app.qpoint.io" + qpoint.io/qtap-api-endpoint: "https://api.qpoint.io" From d704f256fed07a1c92ce2407e1a2f5a92cec9e46 Mon Sep 17 00:00:00 2001 From: Marc Barry <4965634+marc-barry@users.noreply.github.com> Date: Tue, 9 Jan 2024 16:23:19 -0500 Subject: [PATCH 4/4] Bump qtap tag and set log level to error. --- config/webhook/configmap.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/webhook/configmap.yaml b/config/webhook/configmap.yaml index d4d3623..93e785e 100644 --- a/config/webhook/configmap.yaml +++ b/config/webhook/configmap.yaml @@ -22,13 +22,13 @@ data: annotations.yaml: | qpoint.io/inject-ca: "true" qpoint.io/qtap-init-tag: "v0.0.8" - qpoint.io/qtap-tag: "v0.0.10" + qpoint.io/qtap-tag: "v0.0.11" qpoint.io/qtap-init-egress-port-mapping: "10080:80,10443:443" qpoint.io/qtap-init-egress-accept-uids: "1010" qpoint.io/qtap-init-egress-accept-gids: "1010" qpoint.io/qtap-uid: "1010" qpoint.io/qtap-gid: "1010" - qpoint.io/qtap-log-level: "info" + qpoint.io/qtap-log-level: "error" qpoint.io/qtap-log-encoding: "json" qpoint.io/qtap-log-caller: "false" qpoint.io/qtap-egress-http-listen: "0.0.0.0:10080"