From 1fd174b01a3fa83ccf40fe4949c26d01923eced6 Mon Sep 17 00:00:00 2001 From: Damien Degois Date: Fri, 27 Dec 2024 02:04:59 +0100 Subject: [PATCH 1/3] add policy opentelemetry/inject-otel-environment-variable Signed-off-by: Damien Degois --- .../inject-before-preexisting-env.yaml | 14 +++ .../expected/inject-custom-endpoint-ns.yaml | 12 ++ .../expected/inject-custom-endpoint-pod.yaml | 14 +++ .../expected/inject-custom-endpoint.yaml | 14 +++ .../expected/inject-default.yaml | 12 ++ .../.kyverno-test/expected/inject-ns.yaml | 12 ++ .../.kyverno-test/expected/inject-pod.yaml | 14 +++ .../.kyverno-test/kyverno-test.yaml | 79 +++++++++++++ .../.kyverno-test/resource.yaml | 96 ++++++++++++++++ .../.kyverno-test/values.yaml | 24 ++++ .../artifacthub-pkg.yml | 34 ++++++ .../globalcontext.yaml | 9 ++ .../inject-otel-environment-variable.yaml | 105 ++++++++++++++++++ 13 files changed, 439 insertions(+) create mode 100644 opentelemetry/inject-otel-environment-variable/.kyverno-test/expected/inject-before-preexisting-env.yaml create mode 100644 opentelemetry/inject-otel-environment-variable/.kyverno-test/expected/inject-custom-endpoint-ns.yaml create mode 100644 opentelemetry/inject-otel-environment-variable/.kyverno-test/expected/inject-custom-endpoint-pod.yaml create mode 100644 opentelemetry/inject-otel-environment-variable/.kyverno-test/expected/inject-custom-endpoint.yaml create mode 100644 opentelemetry/inject-otel-environment-variable/.kyverno-test/expected/inject-default.yaml create mode 100644 opentelemetry/inject-otel-environment-variable/.kyverno-test/expected/inject-ns.yaml create mode 100644 opentelemetry/inject-otel-environment-variable/.kyverno-test/expected/inject-pod.yaml create mode 100644 opentelemetry/inject-otel-environment-variable/.kyverno-test/kyverno-test.yaml create mode 100644 opentelemetry/inject-otel-environment-variable/.kyverno-test/resource.yaml create mode 100644 opentelemetry/inject-otel-environment-variable/.kyverno-test/values.yaml create mode 100644 opentelemetry/inject-otel-environment-variable/artifacthub-pkg.yml create mode 100644 opentelemetry/inject-otel-environment-variable/globalcontext.yaml create mode 100644 opentelemetry/inject-otel-environment-variable/inject-otel-environment-variable.yaml diff --git a/opentelemetry/inject-otel-environment-variable/.kyverno-test/expected/inject-before-preexisting-env.yaml b/opentelemetry/inject-otel-environment-variable/.kyverno-test/expected/inject-before-preexisting-env.yaml new file mode 100644 index 000000000..f127dc99b --- /dev/null +++ b/opentelemetry/inject-otel-environment-variable/.kyverno-test/expected/inject-before-preexisting-env.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Pod +metadata: + name: inject-before-preexisting-env + namespace: default +spec: + containers: + - name: debdiag + image: ghcr.io/babs/debdiag:0 + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: http://opentelemetry-collector.opentelemetry-collector:4317 + - name: TEST_ENV + value: test diff --git a/opentelemetry/inject-otel-environment-variable/.kyverno-test/expected/inject-custom-endpoint-ns.yaml b/opentelemetry/inject-otel-environment-variable/.kyverno-test/expected/inject-custom-endpoint-ns.yaml new file mode 100644 index 000000000..f295db9da --- /dev/null +++ b/opentelemetry/inject-otel-environment-variable/.kyverno-test/expected/inject-custom-endpoint-ns.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: inject-custom-endpoint-ns + namespace: ns-w-custom-endpoint +spec: + containers: + - name: debdiag + image: ghcr.io/babs/debdiag:0 + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: http://http-collector:4318 diff --git a/opentelemetry/inject-otel-environment-variable/.kyverno-test/expected/inject-custom-endpoint-pod.yaml b/opentelemetry/inject-otel-environment-variable/.kyverno-test/expected/inject-custom-endpoint-pod.yaml new file mode 100644 index 000000000..16cbd5cea --- /dev/null +++ b/opentelemetry/inject-otel-environment-variable/.kyverno-test/expected/inject-custom-endpoint-pod.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Pod +metadata: + annotations: + otel.corp.org/otlp-endpoint: http://http-collector:4318 + name: inject-custom-endpoint-pod + namespace: default +spec: + containers: + - name: debdiag + image: ghcr.io/babs/debdiag:0 + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: http://http-collector:4318 diff --git a/opentelemetry/inject-otel-environment-variable/.kyverno-test/expected/inject-custom-endpoint.yaml b/opentelemetry/inject-otel-environment-variable/.kyverno-test/expected/inject-custom-endpoint.yaml new file mode 100644 index 000000000..a9520b3bb --- /dev/null +++ b/opentelemetry/inject-otel-environment-variable/.kyverno-test/expected/inject-custom-endpoint.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Pod +metadata: + namespace: default + name: inject-custom-endpoint + annotations: + otel.corp.org/otlp-endpoint: http://custom-endpoint:4318 +spec: + containers: + - name: debdiag + image: ghcr.io/babs/debdiag:0 + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: http://custom-endpoint:4318 diff --git a/opentelemetry/inject-otel-environment-variable/.kyverno-test/expected/inject-default.yaml b/opentelemetry/inject-otel-environment-variable/.kyverno-test/expected/inject-default.yaml new file mode 100644 index 000000000..9a81d804b --- /dev/null +++ b/opentelemetry/inject-otel-environment-variable/.kyverno-test/expected/inject-default.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: inject-default + namespace: ns-wo-annotations +spec: + containers: + - name: debdiag + image: ghcr.io/babs/debdiag:0 + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: http://opentelemetry-collector.opentelemetry-collector:4317 diff --git a/opentelemetry/inject-otel-environment-variable/.kyverno-test/expected/inject-ns.yaml b/opentelemetry/inject-otel-environment-variable/.kyverno-test/expected/inject-ns.yaml new file mode 100644 index 000000000..16f91f25d --- /dev/null +++ b/opentelemetry/inject-otel-environment-variable/.kyverno-test/expected/inject-ns.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: inject-ns + namespace: ns-w-inject-true +spec: + containers: + - name: debdiag + image: ghcr.io/babs/debdiag:0 + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: http://opentelemetry-collector.opentelemetry-collector:4317 diff --git a/opentelemetry/inject-otel-environment-variable/.kyverno-test/expected/inject-pod.yaml b/opentelemetry/inject-otel-environment-variable/.kyverno-test/expected/inject-pod.yaml new file mode 100644 index 000000000..0ad326eb0 --- /dev/null +++ b/opentelemetry/inject-otel-environment-variable/.kyverno-test/expected/inject-pod.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Pod +metadata: + annotations: + otel.corp.org/inject-env-var: "true" + name: inject-pod + namespace: ns-wo-annotations +spec: + containers: + - name: debdiag + image: ghcr.io/babs/debdiag:0 + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: http://opentelemetry-collector.opentelemetry-collector:4317 diff --git a/opentelemetry/inject-otel-environment-variable/.kyverno-test/kyverno-test.yaml b/opentelemetry/inject-otel-environment-variable/.kyverno-test/kyverno-test.yaml new file mode 100644 index 000000000..0a481eece --- /dev/null +++ b/opentelemetry/inject-otel-environment-variable/.kyverno-test/kyverno-test.yaml @@ -0,0 +1,79 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Test +metadata: + name: inject-otel-environment-variable +policies: + - ../inject-otel-environment-variable.yaml +resources: + - resource.yaml +variables: values.yaml +results: + - policy: inject-otel-environment-variable + rule: inject-otel-environment-variable + resources: + - no-inject-pod + kind: Pod + result: skip + + - policy: inject-otel-environment-variable + rule: inject-otel-environment-variable + resources: + - no-inject-ns + kind: Pod + result: skip + + - policy: inject-otel-environment-variable + rule: inject-otel-environment-variable + resources: + - inject-ns + patchedResources: expected/inject-ns.yaml + kind: Pod + result: pass + + - policy: inject-otel-environment-variable + rule: inject-otel-environment-variable + resources: + - inject-pod + patchedResources: expected/inject-pod.yaml + kind: Pod + result: pass + + - policy: inject-otel-environment-variable + rule: inject-otel-environment-variable + resources: + - inject-default + patchedResources: expected/inject-default.yaml + kind: Pod + result: pass + + - policy: inject-otel-environment-variable + rule: inject-otel-environment-variable + resources: + - inject-before-preexisting-env + patchedResources: expected/inject-before-preexisting-env.yaml + kind: Pod + result: pass + + - policy: inject-otel-environment-variable + rule: inject-otel-environment-variable + resources: + - inject-custom-endpoint-pod + patchedResources: expected/inject-custom-endpoint-pod.yaml + kind: Pod + result: pass + + - policy: inject-otel-environment-variable + rule: inject-otel-environment-variable + resources: + - inject-custom-endpoint-ns + patchedResources: expected/inject-custom-endpoint-ns.yaml + kind: Pod + result: pass + + ## Enable for ConfigMap namespace filtering (see rule comments ConfigMap-NS-Filtering for detail) + # - policy: inject-otel-environment-variable + # rule: inject-otel-environment-variable + # resources: + # - no-inject-ns-excluded-by-configmap + # kind: Pod + # result: skip diff --git a/opentelemetry/inject-otel-environment-variable/.kyverno-test/resource.yaml b/opentelemetry/inject-otel-environment-variable/.kyverno-test/resource.yaml new file mode 100644 index 000000000..394fb7913 --- /dev/null +++ b/opentelemetry/inject-otel-environment-variable/.kyverno-test/resource.yaml @@ -0,0 +1,96 @@ +apiVersion: v1 +kind: Pod +metadata: + name: no-inject-pod + namespace: ns-wo-annotations + annotations: + otel.corp.org/inject-env-var: "false" +spec: + containers: + - name: debdiag + image: ghcr.io/babs/debdiag:0 +--- +apiVersion: v1 +kind: Pod +metadata: + name: no-inject-ns + namespace: ns-w-inject-false +spec: + containers: + - name: debdiag + image: ghcr.io/babs/debdiag:0 +--- +apiVersion: v1 +kind: Pod +metadata: + name: inject-ns + namespace: ns-w-inject-true +spec: + containers: + - name: debdiag + image: ghcr.io/babs/debdiag:0 +--- +apiVersion: v1 +kind: Pod +metadata: + name: inject-pod + namespace: ns-wo-annotations + annotations: + otel.corp.org/inject-env-var: "true" +spec: + containers: + - name: debdiag + image: ghcr.io/babs/debdiag:0 +--- +apiVersion: v1 +kind: Pod +metadata: + name: inject-default + namespace: ns-wo-annotations +spec: + containers: + - name: debdiag + image: ghcr.io/babs/debdiag:0 +--- +apiVersion: v1 +kind: Pod +metadata: + name: inject-before-preexisting-env +spec: + containers: + - name: debdiag + image: ghcr.io/babs/debdiag:0 + env: + - name: TEST_ENV + value: test +--- +apiVersion: v1 +kind: Pod +metadata: + name: inject-custom-endpoint-pod + annotations: + otel.corp.org/otlp-endpoint: http://http-collector:4318 +spec: + containers: + - name: debdiag + image: ghcr.io/babs/debdiag:0 +--- +apiVersion: v1 +kind: Pod +metadata: + name: inject-custom-endpoint-ns + namespace: ns-w-custom-endpoint +spec: + containers: + - name: debdiag + image: ghcr.io/babs/debdiag:0 +--- +apiVersion: v1 +kind: Pod +metadata: + name: no-inject-ns-excluded-by-configmap + namespace: ns-excluded-by-configmap +spec: + containers: + - name: debdiag + image: ghcr.io/babs/debdiag:0 diff --git a/opentelemetry/inject-otel-environment-variable/.kyverno-test/values.yaml b/opentelemetry/inject-otel-environment-variable/.kyverno-test/values.yaml new file mode 100644 index 000000000..c64497446 --- /dev/null +++ b/opentelemetry/inject-otel-environment-variable/.kyverno-test/values.yaml @@ -0,0 +1,24 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Values +policies: +- name: inject-otel-environment-variable + rules: + - name: inject-otel-environment-variable + values: + namespacefilters.data.exclude: '["ns-excluded-by-configmap"]' + # Global Reference simulation + namespaces: + - metadata: + name: ns-wo-annotations + - metadata: + name: ns-w-inject-true + annotations: + otel.corp.org/inject-env-var: "true" + - metadata: + name: ns-w-inject-false + annotations: + otel.corp.org/inject-env-var: "false" + - metadata: + name: ns-w-custom-endpoint + annotations: + otel.corp.org/otlp-endpoint: "http://http-collector:4318" diff --git a/opentelemetry/inject-otel-environment-variable/artifacthub-pkg.yml b/opentelemetry/inject-otel-environment-variable/artifacthub-pkg.yml new file mode 100644 index 000000000..0c07b71de --- /dev/null +++ b/opentelemetry/inject-otel-environment-variable/artifacthub-pkg.yml @@ -0,0 +1,34 @@ +name: inject-otel-environment-variable +version: 1.0.0 +displayName: Inject OpenTelemetry environment variable +createdAt: "2024-12-27T00:00:00.000Z" +description: >- + Injects OpenTelemetry `OTEL_EXPORTER_OTLP_ENDPOINT` env var in `containers` and `initContainers`. + Injection can be controlled by `otel.corp.org/inject-env-var: "false"` annotation on the Pod or at the Namespace level. + Value of the injected env var can also be overridden per Pod or Namespace via `otel.corp.org/otlp-endpoint: "http://xxxx:4317"` annotation. + The env var will be injected first, meaning that if one is already declared, the later will takes precedence over the injected one. + To avoid stress on the control plane API, a GlobalContextEntry containing Namespaces has to be declared (group: '', version: v1, resource: namespaces) + and `cache.kyverno.io/enabled: "true"` should be set on the `namespacefilters` ConfigMap (see rule comments ConfigMap-NS-Filtering). +install: |- + ```shell + kubectl apply \ + -f https://raw.githubusercontent.com/kyverno/policies/main/opentelemetry/inject-otel-environment-variable/globalcontext.yaml + -f https://raw.githubusercontent.com/kyverno/policies/main/opentelemetry/inject-otel-environment-variable/inject-otel-environment-variable.yaml + ``` +keywords: + - kyverno + - OpenTelemetry +readme: | + Injects OpenTelemetry `OTEL_EXPORTER_OTLP_ENDPOINT` env var in `containers` and `initContainers`. + Injection can be controlled by `otel.corp.org/inject-env-var: "false"` annotation on the Pod or at the Namespace level. + Value of the injected env var can also be overridden per Pod or Namespace via `otel.corp.org/otlp-endpoint: "http://xxxx:4317"` annotation. + The env var will be injected first, meaning that if one is already declared, the later will takes precedence over the injected one. + To avoid stress on the control plane API, a GlobalContextEntry containing Namespaces has to be declared (group: '', version: v1, resource: namespaces) + and `cache.kyverno.io/enabled: "true"` should be set on the `namespacefilters` ConfigMap (see rule comments ConfigMap-NS-Filtering). + + Refer to the documentation for more details on Kyverno annotations: https://artifacthub.io/docs/topics/annotations/kyverno/ +annotations: + kyverno/version: "1.13.0" + kyverno/category: "OpenTelemetry" + kyverno/subject: "Pod" +digest: ffa101097cabaa69e4c3ddfa07d7d4526c5e9a4e32005d794ab37f6c118d6df9 diff --git a/opentelemetry/inject-otel-environment-variable/globalcontext.yaml b/opentelemetry/inject-otel-environment-variable/globalcontext.yaml new file mode 100644 index 000000000..3301386d5 --- /dev/null +++ b/opentelemetry/inject-otel-environment-variable/globalcontext.yaml @@ -0,0 +1,9 @@ +apiVersion: kyverno.io/v2alpha1 +kind: GlobalContextEntry +metadata: + name: namespaces +spec: + kubernetesResource: + group: '' + version: v1 + resource: namespaces diff --git a/opentelemetry/inject-otel-environment-variable/inject-otel-environment-variable.yaml b/opentelemetry/inject-otel-environment-variable/inject-otel-environment-variable.yaml new file mode 100644 index 000000000..0c0a4e44e --- /dev/null +++ b/opentelemetry/inject-otel-environment-variable/inject-otel-environment-variable.yaml @@ -0,0 +1,105 @@ +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: inject-otel-environment-variable + annotations: + policies.kyverno.io/title: Injects OpenTelemetry environment variable + policies.kyverno.io/minversion: 1.13.0 + kyverno.io/kyverno-version: 1.13.0 + policies.kyverno.io/subject: Namespace, Pod + policies.kyverno.io/category: OpenTelemetry + policies.kyverno.io/description: >- + Injects OpenTelemetry `OTEL_EXPORTER_OTLP_ENDPOINT` env var in `containers` and `initContainers`. + Injection can be controlled by `otel.corp.org/inject-env-var: "false"` annotation on the Pod or at the Namespace level. + Value of the injected env var can also be overridden per Pod or Namespace via `otel.corp.org/otlp-endpoint: "http://xxxx:4317"` annotation. + The env var will be injected first, meaning that if one is already declared, the later will takes precedence over the injected one. + To avoid stress on the control plane API, a GlobalContextEntry containing Namespaces has to be declared (group: '', version: v1, resource: namespaces) + and `cache.kyverno.io/enabled: "true"` should be set on the `namespacefilters` ConfigMap (see rule comments ConfigMap-NS-Filtering). + +spec: + rules: + - name: inject-otel-environment-variable + context: + - name: namespaces + globalReference: + # This is the name of the GlobalContextEntry + # + # apiVersion: kyverno.io/v2alpha1 + # kind: GlobalContextEntry + # metadata: + # name: namespaces + # spec: + # kubernetesResource: + # group: '' + # version: v1 + # resource: namespaces + name: namespaces + - name: nsannontations + variable: + jmesPath: namespaces[?metadata.name == '{{ request.object.metadata.namespace }}'] | [0].metadata.annotations + default: {} + - name: otlp_endpoint + variable: + jmesPath: request.object.metadata.annotations."otel.corp.org/otlp-endpoint" || nsannontations."otel.corp.org/otlp-endpoint" + default: 'http://opentelemetry-collector.opentelemetry-collector:4317' + + # ConfigMap-NS-Filtering + # See https://kyverno.io/docs/writing-policies/external-data-sources/#handling-configmap-array-values + # Don't forget to enable kyverno's cache on the ConfigMap to avoid stress on the control plane + # Enable only if the proper config map has been enabled + # + # apiVersion: v1 + # kind: ConfigMap + # metadata: + # namespace: opentelemetry-collector + # name: namespacefilters + # labels: + # cache.kyverno.io/enabled: "true" + # data: + # exclude: '["ns-excluded-by-configmap"]' + + ## Uncomment the following for configmap namespace filtering, make sure the proper ConfigMap exists! + ## Also uncomment bellow in the preconditions section + # - name: namespacefilters + # configMap: + # name: namespacefilters + # namespace: opentelemetry-collector + + preconditions: + all: + # By default inject, expect if namespace or pod annotations says otherwise + - key: "{{ request.object.metadata.annotations.\"otel.corp.org/inject-env-var\" || nsannontations.\"otel.corp.org/inject-env-var\" || 'true' }}" + operator: NotEquals + value: 'false' + + # ConfigMap-NS-Filtering + # Env variable injection can be disabled on per NS basis using ConfigMap (no per pod override possible) + ## Uncomment the following for ConfigMap-based namespace filtering + ## Ensure the corresponding context above is uncommented and the appropriate ConfigMap is configured. + # - key: "{{ request.object.metadata.namespace }}" + # operator: AnyNotIn + # value: "{{ namespacefilters.data.exclude || '[]' }}" + match: + any: + - resources: + kinds: + - Pod + operations: + - CREATE + - UPDATE + mutate: + foreach: + - list: request.object.spec.containers[] + patchesJson6902: |- + - op: add + path: /spec/containers/{{elementIndex}}/env/0 + value: + name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "{{ otlp_endpoint }}" + - list: request.object.spec.initContainers[] + patchesJson6902: |- + - op: add + path: /spec/initContainers/{{elementIndex}}/env/0 + value: + name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "{{ otlp_endpoint }}" From 96b1ccf9a556e3414937f262d4a898ea55ced4f5 Mon Sep 17 00:00:00 2001 From: Damien Degois Date: Sat, 28 Dec 2024 19:21:32 +0100 Subject: [PATCH 2/3] set background to false Signed-off-by: Damien Degois --- .../inject-otel-environment-variable.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/opentelemetry/inject-otel-environment-variable/inject-otel-environment-variable.yaml b/opentelemetry/inject-otel-environment-variable/inject-otel-environment-variable.yaml index 0c0a4e44e..fc3b2da69 100644 --- a/opentelemetry/inject-otel-environment-variable/inject-otel-environment-variable.yaml +++ b/opentelemetry/inject-otel-environment-variable/inject-otel-environment-variable.yaml @@ -17,6 +17,7 @@ metadata: and `cache.kyverno.io/enabled: "true"` should be set on the `namespacefilters` ConfigMap (see rule comments ConfigMap-NS-Filtering). spec: + background: false rules: - name: inject-otel-environment-variable context: From 85fc5484afdf30a40022055e13bca14dd4474a01 Mon Sep 17 00:00:00 2001 From: Damien Degois Date: Sat, 28 Dec 2024 19:27:44 +0100 Subject: [PATCH 3/3] trigger only on Pod creation, not update Signed-off-by: Damien Degois --- .../inject-otel-environment-variable.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/opentelemetry/inject-otel-environment-variable/inject-otel-environment-variable.yaml b/opentelemetry/inject-otel-environment-variable/inject-otel-environment-variable.yaml index fc3b2da69..293e50c2e 100644 --- a/opentelemetry/inject-otel-environment-variable/inject-otel-environment-variable.yaml +++ b/opentelemetry/inject-otel-environment-variable/inject-otel-environment-variable.yaml @@ -87,7 +87,6 @@ spec: - Pod operations: - CREATE - - UPDATE mutate: foreach: - list: request.object.spec.containers[]