From 0b36964a6c54236e84993096214cc0a6810f3f26 Mon Sep 17 00:00:00 2001 From: Gonzalo Reyero Ferreras <87083379+greyerof@users.noreply.github.com> Date: Thu, 26 Oct 2023 17:38:21 +0200 Subject: [PATCH] Added yaml template to deploy the CNF Cert Suite in a kubernetes/Openshift cluster. (#1557) The yaml template and the kustomization file inside the k8s folder allow the deployment of the CNF Cert Suite Pod using: `oc apply -f k8s/cnf-certsuite.yaml` or `oc kustomization k8s | oc apply -f -` See the README.md file inside the k8s folder for more information and some possible configuration changes. As this is a developer's "feature", I decided not to include it in the official CNF Cert Suite documentation markdown files. --- k8s/README.md | 70 ++++++++++++++++++++ k8s/cnf-certsuite.yaml | 146 +++++++++++++++++++++++++++++++++++++++++ k8s/kustomization.yaml | 38 +++++++++++ 3 files changed, 254 insertions(+) create mode 100644 k8s/README.md create mode 100644 k8s/cnf-certsuite.yaml create mode 100644 k8s/kustomization.yaml diff --git a/k8s/README.md b/k8s/README.md new file mode 100644 index 000000000..ba62d78d7 --- /dev/null +++ b/k8s/README.md @@ -0,0 +1,70 @@ + +# How to deploy the CNF Cert Suite App inside a Kubernetes/Openshift cluster + +This is a developer's guide to deploy a Pod in a kubernetes/Openshift cluster that runs the CNF Cert Suite app inside. + +This folder contains two files: + +* [./cnf-certsuite.yaml](cnf-certsuite.yaml) +* [./kustomization.yaml](kustomization.yaml) + +## cnf-certsuite.yaml + +This file contains all the kubernetes templates for deploying the CNF Cert Suite inside a Pod named "cnf-certsuite" in a namespace also named "cnf-certsuite". In order to deploy the pod, just write: + +```console +oc apply -f k8s/cnf-certsuite.yaml +namespace/cnf-certsuite created +clusterrole.rbac.authorization.k8s.io/cnf-certsuite-cr created +clusterrolebinding.rbac.authorization.k8s.io/cnf-certsuite-crb created +configmap/cnf-certsuite-config created +secret/cnf-certsuite-preflight-dockerconfig created +pod/cnf-certsuite created +``` + +The first thing in that yaml is the namespace, so it's the first resource that will be created in the cluster. Then, a cluster role and its cluster role binding will be created. This cluster role is needed because the CNF Cert Suite needs access to all the resources in the whole cluster. + +Then, there's a configMap with the whole config (tnf_config.yaml) that will be used by the pod to create the tnf_config.yaml file inside a volume folder. Also, there's a secret with the preflight's dockerconfig file content that will also be used by the CNF Cert Suitep pod. + +The CNF Cert Suite pod is the last resource defined in the cnf-certsuite.yaml file. It has only one container that uses the [quay.io/testnetworkfunction/cnf-certification-test:latest](latest) tag of the CNF Cert Suite. The command slice of this container has a hardcoded labels to run as many test cases as possible, excluding the intrusive ones. + +## kustomization.yaml + +This kustomization file allows the deployment of the CNF Cert Suite using this command: + +```console +oc kustomize k8s/ | oc apply -f - +``` + +The `kustomization` tool used by `oc` will parse the content of the [./kustomization.yaml](kustomization.yaml) file, which consists of a set of "transformers" over the resources defined in [./cnf-certsuite.yaml](cnf-certsuite.yaml). + +By default, that command will deploy the CNF Cert Suite Pod without any mutation: it will be deployed in the same namespace and with the same configuration than using the `oc apply -f k8s/cnf-certsuite.yaml`. + +But there are the three example of modifications included in [./kustomization.yaml](kustomization.yaml) that can be used out of the box that can be handy: + +1. The namespace and the prefix/suffix of each resource's name. By default, the [./cnf-certsuite.yaml](cnf-certsuite.yaml) uses the namespace "cnf-certsuite" to deploy all the reources (except the cluster role and the cluster role binding), but this can be changed uncommenting the line that starts with `namespace:`. It's highly recommended to uncomment at least one of suffixName/prefixName so unique cluster role & cluster role-bindings can be created for each CNF Cert Pod. This way, you could run more than one CNF Cert Pod in the same cluster!. +2. The (ginkgo) labels expression, in case you want to run different test cases. Uncomment the object that starts with "patches:". The commented example changes the command to use the "preflight" label only. +3. The value of the TNF_NON_INTRUSIVE_ONLY env var. Uncomment the last object that starts with "patches:". The commented example changes the TNF_NON_INTRUSIVE_ONLY to false, so all the intrusive TCs will run in case the lifecycle TCs are selected to run by the appropriate labels. + +In case both (1) and (2) wants to be used, just create a list of patches like this: + +```console +patches: + - target: + version: v1 + kind: Pod + name: cnf-certsuite + patch: | + - op: replace + path: /spec/containers/0/args/1 + value: | + ./run-cnf-suites.sh -l 'preflight' ; sleep inf + - target: + version: v1 + kind: Pod + name: cnf-certsuite + patch: | + - op: replace + path: /spec/containers/0/env/0/value + value: false +``` diff --git a/k8s/cnf-certsuite.yaml b/k8s/cnf-certsuite.yaml new file mode 100644 index 000000000..20804663f --- /dev/null +++ b/k8s/cnf-certsuite.yaml @@ -0,0 +1,146 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: cnf-certsuite + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cnf-certsuite-cr +rules: + - apiGroups: ["*"] + resources: ["*"] + verbs: ["*"] + - nonResourceURLs: ["*"] + verbs: ["*"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cnf-certsuite-crb +subjects: + - kind: ServiceAccount + name: default + namespace: cnf-certsuite +roleRef: + kind: ClusterRole + name: cnf-certsuite-cr + apiGroup: rbac.authorization.k8s.io + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: cnf-certsuite-config + namespace: cnf-certsuite +data: + tnf_config.yaml: | + targetNameSpaces: + - name: tnf + podsUnderTestLabels: + - "test-network-function.com/generic: target" + # deprecated operator label ("test-network-function.com/operator:"") still configured by default, no need to add it here + operatorsUnderTestLabels: + - "test-network-function.com/operator1:new" + targetCrdFilters: + - nameSuffix: "group1.test.com" + scalable: false + - nameSuffix: "test-network-function.com" + scalable: false + - nameSuffix: "tutorial.my.domain" + scalable: true + managedDeployments: + - name: jack + managedStatefulsets: + - name: jack + certifiedcontainerinfo: + - name: rocketchat/rocketchat + repository: registry.connect.redhat.com + tag: 0.56.0-1 # optional, "latest" assumed if empty + digest: # if set, takes precedence over tag. e.g. "sha256:aa34453a6417f8f76423ffd2cf874e9c4a1a5451ac872b78dc636ab54a0ebbc3" + - name: rocketchat/rocketchat + repository: registry.connect.redhat.com + tag: 0.56.0-1 + digest: sha256:03f7f2499233a302351821d6f78f0e813c3f749258184f4133144558097c57b0 + checkDiscoveredContainerCertificationStatus: false + acceptedKernelTaints: + - module: vboxsf + - module: vboxguest + skipScalingTestDeployments: + - name: deployment1 + namespace: tnf + skipScalingTestStatefulsets: + - name: statefulset1 + namespace: tnf + skipHelmChartList: + - name: coredns + validProtocolNames: + - "http3" + - "sctp" + servicesignorelist: + - "hazelcast-platform-controller-manager-service" + - "hazelcast-platform-webhook-service" + - "new-pro-controller-manager-metrics-service" + +--- +apiVersion: v1 +kind: Secret +metadata: + name: cnf-certsuite-preflight-dockerconfig + namespace: cnf-certsuite +type: Opaque +data: + # Sample of empty content, base64-coded: '{ "auths": {} }' + preflight_dockerconfig.json: | + eyAiYXV0aHMiOiB7fSB9Cg== + +--- +apiVersion: v1 +kind: Pod +metadata: + name: cnf-certsuite + namespace: cnf-certsuite + labels: + app: cnf-certsuite +spec: + serviceAccountName: default + restartPolicy: Never + volumes: + - name: config-volume + configMap: + name: cnf-certsuite-config + - name: preflight-dockerconfig + secret: + secretName: cnf-certsuite-preflight-dockerconfig + containers: + - name: cnf-certsuite + imagePullPolicy: Always + image: quay.io/testnetworkfunction/cnf-certification-test:latest + resources: + limits: + memory: 500Mi + cpu: 50m + command: ["sh"] + args: + - "-c" + - | + ./run-cnf-suites.sh -l '!affiliated-certification-container-is-certified-digest && !access-control-security-context' ; sleep inf + volumeMounts: + - name: config-volume + mountPath: /usr/tnf/config + - name: preflight-dockerconfig + mountPath: /usr/tnf/config/preflight + env: + - name: TNF_NON_INTRUSIVE_ONLY + value: "true" + - name: TNF_ALLOW_PREFLIGHT_INSECURE + value: "true" + - name: TNF_LOG_LEVEL + value: trace + - name: PFLT_DOCKERCONFIG + value: /usr/tnf/config/preflight/preflight_dockerconfig.json + - name: TNF_CONFIGURATION_PATH + value: /usr/tnf/config/tnf_config.yaml diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml new file mode 100644 index 000000000..6eabe1939 --- /dev/null +++ b/k8s/kustomization.yaml @@ -0,0 +1,38 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - cnf-certsuite.yaml + +# Uncomment the next line (namespace transformer) to deploy all the cnf-certsuite related +# resources in a different namespace. +# namespace: my-custom-ns-name + +# Uncomment the next lines so each resource's name have a custom prefix and/or suffix appended. +# namePrefix: myprefix- +# nameSuffix: -mysuffix + +# Uncomment the next lines (patches) in order to launch the cnf-certsuite pod with a different +# test cases labels filter/expr. The following example changes the labels to "preflight". +# patches: +# - target: +# version: v1 +# kind: Pod +# name: cnf-certsuite +# patch: | +# - op: replace +# path: /spec/containers/0/args/1 +# value: | +# ./run-cnf-suites.sh -l 'preflight' ; sleep inf + +# Uncomment the next lines (patches) in order to allow intrusive TCs to run. +# patches: +# - target: +# version: v1 +# kind: Pod +# name: cnf-certsuite +# patch: | +# - op: replace +# path: /spec/containers/0/env/0/value +# value: false