From 3a83700f287e73376ba264db1cc75d14dd0da8f8 Mon Sep 17 00:00:00 2001 From: Jaydipkumar Arvindbhai Gabani Date: Thu, 26 Oct 2023 10:46:42 -0700 Subject: [PATCH] chore: adding default helm values for pubsub audit connection and channel (#3097) Signed-off-by: Jaydip Gabani Signed-off-by: Jaydipkumar Arvindbhai Gabani --- cmd/build/helmify/static/values.yaml | 2 + .../charts/gatekeeper/values.yaml | 2 + pkg/audit/manager.go | 4 +- test/pubsub/fake-subscriber/main.go | 18 +- .../fake-subscriber/manifest/subscriber.yaml | 3 + website/docs/pubsub.md | 306 +++++++++--------- .../versioned_docs/version-v3.13.x/pubsub.md | 306 +++++++++--------- 7 files changed, 336 insertions(+), 305 deletions(-) diff --git a/cmd/build/helmify/static/values.yaml b/cmd/build/helmify/static/values.yaml index e1a872959f6..7fe5a94b3c2 100644 --- a/cmd/build/helmify/static/values.yaml +++ b/cmd/build/helmify/static/values.yaml @@ -210,6 +210,8 @@ controllerManager: # cidr: 0.0.0.0/0 audit: enablePubsub: false + connection: audit-connection + channel: audit-channel hostNetwork: false dnsPolicy: ClusterFirst metricsPort: 8888 diff --git a/manifest_staging/charts/gatekeeper/values.yaml b/manifest_staging/charts/gatekeeper/values.yaml index e1a872959f6..7fe5a94b3c2 100644 --- a/manifest_staging/charts/gatekeeper/values.yaml +++ b/manifest_staging/charts/gatekeeper/values.yaml @@ -210,6 +210,8 @@ controllerManager: # cidr: 0.0.0.0/0 audit: enablePubsub: false + connection: audit-connection + channel: audit-channel hostNetwork: false dnsPolicy: ClusterFirst metricsPort: 8888 diff --git a/pkg/audit/manager.go b/pkg/audit/manager.go index dc4311a9307..efa5218b724 100644 --- a/pkg/audit/manager.go +++ b/pkg/audit/manager.go @@ -66,8 +66,8 @@ var ( auditEventsInvolvedNamespace = flag.Bool("audit-events-involved-namespace", false, "emit audit events for each violation in the involved objects namespace, the default (false) generates events in the namespace Gatekeeper is installed in. Audit events from cluster-scoped resources will still follow the default behavior") auditMatchKindOnly = flag.Bool("audit-match-kind-only", false, "only use kinds specified in all constraints for auditing cluster resources. if kind is not specified in any of the constraints, it will audit all resources (same as setting this flag to false)") apiCacheDir = flag.String("api-cache-dir", defaultAPICacheDir, "The directory where audit from api server cache are stored, defaults to /tmp/audit") - auditConnection = flag.String("audit-connection", defaultConnection, "Connection name for publishing audit violation messages") - auditChannel = flag.String("audit-channel", defaultChannel, "Channel name for publishing audit violation messages") + auditConnection = flag.String("audit-connection", defaultConnection, "Connection name for publishing audit violation messages. Defaults to audit-connection") + auditChannel = flag.String("audit-channel", defaultChannel, "Channel name for publishing audit violation messages. Defaults to audit-channel") emptyAuditResults []updateListEntry logStatsAudit = flag.Bool("log-stats-audit", false, "(alpha) log stats metrics for the audit run") ) diff --git a/test/pubsub/fake-subscriber/main.go b/test/pubsub/fake-subscriber/main.go index 9831cbe2b2b..fadd3aac2c9 100644 --- a/test/pubsub/fake-subscriber/main.go +++ b/test/pubsub/fake-subscriber/main.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "log" + "os" "strconv" "github.com/dapr/go-sdk/service/common" @@ -30,15 +31,18 @@ type PubsubMsg struct { ResourceLabels map[string]string `json:"resourceLabels,omitempty"` } -var sub = &common.Subscription{ - PubsubName: "pubsub", - Topic: "audit", - Route: "/checkout", -} - func main() { + auditChannel := os.Getenv("AUDIT_CHANNEL") + if auditChannel == "" { + auditChannel = "audit-channel" + } + sub := &common.Subscription{ + PubsubName: "pubsub", + Topic: auditChannel, + Route: "/checkout", + } s := daprd.NewService(":6002") - log.Printf("Listening...") + log.Printf("Listening on %s...", auditChannel) if err := s.AddTopicEventHandler(sub, eventHandler); err != nil { log.Fatalf("error adding topic subscription: %v", err) } diff --git a/test/pubsub/fake-subscriber/manifest/subscriber.yaml b/test/pubsub/fake-subscriber/manifest/subscriber.yaml index d0372ed6350..4c656ccd577 100644 --- a/test/pubsub/fake-subscriber/manifest/subscriber.yaml +++ b/test/pubsub/fake-subscriber/manifest/subscriber.yaml @@ -41,3 +41,6 @@ spec: - name: go-sub image: fake-subscriber:latest imagePullPolicy: Never + env: + - name: AUDIT_CHANNEL + value: "audit-channel" diff --git a/website/docs/pubsub.md b/website/docs/pubsub.md index ddbbac0d67f..8c1df5fb3c0 100644 --- a/website/docs/pubsub.md +++ b/website/docs/pubsub.md @@ -19,15 +19,15 @@ Install prerequisites such as a pubsub tool, a message broker etc. ### Setting up audit with pubsub enabled -In the audit deployment, set the `--enable-pub-sub` flag to `true` to publish audit violations. Additionally, `--audit-connection` and `--audit-channel` flags must be set to allow audit to publish violations. `--audit-connection` must be set to the name of the connection config, and `--audit-channel` must be set to name of the channel where violations should get published. +In the audit deployment, set the `--enable-pub-sub` flag to `true` to publish audit violations. Additionally, use `--audit-connection` (defaults to `audit-connection`) and `--audit-channel`(defaults to `audit-channel`) flags to allow audit to publish violations using desired connection onto desired channel. `--audit-connection` must be set to the name of the connection config, and `--audit-channel` must be set to name of the channel where violations should get published. -Create a connection configMap that supplies [a provider-specific configuration](pubsub-driver-walkthrough.md#how-to-use-different-providers) for a connection to get established. For instance, to establish a connection that uses Dapr to publish messages this configMap is appropriate: +A ConfigMap that contains `provider` and `config` fields in `data` is required to establish connection for sending violations over the channel. Following is an example ConfigMap to establish a connection that uses Dapr to publish messages: ```yaml apiVersion: v1 kind: ConfigMap metadata: - name: audit-pubsub-connection + name: audit-connection namespace: gatekeeper-system data: provider: "dapr" @@ -43,185 +43,195 @@ data: #### Available Pubsub drivers Dapr: https://dapr.io/ -### Violations - -The audit pod publishes violations in following format: - -```json -{ - "id": "2023-07-18T21:21:52Z", - "details": { - "missing_labels": [ - "test" - ] - }, - "eventType": "violation_audited", - "group": "constraints.gatekeeper.sh", - "version": "v1beta1", - "kind": "K8sRequiredLabels", - "name": "pod-must-have-test", - "message": "you must provide labels: {\"test\"}", - "enforcementAction": "deny", - "resourceAPIVersion": "v1", - "resourceKind": "Pod", - "resourceNamespace": "nginx", - "resourceName": "nginx-deployment-cd55c47f5-2b84x", - "resourceLabels": { - "app": "nginx", - "pod-template-hash": "cd55c47f5" - } -} -``` - ### Quick start with publishing violations using Dapr and Redis -> Redis is used for example purposes only. Dapr supports [many different state store options](https://docs.dapr.io/reference/components-reference/supported-state-stores/). - #### Prerequisites 1. Install Dapr - To install Dapr with specific requirements and configuration, please refer to [Dapr docs](https://docs.dapr.io/operations/hosting/kubernetes/kubernetes-deploy/). - > Dapr is installed with mtls enabled by default, for more details on the same plaase refer to [Dapr security](https://docs.dapr.io/operations/security/mtls/#setting-up-mtls-with-the-configuration-resource). + To install Dapr with specific requirements and configuration, please refer to [Dapr docs](https://docs.dapr.io/operations/hosting/kubernetes/kubernetes-deploy/). + > [!IMPORTANT] + > - Make sure to set `SIDECAR_DROP_ALL_CAPABILITIES` environment variable on `dapr-sidecar` injector pod to `true` to avoid getting `PodSecurity violation` errors for the injected sidecar container as Gatekeeper by default requires workloads to run with [restricted](https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted) policy. If using helm charts to install Dapr, you can use `--set dapr_sidecar_injector.sidecarDropALLCapabilities=true`. + > - Additionally, [configure appropriate seccompProfile for sidecar containers](https://docs.dapr.io/operations/hosting/kubernetes/kubernetes-production/#configure-seccompprofile-for-sidecar-containers) injected by Dapr to avoid getting `PodSecurity violation` errors. We are setting required Dapr annotation for audit pod while deploying Gatekeeper later in this quick start to avoid getting `PodSecurity violation` error. + + > Dapr is installed with mtls enabled by default, for more details on the same please refer to [Dapr security](https://docs.dapr.io/operations/security/mtls/#setting-up-mtls-with-the-configuration-resource). 2. Install Redis Please refer to [this](https://docs.dapr.io/getting-started/tutorials/configure-state-pubsub/#step-1-create-a-redis-store) guide to install Redis. - > To install Redis with TLS, please refer to [this](https://docs.bitnami.com/kubernetes/infrastructure/redis-cluster/administration/enable-tls/) doc. + > Redis is used for example purposes only. Dapr supports [many different state store options](https://docs.dapr.io/reference/components-reference/supported-state-stores/). To install Redis with TLS, please refer to [this](https://docs.bitnami.com/kubernetes/infrastructure/redis-cluster/administration/enable-tls/) doc. #### Configure a sample subscriber to receive violations 1. Create `fake-subscriber` namespace and redis secret -```shell -kubectl create ns fake-subscriber -kubectl get secret redis --namespace=default -o yaml | sed 's/namespace: .*/namespace: fake-subscriber/' | kubectl apply -f - # creating redis secret in subscriber namespace to allow dapr sidecar to connect to redis instance -``` + ```shell + kubectl create ns fake-subscriber + # creating redis secret in subscriber namespace to allow Dapr sidecar to connect to redis instance + kubectl get secret redis --namespace=default -o yaml | sed 's/namespace: .*/namespace: fake-subscriber/' | kubectl apply -f - + ``` 2. Create Dapr pubsub component -```shell -kubectl apply -f < Please use [this guide](https://docs.dapr.io/reference/components-reference/supported-state-stores/setup-redis/) to properly configure Redis pubsub component for Dapr. + + ```shell + kubectl apply -f - < Please use [this guide](https://docs.dapr.io/reference/components-reference/supported-state-stores/setup-redis/) to properly configure Redis pubsub component for Dapr. 3. Deploy subscriber application -```yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: sub - namespace: fake-subscriber - labels: - app: sub -spec: - replicas: 1 - selector: - matchLabels: - app: sub - template: + + ```yaml + apiVersion: apps/v1 + kind: Deployment metadata: + name: sub + namespace: fake-subscriber labels: app: sub - annotations: - dapr.io/enabled: "true" - dapr.io/app-id: "subscriber" - dapr.io/enable-api-logging: "true" - dapr.io/app-port: "6002" spec: - containers: - - name: go-sub - image: fake-subscriber:latest - imagePullPolicy: Never -``` -**Note:** Dockerfile to build image for fake-subscriber is under [gatekeeper/test/fake-subscriber](https://github.com/open-policy-agent/gatekeeper/tree/master/test/pubsub/fake-subscriber). You can find make rule to build and deploy subscriber in [Makefile](https://github.com/open-policy-agent/gatekeeper/blob/master/Makefile) under name `e2e-subscriber-build-load-image` and `e2e-subscriber-deploy`. + replicas: 1 + selector: + matchLabels: + app: sub + template: + metadata: + labels: + app: sub + annotations: + dapr.io/enabled: "true" + dapr.io/app-id: "subscriber" + dapr.io/enable-api-logging: "true" + dapr.io/app-port: "6002" + spec: + containers: + - name: go-sub + image: fake-subscriber:latest + imagePullPolicy: Never + ``` + + > [!IMPORTANT] + > Please make sure `fake-subscriber` image is built and available in your cluster. Dockerfile to build image for `fake-subscriber` is under [gatekeeper/test/fake-subscriber](https://github.com/open-policy-agent/gatekeeper/tree/master/test/pubsub/fake-subscriber). #### Configure Gatekeeper with Pubsub enabled -1. Create Dapr pubsub component and Redis secret in Gatekeeper's namespace (`gatekeeper-system` by default). Please make sure to update `gatekeeper-system` namespace for the next steps if your cluster's Gatekeeper namespace is different. - -```shell -kubectl get secret redis --namespace=default -o yaml | sed 's/namespace: .*/namespace: gatekeeper-system/' | kubectl apply -f - -kubectl apply -f - < .tmp/annotations.yaml # auditPodAnnotations is used to add annotations required by Dapr to inject sidecar to audit pod -helm install gatekeeper/gatekeeper --name-template=gatekeeper --namespace gatekeeper-system \ ---set audit.enablePubsub=true \ ---set audit.connection=audit-pubsub-connection \ ---set audit.channel=audit \ ---values .tmp/annotations.yaml -``` - -**Note:** Verify that after the audit pod is running there is a dapr sidecar injected and running along side `manager` container. + ```shell + kubectl create namespace gatekeeper-system + kubectl get secret redis --namespace=default -o yaml | sed 's/namespace: .*/namespace: gatekeeper-system/' | kubectl apply -f - + kubectl apply -f - < /tmp/annotations.yaml + helm upgrade --install gatekeeper gatekeeper/gatekeeper --namespace gatekeeper-system \ + --set audit.enablePubsub=true \ + --set audit.connection=audit-connection \ + --set audit.channel=audit-channel \ + --values /tmp/annotations.yaml + ``` + + **Note:** Verify that after the audit pod is running there is a Dapr sidecar injected and running along side `manager` container. 3. Create connection config to establish a connection. -```shell -kubectl apply -f - < Redis is used for example purposes only. Dapr supports [many different state store options](https://docs.dapr.io/reference/components-reference/supported-state-stores/). - #### Prerequisites 1. Install Dapr - To install Dapr with specific requirements and configuration, please refer to [Dapr docs](https://docs.dapr.io/operations/hosting/kubernetes/kubernetes-deploy/). - > Dapr is installed with mtls enabled by default, for more details on the same plaase refer to [Dapr security](https://docs.dapr.io/operations/security/mtls/#setting-up-mtls-with-the-configuration-resource). + To install Dapr with specific requirements and configuration, please refer to [Dapr docs](https://docs.dapr.io/operations/hosting/kubernetes/kubernetes-deploy/). + > [!IMPORTANT] + > - Make sure to set `SIDECAR_DROP_ALL_CAPABILITIES` environment variable on `dapr-sidecar` injector pod to `true` to avoid getting `PodSecurity violation` errors for the injected sidecar container as Gatekeeper by default requires workloads to run with [restricted](https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted) policy. If using helm charts to install Dapr, you can use `--set dapr_sidecar_injector.sidecarDropALLCapabilities=true`. + > - Additionally, [configure appropriate seccompProfile for sidecar containers](https://docs.dapr.io/operations/hosting/kubernetes/kubernetes-production/#configure-seccompprofile-for-sidecar-containers) injected by Dapr to avoid getting `PodSecurity violation` errors. We are setting required annotation for audit pod while deploying Gatekeeper later in this quick start to avoid getting `PodSecurity violation` error. + + > Dapr is installed with mtls enabled by default, for more details on the same please refer to [Dapr security](https://docs.dapr.io/operations/security/mtls/#setting-up-mtls-with-the-configuration-resource). 2. Install Redis Please refer to [this](https://docs.dapr.io/getting-started/tutorials/configure-state-pubsub/#step-1-create-a-redis-store) guide to install Redis. - > To install Redis with TLS, please refer to [this](https://docs.bitnami.com/kubernetes/infrastructure/redis-cluster/administration/enable-tls/) doc. + > Redis is used for example purposes only. Dapr supports [many different state store options](https://docs.dapr.io/reference/components-reference/supported-state-stores/). To install Redis with TLS, please refer to [this](https://docs.bitnami.com/kubernetes/infrastructure/redis-cluster/administration/enable-tls/) doc. #### Configure a sample subscriber to receive violations 1. Create `fake-subscriber` namespace and redis secret -```shell -kubectl create ns fake-subscriber -kubectl get secret redis --namespace=default -o yaml | sed 's/namespace: .*/namespace: fake-subscriber/' | kubectl apply -f - # creating redis secret in subscriber namespace to allow dapr sidecar to connect to redis instance -``` + ```shell + kubectl create ns fake-subscriber + # creating redis secret in subscriber namespace to allow Dapr sidecar to connect to redis instance + kubectl get secret redis --namespace=default -o yaml | sed 's/namespace: .*/namespace: fake-subscriber/' | kubectl apply -f - + ``` 2. Create Dapr pubsub component -```shell -kubectl apply -f < Please use [this guide](https://docs.dapr.io/reference/components-reference/supported-state-stores/setup-redis/) to properly configure Redis pubsub component for Dapr. + + ```shell + kubectl apply -f - < Please use [this guide](https://docs.dapr.io/reference/components-reference/supported-state-stores/setup-redis/) to properly configure Redis pubsub component for Dapr. 3. Deploy subscriber application -```yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: sub - namespace: fake-subscriber - labels: - app: sub -spec: - replicas: 1 - selector: - matchLabels: - app: sub - template: + + ```yaml + apiVersion: apps/v1 + kind: Deployment metadata: + name: sub + namespace: fake-subscriber labels: app: sub - annotations: - dapr.io/enabled: "true" - dapr.io/app-id: "subscriber" - dapr.io/enable-api-logging: "true" - dapr.io/app-port: "6002" spec: - containers: - - name: go-sub - image: fake-subscriber:latest - imagePullPolicy: Never -``` -**Note:** Dockerfile to build image for fake-subscriber is under [gatekeeper/test/fake-subscriber](https://github.com/open-policy-agent/gatekeeper/tree/master/test/pubsub/fake-subscriber). You can find make rule to build and deploy subscriber in [Makefile](https://github.com/open-policy-agent/gatekeeper/blob/master/Makefile) under name `e2e-subscriber-build-load-image` and `e2e-subscriber-deploy`. + replicas: 1 + selector: + matchLabels: + app: sub + template: + metadata: + labels: + app: sub + annotations: + dapr.io/enabled: "true" + dapr.io/app-id: "subscriber" + dapr.io/enable-api-logging: "true" + dapr.io/app-port: "6002" + spec: + containers: + - name: go-sub + image: fake-subscriber:latest + imagePullPolicy: Never + ``` + + > [!IMPORTANT] + > Please make sure `fake-subscriber` image is built and available in your cluster. Dockerfile to build image for `fake-subscriber` is under [gatekeeper/test/fake-subscriber](https://github.com/open-policy-agent/gatekeeper/tree/master/test/pubsub/fake-subscriber). #### Configure Gatekeeper with Pubsub enabled -1. Create Dapr pubsub component and Redis secret in Gatekeeper's namespace (`gatekeeper-system` by default). Please make sure to update `gatekeeper-system` namespace for the next steps if your cluster's Gatekeeper namespace is different. - -```shell -kubectl get secret redis --namespace=default -o yaml | sed 's/namespace: .*/namespace: gatekeeper-system/' | kubectl apply -f - -kubectl apply -f - < .tmp/annotations.yaml # auditPodAnnotations is used to add annotations required by Dapr to inject sidecar to audit pod -helm install gatekeeper/gatekeeper --name-template=gatekeeper --namespace gatekeeper-system \ ---set audit.enablePubsub=true \ ---set audit.connection=audit-pubsub-connection \ ---set audit.channel=audit \ ---values .tmp/annotations.yaml -``` - -**Note:** Verify that after the audit pod is running there is a dapr sidecar injected and running along side `manager` container. + ```shell + kubectl create namespace gatekeeper-system + kubectl get secret redis --namespace=default -o yaml | sed 's/namespace: .*/namespace: gatekeeper-system/' | kubectl apply -f - + kubectl apply -f - < /tmp/annotations.yaml + helm upgrade --install gatekeeper gatekeeper/gatekeeper --namespace gatekeeper-system \ + --set audit.enablePubsub=true \ + --set audit.connection=audit-connection \ + --set audit.channel=audit-channel \ + --values /tmp/annotations.yaml + ``` + + **Note:** Verify that after the audit pod is running there is a Dapr sidecar injected and running along side `manager` container. 3. Create connection config to establish a connection. -```shell -kubectl apply -f - <