From 57390d1f7e0f953668c4748c3cf6004708b80d8f Mon Sep 17 00:00:00 2001 From: Anurag Mittal Date: Mon, 4 Nov 2024 17:16:23 +0100 Subject: [PATCH 1/3] COSI-11: generate-docker-image-using-makefile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Updated Makefile - added GO env for `go build` command - CGO_ENABLED = 0 - disable cgo ie. to produce fully static binary that doesn’t rely on system-specific C libraries - GOOS - operating system - GOARCH - architecture - added a new target in Makefile for building container --- Makefile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 613ec5c..78795cc 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,17 @@ APP_NAME = scality-cosi-driver BIN_DIR = ./bin +# 'go env' vars aren't always available in make environments, so get defaults for needed ones +GOARCH ?= $(shell go env GOARCH) +IMAGE_NAME ?= ghcr.io/scality/cosi:latest + .PHONY: all build test clean all: test build build: @echo "Building $(APP_NAME)..." - go build -o $(BIN_DIR)/$(APP_NAME) ./cmd/$(APP_NAME) + CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -o $(BIN_DIR)/$(APP_NAME) ./cmd/$(APP_NAME) test: @echo "Running Ginkgo tests..." @@ -17,3 +21,7 @@ test: clean: @echo "Cleaning up..." rm -rf $(BIN_DIR) + +container: + @echo "Building container image..." + docker build -t $(IMAGE_NAME) . From a62e8d9dd94709dad7e36d298894bd281a997ac2 Mon Sep 17 00:00:00 2001 From: Anurag Mittal Date: Mon, 4 Nov 2024 17:28:37 +0100 Subject: [PATCH 2/3] COSI-11: update-kustomize-files-for-COSI-driver - restructured and re-organized kustomize files for COSI driver - possibility of expandability using overlays in the future - deleted forked kustomize files from resources directory --- kustomization.yaml | 59 +----------------- kustomize/base/deployment.yaml | 45 ++++++++++++++ kustomize/base/kustomization.yaml | 10 ++++ kustomize/base/namespace.yml | 4 ++ kustomize/base/rbac.yaml | 28 +++++++++ kustomize/base/serviceaccount.yaml | 5 ++ kustomize/overlays/kustomization.yaml | 37 ++++++++++++ .../overlays/scality-cosi-driver.properties | 2 + resources/cosi-driver.properties | 4 -- resources/deployment.yaml | 60 ------------------- resources/kustomizeconfig.yaml | 3 - resources/ns.yaml | 10 ---- resources/rbac.yaml | 38 ------------ resources/sa.yaml | 11 ---- 14 files changed, 132 insertions(+), 184 deletions(-) create mode 100644 kustomize/base/deployment.yaml create mode 100644 kustomize/base/kustomization.yaml create mode 100644 kustomize/base/namespace.yml create mode 100644 kustomize/base/rbac.yaml create mode 100644 kustomize/base/serviceaccount.yaml create mode 100644 kustomize/overlays/kustomization.yaml create mode 100644 kustomize/overlays/scality-cosi-driver.properties delete mode 100644 resources/cosi-driver.properties delete mode 100644 resources/deployment.yaml delete mode 100644 resources/kustomizeconfig.yaml delete mode 100644 resources/ns.yaml delete mode 100644 resources/rbac.yaml delete mode 100644 resources/sa.yaml diff --git a/kustomization.yaml b/kustomization.yaml index 607b58e..232d43a 100644 --- a/kustomization.yaml +++ b/kustomization.yaml @@ -1,62 +1,5 @@ ---- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -namespace: scality-cosi-driver - -commonAnnotations: - cosi.storage.k8s.io/authors: "Kubernetes Authors" - cosi.storage.k8s.io/license: "Apache V2" - cosi.storage.k8s.io/support: "https://github.com/kubernetes-sigs/container-object-storage-api" - -commonLabels: - app.kubernetes.io/part-of: container-object-storage-interface - app.kubernetes.io/component: driver-scality - app.kubernetes.io/version: main - app.kubernetes.io/name: cosi-driver-scality - -configMapGenerator: -- name: cosi-driver-scality-config - env: resources/cosi-driver-scality.properties -generatorOptions: - disableNameSuffixHash: true - labels: - generated-by: "kustomize" resources: -- resources/ns.yaml -- resources/sa.yaml -- resources/rbac.yaml -- resources/deployment.yaml - -configurations: - - resources/kustomizeconfig.yaml - -vars: - - name: IMAGE_ORG - objref: - name: cosi-driver-scality-config - kind: ConfigMap - apiVersion: v1 - fieldref: - fieldpath: data.OBJECTSTORAGE_PROVISIONER_IMAGE_ORG - - name: IMAGE_VERSION - objref: - name: cosi-driver-scality-config - kind: ConfigMap - apiVersion: v1 - fieldref: - fieldpath: data.OBJECTSTORAGE_PROVISIONER_IMAGE_VERSION - - name: SCALITY_IMAGE_ORG - objref: - name: cosi-driver-scality-config - kind: ConfigMap - apiVersion: v1 - fieldref: - fieldpath: data.SCALITY_DRIVER_IMAGE_ORG - - name: SCALITY_IMAGE_VERSION - objref: - name: cosi-driver-scality-config - kind: ConfigMap - apiVersion: v1 - fieldref: - fieldpath: data.SCALITY_DRIVER_IMAGE_VERSION +- kustomize/overlays diff --git a/kustomize/base/deployment.yaml b/kustomize/base/deployment.yaml new file mode 100644 index 0000000..1b8a86b --- /dev/null +++ b/kustomize/base/deployment.yaml @@ -0,0 +1,45 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: scality-cosi-driver +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: scality-cosi-driver + template: + metadata: + labels: + app.kubernetes.io/name: scality-cosi-driver + app.kubernetes.io/part-of: container-object-storage-interface + app.kubernetes.io/component: driver + app.kubernetes.io/version: main + app.kubernetes.io/managed-by: kustomize + spec: + serviceAccountName: scality-object-storage-provisioner + containers: + - name: scality-cosi-driver + image: ghcr.io/scality/cosi:latest + imagePullPolicy: IfNotPresent + args: + - "--driver-prefix=cosi" + - "--v=$(COSI_DRIVER_LOG_LEVEL)" + volumeMounts: + - mountPath: /var/lib/cosi + name: socket + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: objectstorage-provisioner-sidecar + image: gcr.io/k8s-staging-sig-storage/objectstorage-sidecar:latest + imagePullPolicy: IfNotPresent + args: + - "--v=$(OBJECTSTORAGE_PROVISIONER_SIDECAR_LOG_LEVEL)" + volumeMounts: + - mountPath: /var/lib/cosi + name: socket + volumes: + - name: socket + emptyDir: {} diff --git a/kustomize/base/kustomization.yaml b/kustomize/base/kustomization.yaml new file mode 100644 index 0000000..366895c --- /dev/null +++ b/kustomize/base/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: scality-object-storage + +resources: + - namespace.yaml + - serviceaccount.yaml + - rbac.yaml + - deployment.yaml diff --git a/kustomize/base/namespace.yml b/kustomize/base/namespace.yml new file mode 100644 index 0000000..41f56d1 --- /dev/null +++ b/kustomize/base/namespace.yml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: scality-object-storage diff --git a/kustomize/base/rbac.yaml b/kustomize/base/rbac.yaml new file mode 100644 index 0000000..45b4475 --- /dev/null +++ b/kustomize/base/rbac.yaml @@ -0,0 +1,28 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: scality-object-storage-provisioner-role +rules: + - apiGroups: ["objectstorage.k8s.io"] + resources: ["buckets", "bucketaccesses", "bucketclaims", "bucketaccessclasses", "buckets/status", "bucketaccesses/status", "bucketclaims/status", "bucketaccessclasses/status"] + verbs: ["get", "list", "watch", "update", "create", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [""] + resources: ["secrets", "events"] + verbs: ["get", "delete", "update", "create"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: scality-object-storage-provisioner-role-binding +subjects: + - kind: ServiceAccount + name: scality-object-storage-provisioner + namespace: default +roleRef: + kind: ClusterRole + name: scality-object-storage-provisioner-role + apiGroup: rbac.authorization.k8s.io diff --git a/kustomize/base/serviceaccount.yaml b/kustomize/base/serviceaccount.yaml new file mode 100644 index 0000000..78f7a7d --- /dev/null +++ b/kustomize/base/serviceaccount.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: scality-object-storage-provisioner + namespace: default diff --git a/kustomize/overlays/kustomization.yaml b/kustomize/overlays/kustomization.yaml new file mode 100644 index 0000000..cfa5ef2 --- /dev/null +++ b/kustomize/overlays/kustomization.yaml @@ -0,0 +1,37 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +configMapGenerator: +- name: scality-cosi-driver-properties + env: scality-cosi-driver.properties +generatorOptions: + disableNameSuffixHash: true + labels: + generated-by: "kustomize" + +resources: + - ../base + +commonLabels: + app.kubernetes.io/version: main + app.kubernetes.io/component: driver + app.kubernetes.io/name: scality-cosi-driver + app.kubernetes.io/part-of: container-object-storage-interface + app.kubernetes.io/managed-by: kustomize + +vars: +- name: COSI_DRIVER_LOG_LEVEL + objref: + name: scality-cosi-driver-properties + kind: ConfigMap + apiVersion: v1 + fieldref: + fieldpath: data.COSI_DRIVER_LOG_LEVEL + +- name: OBJECTSTORAGE_PROVISIONER_SIDECAR_LOG_LEVEL + objref: + name: scality-cosi-driver-properties + kind: ConfigMap + apiVersion: v1 + fieldref: + fieldpath: data.OBJECTSTORAGE_PROVISIONER_SIDECAR_LOG_LEVEL diff --git a/kustomize/overlays/scality-cosi-driver.properties b/kustomize/overlays/scality-cosi-driver.properties new file mode 100644 index 0000000..58af9b7 --- /dev/null +++ b/kustomize/overlays/scality-cosi-driver.properties @@ -0,0 +1,2 @@ +COSI_DRIVER_LOG_LEVEL=5 +OBJECTSTORAGE_PROVISIONER_SIDECAR_LOG_LEVEL=5 diff --git a/resources/cosi-driver.properties b/resources/cosi-driver.properties deleted file mode 100644 index b1f3c88..0000000 --- a/resources/cosi-driver.properties +++ /dev/null @@ -1,4 +0,0 @@ -OBJECTSTORAGE_PROVISIONER_IMAGE_ORG=quay.io/containerobjectstorage -OBJECTSTORAGE_PROVISIONER_IMAGE_VERSION=canary -SCALITY_DRIVER_IMAGE_ORG=ghcr.io/scality/cosi -SCALITY_DRIVER_IMAGE_VERSION=latest diff --git a/resources/deployment.yaml b/resources/deployment.yaml deleted file mode 100644 index 5cb8bf7..0000000 --- a/resources/deployment.yaml +++ /dev/null @@ -1,60 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: objectstorage-provisioner - labels: - app.kubernetes.io/part-of: container-object-storage-interface - app.kubernetes.io/component: driver-scality - app.kubernetes.io/version: main - app.kubernetes.io/name: cosi-driver-scality -spec: - replicas: 1 - minReadySeconds: 30 - progressDeadlineSeconds: 600 - revisionHistoryLimit: 3 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - selector: - matchLabels: - app.kubernetes.io/part-of: container-object-storage-interface - app.kubernetes.io/component: driver-scality - app.kubernetes.io/version: main - app.kubernetes.io/name: cosi-driver-scality - template: - metadata: - labels: - app.kubernetes.io/part-of: container-object-storage-interface - app.kubernetes.io/component: driver-scality - app.kubernetes.io/version: main - app.kubernetes.io/name: cosi-driver-scality - spec: - serviceAccountName: objectstorage-provisioner-sa - volumes: - - name: socket - emptyDir: {} - containers: - - name: scality-cosi-driver - image: $(SCALITY_IMAGE_ORG)/scality-cosi-driver:$(SCALITY_IMAGE_VERSION) - imagePullPolicy: IfNotPresent - args: - - "--driver-prefix=cosi" - - "--v=5" # Verbose mode - volumeMounts: - - mountPath: /var/lib/cosi - name: socket - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: objectstorage-provisioner-sidecar - image: gcr.io/k8s-staging-sig-storage/objectstorage-sidecar:latest - imagePullPolicy: IfNotPresent - args: - - "--v=5" # Verbose mode - volumeMounts: - - mountPath: /var/lib/cosi - name: socket diff --git a/resources/kustomizeconfig.yaml b/resources/kustomizeconfig.yaml deleted file mode 100644 index 3d38939..0000000 --- a/resources/kustomizeconfig.yaml +++ /dev/null @@ -1,3 +0,0 @@ -varReference: -- path: spec/template/spec/containers/image - kind: Deployment diff --git a/resources/ns.yaml b/resources/ns.yaml deleted file mode 100644 index a6aec8e..0000000 --- a/resources/ns.yaml +++ /dev/null @@ -1,10 +0,0 @@ ---- -apiVersion: v1 -kind: Namespace -metadata: - name: scality-cosi-driver - labels: - app.kubernetes.io/part-of: container-object-storage-interface - app.kubernetes.io/component: driver-scality - app.kubernetes.io/version: main - app.kubernetes.io/name: cosi-driver-scality diff --git a/resources/rbac.yaml b/resources/rbac.yaml deleted file mode 100644 index 1e0117a..0000000 --- a/resources/rbac.yaml +++ /dev/null @@ -1,38 +0,0 @@ ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: objectstorage-provisioner-role - labels: - app.kubernetes.io/part-of: container-object-storage-interface - app.kubernetes.io/component: driver-scality - app.kubernetes.io/version: main - app.kubernetes.io/name: cosi-driver-scality -rules: -- apiGroups: ["objectstorage.k8s.io"] - resources: ["buckets", "bucketaccesses", "bucketclaims", "bucketaccessclasses", "buckets/status", "bucketaccesses/status", "bucketclaims/status", "bucketaccessclasses/status"] - verbs: ["get", "list", "watch", "update", "create", "delete"] -- apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] -- apiGroups: [""] - resources: ["secrets", "events"] - verbs: ["get", "delete", "update", "create"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: objectstorage-provisioner-role-binding - labels: - app.kubernetes.io/part-of: container-object-storage-interface - app.kubernetes.io/component: driver-scality - app.kubernetes.io/version: main - app.kubernetes.io/name: cosi-driver-scality -subjects: - - kind: ServiceAccount - name: objectstorage-provisioner-sa - namespace: default # must set to default. see https://github.com/kubernetes-sigs/kustomize/issues/1377#issuecomment-694731163 -roleRef: - kind: ClusterRole - name: objectstorage-provisioner-role - apiGroup: rbac.authorization.k8s.io diff --git a/resources/sa.yaml b/resources/sa.yaml deleted file mode 100644 index 3bf8261..0000000 --- a/resources/sa.yaml +++ /dev/null @@ -1,11 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: objectstorage-provisioner-sa - namespace: default # must set to default. see https://github.com/kubernetes-sigs/kustomize/issues/1377#issuecomment-694731163 - labels: - app.kubernetes.io/part-of: container-object-storage-interface - app.kubernetes.io/component: driver-scality - app.kubernetes.io/version: main - app.kubernetes.io/name: cosi-driver-scality From 737a340867244a234200e4ae06aa4c5d0673b717 Mon Sep 17 00:00:00 2001 From: Anurag Mittal Date: Mon, 4 Nov 2024 17:38:40 +0100 Subject: [PATCH 3/3] COSI-11: deploy-COSI-resources-in-CI - added a script for setting up the COSI driver - added a new workflow step to run the script --- .github/scripts/setup_cosi_resources.sh | 51 +++++++++++++++++++++++++ .github/workflows/ci-e2e-tests.yml | 7 ++++ 2 files changed, 58 insertions(+) create mode 100644 .github/scripts/setup_cosi_resources.sh diff --git a/.github/scripts/setup_cosi_resources.sh b/.github/scripts/setup_cosi_resources.sh new file mode 100644 index 0000000..649395e --- /dev/null +++ b/.github/scripts/setup_cosi_resources.sh @@ -0,0 +1,51 @@ +#!/bin/bash +set -e + +# Define log file for debugging +LOG_FILE=".github/e2e_tests/artifacts/logs/kind_cluster_logs/cosi_deployment/setup_debug.log" +mkdir -p "$(dirname "$LOG_FILE")" # Ensure the log directory exists + +# Error handling function +error_handler() { + echo "An error occurred during the COSI setup. Check the log file for details." | tee -a "$LOG_FILE" + echo "Failed command: $BASH_COMMAND" | tee -a "$LOG_FILE" + exit 1 +} + +# Trap errors and call the error handler +trap 'error_handler' ERR + +# Log command execution to the log file for debugging +log_and_run() { + echo "Running: $*" | tee -a "$LOG_FILE" + "$@" | tee -a "$LOG_FILE" +} + +# Step 1: Install COSI CRDs +log_and_run echo "Installing COSI CRDs..." +log_and_run kubectl create -k github.com/kubernetes-sigs/container-object-storage-interface-api +log_and_run kubectl create -k github.com/kubernetes-sigs/container-object-storage-interface-controller + +# Step 2: Verify COSI Controller Pod Status +log_and_run echo "Verifying COSI Controller Pod status..." +log_and_run kubectl wait --namespace default --for=condition=ready pod -l app.kubernetes.io/name=container-object-storage-interface-controller --timeout=10s +log_and_run kubectl get pods --namespace default + +# Step 3: Build COSI driver Docker image +log_and_run echo "Building COSI driver image..." +log_and_run docker build -t ghcr.io/scality/cosi:latest . + +# Step 4: Load COSI driver image into KIND cluster +log_and_run echo "Loading COSI driver image into KIND cluster..." +log_and_run kind load docker-image ghcr.io/scality/cosi:latest --name object-storage-cluster + +# Step 5: Run COSI driver +log_and_run echo "Applying COSI driver manifests..." +log_and_run kubectl apply -k . + +# Step 6: Verify COSI driver Pod Status +log_and_run echo "Verifying COSI driver Pod status..." +log_and_run kubectl wait --namespace scality-object-storage --for=condition=ready pod --selector=app.kubernetes.io/name=scality-cosi-driver --timeout=20s +log_and_run kubectl get pods -n scality-object-storage + +log_and_run echo "COSI setup completed successfully." diff --git a/.github/workflows/ci-e2e-tests.yml b/.github/workflows/ci-e2e-tests.yml index 860e1de..92ecc23 100644 --- a/.github/workflows/ci-e2e-tests.yml +++ b/.github/workflows/ci-e2e-tests.yml @@ -37,6 +37,13 @@ jobs: with: detached: true + + - name: Setup COSI Controller, CRDs and Driver + run: | + pwd + chmod +x .github/scripts/setup_cosi_resources.sh + .github/scripts/setup_cosi_resources.sh + - name: Capture Kubernetes Logs in artifacts directory run: | chmod +x .github/scripts/capture_k8s_logs.sh