Update Helm Charts with initial support for full config files (#1083)

* Update Helm Charts with initial support for full config files * Temp. disable K8s testing in compliance workflows until cluster is up * Temp. disable `TestWorkerRunKubernetesExecutor` test - Re-enable when in-cluster K8s is up and running! * Temp. disable `kubernetes_test.go`
ohsu-comp-bio · Oct 22, 2024 · 5a8c6b8 · 5a8c6b8
1 parent 2531a4b
commit 5a8c6b8
Show file tree

Hide file tree

Showing 14 changed files with 228 additions and 151 deletions.
diff --git a/.github/workflows/compliance.yaml b/.github/workflows/compliance.yaml
@@ -37,6 +37,8 @@ jobs:
       matrix:
         version: [1.0.0, 1.1.0]
         db: ["boltdb", "mongodb"]
+        # Temporary: Only test local compute backend until we bring up a K8s cluster in Github Actions
+        # compute: ["local", "kubernetes"]
         compute: ["local"]
         storage: ["local", "s3"]
     needs: build

diff --git a/.gitignore b/.gitignore
@@ -20,6 +20,5 @@ storage/_test_download
 docker/funnel
 deployments/kubernetes/**/funnel
 deployments/kubernetes/**/*.json
-funnel
 dist/
 .vscode
diff --git a/compute/kubernetes/backend.go b/compute/kubernetes/backend.go
@@ -147,11 +147,17 @@ func (b *Backend) createJob(task *tes.Task) (*v1.Job, error) {
 		"Cpus":      res.GetCpuCores(),
 		"RamGb":     res.GetRamGb(),
 		"DiskGb":    res.GetDiskGb(),
+		// TODO: How to handle multiple executors/images?
+		"Image": task.Executors[0].Image,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("executing template: %v", err)
 	}
 
+	fmt.Println("DEBUG: task.Id:", task.Id)
+	fmt.Println("DEBUG: b.template:", b.template)
+	fmt.Println("DEBUG: buf.String():", buf.String())
+
 	decode := scheme.Codecs.UniversalDeserializer().Decode
 	obj, _, err := decode(buf.Bytes(), nil, nil)
 	if err != nil {
@@ -165,7 +171,7 @@ func (b *Backend) createJob(task *tes.Task) (*v1.Job, error) {
 	return job, nil
 }
 
-// Submit submits a task to the as a kubernetes v1/batch job.
+// Submit submits a task to the server as a kubernetes v1/batch job.
 func (b *Backend) Submit(ctx context.Context, task *tes.Task) error {
 	job, err := b.createJob(task)
 	if err != nil {
@@ -176,7 +182,7 @@ func (b *Backend) Submit(ctx context.Context, task *tes.Task) error {
 		FieldManager: task.Id,
 	})
 	if err != nil {
-		return fmt.Errorf("creating job: %v", err)
+		return fmt.Errorf("creating job in backend: %v", err)
 	}
 	return nil
 }

diff --git a/config/default-config.yaml b/config/default-config.yaml
@@ -3,7 +3,7 @@
 Database: boltdb
 
 # The name of the active compute backend
-# Available backends: local, htcondor, slurm, pbs, gridengine, manual, aws-batch
+# Available backends: local, htcondor, slurm, pbs, gridengine, manual, aws-batch, kubernetes
 Compute: local
 
 # The name of the active event writer backend(s).

diff --git a/config/internal/bundle.go b/config/internal/bundle.go
diff --git a/config/kubernetes-executor-template.yaml b/config/kubernetes-executor-template.yaml
@@ -11,7 +11,7 @@ spec:
   template:
     spec:
       restartPolicy: Never
-      serviceAccountName: {{.ServiceAccount}}
+      serviceAccountName: funnel-sa
       containers: 
       - name: funnel-worker-{{.TaskId}}
         # TODO: Should this be configurable?
@@ -37,4 +37,4 @@ spec:
       volumes: 
       - name: storage
         persistentVolumeClaim:
-          claimName: storage-pvc 
+          claimName: funnel-pvc 
diff --git a/config/kubernetes-template.yaml b/config/kubernetes-template.yaml
@@ -9,17 +9,17 @@ spec:
   completions: 1
   template:
     spec:
-      serviceAccountName: {{.ServiceAccount}}
+      serviceAccountName: funnel-sa
       restartPolicy: Never
       containers: 
         - name: funnel-worker-{{.TaskId}}
-          image: quay.io/ohsu-comp-bio/funnel:latest
+          image: quay.io/ohsu-comp-bio/funnel:{{.Image}}
           imagePullPolicy: Always
           args:
             - "worker"
             - "run"
             - "--config"
-            - "/etc/config/funnel-worker-config.yml"
+            - "/etc/config/funnel-worker.yaml"
             - "--taskID"
             - {{.TaskId}}
           resources:
@@ -40,4 +40,4 @@ spec:
             name: funnel-config
         - name: funnel-storage-{{.TaskId}}
           persistentVolumeClaim:
-            claimName: storage-pvc 
+            claimName: funnel-pvc 
diff --git a/deployments/kubernetes/README.md b/deployments/kubernetes/README.md
@@ -23,22 +23,24 @@ kubectl apply -f funnel-service.yml
 
 ## 2. Create Funnel config files
 
-> *[funnel-server-config.yml](https://github.com/ohsu-comp-bio/funnel/blob/develop/deployments/kubernetes/funnel-server-config.yml)*
+> *[funnel-server.yaml](https://github.com/ohsu-comp-bio/funnel/blob/develop/deployments/kubernetes/funnel-server.yaml)*
 
-> *[funnel-worker-config.yml](https://github.com/ohsu-comp-bio/funnel/blob/develop/deployments/kubernetes/funnel-worker-config.yml)*
+> *[funnel-worker.yaml](https://github.com/ohsu-comp-bio/funnel/blob/develop/deployments/kubernetes/funnel-worker.yaml)*
 
 Get the clusterIP:
 
 ```sh
 export HOSTNAME=$(kubectl get services funnel --output=jsonpath='{.spec.clusterIP}')
 
-sed -i "s|\${HOSTNAME}|${HOSTNAME}|g" funnel-worker-config.yml
+sed -i "s|\${HOSTNAME}|${HOSTNAME}|g" funnel-worker.yaml
 ```
 
+Use this value to configure the server hostname of the worker config.
+
 ## 3. Create a ConfigMap
 
 ```sh
-kubectl create configmap funnel-config --from-file=funnel-server-config.yml --from-file=funnel-worker-config.yml
+kubectl create configmap funnel-config --from-file=funnel-server.yaml --from-file=funnel-worker.yaml
 ```
 
 ## 4. Create a Service Account for Funnel

diff --git a/deployments/kubernetes/funnel-deployment.yml b/deployments/kubernetes/funnel-deployment.yml
@@ -24,7 +24,7 @@ spec:
             - 'server'
             - 'run'
             - '--config'
-            - '/etc/config/funnel-server-config.yml'
+            - '/etc/config/funnel-server.yaml'
           resources: 
             requests: 
               cpu: 500m

diff --git a/deployments/kubernetes/funnel-server-config.yml b/deployments/kubernetes/funnel-server-config.yml
@@ -12,48 +12,3 @@ Kubernetes:
   DisableReconciler: false
   ReconcileRate: 5m
   Namespace: default
-  Template: |
-    apiVersion: batch/v1
-    kind: Job
-    metadata:
-      ## DO NOT CHANGE NAME
-      name: {{.TaskId}}
-      namespace: {{.Namespace}}
-    spec:
-      backoffLimit: 0
-      completions: 1
-      template:
-        spec:
-          restartPolicy: Never
-          serviceAccountName: funnel-sa
-          containers:
-            - name: {{printf "funnel-worker-%s" .TaskId}}
-              image: quay.io/ohsu-comp-bio/funnel:latest
-              imagePullPolicy: IfNotPresent
-              args:
-                - "worker"
-                - "run"
-                - "--config"
-                - "/etc/config/funnel-worker-config.yml"
-                - "--taskID"
-                - {{.TaskId}}
-              resources:
-                  requests:
-                    cpu: {{if ne .Cpus 0 -}}{{.Cpus}}{{ else }}{{"100m"}}{{end}}
-                    memory: {{if ne .RamGb 0.0 -}}{{printf "%.0fG" .RamGb}}{{else}}{{"16M"}}{{end}}
-                    ephemeral-storage: {{if ne .DiskGb 0.0 -}}{{printf "%.0fG" .DiskGb}}{{else}}{{"100M"}}{{end}}
-              volumeMounts:
-                - name: {{printf "funnel-storage-%s" .TaskId}}
-                  mountPath: {{printf "/opt/funnel/funnel-work-dir/%s" .TaskId}}
-                - name: config-volume
-                  mountPath: /etc/config
-
-              securityContext:
-                privileged: true
-
-          volumes:
-            - name: {{printf "funnel-storage-%s" .TaskId}}
-              emptyDir: {}
-            - name: config-volume
-              configMap:
-                name: funnel-config