Add Helm Charts

compute/kubernetes/backend.go

@@ -147,17 +147,11 @@ func (b *Backend) createJob(task *tes.Task) (*v1.Job, error) {
 		"Cpus":      res.GetCpuCores(),
 		"RamGb":     res.GetRamGb(),
 		"DiskGb":    res.GetDiskGb(),
-		// TODO: How to handle multiple executors/images?
-		"Image": task.Executors[0].Image,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("executing template: %v", err)
 	}
 
-	fmt.Println("DEBUG: task.Id:", task.Id)
-	fmt.Println("DEBUG: b.template:", b.template)
-	fmt.Println("DEBUG: buf.String():", buf.String())
-
 	decode := scheme.Codecs.UniversalDeserializer().Decode
 	obj, _, err := decode(buf.Bytes(), nil, nil)
 	if err != nil {

config/kubernetes-executor-template.yaml

@@ -14,8 +14,7 @@ spec:
       serviceAccountName: funnel-sa
       containers: 
       - name: funnel-worker-{{.TaskId}}
-        # TODO: Should this be configurable?
-        image: quay.io/ohsu-comp-bio/funnel:development
+        image: {{.Image}}
         imagePullPolicy: Always
         command: ["/bin/sh", "-c"]
         args: {{.Command}}
@@ -26,15 +25,15 @@ spec:
             memory: {{if ne .RamGb 0.0 -}}{{printf "%.0fG" .RamGb}}{{else}}{{"16M"}}{{end}}
             ephemeral-storage: {{if ne .DiskGb 0.0 -}}{{printf "%.0fG" .DiskGb}}{{else}}{{"100M"}}{{end}}
 
-        volumeMounts:
-        ### DO NOT CHANGE THIS
-        {{range $idx, $item := .Volumes}}
-        - name: storage 
-          mountPath: {{$item.ContainerPath}}
-          subPath: {{$.TaskId}}{{$item.ContainerPath}}
-        {{end}}
+      #   volumeMounts:
+      #   ### DO NOT CHANGE THIS
+      #   {{range $idx, $item := .Volumes}}
+      #   - name: storage 
+      #     mountPath: {{$item.ContainerPath}}
+      #     subPath: {{$.TaskId}}{{$item.ContainerPath}}
+      #   {{end}}
 
-      volumes: 
-      - name: storage
-        persistentVolumeClaim:
-          claimName: funnel-pvc 
+      # volumes: 
+      # - name: storage
+      #   persistentVolumeClaim:
+      #     claimName: funnel-pvc 

config/kubernetes-template.yaml

@@ -29,16 +29,16 @@ spec:
                 memory: {{if ne .RamGb 0.0 -}}{{printf "%.0fG" .RamGb}}{{else}}{{"16M"}}{{end}}
                 ephemeral-storage: {{if ne .DiskGb 0.0 -}}{{printf "%.0fG" .DiskGb}}{{else}}{{"100M"}}{{end}}
           volumeMounts:
-            - name: funnel-storage-{{.TaskId}}
-              mountPath: /opt/funnel/funnel-work-dir/{{.TaskId}}
-              subPath: {{.TaskId}}
+            # - name: funnel-storage-{{.TaskId}}
+            #   mountPath: /opt/funnel/funnel-work-dir/{{.TaskId}}
+            #   subPath: {{.TaskId}}
             - name: config-volume
               mountPath: /etc/config
 
       volumes: 
         - name: config-volume
           configMap:
             name: funnel-config
-        - name: funnel-storage-{{.TaskId}}
-          persistentVolumeClaim:
-            claimName: funnel-pvc 
+        # - name: funnel-storage-{{.TaskId}}
+        #   persistentVolumeClaim:
+        #     claimName: funnel-pvc 

deployments/kubernetes/helm/funnel/Chart.yaml

@@ -0,0 +1,26 @@
+apiVersion: v2
+name: funnel
+description: A toolkit for distributed task execution ⚙️
+keywords:
+  - funnel
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.10
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.11.0"

deployments/kubernetes/helm/funnel/config/funnel-server.yaml

@@ -0,0 +1,90 @@
+Compute: {{ .Values.Compute }}
+
+Kubernetes:
+  Executor: "kubernetes"
+  Namespace: {{ .Release.Namespace }}
+
+Database: {{ .Values.Database }}
+
+EventWriters:
+  {{- range .Values.EventWriters }}
+  - {{ . }}
+  {{- end }}
+
+Logger:
+  Level: {{ .Values.Logger.Level }}
+  OutputFile: {{ .Values.Logger.OutputFile }}
+
+Server:
+  HostName: {{ .Values.Server.Hostname }}
+  HTTPPort: {{ .Values.Server.HttpPort }}
+  RPCPort: {{ .Values.Server.RpcPort }}
+  DisableHTTPCache: {{ .Values.Server.DisableHttpCache }}
+
+RPCClient:
+  ServerAddress: {{ .Values.RPCClient.ServerAddress }}
+  Timeout: {{ .Values.RPCClient.Timeout }}
+  MaxRetries: {{ .Values.RPCClient.MaxRetries }}
+
+Scheduler:
+  ScheduleRate: {{ .Values.Scheduler.ScheduleRate }}
+  ScheduleChunk: {{ .Values.Scheduler.ScheduleChunk }}
+  NodePingTimeout: {{ .Values.Scheduler.NodePingTimeout }}
+  NodeInitTimeout: {{ .Values.Scheduler.NodeInitTimeout }}
+
+Node:
+  ID: {{ .Values.Node.Id }}
+  Timeout: {{ .Values.Node.Timeout }}
+  UpdateRate: {{ .Values.Node.UpdateRate }}
+  Resources:
+    Cpus: {{ .Values.Node.Resources.cpus }}
+    RamGb: {{ .Values.Node.Resources.ramGb }}
+    DiskGb: {{ .Values.Node.Resources.diskGb }}
+
+Worker:
+  WorkDir: {{ .Values.Worker.WorkDir }}
+  PollingRate: {{ .Values.Worker.PollingRate }}
+  LogUpdateRate: {{ .Values.Worker.LogUpdateRate }}
+  LogTailSize: {{ .Values.Worker.LogTailSize }}
+  LeaveWorkDir: {{ .Values.Worker.LeaveWorkDir }}
+  MaxParallelTransfers: {{ .Values.Worker.MaxParallelTransfers }}
+
+BoltDB:
+  Path: {{ .Values.BoltDB.Path }}
+
+AmazonS3:
+  Disabled: {{ .Values.AmazonS3.Disabled }}
+  MaxRetries: {{ .Values.AmazonS3.MaxRetries }}
+  Key: {{ .Values.AmazonS3.Key }}
+  Secret: {{ .Values.AmazonS3.Secret }}
+
+DynamoDB:
+  TableBasename: {{ .Values.DynamoDB.TableBasename }}
+  Region: {{ .Values.DynamoDB.Region }}
+  Key: {{ .Values.DynamoDB.Key }}
+  Secret: {{ .Values.DynamoDB.Secret }}
+
+Elastic:
+  IndexPrefix: {{ .Values.Elastic.IndexPrefix }}
+  URL: {{ .Values.Elastic.Url }}
+
+Datastore:
+  Project: {{ .Values.Datastore.Project }}
+  CredentialsFile: {{ .Values.Datastore.CredentialsFile }}
+
+MongoDB:
+  Addrs:
+  {{- range .Values.MongoDB.Addrs }}
+  - {{ . }}
+  {{- end }}
+  Database: {{ .Values.MongoDB.Database }}
+  Timeout: {{ .Values.MongoDB.Timeout }}
+  Username: {{ .Values.MongoDB.Username }}
+  Password: {{ .Values.MongoDB.Password }}
+
+Kafka:
+  Servers:
+  {{- range .Values.Kafka.Servers }}
+  - {{ . }}
+  {{- end }}
+  Topic: {{ .Values.Kafka.Topic }}