From 9f87ed84fe0d358e220ed5b7b86247490321e07f Mon Sep 17 00:00:00 2001 From: Taylor Shrauner-Biggs Date: Thu, 16 Jan 2025 14:30:55 -0600 Subject: [PATCH] make changes to allow for "local" vs. "remote" in values, bump to version 1.2.0 --- charts/hauler/Chart.yaml | 4 +- charts/hauler/README.md | 2 +- charts/hauler/app-readme.md | 2 +- .../hauler-fileserver-deployment.yaml | 23 +- .../hauler-registry-deployment.yaml | 39 +-- .../hauler/templates/hauler/hauler-jobs.yaml | 246 ++++++++---------- charts/hauler/values.yaml | 46 ++-- 7 files changed, 147 insertions(+), 215 deletions(-) diff --git a/charts/hauler/Chart.yaml b/charts/hauler/Chart.yaml index 8638442..2f8ddad 100644 --- a/charts/hauler/Chart.yaml +++ b/charts/hauler/Chart.yaml @@ -3,5 +3,5 @@ name: hauler-helm description: Hauler Helm Chart - Airgap Swiss Army Knife icon: https://raw.githubusercontent.com/hauler-dev/hauler/main/static/rgs-hauler-logo-icon.svg type: application -version: 1.1.1 -appVersion: 1.1.1 +version: 1.2.0 +appVersion: 1.2.0 diff --git a/charts/hauler/README.md b/charts/hauler/README.md index 34f0119..8531df2 100644 --- a/charts/hauler/README.md +++ b/charts/hauler/README.md @@ -4,7 +4,7 @@ | Type | Chart Version | App Version | | ----------- | ------------- | ----------- | -| application | `1.1.1` | `1.1.1` | +| application | `1.2.0` | `1.2.0` | ## Installing the Chart diff --git a/charts/hauler/app-readme.md b/charts/hauler/app-readme.md index 34f0119..8531df2 100644 --- a/charts/hauler/app-readme.md +++ b/charts/hauler/app-readme.md @@ -4,7 +4,7 @@ | Type | Chart Version | App Version | | ----------- | ------------- | ----------- | -| application | `1.1.1` | `1.1.1` | +| application | `1.2.0` | `1.2.0` | ## Installing the Chart diff --git a/charts/hauler/templates/hauler-fileserver/hauler-fileserver-deployment.yaml b/charts/hauler/templates/hauler-fileserver/hauler-fileserver-deployment.yaml index 849cb84..b14b5de 100644 --- a/charts/hauler/templates/hauler-fileserver/hauler-fileserver-deployment.yaml +++ b/charts/hauler/templates/hauler-fileserver/hauler-fileserver-deployment.yaml @@ -18,13 +18,11 @@ spec: app: hauler-fileserver {{- include "hauler.selectorLabels" . | nindent 8 }} spec: - {{- if or .Values.haulerJobs.hauls.enabled .Values.haulerJobs.manifests.enabled }} + {{- if and (or .Values.haulerJobs.remote .Values.haulerJobs.local) (or (and .Values.haulerJobs.local .Values.haulerJobs.local.path) (and .Values.haulerJobs.remote .Values.haulerJobs.remote.artifacts (gt (len .Values.haulerJobs.remote.artifacts) 0)) (and .Values.haulerJobs.remote .Values.haulerJobs.remote.manifests (gt (len .Values.haulerJobs.remote.manifests) 0)) (and .Values.haulerJobs.local .Values.haulerJobs.local.artifacts (gt (len .Values.haulerJobs.local.artifacts) 0)) (and .Values.haulerJobs.local .Values.haulerJobs.local.manifests (gt (len .Values.haulerJobs.local.manifests) 0)) ) }} initContainers: - {{- if .Values.haulerJobs.hauls.enabled }} - - name: wait-for-hauler-hauls-job - image: {{ .Values.hauler.initContainers.image.repository }}:{{ .Values.hauler.initContainers.image.tag }} + - name: wait-for-hauler-loader imagePullPolicy: {{ .Values.hauler.initContainers.imagePullPolicy }} - args: ["wait", "--for=condition=complete", "job", "hauler-hauls-job", "--namespace", "{{ .Release.Namespace }}", "--timeout={{ .Values.hauler.initContainers.timeout }}"] + args: ["wait", "--for=condition=complete", "job", "hauler-loader", "--namespace", "{{ .Release.Namespace }}", "--timeout={{ .Values.hauler.initContainers.timeout }}"] securityContext: allowPrivilegeEscalation: false capabilities: @@ -33,21 +31,6 @@ spec: runAsUser: 1001 seccompProfile: type: RuntimeDefault - {{- end }} - {{- if .Values.haulerJobs.manifests.enabled }} - - name: wait-for-hauler-manifests-job - image: {{ .Values.hauler.initContainers.image.repository }}:{{ .Values.hauler.initContainers.image.tag }} - imagePullPolicy: {{ .Values.hauler.initContainers.imagePullPolicy }} - args: ["wait", "--for=condition=complete", "job", "hauler-manifests-job", "--namespace", "{{ .Release.Namespace }}", "--timeout={{ .Values.hauler.initContainers.timeout }}"] - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: ["ALL"] - runAsNonRoot: true - runAsUser: 1001 - seccompProfile: - type: RuntimeDefault - {{- end }} {{- end }} containers: - name: hauler-fileserver diff --git a/charts/hauler/templates/hauler-registry/hauler-registry-deployment.yaml b/charts/hauler/templates/hauler-registry/hauler-registry-deployment.yaml index 8a0474f..f3db745 100644 --- a/charts/hauler/templates/hauler-registry/hauler-registry-deployment.yaml +++ b/charts/hauler/templates/hauler-registry/hauler-registry-deployment.yaml @@ -18,13 +18,11 @@ spec: app: hauler-registry {{- include "hauler.selectorLabels" . | nindent 8 }} spec: - {{- if or .Values.haulerJobs.hauls.enabled .Values.haulerJobs.manifests.enabled .Values.haulerJobs.localhauls.enabled }} + {{- if and (or .Values.haulerJobs.remote .Values.haulerJobs.local) (or (and .Values.haulerJobs.local .Values.haulerJobs.local.path) (and .Values.haulerJobs.remote .Values.haulerJobs.remote.artifacts (gt (len .Values.haulerJobs.remote.artifacts) 0)) (and .Values.haulerJobs.remote .Values.haulerJobs.remote.manifests (gt (len .Values.haulerJobs.remote.manifests) 0)) (and .Values.haulerJobs.local .Values.haulerJobs.local.artifacts (gt (len .Values.haulerJobs.local.artifacts) 0)) (and .Values.haulerJobs.local .Values.haulerJobs.local.manifests (gt (len .Values.haulerJobs.local.manifests) 0)) ) }} initContainers: - {{- if .Values.haulerJobs.hauls.enabled }} - - name: wait-for-hauler-hauls-job - image: {{ .Values.hauler.initContainers.image.repository }}:{{ .Values.hauler.initContainers.image.tag }} + - name: wait-for-hauler-loader imagePullPolicy: {{ .Values.hauler.initContainers.imagePullPolicy }} - args: ["wait", "--for=condition=complete", "job", "hauler-hauls-job", "--namespace", "{{ .Release.Namespace }}", "--timeout={{ .Values.hauler.initContainers.timeout }}"] + args: ["wait", "--for=condition=complete", "job", "hauler-loader", "--namespace", "{{ .Release.Namespace }}", "--timeout={{ .Values.hauler.initContainers.timeout }}"] securityContext: allowPrivilegeEscalation: false capabilities: @@ -33,35 +31,6 @@ spec: runAsUser: 1001 seccompProfile: type: RuntimeDefault - {{- end }} - {{- if .Values.haulerJobs.manifests.enabled }} - - name: wait-for-hauler-manifests-job - image: {{ .Values.hauler.initContainers.image.repository }}:{{ .Values.hauler.initContainers.image.tag }} - imagePullPolicy: {{ .Values.hauler.initContainers.imagePullPolicy }} - args: ["wait", "--for=condition=complete", "job", "hauler-manifests-job", "--namespace", "{{ .Release.Namespace }}", "--timeout={{ .Values.hauler.initContainers.timeout }}"] - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: ["ALL"] - runAsNonRoot: true - runAsUser: 1001 - seccompProfile: - type: RuntimeDefault - {{- end }} - {{- if .Values.haulerJobs.localhauls.enabled }} - - name: wait-for-hauler-localhauls-job - image: {{ .Values.hauler.initContainers.image.repository }}:{{ .Values.hauler.initContainers.image.tag }} - imagePullPolicy: {{ .Values.hauler.initContainers.imagePullPolicy }} - args: ["wait", "--for=condition=complete", "job", "hauler-localhauls-job", "--namespace", "{{ .Release.Namespace }}", "--timeout={{ .Values.hauler.initContainers.timeout }}"] - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: ["ALL"] - runAsNonRoot: true - runAsUser: 1001 - seccompProfile: - type: RuntimeDefault - {{- end }} {{- end }} containers: - name: hauler-registry @@ -87,4 +56,4 @@ spec: - name: hauler-data persistentVolumeClaim: claimName: hauler-data -{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/hauler/templates/hauler/hauler-jobs.yaml b/charts/hauler/templates/hauler/hauler-jobs.yaml index fe6169e..44a660c 100644 --- a/charts/hauler/templates/hauler/hauler-jobs.yaml +++ b/charts/hauler/templates/hauler/hauler-jobs.yaml @@ -1,8 +1,8 @@ -{{- if .Values.haulerJobs.hauls.enabled }} +{{- if and (or .Values.haulerJobs.remote .Values.haulerJobs.local) (or (and .Values.haulerJobs.local .Values.haulerJobs.local.path) (and .Values.haulerJobs.remote .Values.haulerJobs.remote.artifacts (gt (len .Values.haulerJobs.remote.artifacts) 0)) (and .Values.haulerJobs.remote .Values.haulerJobs.remote.manifests (gt (len .Values.haulerJobs.remote.manifests) 0)) (and .Values.haulerJobs.local .Values.haulerJobs.local.artifacts (gt (len .Values.haulerJobs.local.artifacts) 0)) (and .Values.haulerJobs.local .Values.haulerJobs.local.manifests (gt (len .Values.haulerJobs.local.manifests) 0)) ) }} apiVersion: batch/v1 kind: Job metadata: - name: hauler-hauls-job + name: hauler-loader namespace: {{ .Release.Namespace }} labels: {{- include "hauler.labels" . | nindent 4 }} @@ -10,82 +10,64 @@ spec: template: spec: initContainers: - - name: hauler-fetch-hauls + - name: hauler-fetch image: {{ .Values.haulerJobs.image.repository }}:{{ .Values.haulerJobs.image.tag }} imagePullPolicy: {{ .Values.haulerJobs.imagePullPolicy }} command: ["/bin/sh", "-c"] args: - | - {{- range .Values.haulerJobs.hauls.artifacts }} - curl -o /hauls/{{ .name }} {{ .path }} && + {{- if and .Values.haulerJobs.local .Values.haulerJobs.local.store }} + cp -vr /local-store/* /store/ && + chown -R 1001:1001 /store && {{- end }} - echo hauler fetch completed - volumeMounts: - - name: hauler-data - mountPath: /hauls - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: ["ALL"] - runAsNonRoot: true - runAsUser: 1001 - seccompProfile: - type: RuntimeDefault - containers: - - name: hauler-load-hauls - image: {{ .Values.hauler.image.repository }}:{{ .Values.hauler.image.tag }} - imagePullPolicy: {{ .Values.hauler.imagePullPolicy }} - args: - - "store" - - "load" - {{- range .Values.haulerJobs.hauls.artifacts }} - - "/hauls/{{ .name }}" - {{- end }} - volumeMounts: - - name: hauler-data - mountPath: /hauls - - name: hauler-data - mountPath: /store - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: ["ALL"] - runAsNonRoot: true - runAsUser: 1001 - seccompProfile: - type: RuntimeDefault - restartPolicy: OnFailure - volumes: - - name: hauler-data - persistentVolumeClaim: - claimName: hauler-data -{{- end }} ---- -{{- if .Values.haulerJobs.manifests.enabled }} -apiVersion: batch/v1 -kind: Job -metadata: - name: hauler-manifests-job - namespace: {{ .Release.Namespace }} - labels: - {{- include "hauler.labels" . | nindent 4 }} -spec: - template: - spec: - initContainers: - - name: hauler-fetch-manifests - image: {{ .Values.haulerJobs.image.repository }}:{{ .Values.haulerJobs.image.tag }} - imagePullPolicy: {{ .Values.haulerJobs.imagePullPolicy }} - command: ["/bin/sh", "-c"] - args: - - | - {{- range .Values.haulerJobs.manifests.artifacts }} + {{- if and .Values.haulerJobs.local .Values.haulerJobs.local.artifacts }} + {{- range .Values.haulerJobs.local.artifacts }} + cp -v /tmp/local-artifact-{{ .name }} /artifacts/ && + {{- end}} + {{- end}} + {{- if and .Values.haulerJobs.local .Values.haulerJobs.local.manifests }} + {{- range .Values.haulerJobs.local.manifests }} + cp -v /tmp/local-manifest-{{ .name }} /manifests/ && + {{- end}} + {{- end}} + {{- if and .Values.haulerJobs.remote .Values.haulerJobs.remote.artifacts }} + {{- range .Values.haulerJobs.remote.artifacts }} + curl -o /artifacts/{{ .name }} {{ .path }} && + {{- end }} + {{- end }} + {{- if and .Values.haulerJobs.remote .Values.haulerJobs.remote.manifests }} + {{- range .Values.haulerJobs.remote.manifests }} curl -o /manifests/{{ .name }} {{ .path }} && {{- end }} + {{- end }} echo hauler fetch completed volumeMounts: - - name: hauler-data + {{- if or (and .Values.haulerJobs.remote .Values.haulerJobs.remote.artifacts (gt (len .Values.haulerJobs.remote.artifacts) 0)) (and .Values.haulerJobs.local .Values.haulerJobs.local.artifacts (gt (len .Values.haulerJobs.local.artifacts) 0)) }} + - name: artifact-data + mountPath: /artifacts + {{- end }} + {{- if and .Values.haulerJobs.local .Values.haulerJobs.local.artifacts (gt (len .Values.haulerJobs.local.artifacts) 0) }} + {{- range .Values.haulerJobs.local.artifacts }} + - name: local-artifact-{{ .name }} + mountPath: /tmp/local-artifact-{{ .name }} + {{- end }} + {{- end }} + {{- if or (and .Values.haulerJobs.remote .Values.haulerJobs.remote.manifests (gt (len .Values.haulerJobs.remote.manifests) 0)) (and .Values.haulerJobs.local .Values.haulerJobs.local.manifests (gt (len .Values.haulerJobs.local.manifests) 0)) }} + - name: manifest-data mountPath: /manifests + {{- end }} + {{- if and .Values.haulerJobs.local .Values.haulerJobs.local.manifests (gt (len .Values.haulerJobs.local.manifests) 0) }} + {{- range .Values.haulerJobs.local.manifests }} + - name: local-manifest-{{ .name }} + mountPath: /tmp/local-manifest-{{ .name }} + {{- end }} + {{- end }} + {{- if and .Values.haulerJobs.local .Values.haulerJobs.local.store }} + - name: store-data + mountPath: /local-store + {{- end }} + - name: hauler-data + mountPath: /store securityContext: allowPrivilegeEscalation: false capabilities: @@ -95,19 +77,46 @@ spec: seccompProfile: type: RuntimeDefault containers: - - name: hauler-load-manifests + - name: hauler-load image: {{ .Values.hauler.image.repository }}:{{ .Values.hauler.image.tag }} imagePullPolicy: {{ .Values.hauler.imagePullPolicy }} + command: ["/bin/sh", "-c"] args: - {{- range .Values.haulerJobs.manifests.artifacts }} - - "store" - - "sync" - - "--files" - - "/manifests/{{ .name }}" + - | + {{- if and .Values.haulerJobs.remote .Values.haulerJobs.remote.artifacts (gt (len .Values.haulerJobs.remote.artifacts) 0) }} + {{- range .Values.haulerJobs.remote.artifacts }} + hauler store load /artifacts/{{ .name }} && + {{- end }} + {{- end }} + {{- if and .Values.haulerJobs.local .Values.haulerJobs.local.artifacts (gt (len .Values.haulerJobs.local.artifacts) 0) }} + {{- range .Values.haulerJobs.local.artifacts }} + hauler store load /artifacts/{{ .name }} && + {{- end }} + {{- end }} + {{- if and .Values.haulerJobs.remote .Values.haulerJobs.remote.manifests (gt (len .Values.haulerJobs.remote.manifests) 0) }} + {{- range .Values.haulerJobs.remote.manifests }} + hauler store sync --files /manifests/{{ .name }} && {{- end }} + {{- end }} + {{- if and .Values.haulerJobs.local .Values.haulerJobs.local.manifests (gt (len .Values.haulerJobs.local.manifests) 0) }} + {{- range .Values.haulerJobs.local.manifests }} + hauler store sync --files /manifests/{{ .name }} && + {{- end }} + {{- end }} + echo hauler load completed volumeMounts: - - name: hauler-data + {{- if and .Values.haulerJobs.remote .Values.haulerJobs.remote.artifacts (gt (len .Values.haulerJobs.remote.artifacts) 0) }} + - name: artifact-data + mountPath: /artifacts + {{- end }} + {{- if and .Values.haulerJobs.remote .Values.haulerJobs.remote.manifests (gt (len .Values.haulerJobs.remote.manifests) 0) }} + - name: manifest-data mountPath: /manifests + {{- end }} + {{- if and .Values.haulerJobs.local .Values.haulerJobs.local.store }} + - name: store-data + mountPath: /local-store + {{- end }} - name: hauler-data mountPath: /store securityContext: @@ -120,67 +129,34 @@ spec: type: RuntimeDefault restartPolicy: OnFailure volumes: - - name: hauler-data - persistentVolumeClaim: - claimName: hauler-data -{{- end }} ---- -{{- if .Values.haulerJobs.localhauls.enabled }} -apiVersion: batch/v1 -kind: Job -metadata: - name: hauler-localhauls-job - namespace: {{ .Release.Namespace }} - labels: - {{- include "hauler.labels" . | nindent 4 }} -spec: - template: - spec: - containers: - - name: hauler-fetch-localhauls - image: {{ .Values.haulerJobs.image.repository }}:{{ .Values.haulerJobs.image.tag }} - imagePullPolicy: {{ .Values.haulerJobs.imagePullPolicy }} - command: ["/bin/sh", "-c"] - args: - - | - {{- if .Values.haulerJobs.localhauls.storePath }} - ls -lha /stores && - cp -r /stores/* /store/ && - chown -R 1001:1001 /store && - echo hauler whole-store load completed - {{- end }} - {{- if .Values.haulerJobs.localhauls.haulFiles }} - ls -lha /haulfiles/*.zst && - USER=hauler hauler store load -s /store /haulfiles/*.zst --ignore-errors && - chown -R 1001:1001 /store && - echo hauler haul file load completed - {{- end }} - volumeMounts: - - name: hauler-data - mountPath: /store - {{- if .Values.haulerJobs.localhauls.storePath }} - - name: store-data - mountPath: /stores - {{- end }} - {{- if .Values.haulerJobs.localhauls.haulFiles }} - - name: haul-files - mountPath: /haulfiles - {{- end }} - securityContext: - runAsUser: 0 - restartPolicy: OnFailure - volumes: - - name: hauler-data - persistentVolumeClaim: - claimName: hauler-data - {{- if .Values.haulerJobs.localhauls.storePath }} - - name: store-data - hostPath: - path: {{ .Values.haulerJobs.localhauls.storePath }} + {{- if or (and .Values.haulerJobs.remote .Values.haulerJobs.remote.artifacts (gt (len .Values.haulerJobs.remote.artifacts) 0)) (and .Values.haulerJobs.local .Values.haulerJobs.local.artifacts (gt (len .Values.haulerJobs.local.artifacts) 0)) }} + - name: artifact-data + emptyDir: + {{- end }} + {{- if (and .Values.haulerJobs.local .Values.haulerJobs.local.artifacts (gt (len .Values.haulerJobs.local.artifacts) 0)) }} + {{- range .Values.haulerJobs.local.artifacts }} + - name: local-artifact-{{ .name }} + hostPath: + path: {{ .path }} + {{- end }} {{- end }} - {{- if .Values.haulerJobs.localhauls.haulFiles }} - - name: haul-files + {{- if or (and .Values.haulerJobs.remote .Values.haulerJobs.remote.manifests (gt (len .Values.haulerJobs.remote.manifests) 0)) (and .Values.haulerJobs.local .Values.haulerJobs.local.manifests (gt (len .Values.haulerJobs.local.manifests) 0)) }} + - name: manifest-data + emptyDir: + {{- end }} + {{- if (and .Values.haulerJobs.local .Values.haulerJobs.local.manifests (gt (len .Values.haulerJobs.local.manifests) 0)) }} + {{- range .Values.haulerJobs.local.manifests }} + - name: local-manifest-{{ .name }} + hostPath: + path: {{ .path }} + {{- end }} + {{- end }} + {{- if and .Values.haulerJobs.local .Values.haulerJobs.local.store }} + - name: store-data hostPath: - path: {{ .Values.haulerJobs.localhauls.haulFiles }} + path: {{ .Values.haulerJobs.local.store }} {{- end }} -{{- end }} + - name: hauler-data + persistentVolumeClaim: + claimName: hauler-data +{{- end }} \ No newline at end of file diff --git a/charts/hauler/values.yaml b/charts/hauler/values.yaml index 7779d5d..cfacb59 100644 --- a/charts/hauler/values.yaml +++ b/charts/hauler/values.yaml @@ -4,7 +4,7 @@ hauler: image: repository: hauler/hauler - tag: 1.1.1 + tag: 1.2.0 imagePullPolicy: Always initContainers: @@ -22,42 +22,46 @@ hauler: # Helm Chart Values for the Hauler Jobs # Docs: https://rancherfederal.github.io/hauler-docs/docs/introduction/quickstart +# Hauler Jobs are used to populate the Registry and/or Filestore with content +# Use "local" in order to populate the hauler store with locally available stores, artifacts, and/or manifests. +# Use "remote" if these are remotely (HTTP/S/FTP/etc) available instead +# Simply comment out any unneeded values haulerJobs: image: repository: hauler/hauler-debug - tag: 1.1.1 + tag: 1.2.0 imagePullPolicy: Always - - hauls: - enabled: true + # "remote" refers to URL-based hauler artifacts and/or manifests + remote: artifacts: - path: https://raw.githubusercontent.com/hauler-dev/hauler/main/testdata/haul.tar.zst - name: haul.tar.zst + name: remhaul.tar.zst # - path: /path/to/additional-hauls.tar.zst # name: additional-hauls.tar.zst - - manifests: - enabled: true - artifacts: + manifests: - path: https://raw.githubusercontent.com/hauler-dev/hauler/main/testdata/hauler-manifest.yaml + name: remhauler-manifest.yaml + # - path: /path/to/additional-manifests.yaml + # name: additional-manifests.yaml + # "local" refers to the host the container/kubectl is being run on. Use filesystem paths (hostDir) for these values. + local: + artifacts: + - path: /usr/local/testdata/haul.tar.zst + name: haul.tar.zst + # - path: /path/to/additional-hauls.tar.zst + # name: additional-hauls.tar.zst + manifests: + - path: /usr/local/testdata/hauler-manifest.yaml name: hauler-manifest.yaml # - path: /path/to/additional-manifests.yaml # name: additional-manifests.yaml - # - - # use this when copying haulfiles or stores in directly to a host - # storePath is the "-s" argument to hauler store - # haulFiles is the "hauls" directory (containing zst files) to load - # at least one of storePath or haulFiles must be populated, using both will copy in the store then load the haul files - localhauls: - enabled: false - #storePath: /my/previously/created/hauler/store/dir/ # leave commented out if unused - #haulFiles: /haul/files/to/load/dir/ # leave commented out if unused + # "store" refers to a previously created hauler folder structure. See the "-s" argument to hauler. Only one path is acceptable + store: /my/previously/created/hauler/store/dir/ # Helm Chart Values for the Hauler Fileserver # Docs: https://rancherfederal.github.io/hauler-docs/docs/guides-references/command-line/hauler-store#hauler-store-serve-fileserver - +# haulerFileserver: enabled: true port: 8080 # default port for the fileserver