From c868e6ce2885962868677854187b839e237e5f45 Mon Sep 17 00:00:00 2001
From: Anton Dementyev <dementyev@subshell.com>
Date: Wed, 22 Jan 2025 23:05:39 +0700
Subject: [PATCH 01/10] UNSOI-2606: native s3

---
 charts/sophora-importer/Chart.yaml            |  2 +-
 .../templates/statefulset.yaml                | 90 +++++--------------
 charts/sophora-importer/test-values.yaml      |  4 -
 charts/sophora-importer/values.yaml           | 24 +----
 4 files changed, 24 insertions(+), 96 deletions(-)

diff --git a/charts/sophora-importer/Chart.yaml b/charts/sophora-importer/Chart.yaml
index f9be91e..223db02 100644
--- a/charts/sophora-importer/Chart.yaml
+++ b/charts/sophora-importer/Chart.yaml
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.3.3
+version: 2.0.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
diff --git a/charts/sophora-importer/templates/statefulset.yaml b/charts/sophora-importer/templates/statefulset.yaml
index 1e0f821..b810fe3 100644
--- a/charts/sophora-importer/templates/statefulset.yaml
+++ b/charts/sophora-importer/templates/statefulset.yaml
@@ -33,59 +33,6 @@ spec:
       hostAliases: {{- toYaml . | nindent 8 }}
       {{- end }}
       containers:
-      {{- if .Values.sophora.importer.s3Bucket.enabled }}
-      - name: bucket-mount
-        image: "{{ .Values.s3fsImage.repository }}:{{ .Values.s3fsImage.tag }}"
-        env:
-          - name: AWS_S3_BUCKET
-            value: {{ .Values.sophora.importer.s3Bucket.name }}
-          - name: AWS_S3_URL
-            value: {{ .Values.sophora.importer.s3Bucket.url }}
-          - name: S3FS_DEBUG
-            value: "0"
-          - name: AWS_S3_SECRET_ACCESS_KEY
-            valueFrom:
-              secretKeyRef:
-                key: {{ .Values.sophora.importer.s3Bucket.secret.secretAccessKeyKey }}
-                name: {{ .Values.sophora.importer.s3Bucket.secret.name }}
-                optional: false
-          - name: AWS_S3_ACCESS_KEY_ID
-            valueFrom:
-              secretKeyRef:
-                key: {{ .Values.sophora.importer.s3Bucket.secret.accessKeyIdKey }}
-                name: {{ .Values.sophora.importer.s3Bucket.secret.name }}
-                optional: false
-          {{- if .Values.sophora.importer.s3Bucket.extraEnv -}}
-          {{- toYaml .Values.sophora.importer.s3Bucket.extraEnv | nindent 10 }}
-          {{- end }}
-        imagePullPolicy: {{ .Values.s3fsImage.pullPolicy }}
-        lifecycle:
-          postStart:
-            exec:
-              # workaround because the importer can't create the folder by itself.
-              command:
-                - sh
-                - '-c'
-                - |
-                  echo Creating import folders for importer:
-                  {{- range $instance := .Values.sophora.importer.instances }}
-                  {{/* Ensure that every instance folder is available on s3 */}}
-                  mkdir -pv "/opt/s3fs/bucket/{{ $instance }}"
-                  {{- range $folder, $location := $.Values.sophora.importer.createImportFolders }}
-                  mkdir -pv "{{ ((eq $location "s3") | ternary "/opt/s3fs/bucket" "/import") }}/{{ $instance }}/{{ $folder }}"
-                  {{- end }}
-                  {{- end }}
-        resources:
-          {{- toYaml .Values.sophora.importer.s3Bucket.resources | nindent 12 }}
-        volumeMounts:
-          - name: shared-imports
-            mountPath: /opt/s3fs/bucket
-            mountPropagation: Bidirectional
-          - name: local-import-folders
-            mountPath: /import
-        securityContext:
-          privileged: true
-      {{- end }}
       - name: importer
         image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
         imagePullPolicy: {{ .Values.image.pullPolicy }}
@@ -108,6 +55,20 @@ spec:
             {{- else }}
             value: {{ include "sophora-importer.transformationLibsPath" . }}
             {{- end }}
+          {{- if .Values.sophora.importer.s3Bucket.enabled }}
+          - name: AWS_S3_SECRET_ACCESS_KEY
+            valueFrom:
+              secretKeyRef:
+                key: {{ .Values.sophora.importer.s3Bucket.secret.secretAccessKeyKey }}
+                name: {{ .Values.sophora.importer.s3Bucket.secret.name }}
+                optional: false
+          - name: AWS_S3_ACCESS_KEY_ID
+            valueFrom:
+              secretKeyRef:
+                key: {{ .Values.sophora.importer.s3Bucket.secret.accessKeyIdKey }}
+                name: {{ .Values.sophora.importer.s3Bucket.secret.name }}
+                optional: false
+          {{- end }}
           {{ if .Values.sophora.importer.extraEnv -}}
           {{- toYaml .Values.sophora.importer.extraEnv | nindent 10 }}
           {{- end }}
@@ -123,8 +84,6 @@ spec:
         - name: importer-config
           mountPath: /sophora/logback-spring.xml
           subPath: logback-spring.xml
-        - name: shared-imports
-          mountPath: /import/
           mountPropagation: Bidirectional
         - name: local-import-folders
           mountPath: /import-local/
@@ -143,7 +102,6 @@ spec:
         {{- end }}
         securityContext:
           privileged: true
-        {{- if not .Values.sophora.importer.s3Bucket.enabled }}
         lifecycle:
           postStart:
             exec:
@@ -151,16 +109,14 @@ spec:
                 - sh
                 - '-c'
                 - |
-                  echo "Creating import folders for importer:"
-                  {{- range $folder := tuple "import" "import-local" }}
-                  {{- range $instance := $.Values.sophora.importer.instances }}
-                  mkdir -p /{{$folder}}/{{$instance}}/success
-                  mkdir -p /{{$folder}}/{{$instance}}/temp
-                  mkdir -p /{{$folder}}/{{$instance}}/failure
-                  mkdir -p /{{$folder}}/{{$instance}}/incoming
-                  {{ end }}
-                  {{ end }}
-        {{- end }}
+                  echo "Creating local import folders for importer:"
+                  {{- range $.Values.sophora.importer.configuration.importer.instances }}
+                  {{- range $folderName, $folderPath := .folders }}
+                  {{- if and (not (hasPrefix "s3://" $folderPath)) (hasKey (dict "watch" 1 "temp" 1 "success" 1 "failure" 1) $folderName) }}
+                  mkdir -pv /import-local/{{ $folderPath | trimPrefix "/" }}
+                  {{- end }}
+                  {{- end }}
+                  {{- end }}
       initContainers:
       {{/* Transformations Download */}}
       {{- with .Values.transformation }}
@@ -227,8 +183,6 @@ spec:
       - name: importer-config
         configMap:
           name: {{ include "sophora-importer.fullname" . }}
-      - name: shared-imports
-        emptyDir: {}
       - name: local-import-folders
       {{- if not .Values.importPvcSpec }}
         emptyDir: {}
diff --git a/charts/sophora-importer/test-values.yaml b/charts/sophora-importer/test-values.yaml
index 824f7f4..5573ebd 100644
--- a/charts/sophora-importer/test-values.yaml
+++ b/charts/sophora-importer/test-values.yaml
@@ -37,14 +37,10 @@ sophora:
       - name: BBB
         value: b-value
     s3Bucket:
-      name: "sophora-test-importer"
-      url: "https://storage.googleapis.com"
       secret:
         name: "sophora-importer-bucket-credentials"
         secretAccessKeyKey: "secretAccessKey"
         accessKeyIdKey: "accessKeyId"
-    instances:
-      - common
     configuration:
       sophora:
         client:
diff --git a/charts/sophora-importer/values.yaml b/charts/sophora-importer/values.yaml
index fdf5a3e..979610a 100644
--- a/charts/sophora-importer/values.yaml
+++ b/charts/sophora-importer/values.yaml
@@ -14,11 +14,6 @@ downloadViaS3Image:
   tag: "0.0.2"
   pullPolicy: IfNotPresent
 
-s3fsImage:
-  repository: efrecon/s3fs
-  tag: "1.91"
-  pullPolicy: IfNotPresent
-
 nodeSelector: {}
 imagePullSecrets: []
 nameOverride: ""
@@ -45,31 +40,14 @@ sophora:
       usernameKey: "username"
   importer:
     s3Bucket:
-      # If enabled, the importer uses a s3 bucket for incoming imports
+      # enable if importer uses a s3 bucket for imports
       enabled: true
-      name: ""
-      url: "https://storage.googleapis.com"
       secret:
         name: ""
         secretAccessKeyKey: "secretAccessKey"
         accessKeyIdKey: "accessKeyId"
-      extraEnv:
-        - name: S3FS_ARGS
-          value: nonempty
-      resources:
-        requests:
-          cpu: 200m
-          memory: 256Mi
-        limits:
-          memory: 256Mi
     extraEnv:
     loaderPath:
-    instances: []
-    createImportFolders:
-      incoming: s3
-      failure: s3
-      success: local
-      temp: local
     configuration: {}
 
 logbackXml: |

From fb6150642bb0a80978590afac84cec0b69208f95 Mon Sep 17 00:00:00 2001
From: Anton Dementyev <dementyev@subshell.com>
Date: Thu, 23 Jan 2025 16:46:05 +0700
Subject: [PATCH 02/10] UNSOI-2606: use whole path for folders

---
 charts/sophora-importer/templates/statefulset.yaml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/charts/sophora-importer/templates/statefulset.yaml b/charts/sophora-importer/templates/statefulset.yaml
index b810fe3..64d9e04 100644
--- a/charts/sophora-importer/templates/statefulset.yaml
+++ b/charts/sophora-importer/templates/statefulset.yaml
@@ -85,8 +85,6 @@ spec:
           mountPath: /sophora/logback-spring.xml
           subPath: logback-spring.xml
           mountPropagation: Bidirectional
-        - name: local-import-folders
-          mountPath: /import-local/
         {{- if .Values.transformation.data.useSaxon }}
         - name: saxon-licence
           subPath: {{ .Values.transformation.data.saxonLicenceSecretKey }}
@@ -113,7 +111,7 @@ spec:
                   {{- range $.Values.sophora.importer.configuration.importer.instances }}
                   {{- range $folderName, $folderPath := .folders }}
                   {{- if and (not (hasPrefix "s3://" $folderPath)) (hasKey (dict "watch" 1 "temp" 1 "success" 1 "failure" 1) $folderName) }}
-                  mkdir -pv /import-local/{{ $folderPath | trimPrefix "/" }}
+                  mkdir -pv {{ $folderPath }}
                   {{- end }}
                   {{- end }}
                   {{- end }}

From 26ce1856a483b36218e7f8f2c9e374b6de3827e0 Mon Sep 17 00:00:00 2001
From: Anton Dementyev <dementyev@subshell.com>
Date: Thu, 23 Jan 2025 16:49:10 +0700
Subject: [PATCH 03/10] UNSOI-2606: remove remaining mount config

---
 charts/sophora-importer/templates/statefulset.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/charts/sophora-importer/templates/statefulset.yaml b/charts/sophora-importer/templates/statefulset.yaml
index 64d9e04..138bb4f 100644
--- a/charts/sophora-importer/templates/statefulset.yaml
+++ b/charts/sophora-importer/templates/statefulset.yaml
@@ -84,7 +84,6 @@ spec:
         - name: importer-config
           mountPath: /sophora/logback-spring.xml
           subPath: logback-spring.xml
-          mountPropagation: Bidirectional
         {{- if .Values.transformation.data.useSaxon }}
         - name: saxon-licence
           subPath: {{ .Values.transformation.data.saxonLicenceSecretKey }}

From be4928b10f5580b18f4726f8ddf73f7c8a4ac8c8 Mon Sep 17 00:00:00 2001
From: Anton Dementyev <dementyev@subshell.com>
Date: Thu, 23 Jan 2025 16:56:49 +0700
Subject: [PATCH 04/10] UNSOI-2606: test values with s3

---
 charts/sophora-importer/test-values.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/charts/sophora-importer/test-values.yaml b/charts/sophora-importer/test-values.yaml
index 5573ebd..b261a2f 100644
--- a/charts/sophora-importer/test-values.yaml
+++ b/charts/sophora-importer/test-values.yaml
@@ -76,10 +76,10 @@ sophora:
             key: common
             transform: skipTransform
             folders:
-              watch: /import/common/incoming
+              watch: s3://common/incoming
               temp: /import-local/common/temp
-              success: /import-local/common/success
-              failure: /import/common/failure
+              success: s3://common/success
+              failure: s3://common/failure
               xsl: /xsl
             defaultStructureNode: /import
 

From a8e81209069f7b7c65ae204a77716f00f60cb909 Mon Sep 17 00:00:00 2001
From: Jonas Krukenberg <krukenberg@subshell.com>
Date: Thu, 23 Jan 2025 12:29:23 +0100
Subject: [PATCH 05/10] UNSOI-2606: update README

---
 charts/sophora-importer/README.md | 26 ++++----------------------
 1 file changed, 4 insertions(+), 22 deletions(-)

diff --git a/charts/sophora-importer/README.md b/charts/sophora-importer/README.md
index 4ca2d4a..e4a1b8a 100644
--- a/charts/sophora-importer/README.md
+++ b/charts/sophora-importer/README.md
@@ -16,28 +16,10 @@ if you only need the SOAP api. The following directories can be referenced in yo
 
 ## Importer directory paths
 
-On startup, the Sophora Importer assumes that all directories you defined in your `application.yaml` under `folders` already exist.
-These directories will be created automatically by Helm according to your configuration in `sophora.importer.createImportFolders`.
-Use `s3` to create the directory for s3 bucket (`/import/`) or `local` if you don't want to share it (`/import-local/`).
-
-The following example creates directories:
-
-```yaml
-sophora:
-  importer:
-    createImportFolders:
-      temp: local
-      failure: s3
-      incoming: s3
-      success: s3
-```
-
-```
-/import-local/<instance>/temp
-/import/<instance>/failure
-/import/<instance>/incoming
-/import/<instance>/success
-```
+On startup, the Sophora Importer assumes that all directories you defined in your `application.yaml` under `importer.instances[].folders` already exist.
+These directories will be created automatically by Helm for all paths that don't start with `s3://`.
+All paths starting with `/import/` are persisted and contained data will be kept after a restart. 
+Use `s3://` for folders that should be saved in an S3 bucket as configured with `sophora.importer.s3Bucket` configuration options.
 
 ## Import transformation files via S3 or HTTP
 

From 4a111312c24055770f0a2811d4946a22963996a1 Mon Sep 17 00:00:00 2001
From: Jonas Krukenberg <krukenberg@subshell.com>
Date: Thu, 23 Jan 2025 12:29:37 +0100
Subject: [PATCH 06/10] UNSOI-2606: update appVersion

---
 charts/sophora-importer/Chart.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charts/sophora-importer/Chart.yaml b/charts/sophora-importer/Chart.yaml
index 223db02..c5556a7 100644
--- a/charts/sophora-importer/Chart.yaml
+++ b/charts/sophora-importer/Chart.yaml
@@ -20,4 +20,4 @@ version: 2.0.0
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
-appVersion: 4.11.0
+appVersion: 5.0.0

From 3f0f1aa74f7cb43e52aad9bf41c4784e616e867a Mon Sep 17 00:00:00 2001
From: Jonas Krukenberg <krukenberg@subshell.com>
Date: Thu, 23 Jan 2025 12:37:37 +0100
Subject: [PATCH 07/10] UNSOI-2606: simplify s3 bucket configuration with env
 vars

---
 charts/sophora-importer/README.md                  |  2 +-
 charts/sophora-importer/templates/statefulset.yaml | 12 +++++++++---
 charts/sophora-importer/test-values.yaml           |  3 +++
 charts/sophora-importer/values.yaml                |  5 +++--
 4 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/charts/sophora-importer/README.md b/charts/sophora-importer/README.md
index e4a1b8a..4180027 100644
--- a/charts/sophora-importer/README.md
+++ b/charts/sophora-importer/README.md
@@ -6,7 +6,7 @@ Additional environment variables are supported via `sophora.importer.extraEnv`.
 
 ## Importer without s3 bucket
 
-If you don't need a s3 bucket for incoming Sophora documents, you can set `sophora.importer.s3Bucket.enabled` to `false`. This might be useful,
+If you don't need a s3 bucket for incoming Sophora documents, you can omit the configuration of `sophora.importer.s3Bucket.name`. This might be useful,
 if you only need the SOAP api. The following directories can be referenced in your `application.yaml`:
 
 * success: /import/<instance>/success
diff --git a/charts/sophora-importer/templates/statefulset.yaml b/charts/sophora-importer/templates/statefulset.yaml
index 138bb4f..d7de82d 100644
--- a/charts/sophora-importer/templates/statefulset.yaml
+++ b/charts/sophora-importer/templates/statefulset.yaml
@@ -55,19 +55,25 @@ spec:
             {{- else }}
             value: {{ include "sophora-importer.transformationLibsPath" . }}
             {{- end }}
-          {{- if .Values.sophora.importer.s3Bucket.enabled }}
-          - name: AWS_S3_SECRET_ACCESS_KEY
+          {{- if .Values.sophora.importer.s3Bucket.name }}
+          - name: IMPORTER_S3_SECRETACCESSKEY
             valueFrom:
               secretKeyRef:
                 key: {{ .Values.sophora.importer.s3Bucket.secret.secretAccessKeyKey }}
                 name: {{ .Values.sophora.importer.s3Bucket.secret.name }}
                 optional: false
-          - name: AWS_S3_ACCESS_KEY_ID
+          - name: IMPORTER_S3_ACCESSKEYID
             valueFrom:
               secretKeyRef:
                 key: {{ .Values.sophora.importer.s3Bucket.secret.accessKeyIdKey }}
                 name: {{ .Values.sophora.importer.s3Bucket.secret.name }}
                 optional: false
+          - name: IMPORTER_S3_BUCKETNAME
+            value: {{ .Values.sophora.importer.s3Bucket.name }}
+          - name: IMPORTER_S3_HOST
+            value: {{ .Values.sophora.importer.s3Bucket.url }}
+          - name: IMPORTER_S3_REGION
+            value: {{ .Values.sophora.importer.s3Bucket.region }}
           {{- end }}
           {{ if .Values.sophora.importer.extraEnv -}}
           {{- toYaml .Values.sophora.importer.extraEnv | nindent 10 }}
diff --git a/charts/sophora-importer/test-values.yaml b/charts/sophora-importer/test-values.yaml
index b261a2f..8dfedca 100644
--- a/charts/sophora-importer/test-values.yaml
+++ b/charts/sophora-importer/test-values.yaml
@@ -37,6 +37,9 @@ sophora:
       - name: BBB
         value: b-value
     s3Bucket:
+      name: "my-bucket"
+      url: "https://storage.googleapis.com"
+      region: "eu-west-3"
       secret:
         name: "sophora-importer-bucket-credentials"
         secretAccessKeyKey: "secretAccessKey"
diff --git a/charts/sophora-importer/values.yaml b/charts/sophora-importer/values.yaml
index 979610a..97cd510 100644
--- a/charts/sophora-importer/values.yaml
+++ b/charts/sophora-importer/values.yaml
@@ -40,8 +40,9 @@ sophora:
       usernameKey: "username"
   importer:
     s3Bucket:
-      # enable if importer uses a s3 bucket for imports
-      enabled: true
+      name:
+      url:
+      region:
       secret:
         name: ""
         secretAccessKeyKey: "secretAccessKey"

From 8cece8a0e8b61746f743b77502cc6cc0707c6739 Mon Sep 17 00:00:00 2001
From: Jonas Krukenberg <krukenberg@subshell.com>
Date: Thu, 23 Jan 2025 12:39:57 +0100
Subject: [PATCH 08/10] UNSOI-2606: always mount /import-local

---
 charts/sophora-importer/templates/statefulset.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/charts/sophora-importer/templates/statefulset.yaml b/charts/sophora-importer/templates/statefulset.yaml
index d7de82d..93de7b6 100644
--- a/charts/sophora-importer/templates/statefulset.yaml
+++ b/charts/sophora-importer/templates/statefulset.yaml
@@ -90,6 +90,8 @@ spec:
         - name: importer-config
           mountPath: /sophora/logback-spring.xml
           subPath: logback-spring.xml
+        - name: local-import-folders
+          mountPath: /import-local/
         {{- if .Values.transformation.data.useSaxon }}
         - name: saxon-licence
           subPath: {{ .Values.transformation.data.saxonLicenceSecretKey }}

From d2514e52c6ff84fa8934e74e3035de73e6b6564f Mon Sep 17 00:00:00 2001
From: Jonas Krukenberg <krukenberg@subshell.com>
Date: Thu, 23 Jan 2025 12:40:24 +0100
Subject: [PATCH 09/10] UNSOI-2606: fix readme

---
 charts/sophora-importer/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charts/sophora-importer/README.md b/charts/sophora-importer/README.md
index 4180027..2b09a86 100644
--- a/charts/sophora-importer/README.md
+++ b/charts/sophora-importer/README.md
@@ -18,7 +18,7 @@ if you only need the SOAP api. The following directories can be referenced in yo
 
 On startup, the Sophora Importer assumes that all directories you defined in your `application.yaml` under `importer.instances[].folders` already exist.
 These directories will be created automatically by Helm for all paths that don't start with `s3://`.
-All paths starting with `/import/` are persisted and contained data will be kept after a restart. 
+All paths starting with `/import-local/` are persisted and contained data will be kept after a restart. 
 Use `s3://` for folders that should be saved in an S3 bucket as configured with `sophora.importer.s3Bucket` configuration options.
 
 ## Import transformation files via S3 or HTTP

From 91067b677d5e3cb0f47f3b2d1ec9b2d2d07d0fec Mon Sep 17 00:00:00 2001
From: Jonas Krukenberg <krukenberg@subshell.com>
Date: Thu, 23 Jan 2025 12:41:26 +0100
Subject: [PATCH 10/10] UNSOI-2606: rename variable

---
 charts/sophora-importer/templates/statefulset.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/charts/sophora-importer/templates/statefulset.yaml b/charts/sophora-importer/templates/statefulset.yaml
index 93de7b6..4bf6744 100644
--- a/charts/sophora-importer/templates/statefulset.yaml
+++ b/charts/sophora-importer/templates/statefulset.yaml
@@ -116,8 +116,8 @@ spec:
                 - |
                   echo "Creating local import folders for importer:"
                   {{- range $.Values.sophora.importer.configuration.importer.instances }}
-                  {{- range $folderName, $folderPath := .folders }}
-                  {{- if and (not (hasPrefix "s3://" $folderPath)) (hasKey (dict "watch" 1 "temp" 1 "success" 1 "failure" 1) $folderName) }}
+                  {{- range $folderType, $folderPath := .folders }}
+                  {{- if and (not (hasPrefix "s3://" $folderPath)) (hasKey (dict "watch" 1 "temp" 1 "success" 1 "failure" 1) $folderType) }}
                   mkdir -pv {{ $folderPath }}
                   {{- end }}
                   {{- end }}