diff --git a/charts/sophora-importer/Chart.yaml b/charts/sophora-importer/Chart.yaml index f9be91e..c5556a7 100644 --- a/charts/sophora-importer/Chart.yaml +++ b/charts/sophora-importer/Chart.yaml @@ -15,9 +15,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.3.3 +version: 2.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. -appVersion: 4.11.0 +appVersion: 5.0.0 diff --git a/charts/sophora-importer/README.md b/charts/sophora-importer/README.md index 4ca2d4a..2b09a86 100644 --- a/charts/sophora-importer/README.md +++ b/charts/sophora-importer/README.md @@ -6,7 +6,7 @@ Additional environment variables are supported via `sophora.importer.extraEnv`. ## Importer without s3 bucket -If you don't need a s3 bucket for incoming Sophora documents, you can set `sophora.importer.s3Bucket.enabled` to `false`. This might be useful, +If you don't need a s3 bucket for incoming Sophora documents, you can omit the configuration of `sophora.importer.s3Bucket.name`. This might be useful, if you only need the SOAP api. The following directories can be referenced in your `application.yaml`: * success: /import//success @@ -16,28 +16,10 @@ if you only need the SOAP api. The following directories can be referenced in yo ## Importer directory paths -On startup, the Sophora Importer assumes that all directories you defined in your `application.yaml` under `folders` already exist. -These directories will be created automatically by Helm according to your configuration in `sophora.importer.createImportFolders`. -Use `s3` to create the directory for s3 bucket (`/import/`) or `local` if you don't want to share it (`/import-local/`). - -The following example creates directories: - -```yaml -sophora: - importer: - createImportFolders: - temp: local - failure: s3 - incoming: s3 - success: s3 -``` - -``` -/import-local//temp -/import//failure -/import//incoming -/import//success -``` +On startup, the Sophora Importer assumes that all directories you defined in your `application.yaml` under `importer.instances[].folders` already exist. +These directories will be created automatically by Helm for all paths that don't start with `s3://`. +All paths starting with `/import-local/` are persisted and contained data will be kept after a restart. +Use `s3://` for folders that should be saved in an S3 bucket as configured with `sophora.importer.s3Bucket` configuration options. ## Import transformation files via S3 or HTTP diff --git a/charts/sophora-importer/templates/statefulset.yaml b/charts/sophora-importer/templates/statefulset.yaml index 1e0f821..4bf6744 100644 --- a/charts/sophora-importer/templates/statefulset.yaml +++ b/charts/sophora-importer/templates/statefulset.yaml @@ -33,59 +33,6 @@ spec: hostAliases: {{- toYaml . | nindent 8 }} {{- end }} containers: - {{- if .Values.sophora.importer.s3Bucket.enabled }} - - name: bucket-mount - image: "{{ .Values.s3fsImage.repository }}:{{ .Values.s3fsImage.tag }}" - env: - - name: AWS_S3_BUCKET - value: {{ .Values.sophora.importer.s3Bucket.name }} - - name: AWS_S3_URL - value: {{ .Values.sophora.importer.s3Bucket.url }} - - name: S3FS_DEBUG - value: "0" - - name: AWS_S3_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - key: {{ .Values.sophora.importer.s3Bucket.secret.secretAccessKeyKey }} - name: {{ .Values.sophora.importer.s3Bucket.secret.name }} - optional: false - - name: AWS_S3_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - key: {{ .Values.sophora.importer.s3Bucket.secret.accessKeyIdKey }} - name: {{ .Values.sophora.importer.s3Bucket.secret.name }} - optional: false - {{- if .Values.sophora.importer.s3Bucket.extraEnv -}} - {{- toYaml .Values.sophora.importer.s3Bucket.extraEnv | nindent 10 }} - {{- end }} - imagePullPolicy: {{ .Values.s3fsImage.pullPolicy }} - lifecycle: - postStart: - exec: - # workaround because the importer can't create the folder by itself. - command: - - sh - - '-c' - - | - echo Creating import folders for importer: - {{- range $instance := .Values.sophora.importer.instances }} - {{/* Ensure that every instance folder is available on s3 */}} - mkdir -pv "/opt/s3fs/bucket/{{ $instance }}" - {{- range $folder, $location := $.Values.sophora.importer.createImportFolders }} - mkdir -pv "{{ ((eq $location "s3") | ternary "/opt/s3fs/bucket" "/import") }}/{{ $instance }}/{{ $folder }}" - {{- end }} - {{- end }} - resources: - {{- toYaml .Values.sophora.importer.s3Bucket.resources | nindent 12 }} - volumeMounts: - - name: shared-imports - mountPath: /opt/s3fs/bucket - mountPropagation: Bidirectional - - name: local-import-folders - mountPath: /import - securityContext: - privileged: true - {{- end }} - name: importer image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} @@ -108,6 +55,26 @@ spec: {{- else }} value: {{ include "sophora-importer.transformationLibsPath" . }} {{- end }} + {{- if .Values.sophora.importer.s3Bucket.name }} + - name: IMPORTER_S3_SECRETACCESSKEY + valueFrom: + secretKeyRef: + key: {{ .Values.sophora.importer.s3Bucket.secret.secretAccessKeyKey }} + name: {{ .Values.sophora.importer.s3Bucket.secret.name }} + optional: false + - name: IMPORTER_S3_ACCESSKEYID + valueFrom: + secretKeyRef: + key: {{ .Values.sophora.importer.s3Bucket.secret.accessKeyIdKey }} + name: {{ .Values.sophora.importer.s3Bucket.secret.name }} + optional: false + - name: IMPORTER_S3_BUCKETNAME + value: {{ .Values.sophora.importer.s3Bucket.name }} + - name: IMPORTER_S3_HOST + value: {{ .Values.sophora.importer.s3Bucket.url }} + - name: IMPORTER_S3_REGION + value: {{ .Values.sophora.importer.s3Bucket.region }} + {{- end }} {{ if .Values.sophora.importer.extraEnv -}} {{- toYaml .Values.sophora.importer.extraEnv | nindent 10 }} {{- end }} @@ -123,9 +90,6 @@ spec: - name: importer-config mountPath: /sophora/logback-spring.xml subPath: logback-spring.xml - - name: shared-imports - mountPath: /import/ - mountPropagation: Bidirectional - name: local-import-folders mountPath: /import-local/ {{- if .Values.transformation.data.useSaxon }} @@ -143,7 +107,6 @@ spec: {{- end }} securityContext: privileged: true - {{- if not .Values.sophora.importer.s3Bucket.enabled }} lifecycle: postStart: exec: @@ -151,16 +114,14 @@ spec: - sh - '-c' - | - echo "Creating import folders for importer:" - {{- range $folder := tuple "import" "import-local" }} - {{- range $instance := $.Values.sophora.importer.instances }} - mkdir -p /{{$folder}}/{{$instance}}/success - mkdir -p /{{$folder}}/{{$instance}}/temp - mkdir -p /{{$folder}}/{{$instance}}/failure - mkdir -p /{{$folder}}/{{$instance}}/incoming - {{ end }} - {{ end }} - {{- end }} + echo "Creating local import folders for importer:" + {{- range $.Values.sophora.importer.configuration.importer.instances }} + {{- range $folderType, $folderPath := .folders }} + {{- if and (not (hasPrefix "s3://" $folderPath)) (hasKey (dict "watch" 1 "temp" 1 "success" 1 "failure" 1) $folderType) }} + mkdir -pv {{ $folderPath }} + {{- end }} + {{- end }} + {{- end }} initContainers: {{/* Transformations Download */}} {{- with .Values.transformation }} @@ -227,8 +188,6 @@ spec: - name: importer-config configMap: name: {{ include "sophora-importer.fullname" . }} - - name: shared-imports - emptyDir: {} - name: local-import-folders {{- if not .Values.importPvcSpec }} emptyDir: {} diff --git a/charts/sophora-importer/test-values.yaml b/charts/sophora-importer/test-values.yaml index 824f7f4..8dfedca 100644 --- a/charts/sophora-importer/test-values.yaml +++ b/charts/sophora-importer/test-values.yaml @@ -37,14 +37,13 @@ sophora: - name: BBB value: b-value s3Bucket: - name: "sophora-test-importer" + name: "my-bucket" url: "https://storage.googleapis.com" + region: "eu-west-3" secret: name: "sophora-importer-bucket-credentials" secretAccessKeyKey: "secretAccessKey" accessKeyIdKey: "accessKeyId" - instances: - - common configuration: sophora: client: @@ -80,10 +79,10 @@ sophora: key: common transform: skipTransform folders: - watch: /import/common/incoming + watch: s3://common/incoming temp: /import-local/common/temp - success: /import-local/common/success - failure: /import/common/failure + success: s3://common/success + failure: s3://common/failure xsl: /xsl defaultStructureNode: /import diff --git a/charts/sophora-importer/values.yaml b/charts/sophora-importer/values.yaml index fdf5a3e..97cd510 100644 --- a/charts/sophora-importer/values.yaml +++ b/charts/sophora-importer/values.yaml @@ -14,11 +14,6 @@ downloadViaS3Image: tag: "0.0.2" pullPolicy: IfNotPresent -s3fsImage: - repository: efrecon/s3fs - tag: "1.91" - pullPolicy: IfNotPresent - nodeSelector: {} imagePullSecrets: [] nameOverride: "" @@ -45,31 +40,15 @@ sophora: usernameKey: "username" importer: s3Bucket: - # If enabled, the importer uses a s3 bucket for incoming imports - enabled: true - name: "" - url: "https://storage.googleapis.com" + name: + url: + region: secret: name: "" secretAccessKeyKey: "secretAccessKey" accessKeyIdKey: "accessKeyId" - extraEnv: - - name: S3FS_ARGS - value: nonempty - resources: - requests: - cpu: 200m - memory: 256Mi - limits: - memory: 256Mi extraEnv: loaderPath: - instances: [] - createImportFolders: - incoming: s3 - failure: s3 - success: local - temp: local configuration: {} logbackXml: |