diff --git a/.github/workflows/chart-lint-checker.yml b/.github/workflows/chart-lint-checker.yml new file mode 100644 index 000000000..4c520477c --- /dev/null +++ b/.github/workflows/chart-lint-checker.yml @@ -0,0 +1,33 @@ +name: Lint and Test Charts + +on: pull_request + +jobs: + lint-test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up Helm + uses: azure/setup-helm@v1 + with: + version: v3.5.0 + - uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Set up chart-testing + uses: helm/chart-testing-action@v2.0.1 + - name: Run chart-testing (list-changed) + id: list-changed + run: | + changed=$(ct list-changed --config ct.yaml) + if [[ -n "$changed" ]]; then + echo "::set-output name=changed::true" + fi + - name: Run chart-testing (lint) + run: ct lint --config ct.yaml + - name: Create kind cluster + uses: helm/kind-action@v1.1.0 + if: steps.list-changed.outputs.changed == 'true' diff --git a/.github/workflows/publish-chart.yml b/.github/workflows/publish-chart.yml new file mode 100644 index 000000000..7a8ac55db --- /dev/null +++ b/.github/workflows/publish-chart.yml @@ -0,0 +1,64 @@ +# Kindly refer to https://github.com/helm/chart-releaser-action + +name: Publish Charts + +on: + push: + branches: + - master + +jobs: + release: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Configure Git + run: | + git config user.name "$GITHUB_ACTOR" + git config user.email "$GITHUB_ACTOR@users.noreply.github.com" + + - name: Install Helm + uses: azure/setup-helm@v3 + + - name: Run chart-releaser + uses: helm/chart-releaser-action@v1.5.0 # step that writes the latest chart versions (below) depends on this step writing the latest version as the first index in the entries. list in the index.yaml file + env: + CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + + - name: Install Python + uses: actions/setup-python@v4 + + - name: Install pip requirements + uses: BSFishy/pip-action@v1 + with: + packages: | + shyaml==0.6.2 + + - name: Checkout gh-pages + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: gh-pages + + - name: Record the latest chart versions + run: | + releaseDir="latest" + if [[ ! -d ${releaseDir} ]]; then + mkdir -p ${releaseDir} + fi + if [[ -f index.yaml ]]; then + charts=($(cat index.yaml | shyaml keys entries)) + for curChart in "${charts[@]}"; do + curChartVersion=$(cat index.yaml | shyaml get-value entries.${curChart}.0.version) + echo ${curChartVersion} > ${releaseDir}/${curChart} + done + fi + + - uses: EndBug/add-and-commit@v7 + with: + message: 'Set the latest the chart versions' + branch: gh-pages diff --git a/charts/fhir-data-pipes/.helmignore b/charts/fhir-data-pipes/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/fhir-data-pipes/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/fhir-data-pipes/Chart.yaml b/charts/fhir-data-pipes/Chart.yaml new file mode 100644 index 000000000..e71bc5461 --- /dev/null +++ b/charts/fhir-data-pipes/Chart.yaml @@ -0,0 +1,42 @@ +# +# Copyright 2020-2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v2 +name: fhir-data-pipes +description: An open-source pipeline to transform data from a FHIR server (like HAPI, GCP FHIR store, or even OpenMRS) using the FHIR format into a data warehouse based on Apache Parquet files, or another FHIR server. +maintainers: + - name: opensrp + email: techops@ona.io +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "0.1.0" diff --git a/charts/fhir-data-pipes/README.md b/charts/fhir-data-pipes/README.md new file mode 100644 index 000000000..62ba36c1e --- /dev/null +++ b/charts/fhir-data-pipes/README.md @@ -0,0 +1,196 @@ + +# Fhir-data-pipes + +An open-source pipeline to transform data from a FHIR server (like HAPI, GCP FHIR store, or even OpenMRS) using the FHIR format into a data warehouse based on Apache Parquet files, or another FHIR server. + +## TL;DR + +```bash +$ helm repo add google-fhir-data-pipes https://google.github.io/fhir-data-pipes/ +$ helm install fhir-data-pipes google-fhir-data-pipes/fhir-data-pipes +``` + +## Introduction + +This chart bootstraps [fhir-data-pipes](https://github.com/google/fhir-data-pipes) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Prerequisites + +- Kubernetes 1.12+ +- Helm 3.1.0 + +## Configuration + +The following table lists the configurable parameters of the Fhir-data-pipes chart and their default values. + +| Parameter | Description | Default | +|-------------------------------------------------------|-------------|-------------------------------------------------------------------------------------------------------| +| `replicaCount` | | `1` | +| `image.repository` | | `"google/fhir-data-pipes"` | +| `image.pullPolicy` | | `"IfNotPresent"` | +| `image.tag` | | `"master"` | +| `imagePullSecrets` | | `[]` | +| `nameOverride` | | `""` | +| `fullnameOverride` | | `""` | +| `serviceAccount.create` | | `true` | +| `serviceAccount.annotations` | | `{}` | +| `serviceAccount.name` | | `""` | +| `podAnnotations` | | `{}` | +| `podSecurityContext` | | `{}` | +| `securityContext` | | `{}` | +| `service.type` | | `"ClusterIP"` | +| `service.port` | | `8080` | +| `service.extraPorts` | | `null` | +| `service.headless.type` | | `"ClusterIP"` | +| `ingress.enabled` | | `false` | +| `ingress.className` | | `""` | +| `ingress.annotations` | | `{}` | +| `ingress.hosts` | | `[{"host": "fhir-data-pipes.local", "paths": [{"path": "/", "pathType": "ImplementationSpecific"}]}]` | +| `ingress.tls` | | `[]` | +| `ingress.extraRules` | | `null` | +| `resources` | | `null` | +| `autoscaling.enabled` | | `false` | +| `autoscaling.minReplicas` | | `1` | +| `autoscaling.maxReplicas` | | `100` | +| `autoscaling.targetCPUUtilizationPercentage` | | `80` | +| `nodeSelector` | | `{}` | +| `tolerations` | | `[]` | +| `affinity` | | `{}` | +| `recreatePodsWhenConfigMapChange` | | `true` | +| `livenessProbe.httpGet.path` | | `"/"` | +| `livenessProbe.httpGet.port` | | `"http"` | +| `readinessProbe.httpGet.path` | | `"/"` | +| `readinessProbe.httpGet.port` | | `"http"` | +| `flink.execution.attached` | | `false` | +| `hapi.postgres.databaseService` | | `"postgresql"` | +| `hapi.postgres.databaseHostName` | | `""` | +| `hapi.postgres.databasePort` | | `"5432"` | +| `hapi.postgres.databaseUser` | | `""` | +| `hapi.postgres.databasePassword` | | `""` | +| `hapi.postgres.databaseName` | | `""` | +| `thriftserver.hive.databaseService` | | `"hive2"` | +| `thriftserver.hive.databaseHostName` | | `"spark-thriftserver"` | +| `thriftserver.hive.databasePort` | | `"10000"` | +| `thriftserver.hive.databaseUser` | | `"hive"` | +| `thriftserver.hive.databasePassword` | | `""` | +| `thriftserver.hive.databaseName` | | `"default"` | +| `applicationConfig.fhirdata.fhirServerUrl` | | `""` | +| `applicationConfig.fhirdata.dbConfig` | | `"/app/config/hapi-postgres-config.json"` | +| `applicationConfig.fhirdata.dwhRootPrefix` | | `"/dwh/controller_DWH"` | +| `applicationConfig.fhirdata.thriftserverHiveConfig` | | `"/app/config/thrifter-hive-config.json"` | +| `applicationConfig.fhirdata.incrementalSchedule` | | `"* * * * * *"` | +| `applicationConfig.fhirdata.resourceList` | | `"PatientEncounterObservation"` | +| `applicationConfig.fhirdata.maxWorkers` | | `"10"` | +| `applicationConfig.fhirdata.createHiveResourceTables` | | `"true"` | +| `applicationConfig.fhirdata.hiveJdbcDriver` | | `"org.apache.hive.jdbc.HiveDriver"` | +| `initContainers` | | `null` | +| `sidecars` | | `null` | +| `extraVolumes` | | `null` | +| `extraVolumeMounts` | | `null` | +| `extraConfigMaps` | | `null` | +| `env` | | `null` | +| `pdb.enabled` | | `false` | +| `pdb.minAvailable` | | `""` | +| `pdb.maxUnavailable` | | `1` | +| `vpa.enabled` | | `false` | +| `vpa.updatePolicy.updateMode` | | `"Off"` | +| `vpa.resourcePolicy` | | `{}` | +| `pvc.enabled` | | `true` | +| `pvc.volumeMode` | | `"Filesystem"` | +| `pvc.storageClassName` | | `null` | +| `pvc.resources.requests.storage` | | `"20Gi"` | +| `pvc.accessModes` | | `["ReadWriteOnce"]` | +| `pvc.selector` | | `{}` | +| `hiveSiteConfig` | | `null` | + + +## Spark SQL (Thrift Server) as Sidecar +The chart provides the necessary configuration to set up additional containers on the StatefulSet. One such container is the spark thrift server. Below is how one can set it up. +````yaml +--- +sidecars: + - name: spark-thrift-server + image: docker.io/bitnami/spark:3.3.2-debian-11-r12 + imagePullPolicy: IfNotPresent + ports: + - name: hive + containerPort: 10000 + - name: hive-webui + containerPort: 4040 + args: + - "/bin/sh" + - "-c" + - "sbin/start-thriftserver.sh" + - "--master spark://spark-master-svc:7077" + env: + - name: "HIVE_SERVER2_THRIFT_PORT" + value: "10000" + resources: + limits: + memory: 1024Mi + requests: + cpu: 250m + memory: 256Mi + volumeMounts: + - name: dwh-dir + mountPath: /dwh + - name: hive-site-xml + mountPath: /opt/bitnami/spark/conf/hive-site.xml + subPath: hive-site-xml + +# Creates a volume for parquet files. +pvc: + enabled: true + volumeMode: Filesystem + storageClassName: + resources: + requests: + storage: 20Gi + accessModes: + - ReadWriteOnce + selector: {} + +# One can expose the thrift server port using tcp services e.g. for nginx(https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/exposing-tcp-udp-services.md) +service: + type: ClusterIP + port: 8080 + extraPorts: + - port: 4040 + targetPort: hive-webui + protocol: TCP + name: hive-webui + - port: 10000 + targetPort: hive + protocol: TCP + name: hive + +# For LDAP ensure that user has the attribute `uid` on its DN. +# As of now the hive jdbc credentials are not parsed on the pipeline code, as a workaround append the credentials on the `databaseName` credentials as follows: +#thriftserver: +# hive: +# databaseService: "hive2" +# databaseHostName: "fhir-data-pipes" +# databasePort: "10000" +# databaseUser: "user" (not used as of now) +# databasePassword: "password" (not used as of now) +# databaseName: "default;user=;password=" + +hiveSiteConfig: | + + + + + + hive.server2.authentication + LDAP + + + hive.server2.authentication.ldap.url + ldap://openldap.default.svc.cluster.local:389 + + + hive.server2.authentication.ldap.baseDN + ou=users,dc=ldapadmin,dc=labs,dc=example,dc=org + + +```` diff --git a/charts/fhir-data-pipes/templates/NOTES.txt b/charts/fhir-data-pipes/templates/NOTES.txt new file mode 100644 index 000000000..e0ae0a4e6 --- /dev/null +++ b/charts/fhir-data-pipes/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "fhir-data-pipes.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "fhir-data-pipes.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "fhir-data-pipes.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "fhir-data-pipes.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/charts/fhir-data-pipes/templates/_helpers.tpl b/charts/fhir-data-pipes/templates/_helpers.tpl new file mode 100644 index 000000000..299afb0e0 --- /dev/null +++ b/charts/fhir-data-pipes/templates/_helpers.tpl @@ -0,0 +1,74 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "fhir-data-pipes.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "fhir-data-pipes.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "fhir-data-pipes.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "fhir-data-pipes.labels" -}} +helm.sh/chart: {{ include "fhir-data-pipes.chart" . }} +{{ include "fhir-data-pipes.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "fhir-data-pipes.selectorLabels" -}} +app.kubernetes.io/name: {{ include "fhir-data-pipes.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "fhir-data-pipes.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "fhir-data-pipes.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Populate the pod annotations +*/}} +{{- define "fhir-data-pipes.podAnnotations" -}} +{{- range $index, $element:=.Values.podAnnotations }} +{{ $index }}: {{ $element | quote }} +{{- end }} +{{- if .Values.recreatePodsWhenConfigMapChange }} +checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} +{{- end }} +{{- end }} diff --git a/charts/fhir-data-pipes/templates/configmap.yaml b/charts/fhir-data-pipes/templates/configmap.yaml new file mode 100644 index 000000000..19327d87c --- /dev/null +++ b/charts/fhir-data-pipes/templates/configmap.yaml @@ -0,0 +1,40 @@ +# +# Copyright 2020-2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{ $scope := . }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "fhir-data-pipes.fullname" . }} + labels: + {{- include "fhir-data-pipes.labels" . | nindent 4 }} +data: + hapi-postgres-config-json: | + {{ tpl (.Values.hapi.postgres | toJson ) $scope}} + thriftserver-hive-config-json: | + {{ tpl ( .Values.thriftserver.hive | toJson ) $scope }} + application-config-yaml: | + {{ tpl ( .Values.applicationConfig | toYaml) $scope | nindent 8}} + flink-conf-yaml: | + {{ tpl ( .Values.flink | toYaml) $scope | nindent 8}} + {{- if .Values.hiveSiteConfig }} + hive-site-xml: | + {{ tpl (.Values.hiveSiteConfig) $scope | nindent 4 | trim }} + {{- end }} + {{ range $value := .Values.extraConfigMaps -}} + {{ $value.name }}: | + {{ tpl $value.contents $scope | nindent 8 }} + {{ end }} diff --git a/charts/fhir-data-pipes/templates/headless-svc.yaml b/charts/fhir-data-pipes/templates/headless-svc.yaml new file mode 100644 index 000000000..bbcf867aa --- /dev/null +++ b/charts/fhir-data-pipes/templates/headless-svc.yaml @@ -0,0 +1,27 @@ +# +# Copyright 2020-2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v1 +kind: Service +metadata: + name: {{ printf "%s-headless" (include "fhir-data-pipes.fullname" .) }} + labels: + {{- include "fhir-data-pipes.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.headless.type }} + clusterIP: None + selector: + {{- include "fhir-data-pipes.selectorLabels" . | nindent 4 }} diff --git a/charts/fhir-data-pipes/templates/hpa.yaml b/charts/fhir-data-pipes/templates/hpa.yaml new file mode 100644 index 000000000..0a282d3a7 --- /dev/null +++ b/charts/fhir-data-pipes/templates/hpa.yaml @@ -0,0 +1,44 @@ +# +# Copyright 2020-2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{ if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "fhir-data-pipes.fullname" . }} + labels: + {{- include "fhir-data-pipes.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: StatefulSet + name: {{ include "fhir-data-pipes.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/charts/fhir-data-pipes/templates/ingress.yaml b/charts/fhir-data-pipes/templates/ingress.yaml new file mode 100644 index 000000000..c6eca914e --- /dev/null +++ b/charts/fhir-data-pipes/templates/ingress.yaml @@ -0,0 +1,80 @@ +# +# Copyright 2020-2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{ if .Values.ingress.enabled }} +{{- $fullName := include "fhir-data-pipes.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "fhir-data-pipes.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.ingress.extraRules }} + {{- toYaml .Values.ingress.extraRules | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/fhir-data-pipes/templates/pdb.yaml b/charts/fhir-data-pipes/templates/pdb.yaml new file mode 100644 index 000000000..592d4bf22 --- /dev/null +++ b/charts/fhir-data-pipes/templates/pdb.yaml @@ -0,0 +1,38 @@ +# +# Copyright 2020-2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{ if .Values.pdb.enabled }} +{{- if semverCompare "<1.21-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: "policy/v1beta1" +{{- else -}} +apiVersion: "policy/v1" +{{- end }} +kind: PodDisruptionBudget +metadata: + name: {{ include "fhir-data-pipes.fullname" . }} + labels: + {{- include "fhir-data-pipes.labels" . | nindent 4 }} +spec: + {{- if .Values.pdb.minAvailable }} + minAvailable: {{ .Values.pdb.minAvailable }} + {{- end }} + {{- if .Values.pdb.maxUnavailable }} + maxUnavailable: {{ .Values.pdb.maxUnavailable }} + {{- end }} + selector: + matchLabels: + {{- include "fhir-data-pipes.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/charts/fhir-data-pipes/templates/pvc.yaml b/charts/fhir-data-pipes/templates/pvc.yaml new file mode 100644 index 000000000..97f220bc6 --- /dev/null +++ b/charts/fhir-data-pipes/templates/pvc.yaml @@ -0,0 +1,37 @@ +# +# Copyright 2020-2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{ if .Values.pvc.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "fhir-data-pipes.fullname" . }} + labels: + {{- include "fhir-data-pipes.labels" . | nindent 4 }} +spec: + accessModes: + {{- toYaml .Values.pvc.accessModes | nindent 4 }} + volumeMode: {{ .Values.pvc.volumeMode }} + resources: + {{- toYaml .Values.pvc.resources | nindent 4 }} + {{- if .Values.pvc.storageClassName }} + storageClassName: {{ .Values.pvc.storageClassName }} + {{- end }} + {{- if .Values.pvc.selector }} + selector: + {{- toYaml .Values.pvc.selector | nindent 4 }} + {{- end }} +{{- end -}} diff --git a/charts/fhir-data-pipes/templates/service.yaml b/charts/fhir-data-pipes/templates/service.yaml new file mode 100644 index 000000000..51a62f9c2 --- /dev/null +++ b/charts/fhir-data-pipes/templates/service.yaml @@ -0,0 +1,34 @@ +# +# Copyright 2020-2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v1 +kind: Service +metadata: + name: {{ include "fhir-data-pipes.fullname" . }} + labels: + {{- include "fhir-data-pipes.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + {{- if .Values.service.extraPorts }} + {{- toYaml .Values.service.extraPorts | nindent 4 }} + {{- end }} + selector: + {{- include "fhir-data-pipes.selectorLabels" . | nindent 4 }} diff --git a/charts/fhir-data-pipes/templates/serviceaccount.yaml b/charts/fhir-data-pipes/templates/serviceaccount.yaml new file mode 100644 index 000000000..ce8ed6a79 --- /dev/null +++ b/charts/fhir-data-pipes/templates/serviceaccount.yaml @@ -0,0 +1,28 @@ +# +# Copyright 2020-2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{ if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "fhir-data-pipes.serviceAccountName" . }} + labels: + {{- include "fhir-data-pipes.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/fhir-data-pipes/templates/statefulset.yaml b/charts/fhir-data-pipes/templates/statefulset.yaml new file mode 100644 index 000000000..146bf4562 --- /dev/null +++ b/charts/fhir-data-pipes/templates/statefulset.yaml @@ -0,0 +1,131 @@ +# +# Copyright 2020-2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "fhir-data-pipes.fullname" . }} + labels: + {{- include "fhir-data-pipes.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + serviceName: {{ printf "%s-headless" (include "fhir-data-pipes.fullname" .) }} + selector: + matchLabels: + {{- include "fhir-data-pipes.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- include "fhir-data-pipes.podAnnotations" . | indent 8 }} + labels: + {{- include "fhir-data-pipes.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "fhir-data-pipes.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + volumes: + - name: application-config-yaml + configMap: + name: {{ include "fhir-data-pipes.fullname" . }} + - name: hapi-postgres-config-json + configMap: + name: {{ include "fhir-data-pipes.fullname" . }} + - name: thriftserver-hive-config-json + configMap: + name: {{ include "fhir-data-pipes.fullname" . }} + - name: flink-conf-yaml + configMap: + name: {{ include "fhir-data-pipes.fullname" . }} + {{- if .Values.hiveSiteConfig }} + - name: hive-site-xml + configMap: + name: {{ include "fhir-data-pipes.fullname" . }} + {{- end }} + {{- if .Values.pvc.enabled }} + - name: dwh-dir + persistentVolumeClaim: + claimName: {{ include "fhir-data-pipes.fullname" . }} + {{- end }} + {{- if .Values.extraVolumes }} + {{- tpl (toYaml .Values.extraVolumes) . | nindent 12 }} + {{- end }} + {{- if .Values.initContainers }} + initContainers: + {{- toYaml .Values.initContainers | nindent 12 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: SPRING_CONFIG_LOCATION + value: /app/config/application.yaml + {{- if .Values.env }} + {{- tpl (toYaml .Values.env) . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: 8080 + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: application-config-yaml + mountPath: /app/config/application.yaml + subPath: application-config-yaml + - name: hapi-postgres-config-json + mountPath: /app/config/hapi-postgres-config.json + subPath: hapi-postgres-config-json + - name: thriftserver-hive-config-json + mountPath: /app/config/thriftserver-hive-config.json + subPath: thriftserver-hive-config-json + - name: flink-conf-yaml + mountPath: /app/config/flink-conf.yaml + subPath: flink-conf-yaml + {{- if .Values.pvc.enabled }} + - name: dwh-dir + mountPath: /dwh + {{- end }} + {{- if .Values.extraVolumeMounts }} + {{- toYaml .Values.extraVolumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.sidecars }} + {{- toYaml .Values.sidecars | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/fhir-data-pipes/templates/tests/test-connection.yaml b/charts/fhir-data-pipes/templates/tests/test-connection.yaml new file mode 100644 index 000000000..187099169 --- /dev/null +++ b/charts/fhir-data-pipes/templates/tests/test-connection.yaml @@ -0,0 +1,31 @@ +# +# Copyright 2020-2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "fhir-data-pipes.fullname" . }}-test-connection" + labels: + {{- include "fhir-data-pipes.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "fhir-data-pipes.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/charts/fhir-data-pipes/templates/vpa.yaml b/charts/fhir-data-pipes/templates/vpa.yaml new file mode 100644 index 000000000..446e39002 --- /dev/null +++ b/charts/fhir-data-pipes/templates/vpa.yaml @@ -0,0 +1,36 @@ +# +# Copyright 2020-2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +{{ if .Values.vpa.enabled }} +apiVersion: "autoscaling.k8s.io/v1" +kind: VerticalPodAutoscaler +metadata: + name: {{ include "fhir-data-pipes.fullname" . }} + labels: + {{- include "fhir-data-pipes.labels" . | nindent 4 }} +spec: + targetRef: + apiVersion: "apps/v1" + kind: StatefulSet + name: {{ include "fhir-data-pipes.fullname" . }} + updatePolicy: + {{- toYaml .Values.vpa.updatePolicy | nindent 4 }} + {{- if .Values.vpa.resourcePolicy }} + resourcePolicy: + {{- toYaml .Values.vpa.resourcePolicy | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/fhir-data-pipes/values.yaml b/charts/fhir-data-pipes/values.yaml new file mode 100644 index 000000000..1484f005f --- /dev/null +++ b/charts/fhir-data-pipes/values.yaml @@ -0,0 +1,254 @@ +# +# Copyright 2020-2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Default values for fhir-data-pipes. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: google/fhir-data-pipes + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "master" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} +# fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true +# runAsNonRoot: true +# runAsUser: 1000 + +service: + type: ClusterIP + port: 8080 + extraPorts: +# - port: 4040 +# targetPort: hive-webui +# protocol: TCP +# name: hive-webui +# - port: 10000 +# targetPort: hive +# protocol: TCP +# name: hive + headless: + type: ClusterIP +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: fhir-data-pipes.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + extraRules: + + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: +# We usually recommend not to specify default resources and to leave this as a conscious +# choice for the user. This also increases chances charts run on environments with little +# resources such as Minikube. If you do want to specify resources uncomment the following +# lines adjust them as necessary and remove the curly braces after 'resources:'. +# limits: +# memory: 1024Mi +# requests: +# cpu: 100m +# memory: 128Mi + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +recreatePodsWhenConfigMapChange: true + +livenessProbe: + httpGet: + path: / + port: http + +readinessProbe: + httpGet: + path: / + port: http + +flink: + execution.attached: false + +hapi: + postgres: + databaseService: "postgresql" + databaseHostName: "" + databasePort: "5432" + databaseUser: "" + databasePassword: "" + databaseName: "" + +thriftserver: + hive: + databaseService: "hive2" + databaseHostName: "spark-thriftserver" + databasePort: "10000" + databaseUser: "hive" + databasePassword: "" + databaseName: "default" + +applicationConfig: + fhirdata: + fhirServerUrl: "" + dbConfig: "/app/config/hapi-postgres-config.json" + dwhRootPrefix: "/dwh/controller_DWH" + thriftserverHiveConfig: "/app/config/thriftserver-hive-config.json" + incrementalSchedule: "* * * * * *" + resourceList: "Patient,Encounter,Observation" + maxWorkers: "10" + createHiveResourceTables: "true" + hiveJdbcDriver: "org.apache.hive.jdbc.HiveDriver" + +# Override the hive-site.xml by specifying the xml as string, see example below (ensure indentation is correct) refer to README.md. +hiveSiteConfig: +# hiveSiteConfig: | +# +# +# +# +# +# hive.server2.authentication +# LDAP +# +# +# hive.server2.authentication.ldap.url +# ldap://openldap.default.svc.cluster.local:389 +# +# +# hive.server2.authentication.ldap.baseDN +# ou=users,dc=ldapadmin,dc=labs,dc=example,dc=org +# +# + +initContainers: + +sidecars: +# - name: spark-thrift-server +# image: docker.io/bitnami/spark:3.3.2-debian-11-r12 +# imagePullPolicy: IfNotPresent +# ports: +# - name: hive +# containerPort: 10000 +# - name: hive-webui +# containerPort: 4040 +# args: +# - "/bin/sh" +# - "-c" +# - "sbin/start-thriftserver.sh" +# - "--master spark://spark-master-svc:7077" +# env: +# - name: "HIVE_SERVER2_THRIFT_PORT" +# value: "10000" +# resources: +# limits: +# memory: 1024Mi +# requests: +# cpu: 250m +# memory: 256Mi +# volumeMounts: +# - name: dwh-dir +# mountPath: /dwh +# - name: hive-site-xml +# mountPath: /opt/bitnami/spark/conf/hive-site.xml +# subPath: hive-site-xml + +extraVolumes: +# - name: some-name +# configMap: +# name: '{{ include "fhir-data-pipes.fullname" . }}' + +extraVolumeMounts: +# - mountPath: /app/resources/some-file.json +# name: some-name +# subPath: some-configmap.json + +extraConfigMaps: +# - name: some-configmap.json +# contents: | +# { +# "entries": [ +# { +# } +# ] +# } + +env: +# - name: APP_LOGGING_LEVEL +# value: DEBUG + +pdb: + enabled: false + minAvailable: "" + maxUnavailable: 1 + +vpa: + enabled: false + updatePolicy: + updateMode: "Off" + resourcePolicy: {} + +pvc: + enabled: true + volumeMode: Filesystem + storageClassName: + resources: + requests: + # update accordingly + storage: 20Gi + accessModes: + - ReadWriteOnce + selector: {} diff --git a/ct.yaml b/ct.yaml new file mode 100644 index 000000000..4e299ffd4 --- /dev/null +++ b/ct.yaml @@ -0,0 +1,21 @@ +# +# Copyright 2021-2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# See https://github.com/helm/chart-testing#configuration +remote: origin +target-branch: master +chart-dirs: + - charts