From 0ab1fa5ff119e393eafb16946e7635b08b399796 Mon Sep 17 00:00:00 2001 From: Annie Chang Date: Wed, 1 May 2024 08:46:23 -0700 Subject: [PATCH] #24 Create Baton migration for Helm chart name changes and update relevant READMEs --- DRAFT_RELEASE_NOTES.md | 48 +++++++-- .../aissemble-data-access-chart/README.md | 2 +- .../aissemble-elasticsearch-chart/README.md | 2 +- .../aissemble-fastapi-chart/README.md | 2 +- .../aissemble-fastapi-chart/values.yaml | 2 +- .../aissemble-inference-chart/README.md | 2 +- .../aissemble-inference-chart/values.yaml | 2 +- .../aissemble-kafka-chart/README.md | 2 +- .../README.md | 6 +- .../aissemble-localstack-chart/README.md | 26 ++--- .../aissemble-metadata-chart/README.md | 32 +++--- .../aissemble-metadata-chart/values.yaml | 2 +- .../README.md | 22 ++--- .../values.yaml | 2 +- .../aissemble-quarkus-chart/README.md | 4 +- .../aissemble-quarkus-chart/values.yaml | 2 +- .../aissemble-sealed-secrets-chart/README.md | 1 + .../README.md | 16 +-- .../aissemble-spark-operator-chart/README.md | 8 +- .../aissemble-versioning-chart/README.md | 2 +- .../README.md | 2 +- .../values.yaml | 2 +- .../README.md | 2 +- .../values.yaml | 2 +- .../values.yaml | 2 +- .../v1_7_0/HelmChartsNameMigration.java | 98 +++++++++++++++++++ .../src/main/resources/migrations.json | 12 +++ .../upgrade/migration/TiltMigrationSteps.java | 6 +- .../v1_7_0/ChartNameMigrationSteps.java | 97 ++++++++++++++++++ .../v1_7_0/chart-name-migration.feature | 22 +++++ .../migration/kafka-cluster/Chart.yaml | 34 +++++++ .../migration/kafka-cluster/values.yaml | 9 ++ .../skip-migration/kafka-cluster/Chart.yaml | 34 +++++++ .../skip-migration/kafka-cluster/values.yaml | 9 ++ 34 files changed, 431 insertions(+), 85 deletions(-) create mode 100644 foundation/foundation-upgrade/src/main/java/com/boozallen/aissemble/upgrade/migration/v1_7_0/HelmChartsNameMigration.java create mode 100644 foundation/foundation-upgrade/src/test/java/com/boozallen/aissemble/upgrade/migration/v1_7_0/ChartNameMigrationSteps.java create mode 100644 foundation/foundation-upgrade/src/test/resources/specifications/v1_7_0/chart-name-migration.feature create mode 100644 foundation/foundation-upgrade/src/test/resources/test-files/v1_7_0/HelmChartNameMigration/migration/kafka-cluster/Chart.yaml create mode 100644 foundation/foundation-upgrade/src/test/resources/test-files/v1_7_0/HelmChartNameMigration/migration/kafka-cluster/values.yaml create mode 100644 foundation/foundation-upgrade/src/test/resources/test-files/v1_7_0/HelmChartNameMigration/skip-migration/kafka-cluster/Chart.yaml create mode 100644 foundation/foundation-upgrade/src/test/resources/test-files/v1_7_0/HelmChartNameMigration/skip-migration/kafka-cluster/values.yaml diff --git a/DRAFT_RELEASE_NOTES.md b/DRAFT_RELEASE_NOTES.md index dd54ad722..1d516aa97 100644 --- a/DRAFT_RELEASE_NOTES.md +++ b/DRAFT_RELEASE_NOTES.md @@ -1,8 +1,9 @@ # Major Additions * Python modules were renamed to reflect aiSSEMBLE. These include the following. + | Old Python Module | New Python Module | -| --------------------------------------- | ------------------------------------------------- | +| --------------------------------------- |---------------------------------------------------| | foundation-core-python | aissemble-core-python | | foundation-model-training-api | aissemble-foundation-model-training-api | | foundation-versioning-service | aissemble-foundation-versioning-service | @@ -15,7 +16,7 @@ | foundation-transform-core-python | aissemble-Foundation-transform-core-python | | extensions-model-training-api-sagemaker | aissemble-extensions-model-training-api-sagemaker | | extensions-data-delivery-spark-py | aissemble-extensions-data-delivery-spark-py | -| extensions-encryption-valut-python | aissemble-extensions-encryption-valut-python | +| extensions-encryption-valut-python | aissemble-extensions-encryption-vault-python | | extensions-transform-spark-python | aissemble-extensions-transform-spark-python | | test-data-delivery-pyspark-model | aissemble-test-data-delivery-pyspark-model | | test-data-delivery-pyspark-model-basic | aissemble-test-data-delivery-pyspark-model-basic | @@ -24,6 +25,34 @@ | machine-learning-training-base | aissemble-machine-learning-training-base | | machine-learning-sagemaker-training | aissemble-machine-learning-sagemaker-training | +* Helm Charts and their relevant modules have been renamed to the following: + +| Old Module Name | New Helm Chart and Module Name | +|-----------------------------------------|-----------------------------------------| +| extensions-helm-airflow | aissemble-airflow-chart | +| extensions-helm-data-access | aissemble-data-access-chart | +| extensions-helm-elasticsearch | aissemble-elasticsearch-chart | +| extensions-helm-elasticsearch-operator | aissemble-elasticsearch-operator-chart | +| extensions-helm-fastapi | aissemble-fastapi-chart | +| extensions-helm-hive-metastore-db | aissemble-hive-metastore-db-chart | +| extensions-helm-hive-metastore-service | aissemble-hive-metastore-service-chart | +| extensions-helm-inference | aissemble-inference-chart | +| extensions-helm-jenkins | aissemble-jenkins-chart | +| extensions-helm-kafka | aissemble-kafka-chart | +| extensions-helm-keycloak | aissemble-keycloak-chart | +| extensions-helm-lineage-http-consumer | aissemble-lineage-http-consumer-chart | +| extensions-helm-localstack | aissemble-localstack-chart | +| extensions-helm-metadata | aissemble-metadata-chart | +| extensions-helm-mlflow | aissemble-mlflow-chart | +| extensions-helm-pipeline-invocation | aissemble-pipeline-invocation-chart | +| extensions-helm-pipeline-invocation-lib | aissemble-pipeline-invocation-lib-chart | +| extensions-helm-policy-decision-point | aissemble-policy-decision-point-chart | +| extensions-helm-quarkus | aissemble-quarkus-chart | +| extensions-helm-sealed-secrets | aissemble-sealed-secrets-chart | +| extensions-helm-spark-application | aissemble-spark-application-chart | +| extensions-helm-spark-operator | aissemble-spark-operator-chart | +| extensions-helm-vault | aissemble-vault-chart | +| extensions-helm-versioning | aissemble-versioning-chart | ## OpenLineage Namespace Conventions Conventions for setting namespaces when leveraging `Data Lineage` has been updated to better follow [OpenLineage's guidelines](https://openlineage.io/docs/spec/naming/). Moving forward, namespaces should be defined in the `data-lineage.properties` file, such that Jobs are tied to pipelines and Datasets are tied to data sources. This is a departure from the old pattern of one single namespace property (`data.lineage.namespace`) being leveraged for an entire project. Refer to the [GitHub docs](https://boozallen.github.io/aissemble/current-dev/lineage-medatada-capture-overview.html#_configuration) for updated guidance. Usage of the `data.lineage.namespace` property in a project's `data-lineage.properties` file will be supported as a fallback but should not be used in practice. @@ -68,13 +97,14 @@ The following steps will upgrade your project to 1.7. These instructions consist ## Automatic Upgrades To reduce burden of upgrading aiSSEMBLE, the Baton project is used to automate the migration of some files to the new version. These migrations run automatically when you build your project, and are included by default when you update the `build-parent` version in your root POM. Below is a description of all of the Baton migrations that are included with this version of aiSSEMBLE. -| Migration Name | Description | -| -------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| upgrade-tiltfile-aissemble-version-migration | Updates the aiSSEMBLE version within your project's Tiltfile | -| upgrade-v2-chart-files-aissemble-version-migration | Updates the helm chart dependencies within your project's deployment resources (-deploy/src/main/resources/apps/) to use the latest version of the aiSSEMBLE | -| upgrade-v1-chart-files-aissemble-version-migration | Updates the docker image tags within your project's deployment resources (-deploy/src/main/resources/apps/) to use the latest version of the aiSSEMBLE | -| upgrade-mlflow-v2-external-s3-migration | Update the mlflow V2 deployment (if present) in your project to utilize Localstack for local development and SealedSecrets for remote deployments | -| upgrade-foundation-extension-python-package-migration | Updates the pyproject.toml files within your projects pipelines folder (-pipelines) to use the updated aiSSEMBLE foundation and extension Python packages with the latest naming convention | | +| Migration Name | Description | +| -------------------------------------------------- |----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| upgrade-tiltfile-aissemble-version-migration | Updates the aiSSEMBLE version within your project's Tiltfile | +| upgrade-v2-chart-files-aissemble-version-migration | Updates the helm chart dependencies within your project's deployment resources (-deploy/src/main/resources/apps/) to use the latest version of the aiSSEMBLE | +| upgrade-v1-chart-files-aissemble-version-migration | Updates the docker image tags within your project's deployment resources (-deploy/src/main/resources/apps/) to use the latest version of the aiSSEMBLE | +| upgrade-mlflow-v2-external-s3-migration | Update the mlflow V2 deployment (if present) in your project to utilize Localstack for local development and SealedSecrets for remote deployments | +| upgrade-foundation-extension-python-package-migration | Updates the pyproject.toml files within your projects pipelines folder (-pipelines) to use the updated aiSSEMBLE foundation and extension Python packages with the latest naming convention | +|upgrade-helm-chart-files-names-migration | Updates the Chart.yaml and values*.yaml files within your project's deploy folder (-deploy) to use the new Helm chart naming convention (aissemble--chart) | To deactivate any of these migrations, add the following configuration to the `baton-maven-plugin` within your root `pom.xml`: diff --git a/extensions/extensions-helm/aissemble-data-access-chart/README.md b/extensions/extensions-helm/aissemble-data-access-chart/README.md index 963e22f3b..a5a139bbe 100644 --- a/extensions/extensions-helm/aissemble-data-access-chart/README.md +++ b/extensions/extensions-helm/aissemble-data-access-chart/README.md @@ -25,7 +25,7 @@ All properties must be prefixed with the key `aissemble-quarkus-chart` to overri If you are migrating from the v1 version of the data access chart, use the tables below to apply any existing customizations from the old chart to the new v2 chart. ## Property Location -Old properties would have to be prepended with aissemble-data-access.aissemble-quarkus, e.g.: app.name => aissemble-data-access.aissemble-quarkus.app.name +Old properties would have to be prepended with aissemble-data-access-chart.aissemble-quarkus-chart, e.g.: app.name => aissemble-data-access-chart.aissemble-quarkus-chart.app.name If any properties are set to the default value, we recommend removing them from your values file entirely. diff --git a/extensions/extensions-helm/aissemble-elasticsearch-chart/README.md b/extensions/extensions-helm/aissemble-elasticsearch-chart/README.md index c2ee5c2c3..a8ed8eeeb 100644 --- a/extensions/extensions-helm/aissemble-elasticsearch-chart/README.md +++ b/extensions/extensions-helm/aissemble-elasticsearch-chart/README.md @@ -9,7 +9,7 @@ helm install elasticsearch ghcr.io/boozallen/aissemble-elasticsearch-chart --ver **Note**: *the version should match the aiSSEMBLE project version.* # aiSSEMBLE Custom Properties -The following properties are unique to the aissemble-elasticsearch chart and extend the functionality of the base ECK chart. +The following properties are unique to the `aissemble-elasticsearch-chart` chart and extend the functionality of the base ECK chart. | Property | Description | Required Override | Default | |----------------------------|--------------------------------------|-------------------|------------| | basicAuth.enabled | Enable basic http auth | No | true | diff --git a/extensions/extensions-helm/aissemble-fastapi-chart/README.md b/extensions/extensions-helm/aissemble-fastapi-chart/README.md index 85bfe6932..90af7e5fe 100644 --- a/extensions/extensions-helm/aissemble-fastapi-chart/README.md +++ b/extensions/extensions-helm/aissemble-fastapi-chart/README.md @@ -31,7 +31,7 @@ helm install fastapi ghcr.io/boozallen/aissemble-fastapi-chart --version   port: 8082
  targetPort: 8080 | -| deployment.ports | The deployment ports | No | - name: http-1
  containerPort: 8080
  protocol: TCP | -| deployment.restartPolicy | The deployment restart policy | No | Always | -| ingress.enabled | Enable the external access to the service | No | true | -| ingress.metadata.annotations.kubernetes.io/ingress.class | The ingress controller class | No | nginx | +| Property | Description | Required Override | Default | +|----------------------------------------------------------|-------------------------------------------|-------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| app.name | Sets label for app.kubernetes.io/name | No | Chart.Name (aissemble-metadata-chart) | +| app.version | Sets label for app.kubernetes.io/version | No | Chart.AppVersion (aiSSEMBLE project version) | +| hostname | The hostname for the application | No | metadata | +| image.name | The image name | No | boozallen/aissemble-metadata | +| image.imagePullPolicy | The image pull policy | No | Always (ensures local docker image is pulled, rather than from Nexus repo) | +| image.dockerRepo | The image docker repository | No | NB: OSS: update with aissemble docker repository | +| image.tag | The image tag | No | Chart.AppVersion | +| service.spec.ports | The service spec ports | No | - name: rest-api
  port: 8082
  targetPort: 8080 | +| deployment.ports | The deployment ports | No | - name: http-1
  containerPort: 8080
  protocol: TCP | +| deployment.restartPolicy | The deployment restart policy | No | Always | +| ingress.enabled | Enable the external access to the service | No | true | +| ingress.metadata.annotations.kubernetes.io/ingress.class | The ingress controller class | No | nginx | | ingress.hosts | The ingress hosts | No | - host: metadata-ai.boozallen.github.io
  paths:
  - path:  /
    pathType: Prefix
    backend:
      service:
        name: metadata
        port:
          number:8082 | -| ingress.status | The current state of the ingress | No | 10.192.83.167 | +| ingress.status | The current state of the ingress | No | 10.192.83.167 | # Migration from aiSSEMBLE v1 Helm Charts If you are migrating from the v1 version of the metadata chart, use the tables below to apply any existing customizations from the old chart to the new v2 chart. @@ -32,7 +32,7 @@ If you are migrating from the v1 version of the metadata chart, use the tables b ## Property Location All properties listed below have been moved to the parent chart. If any properties are set to the default value, we recommend removing them from your values file entirely. -**Note**: *all new property locations include the prefix `aissemble-metadata.`* +**Note**: *all new property locations include the prefix `aissemble-metadata-chart.`* ## Property Removed The following properties no longer exist. diff --git a/extensions/extensions-helm/aissemble-metadata-chart/values.yaml b/extensions/extensions-helm/aissemble-metadata-chart/values.yaml index 410be8bfe..792b4ee5b 100644 --- a/extensions/extensions-helm/aissemble-metadata-chart/values.yaml +++ b/extensions/extensions-helm/aissemble-metadata-chart/values.yaml @@ -1,6 +1,6 @@ # Default values for metadata app: - # Sets labels for app.kubernetes.io/name Default is Chart.name (aissemble-metadata) + # Sets labels for app.kubernetes.io/name Default is Chart.name (aissemble-metadata-chart) name: '' # Sets the labels for app.kubernetes.io/version. Default is Chart.appVersion version: '' diff --git a/extensions/extensions-helm/aissemble-policy-decision-point-chart/README.md b/extensions/extensions-helm/aissemble-policy-decision-point-chart/README.md index 1d15f835f..936cba885 100644 --- a/extensions/extensions-helm/aissemble-policy-decision-point-chart/README.md +++ b/extensions/extensions-helm/aissemble-policy-decision-point-chart/README.md @@ -11,7 +11,7 @@ helm install policy-decision-point ghcr.io/boozallen/aissemble-policy-decision-p # Properties | Property | Description | Required Override | Default | |-------------------------------|---------------------------------------------|-------------------|------------------------------------------------------------------------------------| -| app.name | Sets label for app.kubernetes.io/name | No | Chart.Name (aissemble-policy-decision-point) | +| app.name | Sets label for app.kubernetes.io/name | No | Chart.Name (aissemble-policy-decision-point-chart) | | app.version | Sets label for app.kubernetes.io/version | No | Chart.AppVersion (aiSSEMBLE project version) | | hostname | The hostname for the application | No | policy-decision-point | | image.name | The image name | Yes | boozallen/aissemble-policy-decision-point | @@ -30,16 +30,16 @@ All properties listed below have been moved to the parent chart. If any properti | Old Property Location | New Property Location | |--------------------------------------------|-------------------------------------------------------------------------| -| app.name | aissemble-policy-decision-point.app.name | -| app.version | aissemble-policy-decision-point.app.version | -| hostname | aissemble-policy-decision-point.hostname | -| image.name | aissemble-policy-decision-point.image.name | -| image.imagePullPolicy | aissemble-policy-decision-point.image.imagePullPolicy | -| image.dockerRepo | aissemble-policy-decision-point.image.dockerRepo | -| image.tag | aissemble-policy-decision-point.image.tag | -| service.spec.ports | aissemble-policy-decision-point.service.spec.ports | -| deployment.ports | aissemble-policy-decision-point.deployment.ports | -| deployment.restartPolicy | aissemble-policy-decision-point.deployment.restartPolicy | +| app.name | aissemble-policy-decision-point-chart.app.name | +| app.version | aissemble-policy-decision-point-chart.app.version | +| hostname | aissemble-policy-decision-point-chart.hostname | +| image.name | aissemble-policy-decision-point-chart.image.name | +| image.imagePullPolicy | aissemble-policy-decision-point-chart.image.imagePullPolicy | +| image.dockerRepo | aissemble-policy-decision-point-chart.image.dockerRepo | +| image.tag | aissemble-policy-decision-point-chart.image.tag | +| service.spec.ports | aissemble-policy-decision-point-chart.service.spec.ports | +| deployment.ports | aissemble-policy-decision-point-chart.deployment.ports | +| deployment.restartPolicy | aissemble-policy-decision-point-chart.deployment.restartPolicy | ## Property Removed The following properties no longer exist. diff --git a/extensions/extensions-helm/aissemble-policy-decision-point-chart/values.yaml b/extensions/extensions-helm/aissemble-policy-decision-point-chart/values.yaml index 5055b1807..e04a8294d 100644 --- a/extensions/extensions-helm/aissemble-policy-decision-point-chart/values.yaml +++ b/extensions/extensions-helm/aissemble-policy-decision-point-chart/values.yaml @@ -1,6 +1,6 @@ # Default values for policy-decision-point app: - # Sets labels for app.kubernetes.io/name Default is Chart.name (aissemble-policy-decision-point) + # Sets labels for app.kubernetes.io/name Default is Chart.name (aissemble-policy-decision-point-chart) name: '' # Sets the labels for app.kubernetes.io/version. Default is Chart.appVersion version: '' diff --git a/extensions/extensions-helm/aissemble-quarkus-chart/README.md b/extensions/extensions-helm/aissemble-quarkus-chart/README.md index 1baaa10ee..0821e9cfd 100644 --- a/extensions/extensions-helm/aissemble-quarkus-chart/README.md +++ b/extensions/extensions-helm/aissemble-quarkus-chart/README.md @@ -11,9 +11,9 @@ helm install my-quarkus-app ghcr.io/boozallen/aissemble-quarkus-chart --version # Properties | Property | Description | Required Override | Default | |-----------------------------------------|----------------------------------------------------------------------------------------------------------------------------------|-------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| - | app.name | Sets label for app.kubernetes.io/name | No | Chart.Name (Quarkus) | + | app.name | Sets label for app.kubernetes.io/name | No | Chart.Name (aissemble-quarkus-chart) | | app.version | Sets label for app.kubernetes.io/version | No | Chart.AppVersion (aiSSEMBLE project version) | -| deployment.image.repo | The image repository | No | NB: OSS: update with aissemble docker repository | +| deployment.image.repo | The image repository | No | NB: OSS: update with aissemble docker repository | | deployment.image.name | The image name | Yes | boozallen/aissemble-quarkus | | deployment.image.tag | The image tag. Will override the default chart AppVersion value | No | Chart.AppVersion | | deployment.image.imagePullPolicy | The image pull policy | No | IfNotPresent | diff --git a/extensions/extensions-helm/aissemble-quarkus-chart/values.yaml b/extensions/extensions-helm/aissemble-quarkus-chart/values.yaml index d92df4c2a..9964d9d56 100644 --- a/extensions/extensions-helm/aissemble-quarkus-chart/values.yaml +++ b/extensions/extensions-helm/aissemble-quarkus-chart/values.yaml @@ -1,6 +1,6 @@ --- app: - # Sets labels for app.kubernetes.io/name Default is Chart.Name (Quarkus) + # Sets labels for app.kubernetes.io/name Default is Chart.Name (aissemble-quarkus-chart) name: '' # Sets the labels for app.kubernetes.io/version. Default is Chart.AppVersion version: '' diff --git a/extensions/extensions-helm/aissemble-sealed-secrets-chart/README.md b/extensions/extensions-helm/aissemble-sealed-secrets-chart/README.md index 45ab4cf9d..83fe156c7 100644 --- a/extensions/extensions-helm/aissemble-sealed-secrets-chart/README.md +++ b/extensions/extensions-helm/aissemble-sealed-secrets-chart/README.md @@ -13,6 +13,7 @@ helm install sealed-secrets ghcr.io/boozallen/aissemble-sealed-secrets-chart --v # Overridden Defaults The following properties are inherited from the [base SealedSecrets chart](https://github.com/bitnami-labs/sealed-secrets/tree/main/helm/sealed-secrets), but with updated default values. + | Property | Description | Required Override | Default | |----------------------|--------------------------------------------------|-------------------|-------------------------------------| | fullnameOverride | String to fully override sealed-secrets.fullname | No | aissemble-sealed-secrets-controller | diff --git a/extensions/extensions-helm/aissemble-spark-application-chart/README.md b/extensions/extensions-helm/aissemble-spark-application-chart/README.md index 3efe84d07..e348c98b6 100644 --- a/extensions/extensions-helm/aissemble-spark-application-chart/README.md +++ b/extensions/extensions-helm/aissemble-spark-application-chart/README.md @@ -114,16 +114,16 @@ If you are migrating from the v1 version of the spark-application chart, use the ## Property Location All properties listed in the **Properties** section have been moved to the parent chart. If any properties are set to the default value, we recommend removing them from your values file entirely. -All new property locations configured for SparkApplication should include the prefix `aissemble-spark-application.sparkApp` except below properties: +All new property locations configured for SparkApplication should include the prefix `aissemble-spark-application-chart.sparkApp` except below properties: -| Old Property Location | New Property Location | -|-----------------------|---------------------------------------------| -| metadata.name | aissemble-spark-application.metadata.name | -| metadata.namespace | aissemble-spark-application.namespace | -| spec.serviceEnabled | aissemble-spark-application.service.enabled | +| Old Property Location | New Property Location | +|-----------------------|---------------------------------------------------| +| metadata.name | aissemble-spark-application-chart.metadata.name | +| metadata.namespace | aissemble-spark-application-chart.namespace | +| spec.serviceEnabled | aissemble-spark-application-chart.service.enabled | -And all new property locations for service configuration should include the prefix `aissemble-spark-application` only. +And all new property locations for service configuration should include the prefix `aissemble-spark-application-chart` only. -*Note: To enable the service configuration, you need to set* `aissemble-spark-application.sparkApp.spec.serviceEnabled=true` +*Note: To enable the service configuration, you need to set* `aissemble-spark-application-chart.sparkApp.spec.serviceEnabled=true` diff --git a/extensions/extensions-helm/aissemble-spark-operator-chart/README.md b/extensions/extensions-helm/aissemble-spark-operator-chart/README.md index 67fefa40e..7cb6308bf 100644 --- a/extensions/extensions-helm/aissemble-spark-operator-chart/README.md +++ b/extensions/extensions-helm/aissemble-spark-operator-chart/README.md @@ -32,7 +32,7 @@ What follows reflects properties in the base spark-operator chart which have bee you must reference the root chart in your `values.yaml` file. For example: ```yaml -aissemble-spark-operator: +aissemble-spark-operator-chart: spark-operator: webhook.enable: false ``` @@ -50,11 +50,11 @@ aissemble-spark-operator: ## Migrated Properties -The following properties have been migrated from the `spark-operator` subchart to the `aissemble-spark-operator` chart. +The following properties have been migrated from the `spark-operator` subchart to the `aissemble-spark-operator-chart` chart. Any required overrides should be cognisant of the alternate path. For example: ```yaml -aissemble-spark-operator: +aissemble-spark-operator-chart: rbac: createClusterRole: false ``` @@ -78,7 +78,7 @@ To disable the shared Ivy cache, set the `ivyCache.enabled` property to `false` Spark Operator to remove the Ivy cache PersistentVolumeClaim. For example: ```yaml -aissemble-spark-operator: +aissemble-spark-operator-chart: ivyCache: enabled: false spark-operator: diff --git a/extensions/extensions-helm/aissemble-versioning-chart/README.md b/extensions/extensions-helm/aissemble-versioning-chart/README.md index e59b98cd9..af24decf3 100644 --- a/extensions/extensions-helm/aissemble-versioning-chart/README.md +++ b/extensions/extensions-helm/aissemble-versioning-chart/README.md @@ -29,4 +29,4 @@ helm install versioning ghcr.io/boozallen/aissemble-versioning-chart --version < | service.spec.ports | The service spec ports | No | - name: http
  port: 8080
  targetPort: 80 | | service.type | The service type | No | ClusterIP | -All properties must be prefixed with the key `aissemble-fastapi-chart` to override any values in the chart. See [helm documentation](https://helm.sh/docs/chart_template_guide/subcharts_and_globals/#overriding-values-from-a-parent-chart) for more info. \ No newline at end of file +All properties must be prefixed with the key `aissemble-versioning-chart` to override any values in the chart. See [helm documentation](https://helm.sh/docs/chart_template_guide/subcharts_and_globals/#overriding-values-from-a-parent-chart) for more info. \ No newline at end of file diff --git a/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-db-chart/README.md b/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-db-chart/README.md index c9c9e34fb..dae725fa0 100644 --- a/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-db-chart/README.md +++ b/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-db-chart/README.md @@ -11,7 +11,7 @@ helm install hive-metastore-db ghcr.io/boozallen/aissemble-hive-metastore-db-cha # Properties | Property | Description | Required Override | Default | |-------------------------------|---------------------------------------------|-------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| app.name | Sets label for app.kubernetes.io/name | No | Chart.Name (aissemble-hive-metastore-db) | +| app.name | Sets label for app.kubernetes.io/name | No | Chart.Name (aissemble-hive-metastore-db-chart) | | app.version | Sets label for app.kubernetes.io/version | No | Chart.AppVersion (aiSSEMBLE project version) | | replicaCount | Sets desired number of replicas (instances) | No | 1 | | hostname | The hostname for the application | No | hive-metastore-db | diff --git a/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-db-chart/values.yaml b/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-db-chart/values.yaml index f1dee095f..4a576ada2 100644 --- a/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-db-chart/values.yaml +++ b/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-db-chart/values.yaml @@ -2,7 +2,7 @@ # This is a YAML-formatted file. # Declare variables to be passed into your templates. app: - # Sets labels for app.kubernetes.io/name Default is Chart.name (aissemble-hive-metastore-db) + # Sets labels for app.kubernetes.io/name Default is Chart.name (aissemble-hive-metastore-db-chart) name: '' # Sets the labels for app.kubernetes.io/version. Default is Chart.appVersion version: '' diff --git a/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/README.md b/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/README.md index 36441af40..15f19de51 100644 --- a/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/README.md +++ b/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/README.md @@ -11,7 +11,7 @@ helm install hive-metastore-service ghcr.io/boozallen/aissemble-hive-metastore-s # Properties | Property | Description | Required Override | Default | |-------------------------------------------------------|----------------------------------------------------|-------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| app.name | Sets label for app.kubernetes.io/name | No | Chart.Name (aissemble-hive-metastore-service) | +| app.name | Sets label for app.kubernetes.io/name | No | Chart.Name (aissemble-hive-metastore-service-chart) | | app.version | Sets label for app.kubernetes.io/version | No | Chart.AppVersion (aiSSEMBLE project version) | | replicaCount | Sets desired number of replicas (instances) | No | 1 | | hostname | The hostname for the application | No | hive-metastore-service | diff --git a/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/values.yaml b/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/values.yaml index ab8caa5ea..f94301f1d 100644 --- a/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/values.yaml +++ b/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/values.yaml @@ -2,7 +2,7 @@ # This is a YAML-formatted file. # Declare variables to be passed into your templates. app: - # Sets labels for app.kubernetes.io/name Default is Chart.name (aissemble-hive-metastore-service) + # Sets labels for app.kubernetes.io/name Default is Chart.name (aissemble-hive-metastore-service-chart) name: '' # Sets the labels for app.kubernetes.io/version. Default is Chart.appVersion version: '' diff --git a/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/values.yaml b/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/values.yaml index 36796f476..0466e1955 100644 --- a/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/values.yaml +++ b/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/values.yaml @@ -1,4 +1,4 @@ -aissemble-quarkus: +aissemble-quarkus-chart: app: name: 'aissemble-pipeline-invocation' deployment: diff --git a/foundation/foundation-upgrade/src/main/java/com/boozallen/aissemble/upgrade/migration/v1_7_0/HelmChartsNameMigration.java b/foundation/foundation-upgrade/src/main/java/com/boozallen/aissemble/upgrade/migration/v1_7_0/HelmChartsNameMigration.java new file mode 100644 index 000000000..d49d8dcfe --- /dev/null +++ b/foundation/foundation-upgrade/src/main/java/com/boozallen/aissemble/upgrade/migration/v1_7_0/HelmChartsNameMigration.java @@ -0,0 +1,98 @@ +package com.boozallen.aissemble.upgrade.migration.v1_7_0; + +/*- + * #%L + * aiSSEMBLE::Foundation::Upgrade + * %% + * Copyright (C) 2021 Booz Allen + * %% + * This software package is licensed under the Booz Allen Public License. All Rights Reserved. + * #L% + */ + +import com.boozallen.aissemble.upgrade.migration.AbstractAissembleMigration; +import com.boozallen.aissemble.upgrade.util.FileUtils; +import org.apache.commons.collections4.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.technologybrewery.baton.BatonException; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import static com.boozallen.aissemble.upgrade.util.FileUtils.getRegExCaptureGroups; + +/** + * Baton migration used to migrate the Chart.yaml and values*.yaml files within a project's deploy folder + * to use the new Helm chart naming convention (aissemble--chart) + */ + +public class HelmChartsNameMigration extends AbstractAissembleMigration { + public static final Logger logger = LoggerFactory.getLogger(HelmChartsNameMigration.class); + private static final String dependencyNameReplaceRegex = "(aissemble-[^\\/\\n:]+)(? dependencyNames = getRegExCaptureGroups(dependencyNameReplaceRegex, file); + shouldExecute = CollectionUtils.isNotEmpty(dependencyNames); + } else { + List appNames = getRegExCaptureGroups(appNameReplaceRegex, file); + shouldExecute = CollectionUtils.isNotEmpty(appNames); + } + } catch (IOException e) { + throw new BatonException("Unable to determine if Helm charts should be migrated!", e); + } + + if (shouldExecute) { + logger.info("Found Helm chart file using old naming convention. Migrating file: {}", file.getName()); + } + return shouldExecute; + } + + /** + * Performs the migration if the shouldExecuteOnFile() returns true. + * @param file file to migrate + * @return performedSuccessfully - Whether the file was migrated successfully. + */ + @Override + protected boolean performMigration(File file) { + boolean performedSuccessfully = false; + + try { + + String appNameReplacementText = FIRST_REGEX_GROUPING.concat(SECOND_REGEX_GROUPING + CHART_POSTFIX + COLON); + String dependencyNameReplacementText = FIRST_REGEX_GROUPING.concat(CHART_POSTFIX); + + if (file.getName().matches("Chart.yaml")){ + performedSuccessfully = FileUtils.replaceInFile( + file, + dependencyNameReplaceRegex, + dependencyNameReplacementText + ); + } else { + performedSuccessfully = FileUtils.replaceInFile( + file, + appNameReplaceRegex, + appNameReplacementText + ); + } + } catch (Exception e) { + logger.error("Unable to perform helm chart name migration due to exception", e); + } + + return performedSuccessfully; + } + +} diff --git a/foundation/foundation-upgrade/src/main/resources/migrations.json b/foundation/foundation-upgrade/src/main/resources/migrations.json index 55960c571..783f9681e 100644 --- a/foundation/foundation-upgrade/src/main/resources/migrations.json +++ b/foundation/foundation-upgrade/src/main/resources/migrations.json @@ -64,5 +64,17 @@ "excludes": ["*-docker/*/target/Dockerfile"] } ] + }, + { + "name": "upgrade-helm-chart-files-names-migration", + "implementation": "com.boozallen.aissemble.upgrade.migration.v1_7_0.HelmChartsNameMigration", + "fileSets": [ + { + "includes": [ + "*-deploy/src/main/resources/apps/*/Chart.yaml", + "*-deploy/src/main/resources/apps/*/values*.yaml" + ] + } + ] } ] diff --git a/foundation/foundation-upgrade/src/test/java/com/boozallen/aissemble/upgrade/migration/TiltMigrationSteps.java b/foundation/foundation-upgrade/src/test/java/com/boozallen/aissemble/upgrade/migration/TiltMigrationSteps.java index 587374c94..ab6c8e477 100644 --- a/foundation/foundation-upgrade/src/test/java/com/boozallen/aissemble/upgrade/migration/TiltMigrationSteps.java +++ b/foundation/foundation-upgrade/src/test/java/com/boozallen/aissemble/upgrade/migration/TiltMigrationSteps.java @@ -63,16 +63,16 @@ public void theTiltfileVersionMigrationExecutes() { @Then("the VERSION_AISSEMBLE is updated to the project version") public void theVERSION_AISSEMBLEIsUpdatedToTheProjectVersion() { - assertTrue("Tiltfile VERSION AISSEMBLE does not line up with required aiSSEMBLE verision", successful); + assertTrue("Tiltfile VERSION AISSEMBLE does not line up with required aiSSEMBLE version", successful); } @Then("the VERSION_AISSEMBLE should not be migrated") public void theVERSION_AISSEMBLEShouldNotBeMigrated() { - assertFalse("Tiltfile VERSION AISSEMBLE is inline with the required aiSSEMBLE verision", shouldExecute); + assertFalse("Tiltfile VERSION AISSEMBLE is inline with the required aiSSEMBLE version", shouldExecute); } @Then("the VERSION_AISSEMBLE is updated to the specified version value from the Baton specifications in the Tiltfile") public void theVERSION_AISSEMBLEIsUpdatedToTheSpecifiedVersionValueFromTheBatonSpecificationsInTheTiltfile() { - assertTrue("Tiltfile VERSION AISSEMBLE does not line up with required aiSSEMBLE verision", shouldExecute); + assertTrue("Tiltfile VERSION AISSEMBLE does not line up with required aiSSEMBLE version", shouldExecute); } } diff --git a/foundation/foundation-upgrade/src/test/java/com/boozallen/aissemble/upgrade/migration/v1_7_0/ChartNameMigrationSteps.java b/foundation/foundation-upgrade/src/test/java/com/boozallen/aissemble/upgrade/migration/v1_7_0/ChartNameMigrationSteps.java new file mode 100644 index 000000000..930875fb5 --- /dev/null +++ b/foundation/foundation-upgrade/src/test/java/com/boozallen/aissemble/upgrade/migration/v1_7_0/ChartNameMigrationSteps.java @@ -0,0 +1,97 @@ +package com.boozallen.aissemble.upgrade.migration.v1_7_0; +/*- + * #%L + * aiSSEMBLE::Foundation::Upgrade + * %% + * Copyright (C) 2021 Booz Allen + * %% + * This software package is licensed under the Booz Allen Public License. All Rights Reserved. + * #L% + */ + +import com.boozallen.aissemble.upgrade.migration.AbstractMigrationTest; +import com.boozallen.aissemble.upgrade.migration.utils.MigrationTestUtils; +import com.boozallen.aissemble.upgrade.migration.v1_7_0.HelmChartsNameMigration; +import io.cucumber.java.en.Given; +import io.cucumber.java.en.Then; +import io.cucumber.java.en.When; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class ChartNameMigrationSteps extends AbstractMigrationTest { + + @Given("a value file using the older Helm chart naming convention") + public void a_value_file_using_the_older_helm_chart_naming_convention() { + testFile = getTestFile("v1_7_0/HelmChartNameMigration/migration/kafka-cluster/values.yaml"); + } + + @When("the Helm chart name change migration executes") + public void the_helm_chart_name_change_migration_executes() { + performMigration(new HelmChartsNameMigration()); + } + + @Then("the value file is migrated to use new naming convention") + public void the_value_file_is_migrated_to_use_new_naming_convention() { + HashMap valueFileContents; + try { + valueFileContents = MigrationTestUtils.extractYamlContents(testFile); + String firstKey = valueFileContents.keySet().iterator().next(); + + assertTrue(firstKey.startsWith("aissemble-") && firstKey.endsWith("-chart")); + } catch (IOException e) { + fail(String.format("Unable to parse yaml contents due to exception: %s", e)); + } + } + + @Given("a Chart file using the older Helm chart naming convention") + public void a_chart_file_using_the_older_helm_chart_naming_convention() { + testFile = getTestFile("v1_7_0/HelmChartNameMigration/migration/kafka-cluster/Chart.yaml"); + } + + @Then("the Chart file is migrated to use new naming convention") + public void the_chart_file_is_migrated_to_use_new_naming_convention() { + HashMap chartFileContents; + try { + chartFileContents = MigrationTestUtils.extractYamlContents(testFile); + Object dependenciesObject = chartFileContents.get("dependencies"); + ArrayList> dependencies = (ArrayList>) dependenciesObject; + + for (HashMap dependency : dependencies) { + String dependencyName = (String) dependency.get("name"); + + assertTrue(dependencyName.startsWith("aissemble-") && dependencyName.endsWith("-chart")); + } + + } catch (IOException e) { + fail(String.format("Unable to parse yaml contents due to exception: %s", e)); + } + } + + @Given("a value file using the new Helm chart naming convention") + public void a_value_file_using_the_new_helm_chart_naming_convention() { + testFile = getTestFile("v1_7_0/HelmChartNameMigration/skip-migration/kafka-cluster/values.yaml"); + } + + @Then("the value file migration is skipped") + public void the_value_file_migration_is_skipped() { + assertFalse("Value file is already using new Helm chart naming convention", shouldExecute); + + } + + @Given("a Chart file using the new Helm chart naming convention") + public void a_chart_file_using_the_new_helm_chart_naming_convention() { + testFile = getTestFile("v1_7_0/HelmChartNameMigration/skip-migration/kafka-cluster/Chart.yaml"); + + } + + @Then("the Chart file migration is skipped") + public void the_chart_file_migration_is_skipped() { + assertFalse("Chart file is already using new Helm chart naming convention", shouldExecute); + } + +} + diff --git a/foundation/foundation-upgrade/src/test/resources/specifications/v1_7_0/chart-name-migration.feature b/foundation/foundation-upgrade/src/test/resources/specifications/v1_7_0/chart-name-migration.feature new file mode 100644 index 000000000..1185ac65c --- /dev/null +++ b/foundation/foundation-upgrade/src/test/resources/specifications/v1_7_0/chart-name-migration.feature @@ -0,0 +1,22 @@ +@chart-name-change-migration +Feature: As an aiSSEMBLE user, I want my aiSSEMBLE Helm charts updated to the latest naming convention automatically + + Scenario: Upgrade aiSSEMBLE Helm chart value file to use to latest naming convention + Given a value file using the older Helm chart naming convention + When the Helm chart name change migration executes + Then the value file is migrated to use new naming convention + + Scenario: Upgrade aiSSEMBLE Helm chart Chart file to use to latest naming convention + Given a Chart file using the older Helm chart naming convention + When the Helm chart name change migration executes + Then the Chart file is migrated to use new naming convention + + Scenario: Skipping Value file migration if values file already uses new naming convention + Given a value file using the new Helm chart naming convention + When the Helm chart name change migration executes + Then the value file migration is skipped + + Scenario: Skipping Chart file migration if Chart file already uses new naming convention + Given a Chart file using the new Helm chart naming convention + When the Helm chart name change migration executes + Then the Chart file migration is skipped \ No newline at end of file diff --git a/foundation/foundation-upgrade/src/test/resources/test-files/v1_7_0/HelmChartNameMigration/migration/kafka-cluster/Chart.yaml b/foundation/foundation-upgrade/src/test/resources/test-files/v1_7_0/HelmChartNameMigration/migration/kafka-cluster/Chart.yaml new file mode 100644 index 000000000..57605cba7 --- /dev/null +++ b/foundation/foundation-upgrade/src/test/resources/test-files/v1_7_0/HelmChartNameMigration/migration/kafka-cluster/Chart.yaml @@ -0,0 +1,34 @@ +apiVersion: v2 +name: kafka-cluster +description: A Helm chart for kafka-cluster + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 1.0.0-SNAPSHOT + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.0.0" + +# To include an aiSSEMBLE chart with ArgoCD we must add it as a dependency until including from multiple sources +# is moved out of beta and into the latest version of Argo. +# https://argo-cd.readthedocs.io/en/stable/user-guide/multiple_sources/ +# Note that this requires prefixing values with the name of the dependency (see test/src/test/resources in the aiSSEMBLE repo +# for example usage) in order to override values. +dependencies: + - name: aissemble-kafka + version: 1.7.0-SNAPSHOT + repository: oci://ghcr.io/boozallen \ No newline at end of file diff --git a/foundation/foundation-upgrade/src/test/resources/test-files/v1_7_0/HelmChartNameMigration/migration/kafka-cluster/values.yaml b/foundation/foundation-upgrade/src/test/resources/test-files/v1_7_0/HelmChartNameMigration/migration/kafka-cluster/values.yaml new file mode 100644 index 000000000..db03236b7 --- /dev/null +++ b/foundation/foundation-upgrade/src/test/resources/test-files/v1_7_0/HelmChartNameMigration/migration/kafka-cluster/values.yaml @@ -0,0 +1,9 @@ +######################################## +## CONFIG | Kafka Configs +######################################## +aissemble-kafka: + kafka: + resources: + limits: + memory: 2G + autoCreateTopicsEnable: true # Enable auto creation of topics \ No newline at end of file diff --git a/foundation/foundation-upgrade/src/test/resources/test-files/v1_7_0/HelmChartNameMigration/skip-migration/kafka-cluster/Chart.yaml b/foundation/foundation-upgrade/src/test/resources/test-files/v1_7_0/HelmChartNameMigration/skip-migration/kafka-cluster/Chart.yaml new file mode 100644 index 000000000..4a15e27ca --- /dev/null +++ b/foundation/foundation-upgrade/src/test/resources/test-files/v1_7_0/HelmChartNameMigration/skip-migration/kafka-cluster/Chart.yaml @@ -0,0 +1,34 @@ +apiVersion: v2 +name: kafka-cluster +description: A Helm chart for kafka-cluster + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 1.0.0-SNAPSHOT + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.0.0" + +# To include an aiSSEMBLE chart with ArgoCD we must add it as a dependency until including from multiple sources +# is moved out of beta and into the latest version of Argo. +# https://argo-cd.readthedocs.io/en/stable/user-guide/multiple_sources/ +# Note that this requires prefixing values with the name of the dependency (see test/src/test/resources in the aiSSEMBLE repo +# for example usage) in order to override values. +dependencies: + - name: aissemble-kafka-chart + version: 1.7.0-SNAPSHOT + repository: oci://ghcr.io/boozallen \ No newline at end of file diff --git a/foundation/foundation-upgrade/src/test/resources/test-files/v1_7_0/HelmChartNameMigration/skip-migration/kafka-cluster/values.yaml b/foundation/foundation-upgrade/src/test/resources/test-files/v1_7_0/HelmChartNameMigration/skip-migration/kafka-cluster/values.yaml new file mode 100644 index 000000000..913a70661 --- /dev/null +++ b/foundation/foundation-upgrade/src/test/resources/test-files/v1_7_0/HelmChartNameMigration/skip-migration/kafka-cluster/values.yaml @@ -0,0 +1,9 @@ +######################################## +## CONFIG | Kafka Configs +######################################## +aissemble-kafka-chart: + kafka: + resources: + limits: + memory: 2G + autoCreateTopicsEnable: true # Enable auto creation of topics \ No newline at end of file