diff --git a/README.md b/README.md index 2a51ad2c..9f1a7a29 100644 --- a/README.md +++ b/README.md @@ -131,13 +131,8 @@ This following information is parsed from the integration: - JUMP_SVR_HOST - JUMP_SVR_USER - JUMP_SVR_RWX_FILESTORE_PATH -- Postgres (When V4_CFG_POSTGRES_TYPE is set to external) - - V4_CFG_POSTGRES_ADMIN_LOGIN - - V4_CFG_POSTGRES_PASSWORD - - V4_CFG_POSTGRES_FQDN - - V4_CFG_POSTGRES_CONNECTION_NAME - - V4_CFG_POSTGRES_SERVICE_ACCOUNT - - V4_CFG_POSTGRES_SSL_ENFORCEMENT +- Postgres + - V4_CFG_POSTGRES_SERVERS (if postgres deployed) - Cluster - KUBECONFIG - V4_CFG_CLUSTER_NODE_POOL_MODE diff --git a/ansible.cfg b/ansible.cfg index 20843c9a..741eebf5 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -18,4 +18,4 @@ display_skipped_hosts = False hash_behaviour=merge library = /usr/share/ansible:./plugins/modules lookup_plugins = ./plugins/lookup -action_plugins = ./plugins/action \ No newline at end of file +action_plugins = ./plugins/action diff --git a/docs/CONFIG-VARS.md b/docs/CONFIG-VARS.md index 99accd8d..ad0fa665 100644 --- a/docs/CONFIG-VARS.md +++ b/docs/CONFIG-VARS.md @@ -22,7 +22,6 @@ Supported configuration variables are listed in the table below. All variables - [TLS](#tls) - [Cert-manager](#cert-manager) - [Postgres](#postgres) - - [External Postgres](#external-postgres) - [CAS](#cas) - [CONNECT](#connect) - [Miscellaneous](#miscellaneous) @@ -191,22 +190,49 @@ When setting V4_CFG_TLS_MODE to a value other than "disabled" and no V4_CFG_TLS_ ## Postgres -| Name | Description | Type | Default | Required | Notes | Tasks | -| :--- | ---: | ---: | ---: | ---: | ---: | ---: | -| V4_CFG_POSTGRES_TYPE | Postgres installation type | string | | true | [internal,external] | viya | +Postgres servers can be defined with the postgres_servers variable which is a map of objects. The variable has the following format: + +```bash +V4_CFG_POSTGRES_SERVERS: + default: {} + ... +``` -### External Postgres +**NOTE**: the `default` elements is always required . This will be the default server. Below is the list of parameters each element can contain. | Name | Description | Type | Default | Required | Notes | Tasks | | :--- | ---: | ---: | ---: | ---: | ---: | ---: | -| V4_CFG_POSTGRES_ADMIN_LOGIN | Existing postgres username | string | | true | | viya | -| V4_CFG_POSTGRES_PASSWORD | Existing postgres password | string | | true | | viya | -| V4_CFG_POSTGRES_FQDN | Existing postgres ip/fqdn | string | | true | | viya | -| V4_CFG_POSTGRES_PORT | Existing postgres port | string | 5432 | false | | viya | -| V4_CFG_POSTGRES_DATABASE | Existing postgres database name | string | "SharedServices" | false | | viya | -| V4_CFG_POSTGRES_SSL_ENFORCEMENT | Require ssl connection to existing postgres | bool | false | false | Ignored on GCP when using cloud sql | viya | -| V4_CFG_POSTGRES_CONNECTION_NAME | Existing postgres database connection name | string | | false | See [ansible cloud authentication](user/AnsibleCloudAuthentication.md) | viya | -| V4_CFG_POSTGRES_SERVICE_ACCOUNT | Existing service account for postgres connectivity | string | | false | See [ansible cloud authentication](user/AnsibleCloudAuthentication.md) | viya | +| internal | Whether the database is internal or external | bool | | true | All servers must but internal or all must be external | viya | +| database | Database name | string | Database server role | false | Default database name for default server is SharedServices | viya | +| admin | External postgres username | string | | false | Required for external postgres servers | viya | +| password | External postgres password | string | | false | Required for external postgres servers | viya | +| fqdn | External postgres ip/fqdn | string | | false | Required for external postgres servers | viya | +| server_port | External postgres port | string | 5432 | false | | viya | +| ssl_enforcement_enabled | Require ssl connection to external postgres | bool | | false | Required for external postgres servers. Ignored on GCP when using cloud sql | viya | +| connection_name | External postgres database connection name | string | | false | Required for using cloud-sql-proxy on gcp. See [ansible cloud authentication](user/AnsibleCloudAuthentication.md) | viya | +| service_account | External service account for postgres connectivity | string | | false | Required for using cloud-sql-proxy on gcp. See [ansible cloud authentication](user/AnsibleCloudAuthentication.md) | viya | + +Example: + +```bash +V4_CFG_POSTGRES_SERVERS: + default: + internal: false + admin: pgadmin + password: "password" + fqdn: mydbserver.local + server_port: 5432 + ssl_enforcement_enabled: true + database: SharedServices + other_db: + internal: false + admin: pgadmin + password: "password" + fqdn: 10.10.10.10 + server_port: 5432 + ssl_enforcement_enabled: true + database: OtherDB +``` ## CAS diff --git a/examples/ansible-vars-iac.yaml b/examples/ansible-vars-iac.yaml index fd08440a..6feb0215 100644 --- a/examples/ansible-vars-iac.yaml +++ b/examples/ansible-vars-iac.yaml @@ -23,7 +23,6 @@ V4_CFG_INGRESS_FQDN: V4_CFG_TLS_MODE: "full-stack" # [full-stack|front-door|disabled] ## Postgres -V4_CFG_POSTGRES_TYPE: external #[internal|external] ## LDAP V4_CFG_EMBEDDED_LDAP_ENABLE: true diff --git a/examples/ansible-vars.yaml b/examples/ansible-vars.yaml index d4768496..6ca8b823 100644 --- a/examples/ansible-vars.yaml +++ b/examples/ansible-vars.yaml @@ -33,11 +33,14 @@ V4_CFG_INGRESS_FQDN: V4_CFG_TLS_MODE: "full-stack" # [full-stack|front-door|disabled] ## Postgres -V4_CFG_POSTGRES_TYPE: external -V4_CFG_POSTGRES_ADMIN_LOGIN: -V4_CFG_POSTGRES_PASSWORD: -V4_CFG_POSTGRES_FQDN: -V4_CFG_POSTGRES_PORT: 5432 +V4_CFG_POSTGRES_SERVERS: + default: + internal: false + admin: + password: + fqdn: + ssl_enforcement_enabled: true + database: ## LDAP V4_CFG_EMBEDDED_LDAP_ENABLE: true @@ -50,4 +53,4 @@ V4_CFG_CONNECT_ENABLE_LOADBALANCER: false ## Monitoring and Logging ## uncomment and update the below values when deploying the viya4-monitoring-kubernetes stack -#V4M_BASE_DOMAIN: \ No newline at end of file +#V4M_BASE_DOMAIN: diff --git a/playbooks/playbook.yaml b/playbooks/playbook.yaml index fab80609..af4c0356 100644 --- a/playbooks/playbook.yaml +++ b/playbooks/playbook.yaml @@ -7,7 +7,7 @@ tags: - install - uninstall - - upgrade + - update - name: common role include_role: name: common @@ -15,7 +15,7 @@ tags: - install - uninstall - - upgrade + - update - name: jump-server role include_role: name: jump-server @@ -46,6 +46,7 @@ - name: monitoring role - namespace include_role: name: monitoring + tasks_from: viya-monitoring tags: - viya-monitoring - name: Delete tmpdir @@ -55,4 +56,4 @@ tags: - install - uninstall - - upgrade + - update diff --git a/requirements.yaml b/requirements.yaml index 03f74026..780f191d 100644 --- a/requirements.yaml +++ b/requirements.yaml @@ -2,3 +2,5 @@ collections: - name: community.kubernetes version: 1.2.1 + - name: ansible.utils + version: 2.3.0 diff --git a/roles/baseline/defaults/main.yml b/roles/baseline/defaults/main.yml index 64c38ac4..307eaab7 100644 --- a/roles/baseline/defaults/main.yml +++ b/roles/baseline/defaults/main.yml @@ -2,6 +2,8 @@ V4_CFG_TLS_MODE: "full-stack" # other valid values are front-door and disabled V4_CFG_RWX_FILESTORE_ENDPOINT: /export V4_CFG_INGRESS_TYPE: ingress +PRIVATE_CLUSTER_ENABLE: false + ## Cert-manager CERT_MANAGER_NAME: cert-manager CERT_MANAGER_NAMESPACE: cert-manager @@ -34,6 +36,8 @@ INGRESS_NGINX_CONFIG: externalTrafficPolicy: Local sessionAffinity: None loadBalancerSourceRanges: "{{ LOADBALANCER_SOURCE_RANGES |default(['0.0.0.0/0'], -1) }}" + annotation: + config: use-forwarded-headers: "true" tcp: {} @@ -91,3 +95,21 @@ CLUSTER_AUTOSCALER_CONFIG: name: cluster-autoscaler annotations: "eks.amazonaws.com/role-arn": "{{ CLUSTER_AUTOSCALER_ACCOUNT }}" + +private_cluster: + aws: + controller: + service: + annotations: + service.beta.kubernetes.io/aws-load-balancer-internal: "true" + service.beta.kubernetes.io/aws-load-balancer-type: nlb + azure: + controller: + service: + annotations: + service.beta.kubernetes.io/azure-load-balancer-internal: "true" + gcp: + controller: + service: + annotations: + networking.gke.io/load-balancer-type: "Internal" diff --git a/roles/baseline/tasks/cert-manager.yaml b/roles/baseline/tasks/cert-manager.yaml index 0bda8fb7..72214f84 100644 --- a/roles/baseline/tasks/cert-manager.yaml +++ b/roles/baseline/tasks/cert-manager.yaml @@ -12,7 +12,7 @@ wait: true tags: - install - - upgrade + - update - name: Remove cert-manager community.kubernetes.helm: diff --git a/roles/baseline/tasks/cluster-autoscaler.yaml b/roles/baseline/tasks/cluster-autoscaler.yaml index fc2438e2..a66a056d 100644 --- a/roles/baseline/tasks/cluster-autoscaler.yaml +++ b/roles/baseline/tasks/cluster-autoscaler.yaml @@ -11,7 +11,7 @@ wait: true tags: - install - - upgrade + - update - name: Remove cluster-autoscaler community.kubernetes.helm: diff --git a/roles/baseline/tasks/contour.yaml b/roles/baseline/tasks/contour.yaml index 640423f6..49b8c4b1 100644 --- a/roles/baseline/tasks/contour.yaml +++ b/roles/baseline/tasks/contour.yaml @@ -12,7 +12,7 @@ wait: true tags: - install - - upgrade + - update - name: Remove contour community.kubernetes.helm: diff --git a/roles/baseline/tasks/ingress-nginx.yaml b/roles/baseline/tasks/ingress-nginx.yaml index fb5595da..1479986b 100644 --- a/roles/baseline/tasks/ingress-nginx.yaml +++ b/roles/baseline/tasks/ingress-nginx.yaml @@ -1,4 +1,13 @@ --- +- set_fact: + INGRESS_NGINX_CONFIG: "{{ INGRESS_NGINX_CONFIG |combine(private_cluster[PROVIDER], recursive=True)}}" + when: + - PRIVATE_CLUSTER_ENABLE + - PROVIDER in private_cluster + tags: + - install + - update + - name: Deploy ingress-nginx community.kubernetes.helm: name: "{{ INGRESS_NGINX_NAME }}" @@ -12,7 +21,7 @@ wait: true tags: - install - - upgrade + - update - name: Remove ingress-nginx community.kubernetes.helm: diff --git a/roles/baseline/tasks/metrics-server.yaml b/roles/baseline/tasks/metrics-server.yaml index 83cfa0ea..f072a070 100644 --- a/roles/baseline/tasks/metrics-server.yaml +++ b/roles/baseline/tasks/metrics-server.yaml @@ -9,7 +9,7 @@ register: metrics_service tags: - install - - upgrade + - update - uninstall - name: Deploy metrics-server @@ -24,7 +24,7 @@ wait: true tags: - install - - upgrade + - update when: - (metrics_service.resources | length) == 0 diff --git a/roles/baseline/tasks/nfs-subdir-external-provisioner.yaml b/roles/baseline/tasks/nfs-subdir-external-provisioner.yaml index 789b84eb..0a540cde 100644 --- a/roles/baseline/tasks/nfs-subdir-external-provisioner.yaml +++ b/roles/baseline/tasks/nfs-subdir-external-provisioner.yaml @@ -8,7 +8,7 @@ tags: - install - uninstall - - upgrade + - update - name: Remove deprecated efs-provisioner community.kubernetes.helm: @@ -19,7 +19,7 @@ tags: - install - uninstall - - upgrade + - update - name: Remove deprecated efs-provisioner namespace community.kubernetes.k8s: @@ -31,7 +31,7 @@ tags: - install - uninstall - - upgrade + - update - name: Deploy nfs-subdir-external-provisioner community.kubernetes.helm: @@ -46,7 +46,7 @@ wait: true tags: - install - - upgrade + - update - name: Remove nfs-subdir-external-provisioner community.kubernetes.helm: diff --git a/roles/common/defaults/main.yml b/roles/common/defaults/main.yml index e69de29b..5e2f69b0 100644 --- a/roles/common/defaults/main.yml +++ b/roles/common/defaults/main.yml @@ -0,0 +1,2 @@ +V4_CFG_POSTGRES_SERVERS: + default: {} diff --git a/roles/common/tasks/main.yaml b/roles/common/tasks/main.yaml index 106f7bb0..770aa72c 100644 --- a/roles/common/tasks/main.yaml +++ b/roles/common/tasks/main.yaml @@ -5,7 +5,7 @@ tags: - install - uninstall - - upgrade + - update - name: Set BASE_DIR default path set_fact: @@ -14,7 +14,7 @@ tags: - install - uninstall - - upgrade + - update - block: - name: Parse tfstate set_fact: @@ -76,56 +76,12 @@ when: - tfstate.cluster_name is defined - tfstate.cluster_name.value|length > 0 - - name: tfstate - postgres admin - set_fact: - V4_CFG_POSTGRES_ADMIN_LOGIN: "{{ tfstate.postgres_admin.value }}" - when: - - V4_CFG_POSTGRES_TYPE is defined - - V4_CFG_POSTGRES_TYPE == "external" - - tfstate.postgres_admin is defined - - tfstate.postgres_server_name is defined - - name: tfstate - postgres port - set_fact: - V4_CFG_POSTGRES_PORT: "{{ tfstate.postgres_server_port.value }}" - when: - - V4_CFG_POSTGRES_TYPE is defined - - V4_CFG_POSTGRES_TYPE == "external" - - tfstate.postgres_server_port is defined - - name: tfstate - postgres password + - name: tfstate - postgres servers set_fact: - V4_CFG_POSTGRES_PASSWORD: "{{ tfstate.postgres_password.value }}" + V4_CFG_POSTGRES_SERVERS: "{{ tfstate.postgres_servers.value |combine(V4_CFG_POSTGRES_SERVERS, recursive=True) }}" when: - - V4_CFG_POSTGRES_TYPE is defined - - V4_CFG_POSTGRES_TYPE == "external" - - tfstate.postgres_password is defined - - name: tfstate - postgres fqdn - set_fact: - V4_CFG_POSTGRES_FQDN: "{{ tfstate.postgres_fqdn.value }}" - when: - - V4_CFG_POSTGRES_TYPE is defined - - V4_CFG_POSTGRES_TYPE == "external" - - tfstate.postgres_fqdn is defined - - name: tfstate - postgres connection name - set_fact: - V4_CFG_POSTGRES_CONNECTION_NAME: "{{ tfstate.postgres_connection_name.value }}" - when: - - V4_CFG_POSTGRES_TYPE is defined - - V4_CFG_POSTGRES_TYPE == "external" - - tfstate.postgres_connection_name is defined - - name: tfstate - postgres service account - set_fact: - V4_CFG_POSTGRES_SERVICE_ACCOUNT: "{{ tfstate.sql_proxy_sa_email.value }}" - when: - - V4_CFG_POSTGRES_TYPE is defined - - V4_CFG_POSTGRES_TYPE == "external" - - tfstate.sql_proxy_sa_email is defined - - name: tfstate - postgres ssl enforcement - set_fact: - V4_CFG_POSTGRES_SSL_ENFORCEMENT: "{{ tfstate.postgres_ssl_enforcement_enabled.value }}" - when: - - V4_CFG_POSTGRES_TYPE is defined - - V4_CFG_POSTGRES_TYPE == "external" - - tfstate.postgres_ssl_enforcement_enabled is defined + - tfstate.postgres_servers is defined + - tfstate.postgres_servers.value|length > 0 - name: tfstate - cluster autoscaler account set_fact: CLUSTER_AUTOSCALER_ACCOUNT: "{{ tfstate.autoscaler_account.value }}" @@ -176,6 +132,57 @@ when: - tfstate.ssh_private_key is defined - tfstate.ssh_private_key.value|length > 0 + ### Deprecations + - name: tfstate - postgres admin + set_fact: + V4_CFG_POSTGRES_ADMIN_LOGIN: "{{ tfstate.postgres_admin.value }}" + when: + - V4_CFG_POSTGRES_TYPE is defined + - V4_CFG_POSTGRES_TYPE == "external" + - tfstate.postgres_admin is defined + - tfstate.postgres_server_name is defined + - name: tfstate - postgres port + set_fact: + V4_CFG_POSTGRES_PORT: "{{ tfstate.postgres_server_port.value }}" + when: + - V4_CFG_POSTGRES_TYPE is defined + - V4_CFG_POSTGRES_TYPE == "external" + - tfstate.postgres_server_port is defined + - name: tfstate - postgres password + set_fact: + V4_CFG_POSTGRES_PASSWORD: "{{ tfstate.postgres_password.value }}" + when: + - V4_CFG_POSTGRES_TYPE is defined + - V4_CFG_POSTGRES_TYPE == "external" + - tfstate.postgres_password is defined + - name: tfstate - postgres fqdn + set_fact: + V4_CFG_POSTGRES_FQDN: "{{ tfstate.postgres_fqdn.value }}" + when: + - V4_CFG_POSTGRES_TYPE is defined + - V4_CFG_POSTGRES_TYPE == "external" + - tfstate.postgres_fqdn is defined + - name: tfstate - postgres connection name + set_fact: + V4_CFG_POSTGRES_CONNECTION_NAME: "{{ tfstate.postgres_connection_name.value }}" + when: + - V4_CFG_POSTGRES_TYPE is defined + - V4_CFG_POSTGRES_TYPE == "external" + - tfstate.postgres_connection_name is defined + - name: tfstate - postgres service account + set_fact: + V4_CFG_POSTGRES_SERVICE_ACCOUNT: "{{ tfstate.sql_proxy_sa_email.value }}" + when: + - V4_CFG_POSTGRES_TYPE is defined + - V4_CFG_POSTGRES_TYPE == "external" + - tfstate.sql_proxy_sa_email is defined + - name: tfstate - postgres ssl enforcement + set_fact: + V4_CFG_POSTGRES_SSL_ENFORCEMENT: "{{ tfstate.postgres_ssl_enforcement_enabled.value }}" + when: + - V4_CFG_POSTGRES_TYPE is defined + - V4_CFG_POSTGRES_TYPE == "external" + - tfstate.postgres_ssl_enforcement_enabled is defined - set_fact: tfstate: "" when: @@ -183,7 +190,7 @@ tags: - install - uninstall - - upgrade + - update - name: Set DEPLOY_DIR set_fact: @@ -192,4 +199,12 @@ tags: - install - uninstall - - upgrade + - update + +- name: migrations + include_tasks: + file: migrations.yaml + tags: + - install + - uninstall + - update diff --git a/roles/common/tasks/migrations.yaml b/roles/common/tasks/migrations.yaml new file mode 100644 index 00000000..405b7fb3 --- /dev/null +++ b/roles/common/tasks/migrations.yaml @@ -0,0 +1,146 @@ + + +- block: + - ansible.utils.update_fact: + updates: + - path: "V4_CFG_POSTGRES_SERVERS['default']['fqdn']" + value: "{{ V4_CFG_POSTGRES_FQDN }}" + register: updated + - set_fact: + V4_CFG_POSTGRES_SERVERS: "{{ updated.V4_CFG_POSTGRES_SERVERS }}" + when: + - V4_CFG_POSTGRES_FQDN is defined + - "'fqdn' not in V4_CFG_POSTGRES_SERVERS['default']" + tags: + - install + - uninstall + - update + +- block: + - ansible.utils.update_fact: + updates: + - path: "V4_CFG_POSTGRES_SERVERS['default']['admin']" + value: "{{ V4_CFG_POSTGRES_ADMIN_LOGIN }}" + register: updated + - set_fact: + V4_CFG_POSTGRES_SERVERS: "{{ updated.V4_CFG_POSTGRES_SERVERS }}" + when: + - V4_CFG_POSTGRES_ADMIN_LOGIN is defined + - "'admin' not in V4_CFG_POSTGRES_SERVERS['default']" + tags: + - install + - uninstall + - update + +- block: + - ansible.utils.update_fact: + updates: + - path: "V4_CFG_POSTGRES_SERVERS['default']['server_port']" + value: "{{ V4_CFG_POSTGRES_PORT }}" + register: updated + - set_fact: + V4_CFG_POSTGRES_SERVERS: "{{ updated.V4_CFG_POSTGRES_SERVERS }}" + when: + - V4_CFG_POSTGRES_PORT is defined + - "'server_port' not in V4_CFG_POSTGRES_SERVERS['default']" + tags: + - install + - uninstall + - update + +- block: + - ansible.utils.update_fact: + updates: + - path: "V4_CFG_POSTGRES_SERVERS['default']['password']" + value: "{{ V4_CFG_POSTGRES_PASSWORD }}" + register: updated + - set_fact: + V4_CFG_POSTGRES_SERVERS: "{{ updated.V4_CFG_POSTGRES_SERVERS }}" + when: + - V4_CFG_POSTGRES_PASSWORD is defined + - "'password' not in V4_CFG_POSTGRES_SERVERS['default']" + tags: + - install + - uninstall + - update + +- block: + - ansible.utils.update_fact: + updates: + - path: "V4_CFG_POSTGRES_SERVERS['default']['connection_name']" + value: "{{ V4_CFG_POSTGRES_CONNECTION_NAME }}" + register: updated + - set_fact: + V4_CFG_POSTGRES_SERVERS: "{{ updated.V4_CFG_POSTGRES_SERVERS }}" + when: + - V4_CFG_POSTGRES_CONNECTION_NAME is defined + - "'connection_name' not in V4_CFG_POSTGRES_SERVERS['default']" + tags: + - install + - uninstall + - update + +- block: + - ansible.utils.update_fact: + updates: + - path: "V4_CFG_POSTGRES_SERVERS['default']['service_account']" + value: "{{ V4_CFG_POSTGRES_SERVICE_ACCOUNT }}" + register: updated + - set_fact: + V4_CFG_POSTGRES_SERVERS: "{{ updated.V4_CFG_POSTGRES_SERVERS }}" + when: + - V4_CFG_POSTGRES_SERVICE_ACCOUNT is defined + - "'service_account' not in V4_CFG_POSTGRES_SERVERS['default']" + tags: + - install + - uninstall + - update + +- block: + - ansible.utils.update_fact: + updates: + - path: "V4_CFG_POSTGRES_SERVERS['default']['ssl_enforcement_enabled']" + value: "{{ V4_CFG_POSTGRES_SSL_ENFORCEMENT }}" + register: updated + - set_fact: + V4_CFG_POSTGRES_SERVERS: "{{ updated.V4_CFG_POSTGRES_SERVERS }}" + when: + - V4_CFG_POSTGRES_SSL_ENFORCEMENT is defined + - "'ssl_enforcement_enabled' not in V4_CFG_POSTGRES_SERVERS['default']" + tags: + - install + - uninstall + - update + +- block: + - ansible.utils.update_fact: + updates: + - path: "V4_CFG_POSTGRES_SERVERS['default']['internal']" + value: "{{ V4_CFG_POSTGRES_TYPE == 'internal' }}" + register: updated + - set_fact: + V4_CFG_POSTGRES_SERVERS: "{{ updated.V4_CFG_POSTGRES_SERVERS }}" + when: + - V4_CFG_POSTGRES_TYPE is defined + - "'internal' not in V4_CFG_POSTGRES_SERVERS['default']" + tags: + - install + - uninstall + - update + +- block: + - ansible.utils.update_fact: + updates: + - path: "V4_CFG_POSTGRES_SERVERS['default']['database']" + value: "{{ V4_CFG_POSTGRES_DATABASE }}" + register: updated + - set_fact: + V4_CFG_POSTGRES_SERVERS: "{{ updated.V4_CFG_POSTGRES_SERVERS }}" + when: + - V4_CFG_POSTGRES_DATABASE is defined + - "'database' not in V4_CFG_POSTGRES_SERVERS['default']" + tags: + - install + - uninstall + - update + diff --git a/roles/istio/tasks/main.yml b/roles/istio/tasks/main.yml index c4552445..9138d9d6 100644 --- a/roles/istio/tasks/main.yml +++ b/roles/istio/tasks/main.yml @@ -6,7 +6,7 @@ tags: - install - uninstall - - upgrade + - update - name: Create namespace community.kubernetes.k8s: @@ -17,7 +17,7 @@ wait: true tags: - install - - upgrade + - update - name: Set kiali secret community.kubernetes.k8s: @@ -26,7 +26,7 @@ definition: "{{ lookup('template', 'kiali-secret.yaml') }}" tags: - install - - upgrade + - update - name: Set grafana secret community.kubernetes.k8s: @@ -35,13 +35,13 @@ definition: "{{ lookup('template', 'grafana-secret.yaml') }}" tags: - install - - upgrade + - update - name: Install base Istio command: "{{ tmpdir.path }}/istio-{{ istio_ver }}/bin/istioctl --kubeconfig {{ KUBECONFIG }} manifest apply {{ istio_config }}" tags: - install - - upgrade + - update - name: Lockdown ingress community.kubernetes.k8s: @@ -64,7 +64,7 @@ ipBlocks: "{{ LOADBALANCER_SOURCE_RANGES }}" tags: - install - - upgrade + - update - name: Grafana ingress community.kubernetes.k8s: @@ -90,7 +90,7 @@ when: grafana_ingress_enabled tags: - install - - upgrade + - update - name: Prometheus ingress community.kubernetes.k8s: @@ -116,7 +116,7 @@ when: prometheus_ingress_enabled tags: - install - - upgrade + - update - name: Kiali ingress community.kubernetes.k8s: @@ -142,7 +142,7 @@ when: kiali_ingress_enabled tags: - install - - upgrade + - update - name: Tracing ingress community.kubernetes.k8s: @@ -168,7 +168,7 @@ when: tracing_ingress_enabled tags: - install - - upgrade + - update - name: Lookup ingress community.kubernetes.k8s_info: @@ -181,7 +181,7 @@ register: ingress_config tags: - install - - upgrade + - update - uninstall - set_fact: @@ -190,7 +190,7 @@ cacheable: yes tags: - install - - upgrade + - update - uninstall - name: Uninstall base Istio @@ -221,5 +221,5 @@ state: absent tags: - install - - upgrade + - update - uninstall diff --git a/roles/monitoring/tasks/cluster-logging.yaml b/roles/monitoring/tasks/cluster-logging.yaml index 1d9b861c..79b008e6 100644 --- a/roles/monitoring/tasks/cluster-logging.yaml +++ b/roles/monitoring/tasks/cluster-logging.yaml @@ -7,7 +7,7 @@ tags: - install - uninstall - - upgrade + - update - name: cluster-logging - lookup existing credentials community.kubernetes.k8s_info: diff --git a/roles/monitoring/tasks/cluster-monitoring.yaml b/roles/monitoring/tasks/cluster-monitoring.yaml index d119be6c..61205ccb 100644 --- a/roles/monitoring/tasks/cluster-monitoring.yaml +++ b/roles/monitoring/tasks/cluster-monitoring.yaml @@ -7,7 +7,7 @@ tags: - install - uninstall - - upgrade + - update - name: cluster-monitoring - lookup existing credentials community.kubernetes.k8s_info: @@ -19,14 +19,14 @@ register: monitoring_creds tags: - install - - upgrade + - update - name: cluster-monitoring - save credentials set_fact: V4M_GRAFANA_PASSWORD: "{{ monitoring_creds.resources[0].data['admin-password']|b64decode }}" tags: - install - - upgrade + - update when: - (monitoring_creds.resources | length) == 1 diff --git a/roles/monitoring/tasks/main.yaml b/roles/monitoring/tasks/main.yaml index 8564d3f6..7a50684b 100644 --- a/roles/monitoring/tasks/main.yaml +++ b/roles/monitoring/tasks/main.yaml @@ -7,7 +7,7 @@ tags: - install - uninstall - - upgrade + - update - name: v4m - add storageclass community.kubernetes.k8s: diff --git a/roles/monitoring/tasks/viya-monitoring.yaml b/roles/monitoring/tasks/viya-monitoring.yaml index 5c34e8a0..e434d7db 100644 --- a/roles/monitoring/tasks/viya-monitoring.yaml +++ b/roles/monitoring/tasks/viya-monitoring.yaml @@ -7,7 +7,7 @@ tags: - install - uninstall - - upgrade + - update when: - "'cluster-logging' not in ansible_run_tags" - "'cluster-monitoring' not in ansible_run_tags" diff --git a/roles/vdm/defaults/main.yaml b/roles/vdm/defaults/main.yaml index d5b601f5..00e1b9e6 100644 --- a/roles/vdm/defaults/main.yaml +++ b/roles/vdm/defaults/main.yaml @@ -27,16 +27,6 @@ V4_CFG_RWX_FILESTORE_BIN_PATH: "{{ V4_CFG_RWX_FILESTORE_PATH | replace('/$', '') V4_CFG_STORAGECLASS: sas V4_CFG_MANAGE_STORAGE: false -V4_CFG_POSTGRES_TYPE: "internal" -V4_CFG_POSTGRES_FQDN: null -V4_CFG_POSTGRES_ADMIN_LOGIN: null -V4_CFG_POSTGRES_PASSWORD: null -V4_CFG_POSTGRES_DATABASE: 'SharedServices' -V4_CFG_POSTGRES_PORT: '5432' -V4_CFG_POSTGRES_CONNECTION_NAME: null -V4_CFG_POSTGRES_SERVICE_ACCOUNT: null -V4_CFG_POSTGRES_SSL_ENFORCEMENT: false - V4_CFG_DEPLOYMENT_URL_PREFIX: null V4_CFG_DEPLOYMENT_URL_PORT: null V4_CFG_INGRESS_FQDN: null @@ -78,3 +68,6 @@ V4_CFG_ELASTICSEARCH_ENABLE: true ## Cloud V4_CFG_CLOUD_SERVICE_ACCOUNT_NAME: null V4_CFG_CLOUD_SERVICE_ACCOUNT_AUTH: null + + +internal_postgres: false diff --git a/roles/vdm/library/overlay_facts.py b/roles/vdm/library/overlay_facts.py index bb3e9d3e..c79e5550 100644 --- a/roles/vdm/library/overlay_facts.py +++ b/roles/vdm/library/overlay_facts.py @@ -51,7 +51,8 @@ def main(): module.params['existing'][overlay_type].setdefault(phase, {}) if priority in module.params['existing'][overlay_type][phase]: - module.params['existing'][overlay_type][phase][priority].append(overlay_path) + if overlay_path not in module.params['existing'][overlay_type][phase][priority]: + module.params['existing'][overlay_type][phase][priority].append(overlay_path) else: module.params['existing'][overlay_type][phase].update({priority: [overlay_path]}) diff --git a/roles/vdm/tasks/assets.yaml b/roles/vdm/tasks/assets.yaml index 21ce6b7b..2844eb62 100644 --- a/roles/vdm/tasks/assets.yaml +++ b/roles/vdm/tasks/assets.yaml @@ -8,7 +8,7 @@ tags: - install - uninstall - - upgrade + - update - name: assets - Get License command: @@ -21,7 +21,7 @@ tags: - install - uninstall - - upgrade + - update - name: assets - Download command: @@ -34,7 +34,7 @@ tags: - install - uninstall - - upgrade + - update register: res - set_fact: @@ -44,7 +44,7 @@ tags: - install - uninstall - - upgrade + - update - name: assets - Remove old files file: @@ -56,7 +56,7 @@ tags: - install - uninstall - - upgrade + - update - name: assets - Extract downloaded assets unarchive: @@ -67,7 +67,7 @@ tags: - install - uninstall - - upgrade + - update - name: assets - Extract user-provided assets unarchive: @@ -78,4 +78,4 @@ tags: - install - uninstall - - upgrade + - update diff --git a/roles/vdm/tasks/cas.yaml b/roles/vdm/tasks/cas.yaml index 3c1cb6be..494ec00c 100644 --- a/roles/vdm/tasks/cas.yaml +++ b/roles/vdm/tasks/cas.yaml @@ -9,7 +9,7 @@ tags: - install - uninstall - - upgrade + - update - name: cas - user defined sssd block: @@ -34,7 +34,7 @@ tags: - install - uninstall - - upgrade + - update - name: "cas - backup controller" overlay_facts: @@ -48,7 +48,7 @@ tags: - install - uninstall - - upgrade + - update - name: "cas - auto resources" overlay_facts: @@ -63,7 +63,7 @@ tags: - install - uninstall - - upgrade + - update - name: "cas - user-defined resources" overlay_facts: @@ -78,7 +78,7 @@ tags: - install - uninstall - - upgrade + - update - name: "cas - MPP workers" overlay_facts: @@ -92,7 +92,7 @@ tags: - install - uninstall - - upgrade + - update - name: "cas - External services" overlay_facts: @@ -106,4 +106,4 @@ tags: - install - uninstall - - upgrade + - update diff --git a/roles/vdm/tasks/connect.yaml b/roles/vdm/tasks/connect.yaml index 0c6cfd2c..47bfcc74 100644 --- a/roles/vdm/tasks/connect.yaml +++ b/roles/vdm/tasks/connect.yaml @@ -11,7 +11,7 @@ tags: - install - uninstall - - upgrade + - update - name: connect - SAS/CONNECT FQDN to the SAN DNS list for cert-manager set_fact: @@ -21,7 +21,7 @@ tags: - install - uninstall - - upgrade + - update - name: connect - Configure Customer Provided Server Certificates overlay_facts: @@ -36,4 +36,4 @@ tags: - install - uninstall - - upgrade + - update diff --git a/roles/vdm/tasks/copy_overlay.yaml b/roles/vdm/tasks/copy_overlay.yaml index c3c1655c..3d0615e1 100644 --- a/roles/vdm/tasks/copy_overlay.yaml +++ b/roles/vdm/tasks/copy_overlay.yaml @@ -8,7 +8,7 @@ tags: - install - uninstall - - upgrade + - update - name: copy - VDM {{ resource }} template: @@ -22,4 +22,4 @@ tags: - install - uninstall - - upgrade + - update diff --git a/roles/vdm/tasks/deploy.yaml b/roles/vdm/tasks/deploy.yaml index 063710da..3cc8435a 100644 --- a/roles/vdm/tasks/deploy.yaml +++ b/roles/vdm/tasks/deploy.yaml @@ -9,7 +9,7 @@ kubeconfig: "{{ KUBECONFIG }}" tags: - install - - upgrade + - update - name: prereqs - cluster-wide shell: | @@ -22,7 +22,7 @@ - result["stderr"] is not regex(".*Warning.*") tags: - install - - upgrade + - update - name: prereqs - cluster-local deploy command: | @@ -34,7 +34,7 @@ - result["stderr"] is not regex(".*Warning.*") tags: - install - - upgrade + - update - name: manifest - deploy command: | @@ -46,21 +46,19 @@ - result["stderr"] is not regex(".*Warning.*") tags: - install - - upgrade + - update -- name: manifest - deploy istio +- name: manifest - deploy update command: | - kubectl --kubeconfig {{ KUBECONFIG }} apply -n {{ NAMESPACE }} --selector="sas.com/admin=namespace" --prune --prune-whitelist=networking.istio.io/v1alpha3/DestinationRule --prune-whitelist=networking.istio.io/v1alpha3/VirtualService -f {{ DEPLOY_DIR }}/site.yaml + kubectl --kubeconfig {{ KUBECONFIG }} apply -n {{ NAMESPACE }} --selector="sas.com/admin=namespace" -f {{ DEPLOY_DIR }}/site.yaml --prune --prune-whitelist=autoscaling/v2beta2/HorizontalPodAutoscaler register: result - when: - - V4_CFG_INGRESS_TYPE == "istio" failed_when: - result["stderr"]|length > 0 - result["stderr"] is not regex(".* no matches for kind .* in version .*") - result["stderr"] is not regex(".*Warning.*") tags: - install - - upgrade + - update - name: Remove Viya community.kubernetes.k8s: diff --git a/roles/vdm/tasks/elasticsearch.yaml b/roles/vdm/tasks/elasticsearch.yaml index e5a62787..f9402887 100644 --- a/roles/vdm/tasks/elasticsearch.yaml +++ b/roles/vdm/tasks/elasticsearch.yaml @@ -6,7 +6,7 @@ tags: - install - uninstall - - upgrade + - update - name: elasticsearch - add overlays overlay_facts: @@ -23,4 +23,4 @@ tags: - install - uninstall - - upgrade + - update diff --git a/roles/vdm/tasks/ingress.yaml b/roles/vdm/tasks/ingress.yaml index 4eee4240..44f24854 100644 --- a/roles/vdm/tasks/ingress.yaml +++ b/roles/vdm/tasks/ingress.yaml @@ -14,7 +14,7 @@ tags: - install - uninstall - - upgrade + - update - name: ingress - http facts set_fact: @@ -25,7 +25,7 @@ tags: - install - uninstall - - upgrade + - update - block: - name: ingress - https facts @@ -44,7 +44,7 @@ tags: - install - uninstall - - upgrade + - update - name: ingress - Full-stack TLS overlay_facts: @@ -62,7 +62,7 @@ tags: - install - uninstall - - upgrade + - update - name: ingress - Front-door TLS overlay_facts: @@ -78,7 +78,7 @@ tags: - install - uninstall - - upgrade + - update - name: ingress - Truststores only overlay_facts: @@ -94,7 +94,7 @@ tags: - install - uninstall - - upgrade + - update - name: ingress - customer provided ca certificates block: @@ -113,13 +113,11 @@ add: - { generators: "customer-provided-ca-certificates.yaml", vdm: true } when: - # NOTE: These can be provided to add to the trust store even when TLS mode is "disabled" to establish outbound trust - - V4_CFG_TLS_MODE != "disabled" - V4_CFG_TLS_TRUSTED_CA_CERTS is not none tags: - install - uninstall - - upgrade + - update - name: ingress - customer provided server certificates block: @@ -145,7 +143,7 @@ tags: - install - uninstall - - upgrade + - update - name: ingress - Cert-manager Certificate Generation overlay_facts: @@ -162,7 +160,7 @@ tags: - install - uninstall - - upgrade + - update - name: ingress - Consul UI @@ -177,7 +175,7 @@ tags: - install - uninstall - - upgrade + - update - name: add ingress class overlay_facts: diff --git a/roles/vdm/tasks/kustomize.yaml b/roles/vdm/tasks/kustomize.yaml index 9aec0f09..94ba1430 100644 --- a/roles/vdm/tasks/kustomize.yaml +++ b/roles/vdm/tasks/kustomize.yaml @@ -7,7 +7,7 @@ tags: - install - uninstall - - upgrade + - update - set_fact: timestamp: "{{ ansible_date_time.iso8601 }}" @@ -16,7 +16,7 @@ tags: - install - uninstall - - upgrade + - update - name: kustomize - buildinfo overlay_facts: @@ -28,7 +28,7 @@ tags: - install - uninstall - - upgrade + - update - name: kustomize - Get user's customizations siteconfig_info: @@ -39,7 +39,7 @@ tags: - install - uninstall - - upgrade + - update - name: kustomize - Ordered overlays overlay_facts: @@ -48,7 +48,7 @@ tags: - install - uninstall - - upgrade + - update - name: kustomize - Copy VDM overlays include_tasks: copy_overlay.yaml @@ -59,7 +59,7 @@ tags: - install - uninstall - - upgrade + - update - name: kustomize - Generate kustomization.yaml template: @@ -69,7 +69,7 @@ tags: - install - uninstall - - upgrade + - update - name: kustomize - Generate deployment manifest command: | @@ -77,4 +77,4 @@ tags: - install - uninstall - - upgrade + - update diff --git a/roles/vdm/tasks/main.yaml b/roles/vdm/tasks/main.yaml index c13d7bd0..f4cf4a0d 100644 --- a/roles/vdm/tasks/main.yaml +++ b/roles/vdm/tasks/main.yaml @@ -6,7 +6,7 @@ tags: - install - uninstall - - upgrade + - update - stat: path: "{{ DEPLOY_DIR }}/site-config/sitedefault.yaml" @@ -14,7 +14,7 @@ tags: - install - uninstall - - upgrade + - update - name: Sitedefault block: @@ -27,14 +27,14 @@ tags: - install - uninstall - - upgrade + - update - name: Include Deployment assets include_tasks: assets.yaml tags: - install - uninstall - - upgrade + - update - name: Base overlays overlay_facts: @@ -53,7 +53,7 @@ tags: - install - uninstall - - upgrade + - update - name: CR access block: @@ -72,7 +72,7 @@ tags: - install - uninstall - - upgrade + - update - name: Include Mirror include_tasks: mirror.yaml @@ -81,21 +81,21 @@ tags: - install - uninstall - - upgrade + - update - name: Include CAS include_tasks: cas.yaml tags: - install - uninstall - - upgrade + - update - name: Include Connect include_tasks: connect.yaml tags: - install - uninstall - - upgrade + - update - name: Include Openldap overlay_facts: @@ -111,49 +111,49 @@ tags: - install - uninstall - - upgrade + - update - name: Include Postgres - include_tasks: postgres.yaml + include_tasks: postgres/postgres.yaml tags: - install - uninstall - - upgrade + - update - name: Include Ingress include_tasks: ingress.yaml tags: - install - uninstall - - upgrade + - update - name: Include Elasticsearch include_tasks: elasticsearch.yaml tags: - install - uninstall - - upgrade + - update - name: Include Storage include_tasks: storage.yaml tags: - install - uninstall - - upgrade + - update - name: Include Sizing include_tasks: sizing.yaml tags: - install - uninstall - - upgrade + - update - name: Include Kustomize include_tasks: kustomize.yaml tags: - install - uninstall - - upgrade + - update - name: Include Deploy include_tasks: deploy.yaml @@ -162,4 +162,4 @@ tags: - install - uninstall - - upgrade + - update diff --git a/roles/vdm/tasks/mirror.yaml b/roles/vdm/tasks/mirror.yaml index cb9479f9..06b26969 100644 --- a/roles/vdm/tasks/mirror.yaml +++ b/roles/vdm/tasks/mirror.yaml @@ -6,7 +6,7 @@ tags: - install - uninstall - - upgrade + - update - name: mirror - update example replace: @@ -16,7 +16,7 @@ tags: - install - uninstall - - upgrade + - update - name: mirror - overlay overlay_facts: @@ -29,4 +29,4 @@ tags: - install - uninstall - - upgrade + - update diff --git a/roles/vdm/tasks/postgres.yaml b/roles/vdm/tasks/postgres.yaml deleted file mode 100644 index f9868ec2..00000000 --- a/roles/vdm/tasks/postgres.yaml +++ /dev/null @@ -1,56 +0,0 @@ -- name: postgres - internal - overlay_facts: - cadence_name: "{{ V4_CFG_CADENCE_NAME }}" - cadence_number: "{{ V4_CFG_CADENCE_VERSION }}" - existing: "{{ vdm_overlays }}" - add: - - { resources: "overlays/internal-postgres" } - - { resources: "overlays/crunchydata" } - - { transformers: "overlays/internal-postgres/internal-postgres-transformer.yaml" } - - { transformers: "postgres-storage-transformer.yaml", vdm: true, max: "2020.1.3" } - - { transformers: "postgres-storage-transformer.v2.yaml", vdm: true, min: "2020.1.4"} - when: - - V4_CFG_POSTGRES_TYPE == 'internal' - tags: - - install - - uninstall - - upgrade - -- name: postgres - gcp sql-proxy - block: - - set_fact: - V4_CFG_POSTGRES_FQDN: "sql-proxy" - - shell: | - gcloud auth activate-service-account '{{ V4_CFG_CLOUD_SERVICE_ACCOUNT_NAME }}' --key-file={{ V4_CFG_CLOUD_SERVICE_ACCOUNT_AUTH }} - gcloud iam service-accounts add-iam-policy-binding '{{ V4_CFG_POSTGRES_SERVICE_ACCOUNT }}' --role='roles/iam.workloadIdentityUser' --member='serviceAccount:{{ PROVIDER_ACCOUNT }}.svc.id.goog[{{ NAMESPACE }}/{{ PROXY_SQL_K8S_SERVICE_ACCOUNT }}]' --project='{{ PROVIDER_ACCOUNT }}' - - overlay_facts: - cadence_name: "{{ V4_CFG_CADENCE_NAME }}" - cadence_number: "{{ V4_CFG_CADENCE_VERSION }}" - existing: "{{ vdm_overlays }}" - add: - - { resources: "cloud-sql-proxy.yaml", vdm: true } - - { transformers: "overlays/external-postgres/googlecloud-full-stack-tls-transformer.yaml", priority: 55 } - when: - - V4_CFG_POSTGRES_TYPE == 'external' - - PROVIDER == "gcp" - tags: - - install - - upgrade - -- name: postgres - external - overlay_facts: - cadence_name: "{{ V4_CFG_CADENCE_NAME }}" - cadence_number: "{{ V4_CFG_CADENCE_VERSION }}" - existing: "{{ vdm_overlays }}" - add: - - { transformers: "overlays/external-postgres/external-postgres-transformer.yaml" } - - { generators: "postgres-sas-user.yaml", vdm: true } - - { generators: "sas-go-config.yaml", vdm: true } - - { generators: "sas-postgres-config.yaml", max: "2020.0.4", vdm: true } - - { generators: "sas-postgres-config.v2.yaml", min: "2020.0.5", vdm: true } - when: - - V4_CFG_POSTGRES_TYPE == 'external' - tags: - - install - - uninstall - - upgrade diff --git a/roles/vdm/tasks/postgres/gcp-cloud-sql-proxy.yaml b/roles/vdm/tasks/postgres/gcp-cloud-sql-proxy.yaml new file mode 100644 index 00000000..c601c29b --- /dev/null +++ b/roles/vdm/tasks/postgres/gcp-cloud-sql-proxy.yaml @@ -0,0 +1,40 @@ +- name: postgres - update cloud-sql-proxy fqdn + block: + - ansible.utils.update_fact: + updates: + - path: "V4_CFG_POSTGRES_SERVERS[{{ role }}][fqdn]" + value: "sql-proxy-{{ role }}" + register: updated + - set_fact: + V4_CFG_POSTGRES_SERVERS: "{{ updated.V4_CFG_POSTGRES_SERVERS }}" + tags: + - install + - uninstall + - update + +- name: postgres - setup cloud-sql-proxy account + shell: | + gcloud auth activate-service-account '{{ V4_CFG_CLOUD_SERVICE_ACCOUNT_NAME }}' --key-file={{ V4_CFG_CLOUD_SERVICE_ACCOUNT_AUTH }} + gcloud iam service-accounts add-iam-policy-binding '{{ settings.service_account }}' --role='roles/iam.workloadIdentityUser' --member='serviceAccount:{{ PROVIDER_ACCOUNT }}.svc.id.goog[{{ NAMESPACE }}/sql-proxy-{{ role }}]' --project='{{ PROVIDER_ACCOUNT }}' + tags: + - install + - uninstall + - update + +- name: postgres - cloud-sql-proxy overlays + block: + - template: + src: "{{ role_path }}/templates/resources/cloud-sql-proxy-instance.yaml" + dest: "{{ role_path }}/templates/resources/cloud-sql-proxy-{{ role }}-instance.yaml" + mode: "0660" + - overlay_facts: + cadence_name: "{{ V4_CFG_CADENCE_NAME }}" + cadence_number: "{{ V4_CFG_CADENCE_VERSION }}" + existing: "{{ vdm_overlays }}" + add: + - { resources: "cloud-sql-proxy-{{ role }}-instance.yaml", vdm: true } + - { transformers: "overlays/external-postgres/googlecloud-full-stack-tls-transformer.yaml", priority: 55 } + tags: + - install + - uninstall + - update diff --git a/roles/vdm/tasks/postgres/postgres-instance.yaml b/roles/vdm/tasks/postgres/postgres-instance.yaml new file mode 100644 index 00000000..2a2ed918 --- /dev/null +++ b/roles/vdm/tasks/postgres/postgres-instance.yaml @@ -0,0 +1,68 @@ +- name: postgres instance - ensure all are internal/external + fail: + msg: All database must either be internal or external. Mix-n-match is not supported + when: + - settings.internal != internal + tags: + - install + - uninstall + - update + +- name: postgres - internal default + overlay_facts: + cadence_name: "{{ V4_CFG_CADENCE_NAME }}" + cadence_number: "{{ V4_CFG_CADENCE_VERSION }}" + existing: "{{ vdm_overlays }}" + add: + - { resources: "overlays/internal-postgres" } + when: + - settings.internal + - role == "default" + tags: + - install + - uninstall + - update + +- name: postgres - internal folder check + stat: + path: "{{ DEPLOY_DIR }}/sas-bases/overlays/internal-postgres/{{ role }}" + register: result + when: settings.internal + tags: + - install + - uninstall + - update + +- name: postgres - internal cds + overlay_facts: + cadence_name: "{{ V4_CFG_CADENCE_NAME }}" + cadence_number: "{{ V4_CFG_CADENCE_VERSION }}" + existing: "{{ vdm_overlays }}" + add: + - { resources: "overlays/internal-postgres/{{ role }}" } + when: + - settings.internal + - result.stat.exists + tags: + - install + - uninstall + - update + +- block: + - name: postgres instance - crd + template: + src: "{{ role_path }}/templates/resources/postgres-instance.yaml" + dest: "{{ role_path }}/templates/resources/postgres-{{ role }}-instance.yaml" + mode: "0660" + - overlay_facts: + cadence_name: "{{ V4_CFG_CADENCE_NAME }}" + cadence_number: "{{ V4_CFG_CADENCE_VERSION }}" + existing: "{{ vdm_overlays }}" + add: + - { resources: "postgres-{{ role }}-instance.yaml", vdm: true } + when: + - not settings.internal or ( settings.internal and role != "default" and not result.stat.exists ) + tags: + - install + - uninstall + - update diff --git a/roles/vdm/tasks/postgres/postgres.yaml b/roles/vdm/tasks/postgres/postgres.yaml new file mode 100644 index 00000000..2a35ffe2 --- /dev/null +++ b/roles/vdm/tasks/postgres/postgres.yaml @@ -0,0 +1,93 @@ +- set_fact: + internal_postgres: "{{ V4_CFG_POSTGRES_SERVERS.default.internal }}" + tags: + - install + - uninstall + - update + +- name: postgres - gcp cloud-sql-proxy + include_tasks: gcp-cloud-sql-proxy.yaml + vars: + role: "{{ item.key }}" + settings: "{{ item.value }}" + with_dict: "{{ V4_CFG_POSTGRES_SERVERS }}" + when: + - not item.value.internal + - "'service_account' in item.value" + - item.value.service_account is defined + - V4_CFG_CLOUD_SERVICE_ACCOUNT_NAME is defined + - PROVIDER == "gcp" + tags: + - install + - uninstall + - update + +- name: postgres - pre 2021.1.4 + block: + - name: postgres - internal + overlay_facts: + cadence_name: "{{ V4_CFG_CADENCE_NAME }}" + cadence_number: "{{ V4_CFG_CADENCE_VERSION }}" + existing: "{{ vdm_overlays }}" + add: + - { resources: "overlays/internal-postgres" } + - { resources: "overlays/crunchydata" } + - { transformers: "overlays/internal-postgres/internal-postgres-transformer.yaml" } + - { transformers: "postgres-storage-transformer.yaml", vdm: true, max: "2020.1.3" } + - { transformers: "postgres-storage-transformer.v2.yaml", vdm: true, min: "2020.1.3"} + when: + - internal_postgres + - name: postgres - external + overlay_facts: + cadence_name: "{{ V4_CFG_CADENCE_NAME }}" + cadence_number: "{{ V4_CFG_CADENCE_VERSION }}" + existing: "{{ vdm_overlays }}" + add: + - { transformers: "overlays/external-postgres/external-postgres-transformer.yaml" } + - { generators: "postgres-sas-user.yaml", vdm: true } + - { generators: "sas-go-config.yaml", vdm: true } + - { generators: "sas-postgres-config.yaml", max: "2020.0.4", vdm: true } + - { generators: "sas-postgres-config.v2.yaml", min: "2020.0.5", vdm: true } + when: + - not internal_postgres + when: + - V4_CFG_CADENCE_VERSION is version('2021.1.4', "<") + - V4_CFG_CADENCE_NAME != "fast" + tags: + - install + - uninstall + - update + +- name: postgres - post 2021.1.4 + block: + - name: postgres - internal + overlay_facts: + cadence_name: "{{ V4_CFG_CADENCE_NAME }}" + cadence_number: "{{ V4_CFG_CADENCE_VERSION }}" + existing: "{{ vdm_overlays }}" + add: + - { transformers: "postgres-storage-transformer.v3.yaml", vdm: true } + when: + - internal_postgres + - name: postgres - external + overlay_facts: + cadence_name: "{{ V4_CFG_CADENCE_NAME }}" + cadence_number: "{{ V4_CFG_CADENCE_VERSION }}" + existing: "{{ vdm_overlays }}" + add: + - { transformers: "overlays/external-postgres/external-postgres-transformer.yaml" } + when: + - not internal_postgres + - name: postgres - instance + include_tasks: postgres-instance.yaml + vars: + role: "{{ item.key }}" + settings: "{{ item.value }}" + internal: "{{ internal_postgres }}" + with_dict: "{{ V4_CFG_POSTGRES_SERVERS }}" + when: + - V4_CFG_CADENCE_VERSION is version('2021.1.4', ">=") or V4_CFG_CADENCE_NAME == "fast" + tags: + - install + - uninstall + - update diff --git a/roles/vdm/tasks/sizing.yaml b/roles/vdm/tasks/sizing.yaml index c4605579..ddf946e7 100644 --- a/roles/vdm/tasks/sizing.yaml +++ b/roles/vdm/tasks/sizing.yaml @@ -11,4 +11,4 @@ tags: - install - uninstall - - upgrade + - update diff --git a/roles/vdm/tasks/storage.yaml b/roles/vdm/tasks/storage.yaml index dcdebb7d..01226423 100644 --- a/roles/vdm/tasks/storage.yaml +++ b/roles/vdm/tasks/storage.yaml @@ -9,7 +9,7 @@ tags: - install - uninstall - - upgrade + - update - name: storage - nfs overlay_facts: @@ -26,4 +26,4 @@ tags: - install - uninstall - - upgrade \ No newline at end of file + - update diff --git a/roles/vdm/templates/generators/mirror.yaml b/roles/vdm/templates/generators/mirror.yaml index c8b514a8..5b4d7cd6 100644 --- a/roles/vdm/templates/generators/mirror.yaml +++ b/roles/vdm/templates/generators/mirror.yaml @@ -6,7 +6,7 @@ metadata: behavior: merge literals: - IMAGE_REGISTRY={{ V4_CFG_CR_HOST }} -{% if V4_CFG_POSTGRES_TYPE is defined and V4_CFG_POSTGRES_TYPE == "internal" %} +{% if internal_postgres|bool %} --- apiVersion: builtin kind: ConfigMapGenerator @@ -16,4 +16,4 @@ behavior: merge literals: - CCP_IMAGE_REPO={{ V4_CFG_CR_HOST }} - CCP_IMAGE_PATH={{ V4_CFG_CR_HOST }} -{% endif %} \ No newline at end of file +{% endif %} diff --git a/roles/vdm/templates/generators/postgres-config.yaml b/roles/vdm/templates/generators/postgres-config.yaml index 8e0e3217..01754e81 100644 --- a/roles/vdm/templates/generators/postgres-config.yaml +++ b/roles/vdm/templates/generators/postgres-config.yaml @@ -4,7 +4,7 @@ kind: ConfigMapGenerator metadata: name: postgres-config literals: - - DATABASE_HOST={{ V4_CFG_POSTGRES_FQDN }} - - DATABASE_PORT=5432 + - DATABASE_HOST={{ V4_CFG_POSTGRES_SERVERS.default.fqdn }} + - DATABASE_PORT={{ V4_CFG_POSTGRES_SERVERS.default.server_port }} - DATABASE_NAME={{ V4_CFG_POSTGRES_DATABASE | default('SharedServices', true) }} - - EXTERNAL_DATABASE="true" \ No newline at end of file + - EXTERNAL_DATABASE="true" diff --git a/roles/vdm/templates/generators/postgres-sas-user.yaml b/roles/vdm/templates/generators/postgres-sas-user.yaml index ffb967cf..64bd22e9 100644 --- a/roles/vdm/templates/generators/postgres-sas-user.yaml +++ b/roles/vdm/templates/generators/postgres-sas-user.yaml @@ -3,5 +3,5 @@ kind: SecretGenerator metadata: name: postgres-sas-user literals: - - username={{ V4_CFG_POSTGRES_ADMIN_LOGIN }} - - password={{ V4_CFG_POSTGRES_PASSWORD }} + - username={{ V4_CFG_POSTGRES_SERVERS.default.admin }} + - password={{ V4_CFG_POSTGRES_SERVERS.default.password }} diff --git a/roles/vdm/templates/generators/sas-go-config.yaml b/roles/vdm/templates/generators/sas-go-config.yaml index 7522d5cb..426fbe4c 100644 --- a/roles/vdm/templates/generators/sas-go-config.yaml +++ b/roles/vdm/templates/generators/sas-go-config.yaml @@ -5,4 +5,4 @@ metadata: name: sas-go-config behavior: merge literals: - - SAS_DATABASE_DATABASE={{ V4_CFG_POSTGRES_DATABASE }} \ No newline at end of file + - SAS_DATABASE_DATABASE={{ V4_CFG_POSTGRES_SERVERS.default.database if 'database' in V4_CFG_POSTGRES_SERVERS.default else 'SharedServices' }} diff --git a/roles/vdm/templates/generators/sas-postgres-config.v2.yaml b/roles/vdm/templates/generators/sas-postgres-config.v2.yaml index 4c5fedbd..8ec89437 100644 --- a/roles/vdm/templates/generators/sas-postgres-config.v2.yaml +++ b/roles/vdm/templates/generators/sas-postgres-config.v2.yaml @@ -5,9 +5,9 @@ metadata: name: sas-postgres-config behavior: merge literals: - - DATABASE_HOST={{ V4_CFG_POSTGRES_FQDN }} - - DATABASE_PORT={{ V4_CFG_POSTGRES_PORT }} - - DATABASE_SSL_ENABLED="{{ V4_CFG_POSTGRES_SSL_ENFORCEMENT|bool|lower }}" - - DATABASE_NAME={{ V4_CFG_POSTGRES_DATABASE }} + - DATABASE_HOST={{ V4_CFG_POSTGRES_SERVERS.default.fqdn }} + - DATABASE_PORT={{ V4_CFG_POSTGRES_SERVERS.default.server_port }} + - DATABASE_SSL_ENABLED="{{ V4_CFG_POSTGRES_SERVERS.default.ssl_enforcement_enabled|bool|lower }}" + - DATABASE_NAME={{ V4_CFG_POSTGRES_SERVERS.default.database|default('SharedServices', true) }} - EXTERNAL_DATABASE="true" - SAS_DATABASE_DATABASESERVERNAME="postgres" diff --git a/roles/vdm/templates/generators/sas-postgres-config.yaml b/roles/vdm/templates/generators/sas-postgres-config.yaml index 4051f19c..f30105ea 100644 --- a/roles/vdm/templates/generators/sas-postgres-config.yaml +++ b/roles/vdm/templates/generators/sas-postgres-config.yaml @@ -5,10 +5,10 @@ metadata: name: sas-postgres-config behavior: merge literals: - - DATABASE_HOST={{ V4_CFG_POSTGRES_FQDN }} - - DATABASE_PORT={{ V4_CFG_POSTGRES_PORT }} + - DATABASE_HOST={{ V4_CFG_POSTGRES_SERVERS.default.fqdn }} + - DATABASE_PORT={{ V4_CFG_POSTGRES_SERVERS.default.server_port }} - DATABASE_SSL_ENABLED="false" - - DATABASE_NAME={{ V4_CFG_POSTGRES_DATABASE }} + - DATABASE_NAME={{ V4_CFG_POSTGRES_SERVERS.default.database|default('SharedServices', true) }} - EXTERNAL_DATABASE="true" - SAS_DATABASE_DATABASESERVERNAME="postgres" - - SPRING_DATASOURCE_URL=jdbc:postgresql://{{ V4_CFG_POSTGRES_FQDN }}:{{ 5432 }}/{{ V4_CFG_POSTGRES_DATABASE }}?currentSchema=${application.schema} \ No newline at end of file + - SPRING_DATASOURCE_URL=jdbc:postgresql://{{ V4_CFG_POSTGRES_SERVERS.default.fqdn }}:{{ V4_CFG_POSTGRES_SERVERS.default.port }}/{{ V4_CFG_POSTGRES_SERVERS.default.database if 'database' in V4_CFG_POSTGRES_SERVERS.default else 'SharedServices' }}?currentSchema=${application.schema} diff --git a/roles/vdm/templates/resources/cloud-sql-proxy.yaml b/roles/vdm/templates/resources/cloud-sql-proxy-instance.yaml similarity index 61% rename from roles/vdm/templates/resources/cloud-sql-proxy.yaml rename to roles/vdm/templates/resources/cloud-sql-proxy-instance.yaml index eacc5e8d..e625b92b 100644 --- a/roles/vdm/templates/resources/cloud-sql-proxy.yaml +++ b/roles/vdm/templates/resources/cloud-sql-proxy-instance.yaml @@ -1,46 +1,46 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: {{ PROXY_SQL_K8S_SERVICE_ACCOUNT }} + name: sql-proxy-{{ role }} annotations: - iam.gke.io/gcp-service-account: {{ V4_CFG_POSTGRES_SERVICE_ACCOUNT }} + iam.gke.io/gcp-service-account: {{ settings.service_account }} --- apiVersion: apps/v1 kind: Deployment metadata: - name: sql-proxy + name: sql-proxy-{{ role }} labels: - app: sql-proxy + app: sql-proxy-{{ role }} spec: replicas: 1 selector: matchLabels: - app: sql-proxy + app: sql-proxy-{{ role }} template: metadata: labels: - app: sql-proxy + app: sql-proxy-{{ role }} spec: containers: - name: sql-proxy image: '{{ PROXY_SQL_IMAGE }}' env: - name: POSTGRES_CONNECTION_NAME - value: {{ V4_CFG_POSTGRES_CONNECTION_NAME }} + value: {{ settings.connection_name }} command: - "/cloud_sql_proxy" args: - "-ip_address_types=PRIVATE" - - "-instances=$(POSTGRES_CONNECTION_NAME)=tcp:0.0.0.0:5432" + - "-instances=$(POSTGRES_CONNECTION_NAME)=tcp:0.0.0.0:{{ settings.server_port|default(5432, true) }}" imagePullSecrets: [] - serviceAccountName: {{ PROXY_SQL_K8S_SERVICE_ACCOUNT }} + serviceAccountName: sql-proxy-{{ role }} --- kind: Service apiVersion: v1 metadata: - name: sql-proxy + name: sql-proxy-{{ role }} labels: - app: sql-proxy + app: sql-proxy-{{ role }} spec: ports: - name: sql @@ -48,5 +48,5 @@ spec: port: 5432 targetPort: 5432 selector: - app: sql-proxy + app: sql-proxy-{{ role }} type: ClusterIP diff --git a/roles/vdm/templates/resources/postgres-instance.yaml b/roles/vdm/templates/resources/postgres-instance.yaml new file mode 100644 index 00000000..95ee854e --- /dev/null +++ b/roles/vdm/templates/resources/postgres-instance.yaml @@ -0,0 +1,35 @@ +{%- set db = db_default_name_map[role] if ('database' not in settings and role in db_default_name_map) else settings.database|default(role, true) -%} +{%- set server_name = role if role != "default" else "postgres" -%} +apiVersion: webinfdsvr.sas.com/v1 +kind: Pgcluster +metadata: + name: {{ server_name }} + annotations: + sas.com/default-database: "{{ (role == "default") | bool|lower}}" +{% if settings.internal|bool %} + sas.com/component-name: sas-crunchy-data-postgres-12 +{% endif %} +spec: + internal: {{ settings.internal|bool|lower }} + database: {{ db }} +{% if settings.internal|bool %} + storage: + storageclass: "{{ V4_CFG_STORAGECLASS }}" +{% endif %} + connection: + ssl: {{ (settings.ssl_enforcement_enabled if 'ssl_enforcement_enabled' in settings else True)|bool|lower }} +{% if not settings.internal %} + host: {{ settings.fqdn }} + port: {{ settings.server_port|default(5432, true) }} + rolesecret: postgres-{{ role }}-user +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgres-{{ role }}-user + labels: + pg-cluster: "{{ server_name }}" +stringData: + username: {{ settings.admin }} + password: {{ settings.password }} +{% endif %} diff --git a/roles/vdm/templates/transformers/postgres-storage-transformer.v3.yaml b/roles/vdm/templates/transformers/postgres-storage-transformer.v3.yaml new file mode 100644 index 00000000..936f3569 --- /dev/null +++ b/roles/vdm/templates/transformers/postgres-storage-transformer.v3.yaml @@ -0,0 +1,13 @@ +apiVersion: builtin +kind: PatchTransformer +metadata: + name: postgres-storage-transformer +patch: |- + - op: replace + path: /spec/storage/storageclass + value: {{ V4_CFG_STORAGECLASS }} +target: + group: webinfdsvr.sas.com + kind: Pgcluster + name: .* # By default, target all Pgclusters. Change to a specific name to target just one. + version: v1 diff --git a/roles/vdm/vars/main.yaml b/roles/vdm/vars/main.yaml index 4409b24c..744b6f39 100644 --- a/roles/vdm/vars/main.yaml +++ b/roles/vdm/vars/main.yaml @@ -1,4 +1,7 @@ -PROXY_SQL_K8S_SERVICE_ACCOUNT: sql-proxy PROXY_SQL_IMAGE: gcr.io/cloudsql-docker/gce-proxy:1.20.2 vdm_overlays: {} + +db_default_name_map: { + default: "SharedServices" +}