起動時にbleveのインデックスを貼るように変更

kubernetes/mattermost/external-secret.yml

@@ -0,0 +1,34 @@
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: gcp-secret-mattermost-config
+  namespace: mattermost
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: gcp-secret-store
+  target:
+    name: gcp-secret-config
+    creationPolicy: Owner
+  data:
+  - secretKey: config.json
+    remoteRef:
+      key: mattermost-config
+---
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: gcp-secret-mattermost-env
+  namespace: mattermost
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: gcp-secret-store
+  target:
+    name: gcp-secret-env
+    creationPolicy: Owner
+  dataFrom:
+  - extract:
+    key: mattermost-env

kubernetes/mattermost/mattermost.yml

@@ -23,6 +23,8 @@ spec:
       securityContext: 
         fsGroup: 2000
       serviceAccountName: mattermost-primary
+      readinessGates:
+      - conditionType: bleve-ready
       initContainers:
         - name: secret-placer
           image: gcr.io/google.com/cloudsdktool/cloud-sdk:latest
@@ -39,6 +41,12 @@ spec:
       - name: mattermost
         image: mattermost/mattermost-team-edition:release-9.1
         imagePullPolicy: Always
+        env:
+          - name: MATTERMOST_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: gcp-secret-env
+                key: MATTERMOST_TOKEN
         ports:
         - name: http
           containerPort: 8000
@@ -50,6 +58,14 @@ spec:
           requests:
             cpu: 500m
             memory: 2Gi
+        lifecycle:
+          postStart:
+            exec:
+              command: [
+                "/usr/bin/bash",
+                "-c",
+                '(max_retries=60; retry_delay=10; retries=0; result=1; while [[ $retries -lt $max_retries && $result -ne 0 ]]; do ((retries++)); echo "retry $retries/$max_retries..."; curl http://localhost:8000/api/v4/system/ping -s; result=$?; if [[ $result -ne 0 ]]; then echo "retry"; sleep $retry_delay; fi; done; if [[ $result -ne 0 ]]; then echo "startup timeout";exit 1;fi;echo "ping success"; response=$(curl -s -i -H "Authorization: Bearer $MATTERMOST_TOKEN" http://localhost:8000/api/v4/jobs -X POST -d ''{"type":"bleve_post_indexing"}'' | grep bleve_post_indexing);if [[ "$response" == "" ]]; then echo "failed to create job";echo "$response" exit 1;fi;echo "job created"; max_retries=864; retry_delay=10; retries=0; result=""; while [[ $retries -lt $max_retries && "$result" == "" ]]; do ((retries++)); echo "wait for index $retries/$max_retries..."; response=$(curl -s "http://localhost:8000/api/v4/jobs/type/bleve_post_indexing?page=0&per_page=1" -H "Authorization: Bearer $MATTERMOST_TOKEN" | grep success); result=$response; if [[ "$result" == "" ]]; then echo "index not completed"; sleep $retry_delay; fi; done; if [[ "$result" == "" ]]; then echo "index timeout"; exit 1;else echo "index success"; fi; curl -s https://kubernetes.default.svc/api/v1/namespaces/$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)/pods/$(hostname)/status -X PATCH -H "Content-Type: application/json-patch+json"  -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" -d ''[{"op": "add", "path": "/status/conditions/-", "value": {"type": "bleve-ready", "status": "True", "lastTransitionTime": "2018-10-16T06:59:45Z", "lastProbeTime": null}}]'' -k) > /var/tmp/post-start.log 2>&1'
+              ]
         readinessProbe:
           timeoutSeconds: 5
           periodSeconds: 5
@@ -74,14 +90,14 @@ spec:
         volumeMounts:
           - name: config
             mountPath: /mattermost/config/
-          - name: bleve-cache
+          - name: bleve
             mountPath: /mattermost/bleve/
       volumes:
         - name: config
           emptyDir: {}
+        - name: bleve
+          emptyDir:
+            sizeLimit: 2Gi
         - name: secret
           secret:
-            secretName: secret-configs
-        - name: bleve-cache
-          persistentVolumeClaim:
-            claimName: mattermost-bleve-pvc
+            secretName: gcp-secret-config

kubernetes/mattermost/sa.yaml

@@ -5,3 +5,11 @@ metadata:
   name: mattermost-primary
   annotations:
     iam.gke.io/gcp-service-account: "[email protected]"
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  namespace: mattermost
+  name: secret-mattermost-primary
+  annotations:
+    iam.gke.io/gcp-service-account: "[email protected]"

kubernetes/mattermost/secret-store.yml

@@ -0,0 +1,17 @@
+apiVersion: external-secrets.io/v1beta1
+kind: ClusterSecretStore
+metadata:
+  name: gcp-secret-store
+  namespace: mattermost
+spec:
+  provider:
+    gcpsm:
+      projectID: mitou-jr
+      auth:
+        workloadIdentity:
+          clusterLocation: asia-northeast1
+          clusterName: primary-cluster
+          clusterProjectID: primary-cluster
+          serviceAccountRef:
+            name: secret-mattermost-primary
+            namespace: mattermost

terraform/basic/uptime-check.tf

@@ -1,5 +1,5 @@
 resource "google_monitoring_uptime_check_config" "https_check" {
-  provider = google-beta
+  provider     = google-beta
   display_name = "Health check for Mattermost server"
   timeout      = "10s"
   period       = "60s"
@@ -23,7 +23,7 @@ resource "google_monitoring_uptime_check_config" "https_check" {
     type = "uptime_url"
     labels = {
       project_id = "mitou-jr"
-      host       = "mattermost.jr.mitou.org"  # 監視対象のホスト名またはIPアドレスを記載します
+      host       = "mattermost.jr.mitou.org" # 監視対象のホスト名またはIPアドレスを記載します
     }
   }
 
@@ -45,7 +45,7 @@ resource "google_monitoring_alert_policy" "https_check_alert" {
       # filter に 稼働時間チェックの指標を指定して、関連づけています。
       filter          = "resource.type = \"uptime_url\" AND metric.type = \"monitoring.googleapis.com/uptime_check/check_passed\" AND metric.labels.check_id = \"${google_monitoring_uptime_check_config.https_check.uptime_check_id}\""
       duration        = "0s"
-      threshold_value = 1  # 失敗が1回より多くなったときにアラートを通知
+      threshold_value = 1 # 失敗が1回より多くなったときにアラートを通知
       comparison      = "COMPARISON_GT"
       aggregations {
         alignment_period     = "1200s"
@@ -63,7 +63,7 @@ resource "google_monitoring_alert_policy" "https_check_alert" {
     google_monitoring_notification_channel.default.id,
     google_monitoring_notification_channel.email.id
   ]
-  enabled     = true
+  enabled = true
 
   depends_on = [
     google_monitoring_uptime_check_config.https_check

terraform/iam/iam.tf

@@ -30,6 +30,11 @@ locals {
       gsa           = "${google_service_account.wi-mattermost-primary.name}",
       ksa_namespace = "mattermost",
       ksa_name      = "mattermost-primary"
+    },
+    {
+      gsa           = "${google_service_account.wi-secret-mattermost-primary.name}",
+      ksa_namespace = "mattermost",
+      ksa_name      = "secret-mattermost-primary"
     }
   ]
 }
@@ -64,9 +69,9 @@ resource "google_project_iam_binding" "iam-binding-iam-applier" {
 }
 
 resource "google_project_iam_binding" "iam-binding-k8s-operation" {
-   role    = "roles/container.developer"
-   project = "mitou-jr"
-   members = local.admin-access
+  role    = "roles/container.developer"
+  project = "mitou-jr"
+  members = local.admin-access
 }
 
 variable "basic-sa-iam-roles" {
@@ -103,9 +108,17 @@ resource "google_service_account_iam_binding" "wi-bindings" {
 }
 
 resource "google_project_iam_binding" "monitoring-pubsub" {
-  role               = "roles/pubsub.publisher"
-  project  = "mitou-jr"
+  role    = "roles/pubsub.publisher"
+  project = "mitou-jr"
   members = [
     "serviceAccount:service-233207969476@gcp-sa-monitoring-notification.iam.gserviceaccount.com"
   ]
 }
+
+resource "google_service_account_iam_binding" "secret-manager" {
+  service_account_id = google_service_account.wi-secret-mattermost-primary.id
+  role               = "roles/secretmanager.secretAccessor"
+  members = [
+    "serviceAccount:mitou-jr.svc.id.goog[mattermost/mattermost-primary]"
+  ]
+}

terraform/iam/sa.tf

@@ -17,4 +17,9 @@ resource "google_service_account" "sa-ga-planner" {
 resource "google_service_account" "wi-mattermost-primary" {
   account_id   = "wi-mattermost-primary"
   display_name = "Mattermostのワークロードで使うWorkload Identity"
-}
+}
+
+resource "google_service_account" "wi-secret-mattermost-primary" {
+  account_id   = "wi-secret-mattermost-primary"
+  display_name = "MattermostのExternalSecretOperatorで使うWorkload Identity"
+}