Feat: deployed partner mainnet node, added fixes for partner code (#732)

* deployed partner mainnet node, added fixes for partner code

* added logging to backend service from LB

* swapped role binding for non-authoritive member

* swapped echo command for secret creation to printf to avoid newline

infra/.gitignore

@@ -20,6 +20,7 @@ crash.*.log
 !terraform-testnet-example.tfvars
 !backend.tfvars
 !terraform-mainnet-example.tfvars
+secrets.txt
 
 # Ignore override files as they are usually used to override resources locally and so
 # are not checked in

infra/multichain-mainnet/main.tf

@@ -0,0 +1,233 @@
+provider "google" {
+  project = var.project_id
+}
+provider "google-beta" {
+  project = var.project_id
+}
+module "gce-container" {
+  count   = length(var.node_configs)
+  source  = "terraform-google-modules/container-vm/google"
+  version = "~> 3.0"
+
+  container = {
+    image = var.image
+    args  = ["start"]
+    port  = "3000"
+
+    env = concat(var.static_env, [
+      {
+        name  = "MPC_RECOVERY_NODE_ID"
+        value = "${count.index}"
+      },
+      {
+        name  = "MPC_RECOVERY_ACCOUNT_ID"
+        value = var.node_configs["${count.index}"].account
+      },
+      {
+        name  = "MPC_RECOVERY_CIPHER_PK"
+        value = var.node_configs["${count.index}"].cipher_pk
+      },
+      {
+        name  = "MPC_RECOVERY_ACCOUNT_SK"
+        value = data.google_secret_manager_secret_version.account_sk_secret_id[count.index].secret_data
+      },
+      {
+        name  = "MPC_RECOVERY_CIPHER_SK"
+        value = data.google_secret_manager_secret_version.cipher_sk_secret_id[count.index].secret_data
+      },
+      {
+        name  = "MPC_RECOVERY_SIGN_SK"
+        value = data.google_secret_manager_secret_version.sign_sk_secret_id[count.index] != null ? data.google_secret_manager_secret_version.sign_sk_secret_id[count.index].secret_data : data.google_secret_manager_secret_version.account_sk_secret_id[count.index].secret_data
+      },
+      {
+        name  = "AWS_ACCESS_KEY_ID"
+        value = data.google_secret_manager_secret_version.aws_access_key_secret_id.secret_data
+      },
+      {
+        name  = "AWS_SECRET_ACCESS_KEY"
+        value = data.google_secret_manager_secret_version.aws_secret_key_secret_id.secret_data
+      },
+      {
+        name  = "MPC_RECOVERY_LOCAL_ADDRESS"
+        value = "https://${var.node_configs[count.index].domain}"
+      },
+      {
+        name = "MPC_RECOVERY_SK_SHARE_SECRET_ID"
+        value = var.node_configs["${count.index}"].sk_share_secret_id
+      },
+      {
+        name = "MPC_RECOVERY_ENV",
+        value = var.env
+      }
+    ])
+  }
+}
+
+resource "google_service_account" "service_account" {
+  account_id   = "multichain-partner-${var.env}"
+  display_name = "Multichain ${var.env} Account"
+}
+
+resource "google_project_iam_member" "sa-roles" {
+  for_each = toset([
+      "roles/datastore.user",
+      "roles/secretmanager.admin",
+      "roles/storage.objectAdmin",
+      "roles/iam.serviceAccountAdmin",
+  ])
+
+  role    = each.key
+  member  = "serviceAccount:${google_service_account.service_account.email}"
+  project = var.project_id
+}
+
+resource "google_compute_global_address" "external_ips" {
+  count        = length(var.node_configs)
+  name         = "multichain-partner-mainnet-${count.index}"
+  address_type = "EXTERNAL"
+
+  lifecycle {
+    prevent_destroy = true
+  }
+}
+
+resource "google_compute_managed_ssl_certificate" "mainnet_ssl" {
+  count = length(var.node_configs)
+  name = "multichain-partner-mainnet-ssl-${count.index}"
+
+  managed {
+    domains = [var.node_configs[count.index].domain]
+  }
+}
+
+module "ig_template" {
+  count      = length(var.node_configs)
+  source     = "../modules/mig_template"
+  network    = var.network
+  subnetwork = var.subnetwork
+  region     = var.region
+  service_account = {
+    email  = google_service_account.service_account.email,
+    scopes = ["cloud-platform"]
+  }
+  name_prefix          = "multichain-partner-mainnet-${count.index}"
+  source_image_family  = "cos-113-lts"
+  source_image_project = "cos-cloud"
+  machine_type         = "n2d-standard-2"
+
+  startup_script = "docker rm watchtower ; docker run -d --name watchtower -v /var/run/docker.sock:/var/run/docker.sock containrrr/watchtower --debug --interval 30"
+
+  source_image = reverse(split("/", module.gce-container[count.index].source_image))[0]
+  metadata     = merge(var.additional_metadata, { "gce-container-declaration" = module.gce-container["${count.index}"].metadata_value })
+  tags = [
+    "multichain",
+    "allow-ssh"
+  ]
+  labels = {
+    "container-vm" = module.gce-container[count.index].vm_container_label
+  }
+
+  depends_on = [google_compute_global_address.external_ips]
+}
+
+
+module "instances" {
+  count      = length(var.node_configs)
+  source     = "../modules/instance-from-tpl"
+  region     = var.region
+  project_id = var.project_id
+  hostname   = "multichain-mainnet-partner-${count.index}"
+  network    = var.network
+  subnetwork = var.subnetwork
+
+  instance_template = module.ig_template[count.index].self_link_unique
+
+}
+
+resource "google_compute_health_check" "multichain_healthcheck" {
+  name = "multichain-mainnet-partner-healthcheck"
+
+  http_health_check {
+    port         = 3000
+    request_path = "/"
+  }
+
+}
+
+resource "google_compute_global_forwarding_rule" "http_fw" {
+  count      = length(var.node_configs)
+  name       = "multichain-partner-mainnet-http-rule-${count.index}"
+  target     = google_compute_target_http_proxy.default[count.index].id
+  port_range = "80"
+  ip_protocol = "TCP"
+  load_balancing_scheme = "EXTERNAL"
+  ip_address = google_compute_global_address.external_ips[count.index].address
+}
+
+resource "google_compute_global_forwarding_rule" "https_fw" {
+  count      = length(var.node_configs)
+  name       = "multichain-partner-mainnet-https-rule-${count.index}"
+  target     = google_compute_target_https_proxy.default_https[count.index].id
+  port_range = "443"
+  ip_protocol = "TCP"
+  load_balancing_scheme = "EXTERNAL"
+  ip_address = google_compute_global_address.external_ips[count.index].address
+}
+
+resource "google_compute_target_http_proxy" "default" {
+  count      = length(var.node_configs)
+  name        = "multichain-partner-mainnet-http-target-proxy-${count.index}"
+  description = "a description"
+  url_map     = google_compute_url_map.redirect_default[count.index].id
+}
+
+resource "google_compute_target_https_proxy" "default_https" {
+  count      = length(var.node_configs)
+  name        = "multichain-partner-mainnet-https-target-proxy-${count.index}"
+  description = "a description"
+  ssl_certificates = [ google_compute_managed_ssl_certificate.mainnet_ssl[count.index].self_link ]
+  url_map     = google_compute_url_map.default[count.index].id
+}
+
+resource "google_compute_url_map" "default" {
+  count           = length(var.node_configs)
+  name            = "multichain-partner-mainnet-url-map-${count.index}"
+  default_service = google_compute_backend_service.multichain_backend[count.index].id
+}
+
+resource "google_compute_url_map" "redirect_default" {
+  count           = length(var.node_configs)
+  name            = "multichain-partner-mainnet-redirect-url-map-${count.index}"
+  default_url_redirect {
+    strip_query    = false
+    https_redirect = true
+  }
+}
+
+resource "google_compute_backend_service" "multichain_backend" {
+  count                 = length(var.node_configs)
+  name                  = "multichain-partner-mainnet-backend-service-${count.index}"
+  load_balancing_scheme = "EXTERNAL"
+
+  log_config {
+    enable = true
+    sample_rate = 0.5
+  }
+  backend {
+    group = google_compute_instance_group.multichain_group[count.index].id
+  }
+
+  health_checks = [google_compute_health_check.multichain_healthcheck.id]
+}
+
+resource "google_compute_instance_group" "multichain_group" {
+  count     = length(var.node_configs)
+  name      = "multichain-partner-mainnet-instance-group-${count.index}"
+  instances = [module.instances[count.index].self_links[0]]
+
+  zone = var.zone
+  named_port {
+    name = "http"
+    port = 3000
+  }
+}

infra/multichain-mainnet/outputs.tf

@@ -0,0 +1,3 @@
+output "node_public_ip" {
+  value = google_compute_global_address.external_ips[*].address
+}

infra/multichain-mainnet/resources.tf

@@ -0,0 +1,47 @@
+terraform {
+  backend "gcs" {
+    bucket = "terraform-prod-multichain"
+    prefix = "state/multichain-partner-vm-mainnet"
+  }
+
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "4.73.0"
+    }
+  }
+}
+
+# These data blocks grab the values from your GCP secret manager, please adjust secret names as desired
+data "google_secret_manager_secret_version" "account_sk_secret_id" {
+  count   = length(var.node_configs)
+  secret  = var.node_configs[0].account_sk_secret_id
+  project = var.project_id
+}
+
+data "google_secret_manager_secret_version" "cipher_sk_secret_id" {
+  count   = length(var.node_configs)
+  secret  = var.node_configs[0].cipher_sk_secret_id
+  project = var.project_id
+}
+
+data "google_secret_manager_secret_version" "sign_sk_secret_id" {
+  count   = length(var.node_configs)
+  secret  = var.node_configs[0].sign_sk_secret_id
+  project = var.project_id
+}
+
+data "google_secret_manager_secret_version" "sk_share_secret_id" {
+  count   = length(var.node_configs)
+  secret  = var.node_configs[0].sk_share_secret_id
+  project = var.project_id
+}
+
+# This is the AWS access key and secret key for our public S3 bucket with Lake data
+data "google_secret_manager_secret_version" "aws_access_key_secret_id" {
+  secret = "multichain-indexer-aws-access-key"
+}
+
+data "google_secret_manager_secret_version" "aws_secret_key_secret_id" {
+  secret = "multichain-indexer-aws-secret-key"
+}