From a6dd7887e96ce3ee2feaa52ec1df8c18c3d401c5 Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Thu, 2 Jan 2025 10:11:21 -0500 Subject: [PATCH 1/4] Gem updates --- Gemfile | 4 ++-- Gemfile.lock | 36 ++++++++++++++++++------------------ 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Gemfile b/Gemfile index 5e29e17..17ea354 100644 --- a/Gemfile +++ b/Gemfile @@ -63,14 +63,14 @@ gem "faraday-typhoeus", "~> 1.1" group :development, :test do gem "rspec-rails", "~> 7.1" gem "dotenv-rails", "~> 3.1" - gem "brakeman", "~> 6.1" + gem "brakeman", "~> 7.0" gem "bundler-audit", "~> 0.9" gem "standard", "~> 1.36" gem "i18n-tasks", "~> 1.0" end group :development do - gem "rails_template_18f", github: "gsa-tts/rails-template", branch: "next-gen-terraform" + gem "rails_template_18f", github: "gsa-tts/rails-template", branch: "easy-sandbox-deploys" gem "rails-erd", "~> 1.7" end diff --git a/Gemfile.lock b/Gemfile.lock index fe187de..91dff99 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,9 +1,9 @@ GIT remote: https://github.com/gsa-tts/rails-template.git - revision: 2a9e85dbfe124f31d83e4f423a7b90714a0ebc68 - branch: next-gen-terraform + revision: a6b07744014782e750296dd826de762c094800d7 + branch: easy-sandbox-deploys specs: - rails_template_18f (1.3.0) + rails_template_18f (2.0.0) activesupport (~> 8.0.1) colorize (~> 1.1) railties (~> 8.0.1) @@ -101,7 +101,7 @@ GEM bindex (0.8.1) bootsnap (1.18.4) msgpack (~> 1.2) - brakeman (6.2.2) + brakeman (7.0.0) racc builder (3.3.0) bundler-audit (0.9.2) @@ -144,12 +144,12 @@ GEM faraday-typhoeus (1.1.0) faraday (~> 2.0) typhoeus (~> 1.4) - ffi (1.17.0-aarch64-linux-gnu) - ffi (1.17.0-arm-linux-gnu) - ffi (1.17.0-arm64-darwin) - ffi (1.17.0-x86-linux-gnu) - ffi (1.17.0-x86_64-darwin) - ffi (1.17.0-x86_64-linux-gnu) + ffi (1.17.1-aarch64-linux-gnu) + ffi (1.17.1-arm-linux-gnu) + ffi (1.17.1-arm64-darwin) + ffi (1.17.1-x86-linux-gnu) + ffi (1.17.1-x86_64-darwin) + ffi (1.17.1-x86_64-linux-gnu) glob (0.4.1) globalid (1.2.1) activesupport (>= 6.1) @@ -186,7 +186,7 @@ GEM rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) logger (1.6.4) - loofah (2.23.1) + loofah (2.24.0) crass (~> 1.0.2) nokogiri (>= 1.12.0) mail (2.8.1) @@ -212,18 +212,18 @@ GEM net-smtp (0.5.0) net-protocol nio4r (2.7.4) - nokogiri (1.18.0) + nokogiri (1.18.1) mini_portile2 (~> 2.8.2) racc (~> 1.4) - nokogiri (1.18.0-aarch64-linux-gnu) + nokogiri (1.18.1-aarch64-linux-gnu) racc (~> 1.4) - nokogiri (1.18.0-arm-linux-gnu) + nokogiri (1.18.1-arm-linux-gnu) racc (~> 1.4) - nokogiri (1.18.0-arm64-darwin) + nokogiri (1.18.1-arm64-darwin) racc (~> 1.4) - nokogiri (1.18.0-x86_64-darwin) + nokogiri (1.18.1-x86_64-darwin) racc (~> 1.4) - nokogiri (1.18.0-x86_64-linux-gnu) + nokogiri (1.18.1-x86_64-linux-gnu) racc (~> 1.4) orm_adapter (0.5.0) oscal (0.2.3) @@ -402,7 +402,7 @@ PLATFORMS DEPENDENCIES bootsnap - brakeman (~> 6.1) + brakeman (~> 7.0) bundler-audit (~> 0.9) byebug (~> 11.1) climate_control (~> 1.0) From eb98321db70356a185f468d98f578f545d28c974 Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Thu, 2 Jan 2025 10:31:39 -0500 Subject: [PATCH 2/4] Terraform updates from easy-sandbox --- .gitignore | 1 - terraform/README.md | 108 +++++++++++---------------------- terraform/app.tf | 8 +-- terraform/bootstrap/apply.sh | 3 + terraform/bootstrap/main.tf | 10 +-- terraform/main.tf | 21 ++++--- terraform/sandbox.tfvars | 10 +++ terraform/sandbox_bot/apply.sh | 17 ------ terraform/sandbox_bot/main.tf | 3 +- terraform/sandbox_bot/run.sh | 17 ++++++ terraform/set_space_egress.sh | 4 +- terraform/terraform.sh | 44 +++++++++++--- 12 files changed, 126 insertions(+), 120 deletions(-) create mode 100644 terraform/sandbox.tfvars delete mode 100755 terraform/sandbox_bot/apply.sh create mode 100755 terraform/sandbox_bot/run.sh diff --git a/.gitignore b/.gitignore index 0c67913..87e4612 100644 --- a/.gitignore +++ b/.gitignore @@ -67,7 +67,6 @@ doc/compliance/oscal/ssp-render/continuous_monitoring_ssp.* .terraform.lock.hcl **/.terraform/* secrets.*.tfvars -sandbox*.tfvars terraform.tfstate terraform.tfstate.backup terraform/dist diff --git a/terraform/README.md b/terraform/README.md index ef31bbe..64f61c8 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -2,101 +2,56 @@ This directory holds the terraform module for maintaining the system infrastructure and deploying the application. -## Initial project setup - -These steps only need to be run once per project. - -1. Manually [bootstrap the state storage bucket](#bootstrapping-the-state-storage-s3-buckets-for-the-first-time) within the `bootstrap` directory -1. Setup CI/CD Pipeline to run Terraform - 1. Copy backend credentials from `secrets.backend.tfvars` to your CI/CD secrets using the instructions in the base README - 1. Copy the cf_user and cf_password credentials from `secrets.auto.tfvars` to your CI/CD secrets using the instructions in the base README -1. Manually Running Terraform - 1. Follow instructions under `Set up a new environment` to create your infrastructure - -## Initial developer setup - -These steps should be run for any developer that needs to start running terraform or who just moved to a new machine. - -They are not necessary for the developer who runs the [initial project setup](#initial-project-setup) - -1. Import the existing bootstrap resources to your local state with `./apply.sh` -1. Follow instructions under [Use backend credentials](#use-backend-credentials) -1. Follow instructions under `Set up a new environment` to create your infrastructure - - ## Terraform State Credentials -The `bootstrap` module is used to create an s3 bucket for later terraform runs to store their state in. +The `bootstrap` module is used to create an s3 bucket for later terraform runs to store their state in as well as +create credentials files so developers can use that s3 bucket to create their own sandbox environments. -### Bootstrapping the state storage s3 buckets for the first time +### Initial project setup -These steps are run once per project. +These steps only need to be run once per project. +1. `cd bootstrap` 1. Add any users who should have access to the terraform state bucket to `users.auto.tfvars` -1. Run `cf login --sso` to log in to cloud.gov 1. Run `./apply.sh -var create_bot_secrets_file=true` 1. Add `imports.tf` to git and commit the changes -1. Follow instructions under [Use backend credentials](#use-backend-credentials) +1. Setup your CI/CD Pipeline to run terraform and deploy your staging and production environments + 1. Copy backend credentials from `/terraform/secrets.backend.tfvars` to your CI/CD secrets using the instructions in the base README + 1. Copy the cf_user and cf_password credentials from `/terraform/secrets.cicd.tfvars` to your CI/CD secrets using the instructions in the base README +1. Delete the two secrets files ### To make changes to the bootstrap module -*This should not be necessary in most cases, other than adding or removing users who should have access to the state bucket* +*This should not be necessary in most cases, other than adding or removing users who should have access to the state bucket in `bootstrap/users.auto.tfvars`* 1. Make your changes 1. Run `./apply.sh` and verify the plan before entering `yes` 1. Commit any changes to `imports.tf` -### Use backend credentials - -1. In the main `terraform` module, run: `terraform init -backend-config=secrets.backend.tfvars` -1. When prompted for the key, enter a keyfile for your environment: for example `terraform.tfstate.my_name` for a per-developer sandbox -1. Delete the `secrets.backend.tfvars` file. - -## SpaceDeployers +## Set up a sandbox environment or review app -A [SpaceDeployer](https://cloud.gov/docs/services/cloud-gov-service-account/) account is required to run the main terraform module or -deploy the application from the CI/CD pipeline. Create a new account by running: +### Pre-requisites: -`../bin/ops/create_service_account.sh -s -u -m` +1. Someone on the team has run the [Initial project setup](#initial-project-setup) steps and `imports.tf` is up-to-date on your branch. +1. You are included in the list of users in `bootstrap/users.auto.tfvars` and `bootstrap/imports.tf` -## Set up a new environment manually +### Steps: -The below steps rely on you first configuring access to the Terraform state in s3 as described in [initial project setup](#initial-project-setup) or [initial developer setup](#initial-developer-setup). +1. Create a new `sandbox-.tfvars` file to hold variable values for your environment. A good starting point is copying `staging.tfvars` and editing it with your values. -1. Initialize a new terraform state file: `terraform init -backend-config=secrets.backend.tfvars -backend-config="key=terraform.tfstate.sandbox-name" -reconfigure` - -1. Set up a SpaceDeployer and save the credentials in a file named `secrets.auto.tfvars` +1. Run terraform plan with: ```bash - # create a space deployer service instance that can log in with just a username and password - # the value for < SPACE_NAME > should be your management space, by default "capoc-mgmt" - # the value for < ACCOUNT_NAME > can be anything, although we recommend - # something that communicates the purpose of the deployer - # for example: circleci-deployer for the credentials CircleCI uses to - # deploy the application or -terraform for credentials to run terraform manually - ../../bin/ops/create_service_account.sh -s -u -m > secrets.auto.tfvars + ./terraform.sh -e sandbox- ``` - The script will output the `username` (as `cf_user`) and `password` (as `cf_password`) for your ``. Read more in the [cloud.gov service account documentation](https://cloud.gov/docs/services/cloud-gov-service-account/). - - The easiest way to use this script locally is to redirect the output directly to the `secrets.auto.tfvars` file it needs to be used in - -1. Create a new `sandbox.tfvars` file to hold variable values for your environment. - -1. Run terraform from your new environment directory with +1. Apply changes with: ```bash - ./tfm_sandbox.sh plan + ./terraform.sh -e sandbox- -c apply ``` -1. Apply changes with +1. Optional: tear down the sandbox if it does not need to be used anymore ```bash - ./tfm_sandbox.sh apply - ``` - -1. Remove the space deployer service instance if it doesn't need to be used again, such as when manually running terraform plan before letting CI/CD apply the changes. - ```bash - # and have the same values as used above. - ../../bin/ops/destroy_service_account.sh -s -u - rm secrets.auto.tfvars + ./terraform.sh -e sandbox- -c destroy ``` ## Structure @@ -107,11 +62,16 @@ The below steps rely on you first configuring access to the Terraform state in s | |- apply.sh | |- imports.tf (automatically generated) | |- users.auto.tfvars -| |- templates/ -| | |- backend_config.tftpl -| | |- bot_secrets.tftpl -| | |- imports.tf.tftpl | |- terraform.tfstate(.backup) (automatically generated) +| |- templates/ +| |- backend_config.tftpl +| |- bot_secrets.tftpl +| |- imports.tf.tftpl +|- sandbox_bot/ +| |- main.tf +| |- run.sh +| |- / (automatically generated) +| |- terraform.tfstate(.backup) (automatically generated) |- dist/ | |- src.zip (automatically generated) |- README.md @@ -119,12 +79,14 @@ The below steps rely on you first configuring access to the Terraform state in s |- main.tf |- providers.tf |- set_space_egress.sh +|- terraform.sh |- variables.tf |- .tfvars ``` In the root module: - `.tfvars` is where to set variable values for the given environment name +- `terraform.sh` Helper script to setup terraform to point to the correct state file, create a service account to run the root module, and apply the root module. - `app.tf` defines the application resource and configuration - `main.tf` defines the persistent infrastructure - `providers.tf` lists the required providers and shell backend config @@ -136,3 +98,7 @@ In the bootstrap module: - `apply.sh` Helper script to either recreate the state locally or call `terraform apply` Any arguments are passed through to the `apply` call - `imports.tf` import blocks to create a new local state file when new developers need to access the state file. This file is automatically generated by calling `./apply.sh` and should be checked into git on any changes - `users.auto.tfvars` this file defines the list of cloud.gov accounts that should have access to the terraform state bucket + +In the sandbox_bot module: +- `main.tf` sets up a cloud.gov SpaceDeployer to manage the sandbox environment and outputs its credentials into the main module `secrets.auto.tfvars` +- `run.sh` Helper script to set up a separate local state file for each sandbox name. In normal use this will only ever be called by `./terraform.sh` diff --git a/terraform/app.tf b/terraform/app.tf index 643c1ca..a24496d 100644 --- a/terraform/app.tf +++ b/terraform/app.tf @@ -19,7 +19,7 @@ locals { resource "cloudfoundry_app" "app" { name = "${local.app_name}-${var.env}" - space_name = module.app_space.space_name + space_name = var.cf_space_name org_name = local.cf_org_name path = data.archive_file.src.output_path @@ -47,7 +47,7 @@ resource "cloudfoundry_app" "app" { type = "web" instances = var.web_instances memory = var.web_memory - command = "bundle exec rake cf:on_first_instance db:migrate && exec env HTTP_PORT=$PORT ./bin/thrust ./bin/rails server" + command = "./bin/rake cf:on_first_instance db:migrate && exec env HTTP_PORT=$PORT ./bin/thrust ./bin/rails server" } ] @@ -60,7 +60,7 @@ resource "cloudfoundry_app" "app" { depends_on = [ cloudfoundry_service_instance.egress_proxy_credentials, # module.redis, - module.database, - module.app_space + module.app_space, + module.database ] } diff --git a/terraform/bootstrap/apply.sh b/terraform/bootstrap/apply.sh index ca7896b..05126a5 100755 --- a/terraform/bootstrap/apply.sh +++ b/terraform/bootstrap/apply.sh @@ -8,5 +8,8 @@ fi set -e +# ensure we're logged in via cli +cf spaces &> /dev/null || cf login -a api.fr.cloud.gov --sso + terraform init terraform apply "$@" diff --git a/terraform/bootstrap/main.tf b/terraform/bootstrap/main.tf index 6ffb285..92af772 100644 --- a/terraform/bootstrap/main.tf +++ b/terraform/bootstrap/main.tf @@ -27,11 +27,13 @@ variable "mgmt_space_name" { variable "create_bot_secrets_file" { type = bool default = false - description = "Flag whether to create secrets.auto.tfvars file" + description = "Flag whether to create secrets.cicd.tfvars file" } locals { org_name = "gsa-tts-devtools-prototyping" + # s3_plan_name should be basic when holding production data, though basic-sandbox will make early iterations easier + s3_plan_name = "basic-sandbox" } module "mgmt_space" { source = "github.com/gsa-tts/terraform-cloudgov//cg_space?ref=v2.0.2" @@ -42,11 +44,11 @@ module "mgmt_space" { } module "s3" { - source = "github.com/gsa-tts/terraform-cloudgov//s3?ref=v2.0.0" + source = "github.com/gsa-tts/terraform-cloudgov//s3?ref=v2.0.2" cf_space_id = module.mgmt_space.space_id name = "continuous_monitoring-terraform-state" - s3_plan_name = "basic-sandbox" + s3_plan_name = local.s3_plan_name depends_on = [module.mgmt_space] } @@ -138,7 +140,7 @@ resource "local_sensitive_file" "bucket_creds" { resource "local_sensitive_file" "bot_secrets_file" { count = (var.create_bot_secrets_file ? 1 : 0) - filename = "${path.module}/../secrets.auto.tfvars" + filename = "${path.module}/../secrets.cicd.tfvars" file_permission = "0600" content = templatefile("${path.module}/templates/bot_secrets.tftpl", { diff --git a/terraform/main.tf b/terraform/main.tf index 17381db..7099022 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -5,7 +5,7 @@ locals { } module "app_space" { - source = "github.com/gsa-tts/terraform-cloudgov//cg_space?ref=v2.0.0" + source = "github.com/gsa-tts/terraform-cloudgov//cg_space?ref=v2.0.2" cf_org_name = local.cf_org_name cf_space_name = var.cf_space_name @@ -15,28 +15,29 @@ module "app_space" { } # temporary method for setting egress rules until terraform provider supports it and cg_space module is updated data "external" "set-app-space-egress" { - program = ["/bin/sh", "${path.module}/set_space_egress.sh", "-t", "-s", module.app_space.space_name, "-o", local.cf_org_name] + program = ["/bin/sh", "set_space_egress.sh", "-t", "-s", var.cf_space_name, "-o", local.cf_org_name] working_dir = path.module - depends_on = [module.app_space] + # depends_on line is required only for initial creation and destruction. It can be commented out for updates if you see unwanted cascading effects + depends_on = [module.app_space] } module "database" { - source = "github.com/gsa-tts/terraform-cloudgov//database?ref=v2.0.0" + source = "github.com/gsa-tts/terraform-cloudgov//database?ref=v2.0.2" cf_space_id = module.app_space.space_id name = "${local.app_name}-rds-${var.env}" rds_plan_name = var.rds_plan_name - # depends_on line is needed only for initial creation and destruction. It should be commented out for updates to prevent unwanted cascading effects + # depends_on line is required only for initial creation and destruction. It can be commented out for updates if you see unwanted cascading effects depends_on = [module.app_space] } # module "redis" { -# source = "github.com/gsa-tts/terraform-cloudgov//redis?ref=v2.0.0" +# source = "github.com/gsa-tts/terraform-cloudgov//redis?ref=v2.0.2" # cf_space_id = module.app_space.space_id # name = "${local.app_name}-redis-${var.env}" # redis_plan_name = var.redis_plan_name -# # depends_on line is needed only for initial creation and destruction. It should be commented out for updates to prevent unwanted cascading effects +# # depends_on line is required only for initial creation and destruction. It can be commented out for updates if you see unwanted cascading effects # depends_on = [module.app_space] # } @@ -51,14 +52,14 @@ module "database" { ########################################################################### module "domain" { count = (var.custom_domain_name == null ? 0 : 1) - source = "github.com/gsa-tts/terraform-cloudgov//domain?ref=v2.0.0" + source = "github.com/gsa-tts/terraform-cloudgov//domain?ref=v2.0.2" cf_org_name = local.cf_org_name cf_space = module.app_space.space cdn_plan_name = "domain" domain_name = var.custom_domain_name host_name = var.host_name - # depends_on line is needed only for initial creation and destruction. It should be commented out for updates to prevent unwanted cascading effects + # depends_on line is required only for initial creation and destruction. It can be commented out for updates if you see unwanted cascading effects depends_on = [module.app_space] } @@ -73,7 +74,7 @@ module "egress_space" { } # temporary method for setting egress rules until terraform provider supports it and cg_space module is updated data "external" "set-egress-space-egress" { - program = ["/bin/sh", "${path.module}/set_space_egress.sh", "-p", "-s", module.egress_space.space_name, "-o", local.cf_org_name] + program = ["/bin/sh", "set_space_egress.sh", "-p", "-s", module.egress_space.space_name, "-o", local.cf_org_name] working_dir = path.module depends_on = [module.egress_space] } diff --git a/terraform/sandbox.tfvars b/terraform/sandbox.tfvars new file mode 100644 index 0000000..60cb4ba --- /dev/null +++ b/terraform/sandbox.tfvars @@ -0,0 +1,10 @@ +cf_space_name = "capoc-rahearn" +env = "staging" +host_name = "capoc-rahearn" +allow_space_ssh = true +space_developers = [ + "ryan.ahearn@gsa.gov" +] +egress_allowlist = [ + "raw.githubusercontent.com" # download content previews +] diff --git a/terraform/sandbox_bot/apply.sh b/terraform/sandbox_bot/apply.sh deleted file mode 100755 index 5891e54..0000000 --- a/terraform/sandbox_bot/apply.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -set -e - -if [[ -z "$1" ]]; then - echo "You must pass the sandbox_name as the first argument" - echo "usage: $0 SANDBOX_NAME [TERRAFORM_CMD]" - exit 1 -fi - -cmd="apply" -if [[ -n "$2" ]]; then - cmd="$2" -fi - -terraform init -backend-config="path=$1/terraform.tfstate" -reconfigure -terraform "$cmd" -var sandbox_name="$1" diff --git a/terraform/sandbox_bot/main.tf b/terraform/sandbox_bot/main.tf index c18079f..d202054 100644 --- a/terraform/sandbox_bot/main.tf +++ b/terraform/sandbox_bot/main.tf @@ -6,7 +6,6 @@ terraform { version = "1.1.0" } } - backend "local" {} } # empty config will let terraform borrow cf-cli's auth @@ -67,7 +66,7 @@ resource "local_sensitive_file" "bot_secrets_file" { content = <<-EOT # "${local.sa_service_name}"/"${local.sa_key_name}" generated by sandbox_bot terraform module. - # Run `./apply ${var.sandbox_name} destroy` in that directory to clean up + # Run `./run.sh ${var.sandbox_name} destroy` in that directory to clean up cf_user = "${local.sa_cf_username}" cf_password = "${local.sa_bot_credentials.password}" diff --git a/terraform/sandbox_bot/run.sh b/terraform/sandbox_bot/run.sh new file mode 100755 index 0000000..e2c25cb --- /dev/null +++ b/terraform/sandbox_bot/run.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -e + +if [[ $# -lt 2 ]]; then + echo "usage: $0 SANDBOX_NAME TERRAFORM_CMD [TERRAFORM_ARGS]" + echo "You must pass the sandbox_name as the first argument and terraform command as the second" + echo "All other arguments are passed as-is to terraform" + exit 1 +fi + +sandbox_name="$1" +cmd="$2" +shift 2 + +terraform init -backend-config="path=$sandbox_name/terraform.tfstate" -reconfigure +terraform "$cmd" -var sandbox_name="$sandbox_name" "$@" diff --git a/terraform/set_space_egress.sh b/terraform/set_space_egress.sh index 007eadf..c44b6ea 100755 --- a/terraform/set_space_egress.sh +++ b/terraform/set_space_egress.sh @@ -49,11 +49,11 @@ if [[ "$space" = "" ]] || [[ "$org" = "" ]]; then fi if [[ $public = true ]]; then - cf bind-security-group public_networks_egress "$org" --space "$space" 1>&2 + cf bind-security-group public_networks_egress "$org" --space "$space" > /dev/null fi if [[ $trusted = true ]]; then - cf bind-security-group trusted_local_networks_egress "$org" --space "$space" 1>&2 + cf bind-security-group trusted_local_networks_egress "$org" --space "$space" > /dev/null fi cat << EOF diff --git a/terraform/terraform.sh b/terraform/terraform.sh index c7eafc0..e5a5d2d 100755 --- a/terraform/terraform.sh +++ b/terraform/terraform.sh @@ -8,12 +8,13 @@ $0: Run terraform commands against a given environment Usage: $0 -h - $0 -e [-k ] [-c ] [] + $0 -e [-k ] [-f] [-c ] [-- ] Options: -h: show help and exit -e ENV_NAME: The name of the environment to run terraform against -k RAILS_MASTER_KEY: RAILS_MASTER_KEY value. Defaults to contents of $rmk_file +-f: Force, pass -auto-approve to all invocations of terraform -c TERRAFORM-CMD: command to run. Defaults to $cmd []: arguments to pass as-is to terraform " @@ -21,10 +22,11 @@ Options: rmk=`cat $rmk_file || echo -n ""` env="" +force="" args_to_shift=0 set -e -while getopts ":he:k:c:" opt; do +while getopts ":he:k:fc:" opt; do case "$opt" in e) env=${OPTARG} @@ -34,6 +36,10 @@ while getopts ":he:k:c:" opt; do rmk=${OPTARG} args_to_shift=$((args_to_shift + 2)) ;; + f) + force="-auto-approve" + args_to_shift=$((args_to_shift + 1)) + ;; c) cmd=${OPTARG} args_to_shift=$((args_to_shift + 2)) @@ -46,6 +52,9 @@ while getopts ":he:k:c:" opt; do done shift $args_to_shift +if [[ "$1" = "--" ]]; then + shift 1 +fi if [[ -z "$env" ]]; then echo "-e is required" @@ -54,7 +63,7 @@ if [[ -z "$env" ]]; then fi if [[ ! -f "$env.tfvars" ]]; then - echo "$env.tfvars file is missing" + echo "$env.tfvars file is missing. Create it first" exit 1 fi @@ -74,21 +83,38 @@ if [[ $tfm_needs_init = true ]]; then echo "==============================================================================================================" echo "= Recreating backend config file. It is fine if this step wants to delete any local_sensitive_file resources" echo "==============================================================================================================" - (cd bootstrap && ./apply.sh) + (cd bootstrap && ./apply.sh $force) fi terraform init -backend-config=secrets.backend.tfvars -backend-config="key=terraform.tfstate.$env" -reconfigure fi echo "==============================================================================================================" -echo "= Creating or finding a bot deployer for $env" +echo "= Creating a bot deployer for $env" echo "==============================================================================================================" if [[ "$env" = "staging" ]] || [[ "$env" = "production" ]]; then - (cd bootstrap && ./apply.sh -var create_bot_secrets_file=true) + (cd bootstrap && ./apply.sh -var create_bot_secrets_file=true $force) else - (cd sandbox_bot && ./apply.sh "$env") + (cd sandbox_bot && ./run.sh "$env" apply $force) +fi + +if [[ -f secrets.backend.tfvars ]]; then + rm secrets.backend.tfvars fi echo "==============================================================================================================" -echo "= Calling $cmd on the application infrastructure" +echo "= Calling $cmd $force on the application infrastructure" echo "==============================================================================================================" -terraform "$cmd" -var-file="$env.tfvars" -var rails_master_key="$rmk" "$@" +terraform "$cmd" -var-file="$env.tfvars" -var rails_master_key="$rmk" $force "$@" + +if [[ "$cmd" = "destroy" ]] && [[ "$env" != "staging" ]] && [[ "$env" != "production" ]]; then + if [[ -z "$force" ]]; then + read -p "Destroy the sandbox_bot user? (y/n) " confirm + if [[ "$confirm" != "y" ]]; then + exit 0 + fi + fi + echo "==============================================================================================================" + echo "= Destroying the sandbox_bot user" + echo "==============================================================================================================" + (cd sandbox_bot && ./run.sh "$env" destroy -auto-approve) +fi From ca4ffa3fbc6965c3c12780c5ef455709c6cbb0c6 Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Thu, 2 Jan 2025 12:21:19 -0500 Subject: [PATCH 3/4] Dont use processes to workaround bug in rolling deploy --- terraform/app.tf | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/terraform/app.tf b/terraform/app.tf index a24496d..a6a659f 100644 --- a/terraform/app.tf +++ b/terraform/app.tf @@ -22,11 +22,16 @@ resource "cloudfoundry_app" "app" { space_name = var.cf_space_name org_name = local.cf_org_name - path = data.archive_file.src.output_path - source_code_hash = data.archive_file.src.output_base64sha256 - buildpacks = ["ruby_buildpack"] - strategy = "blue-green" - routes = [{ route = "${local.host_name}.${local.domain}" }] + path = data.archive_file.src.output_path + source_code_hash = data.archive_file.src.output_base64sha256 + buildpacks = ["ruby_buildpack"] + strategy = "rolling" + instances = var.web_instances + memory = var.web_memory + command = "./bin/rake cf:on_first_instance db:migrate && exec env HTTP_PORT=$PORT ./bin/thrust ./bin/rails server" + health_check_http_endpoint = "/up" + health_check_type = "http" + routes = [{ route = "${local.host_name}.${local.domain}" }] environment = { no_proxy = "apps.internal,s3-fips.us-gov-west-1.amazonaws.com" @@ -36,20 +41,20 @@ resource "cloudfoundry_app" "app" { RAILS_SERVE_STATIC_FILES = "true" } - processes = [ - # { - # type = "worker" - # instances = var.worker_instances - # memory = var.worker_memory - # command = "bundle exec sidekiq" - # }, - { - type = "web" - instances = var.web_instances - memory = var.web_memory - command = "./bin/rake cf:on_first_instance db:migrate && exec env HTTP_PORT=$PORT ./bin/thrust ./bin/rails server" - } - ] + # processes = [ + # # { + # # type = "worker" + # # instances = var.worker_instances + # # memory = var.worker_memory + # # command = "bundle exec sidekiq" + # # }, + # { + # type = "web" + # instances = var.web_instances + # memory = var.web_memory + # command = "./bin/rake cf:on_first_instance db:migrate && exec env HTTP_PORT=$PORT ./bin/thrust ./bin/rails server" + # } + # ] service_bindings = [ { service_instance = "egress-proxy-${var.env}-credentials" }, From b347a6ae88d7e9650b72cac543a11fbb9cef1ab9 Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Thu, 2 Jan 2025 14:41:32 -0500 Subject: [PATCH 4/4] Precompile assets and cache them as part of staging deploy --- .github/workflows/deploy-staging.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index 15d7a8f..06b8d80 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -20,6 +20,21 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - uses: actions/checkout@v4 + - uses: ./.github/actions/setup-languages + + - name: Restore asset cache + uses: actions/cache@v4 + with: + key: cache-staging-assets + path: | + public/assets + + - name: Precompile assets and clean old versions + env: + RAILS_ENV: staging + SECRET_KEY_BASE_DUMMY: 1 + run: ./bin/rake assets:precompile && ./bin/rake assets:clean + - name: Terraform apply uses: dflook/terraform-apply@v1 env: