Merge pull request #86 from GSA-TTS/merged-assets

Merged assets
GSA-TTS · Jan 2, 2025 · 4763f9a · 4763f9a
2 parents c3723c1 + b347a6a
commit 4763f9a
Show file tree

Hide file tree

Showing 15 changed files with 184 additions and 158 deletions.
diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml
@@ -20,6 +20,21 @@ jobs:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-languages
+
+      - name: Restore asset cache
+        uses: actions/cache@v4
+        with:
+          key: cache-staging-assets
+          path: |
+            public/assets
+
+      - name: Precompile assets and clean old versions
+        env:
+          RAILS_ENV: staging
+          SECRET_KEY_BASE_DUMMY: 1
+        run: ./bin/rake assets:precompile && ./bin/rake assets:clean
+
       - name: Terraform apply
         uses: dflook/terraform-apply@v1
         env:

diff --git a/.gitignore b/.gitignore
@@ -67,7 +67,6 @@ doc/compliance/oscal/ssp-render/continuous_monitoring_ssp.*
 .terraform.lock.hcl
 **/.terraform/*
 secrets.*.tfvars
-sandbox*.tfvars
 terraform.tfstate
 terraform.tfstate.backup
 terraform/dist
diff --git a/Gemfile b/Gemfile
@@ -63,14 +63,14 @@ gem "faraday-typhoeus", "~> 1.1"
 group :development, :test do
   gem "rspec-rails", "~> 7.1"
   gem "dotenv-rails", "~> 3.1"
-  gem "brakeman", "~> 6.1"
+  gem "brakeman", "~> 7.0"
   gem "bundler-audit", "~> 0.9"
   gem "standard", "~> 1.36"
   gem "i18n-tasks", "~> 1.0"
 end
 
 group :development do
-  gem "rails_template_18f", github: "gsa-tts/rails-template", branch: "next-gen-terraform"
+  gem "rails_template_18f", github: "gsa-tts/rails-template", branch: "easy-sandbox-deploys"
   gem "rails-erd", "~> 1.7"
 end
 

diff --git a/Gemfile.lock b/Gemfile.lock
@@ -1,9 +1,9 @@
 GIT
   remote: https://github.com/gsa-tts/rails-template.git
-  revision: 2a9e85dbfe124f31d83e4f423a7b90714a0ebc68
-  branch: next-gen-terraform
+  revision: a6b07744014782e750296dd826de762c094800d7
+  branch: easy-sandbox-deploys
   specs:
-    rails_template_18f (1.3.0)
+    rails_template_18f (2.0.0)
       activesupport (~> 8.0.1)
       colorize (~> 1.1)
       railties (~> 8.0.1)
@@ -101,7 +101,7 @@ GEM
     bindex (0.8.1)
     bootsnap (1.18.4)
       msgpack (~> 1.2)
-    brakeman (6.2.2)
+    brakeman (7.0.0)
       racc
     builder (3.3.0)
     bundler-audit (0.9.2)
@@ -144,12 +144,12 @@ GEM
     faraday-typhoeus (1.1.0)
       faraday (~> 2.0)
       typhoeus (~> 1.4)
-    ffi (1.17.0-aarch64-linux-gnu)
-    ffi (1.17.0-arm-linux-gnu)
-    ffi (1.17.0-arm64-darwin)
-    ffi (1.17.0-x86-linux-gnu)
-    ffi (1.17.0-x86_64-darwin)
-    ffi (1.17.0-x86_64-linux-gnu)
+    ffi (1.17.1-aarch64-linux-gnu)
+    ffi (1.17.1-arm-linux-gnu)
+    ffi (1.17.1-arm64-darwin)
+    ffi (1.17.1-x86-linux-gnu)
+    ffi (1.17.1-x86_64-darwin)
+    ffi (1.17.1-x86_64-linux-gnu)
     glob (0.4.1)
     globalid (1.2.1)
       activesupport (>= 6.1)
@@ -186,7 +186,7 @@ GEM
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
     logger (1.6.4)
-    loofah (2.23.1)
+    loofah (2.24.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
     mail (2.8.1)
@@ -212,18 +212,18 @@ GEM
     net-smtp (0.5.0)
       net-protocol
     nio4r (2.7.4)
-    nokogiri (1.18.0)
+    nokogiri (1.18.1)
       mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
-    nokogiri (1.18.0-aarch64-linux-gnu)
+    nokogiri (1.18.1-aarch64-linux-gnu)
       racc (~> 1.4)
-    nokogiri (1.18.0-arm-linux-gnu)
+    nokogiri (1.18.1-arm-linux-gnu)
       racc (~> 1.4)
-    nokogiri (1.18.0-arm64-darwin)
+    nokogiri (1.18.1-arm64-darwin)
       racc (~> 1.4)
-    nokogiri (1.18.0-x86_64-darwin)
+    nokogiri (1.18.1-x86_64-darwin)
       racc (~> 1.4)
-    nokogiri (1.18.0-x86_64-linux-gnu)
+    nokogiri (1.18.1-x86_64-linux-gnu)
       racc (~> 1.4)
     orm_adapter (0.5.0)
     oscal (0.2.3)
@@ -402,7 +402,7 @@ PLATFORMS
 
 DEPENDENCIES
   bootsnap
-  brakeman (~> 6.1)
+  brakeman (~> 7.0)
   bundler-audit (~> 0.9)
   byebug (~> 11.1)
   climate_control (~> 1.0)

diff --git a/terraform/README.md b/terraform/README.md
@@ -2,101 +2,56 @@
 
 This directory holds the terraform module for maintaining the system infrastructure and deploying the application.
 
-## Initial project setup
-
-These steps only need to be run once per project.
-
-1. Manually [bootstrap the state storage bucket](#bootstrapping-the-state-storage-s3-buckets-for-the-first-time) within the `bootstrap` directory
-1. Setup CI/CD Pipeline to run Terraform
-    1. Copy backend credentials from `secrets.backend.tfvars` to your CI/CD secrets using the instructions in the base README
-    1. Copy the cf_user and cf_password credentials from `secrets.auto.tfvars` to your CI/CD secrets using the instructions in the base README
-1. Manually Running Terraform
-    1. Follow instructions under `Set up a new environment` to create your infrastructure
-
-## Initial developer setup
-
-These steps should be run for any developer that needs to start running terraform or who just moved to a new machine.
-
-They are not necessary for the developer who runs the [initial project setup](#initial-project-setup)
-
-1. Import the existing bootstrap resources to your local state with `./apply.sh`
-1. Follow instructions under [Use backend credentials](#use-backend-credentials)
-1. Follow instructions under `Set up a new environment` to create your infrastructure
-
-
 ## Terraform State Credentials
 
-The `bootstrap` module is used to create an s3 bucket for later terraform runs to store their state in.
+The `bootstrap` module is used to create an s3 bucket for later terraform runs to store their state in as well as
+create credentials files so developers can use that s3 bucket to create their own sandbox environments.
 
-### Bootstrapping the state storage s3 buckets for the first time
+### Initial project setup
 
-These steps are run once per project.
+These steps only need to be run once per project.
 
+1. `cd bootstrap`
 1. Add any users who should have access to the terraform state bucket to `users.auto.tfvars`
-1. Run `cf login --sso` to log in to cloud.gov
 1. Run `./apply.sh -var create_bot_secrets_file=true`
 1. Add `imports.tf` to git and commit the changes
-1. Follow instructions under [Use backend credentials](#use-backend-credentials)
+1. Setup your CI/CD Pipeline to run terraform and deploy your staging and production environments
+    1. Copy backend credentials from `/terraform/secrets.backend.tfvars` to your CI/CD secrets using the instructions in the base README
+    1. Copy the cf_user and cf_password credentials from `/terraform/secrets.cicd.tfvars` to your CI/CD secrets using the instructions in the base README
+1. Delete the two secrets files
 
 ### To make changes to the bootstrap module
 
-*This should not be necessary in most cases, other than adding or removing users who should have access to the state bucket*
+*This should not be necessary in most cases, other than adding or removing users who should have access to the state bucket in `bootstrap/users.auto.tfvars`*
 
 1. Make your changes
 1. Run `./apply.sh` and verify the plan before entering `yes`
 1. Commit any changes to `imports.tf`
 
-### Use backend credentials
-
-1. In the main `terraform` module, run: `terraform init -backend-config=secrets.backend.tfvars`
-1. When prompted for the key, enter a keyfile for your environment: for example `terraform.tfstate.my_name` for a per-developer sandbox
-1. Delete the `secrets.backend.tfvars` file.
-
-## SpaceDeployers
+## Set up a sandbox environment or review app
 
-A [SpaceDeployer](https://cloud.gov/docs/services/cloud-gov-service-account/) account is required to run the main terraform module or
-deploy the application from the CI/CD pipeline. Create a new account by running:
+### Pre-requisites:
 
-`../bin/ops/create_service_account.sh -s <SPACE_NAME> -u <ACCOUNT_NAME> -m`
+1. Someone on the team has run the [Initial project setup](#initial-project-setup) steps and `imports.tf` is up-to-date on your branch.
+1. You are included in the list of users in `bootstrap/users.auto.tfvars` and `bootstrap/imports.tf`
 
-## Set up a new environment manually
+### Steps:
 
-The below steps rely on you first configuring access to the Terraform state in s3 as described in [initial project setup](#initial-project-setup) or [initial developer setup](#initial-developer-setup).
+1. Create a new `sandbox-<NAME>.tfvars` file to hold variable values for your environment. A good starting point is copying `staging.tfvars` and editing it with your values.
 
-1. Initialize a new terraform state file: `terraform init -backend-config=secrets.backend.tfvars -backend-config="key=terraform.tfstate.sandbox-name" -reconfigure`
-
-1. Set up a SpaceDeployer and save the credentials in a file named `secrets.auto.tfvars`
+1. Run terraform plan with:
     ```bash
-    # create a space deployer service instance that can log in with just a username and password
-    # the value for < SPACE_NAME > should be your management space, by default "capoc-mgmt"
-    # the value for < ACCOUNT_NAME > can be anything, although we recommend
-    # something that communicates the purpose of the deployer
-    # for example: circleci-deployer for the credentials CircleCI uses to
-    # deploy the application or <your_name>-terraform for credentials to run terraform manually
-    ../../bin/ops/create_service_account.sh -s <SPACE_NAME> -u <ACCOUNT_NAME> -m > secrets.auto.tfvars
+    ./terraform.sh -e sandbox-<NAME>
     ```
 
-    The script will output the `username` (as `cf_user`) and `password` (as `cf_password`) for your `<ACCOUNT_NAME>`. Read more in the [cloud.gov service account documentation](https://cloud.gov/docs/services/cloud-gov-service-account/).
-
-    The easiest way to use this script locally is to redirect the output directly to the `secrets.auto.tfvars` file it needs to be used in
-
-1. Create a new `sandbox.tfvars` file to hold variable values for your environment.
-
-1. Run terraform from your new environment directory with
+1. Apply changes with:
     ```bash
-    ./tfm_sandbox.sh plan
+    ./terraform.sh -e sandbox-<NAME> -c apply
     ```
 
-1. Apply changes with
+1. Optional: tear down the sandbox if it does not need to be used anymore
     ```bash
-    ./tfm_sandbox.sh apply
-    ```
-
-1. Remove the space deployer service instance if it doesn't need to be used again, such as when manually running terraform plan before letting CI/CD apply the changes.
-    ```bash
-    # <SPACE_NAME> and <ACCOUNT_NAME> have the same values as used above.
-    ../../bin/ops/destroy_service_account.sh -s <SPACE_NAME> -u <ACCOUNT_NAME>
-    rm secrets.auto.tfvars
+    ./terraform.sh -e sandbox-<NAME> -c destroy
     ```
 
 ## Structure
@@ -107,24 +62,31 @@ The below steps rely on you first configuring access to the Terraform state in s
 |  |- apply.sh
 |  |- imports.tf (automatically generated)
 |  |- users.auto.tfvars
-|  |- templates/
-|  |  |- backend_config.tftpl
-|  |  |- bot_secrets.tftpl
-|  |  |- imports.tf.tftpl
 |  |- terraform.tfstate(.backup) (automatically generated)
+|  |- templates/
+|     |- backend_config.tftpl
+|     |- bot_secrets.tftpl
+|     |- imports.tf.tftpl
+|- sandbox_bot/
+|  |- main.tf
+|  |- run.sh
+|  |- <sandbox_name>/ (automatically generated)
+|     |- terraform.tfstate(.backup) (automatically generated)
 |- dist/
 |  |- src.zip (automatically generated)
 |- README.md
 |- app.tf
 |- main.tf
 |- providers.tf
 |- set_space_egress.sh
+|- terraform.sh
 |- variables.tf
 |- <env>.tfvars
 ```
 
 In the root module:
 - `<env>.tfvars` is where to set variable values for the given environment name
+- `terraform.sh` Helper script to setup terraform to point to the correct state file, create a service account to run the root module, and apply the root module.
 - `app.tf` defines the application resource and configuration
 - `main.tf` defines the persistent infrastructure
 - `providers.tf` lists the required providers and shell backend config
@@ -136,3 +98,7 @@ In the bootstrap module:
 - `apply.sh` Helper script to either recreate the state locally or call `terraform apply` Any arguments are passed through to the `apply` call
 - `imports.tf` import blocks to create a new local state file when new developers need to access the state file. This file is automatically generated by calling `./apply.sh` and should be checked into git on any changes
 - `users.auto.tfvars` this file defines the list of cloud.gov accounts that should have access to the terraform state bucket
+
+In the sandbox_bot module:
+- `main.tf` sets up a cloud.gov SpaceDeployer to manage the sandbox environment and outputs its credentials into the main module `secrets.auto.tfvars`
+- `run.sh` Helper script to set up a separate local state file for each sandbox name. In normal use this will only ever be called by `./terraform.sh`
diff --git a/terraform/app.tf b/terraform/app.tf
@@ -19,14 +19,19 @@ locals {
 
 resource "cloudfoundry_app" "app" {
   name       = "${local.app_name}-${var.env}"
-  space_name = module.app_space.space_name
+  space_name = var.cf_space_name
   org_name   = local.cf_org_name
 
-  path             = data.archive_file.src.output_path
-  source_code_hash = data.archive_file.src.output_base64sha256
-  buildpacks       = ["ruby_buildpack"]
-  strategy         = "blue-green"
-  routes           = [{ route = "${local.host_name}.${local.domain}" }]
+  path                       = data.archive_file.src.output_path
+  source_code_hash           = data.archive_file.src.output_base64sha256
+  buildpacks                 = ["ruby_buildpack"]
+  strategy                   = "rolling"
+  instances                  = var.web_instances
+  memory                     = var.web_memory
+  command                    = "./bin/rake cf:on_first_instance db:migrate && exec env HTTP_PORT=$PORT ./bin/thrust ./bin/rails server"
+  health_check_http_endpoint = "/up"
+  health_check_type          = "http"
+  routes                     = [{ route = "${local.host_name}.${local.domain}" }]
 
   environment = {
     no_proxy                 = "apps.internal,s3-fips.us-gov-west-1.amazonaws.com"
@@ -36,20 +41,20 @@ resource "cloudfoundry_app" "app" {
     RAILS_SERVE_STATIC_FILES = "true"
   }
 
-  processes = [
-    # {
-    #   type      = "worker"
-    #   instances = var.worker_instances
-    #   memory    = var.worker_memory
-    #   command   = "bundle exec sidekiq"
-    # },
-    {
-      type      = "web"
-      instances = var.web_instances
-      memory    = var.web_memory
-      command   = "bundle exec rake cf:on_first_instance db:migrate && exec env HTTP_PORT=$PORT ./bin/thrust ./bin/rails server"
-    }
-  ]
+  # processes = [
+  #   # {
+  #   #   type      = "worker"
+  #   #   instances = var.worker_instances
+  #   #   memory    = var.worker_memory
+  #   #   command   = "bundle exec sidekiq"
+  #   # },
+  #   {
+  #     type      = "web"
+  #     instances = var.web_instances
+  #     memory    = var.web_memory
+  #     command   = "./bin/rake cf:on_first_instance db:migrate && exec env HTTP_PORT=$PORT ./bin/thrust ./bin/rails server"
+  #   }
+  # ]
 
   service_bindings = [
     { service_instance = "egress-proxy-${var.env}-credentials" },
@@ -60,7 +65,7 @@ resource "cloudfoundry_app" "app" {
   depends_on = [
     cloudfoundry_service_instance.egress_proxy_credentials,
     # module.redis,
-    module.database,
-    module.app_space
+    module.app_space,
+    module.database
   ]
 }
diff --git a/terraform/bootstrap/apply.sh b/terraform/bootstrap/apply.sh
@@ -8,5 +8,8 @@ fi
 
 set -e
 
+# ensure we're logged in via cli
+cf spaces &> /dev/null || cf login -a api.fr.cloud.gov --sso
+
 terraform init
 terraform apply "$@"