From 496cec1ab283f18de298517c46cafbda8cc77ac8 Mon Sep 17 00:00:00 2001 From: Aidan Feldman Date: Thu, 18 Jan 2018 00:32:00 -0500 Subject: [PATCH 1/6] separate AMI building and full deployment into their own jobs --- .circleci/config.yml | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4a4bf21..b0b9589 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -50,18 +50,13 @@ jobs: paths: - ./* - deploy_env: + build_ami: docker: - image: 18fgsa/devsecops-builder:alpine steps: - attach_workspace: at: . - - add_ssh_keys - - run: - name: Disable host key checking - command: printf "\nHost *\n\tStrictHostKeyChecking no" >> ~/.ssh/config - - run: name: Bootstrap the environment command: terraform apply -input=false -auto-approve -target=aws_route53_record.db @@ -83,6 +78,19 @@ jobs: ../../packer/wordpress.json working_directory: ~/project/terraform/env + # AMI builds are slow, so keeping the environment deploy separate so it can be re-run separately if needed + deploy_env: + docker: + - image: 18fgsa/devsecops-builder:alpine + steps: + - attach_workspace: + at: . + + - add_ssh_keys + - run: + name: Disable host key checking + command: printf "\nHost *\n\tStrictHostKeyChecking no" >> ~/.ssh/config + - run: name: Deploy the full environment command: terraform apply -input=false -auto-approve @@ -95,10 +103,16 @@ workflows: jobs: - validate_packer - validate_terraform - - deploy_env: + - build_ami: filters: branches: only: master requires: - validate_packer - validate_terraform + - deploy_env: + filters: + branches: + only: master + requires: + - build_ami From 958a70b4d940fdaaae1e255756811bf533ac8a1a Mon Sep 17 00:00:00 2001 From: Aidan Feldman Date: Thu, 18 Jan 2018 00:40:56 -0500 Subject: [PATCH 2/6] combine validation jobs in CircleCI so they run faster and using the same dependencies --- .circleci/config.yml | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b0b9589..ad96c21 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,20 +1,21 @@ version: 2 jobs: - validate_packer: + install_and_validate: docker: - - image: hashicorp/packer:light + - image: 18fgsa/devsecops-builder:alpine steps: - checkout + + # this command fails if the requirements.yml is empty + # - run: + # name: Download Ansible dependencies + # command: ansible-galaxy install -p roles -r requirements.yml + # working_directory: ~/project/ansible + - run: name: Validate WordPress template command: packer validate -syntax-only packer/wordpress.json - validate_terraform: - docker: - - image: hashicorp/terraform:light - steps: - - checkout - - run: name: bootstrap - Set up Terraform command: terraform init -backend=false -input=false @@ -45,6 +46,7 @@ jobs: name: env - Validate Terraform command: terraform validate working_directory: ~/project/terraform/env + - persist_to_workspace: root: . paths: @@ -62,11 +64,6 @@ jobs: command: terraform apply -input=false -auto-approve -target=aws_route53_record.db working_directory: ~/project/terraform/env - # this command fails if the requirements.yml is empty - # - run: - # name: Download Ansible dependencies - # command: ansible-galaxy install -p ansible/roles -r ansible/requirements.yml - - run: name: Build WordPress AMI command: | @@ -101,15 +98,13 @@ workflows: validate_and_deploy: jobs: - - validate_packer - - validate_terraform + - install_and_validate - build_ami: filters: branches: only: master requires: - - validate_packer - - validate_terraform + - install_and_validate - deploy_env: filters: branches: From 5cfcee0d1b9f586c0393a19877f0e2f954549052 Mon Sep 17 00:00:00 2001 From: Aidan Feldman Date: Thu, 18 Jan 2018 00:43:04 -0500 Subject: [PATCH 3/6] validate Ansible playbook in CircleCI --- .circleci/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index ad96c21..b91f968 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,6 +11,10 @@ jobs: # name: Download Ansible dependencies # command: ansible-galaxy install -p roles -r requirements.yml # working_directory: ~/project/ansible + - run: + name: Validate Ansible playbook + command: ansible-playbook --syntax-check -i "localhost," -c local wordpress.yml + working_directory: ~/project/ansible - run: name: Validate WordPress template From a55c9e13ea50c5f9f28ffa913838d08ac4b82842 Mon Sep 17 00:00:00 2001 From: Aidan Feldman Date: Thu, 18 Jan 2018 01:29:10 -0500 Subject: [PATCH 4/6] move validation commands into a Makefile --- .circleci/config.yml | 28 +++------------------------- Makefile | 10 ++++++++++ README.md | 3 ++- 3 files changed, 15 insertions(+), 26 deletions(-) create mode 100644 Makefile diff --git a/.circleci/config.yml b/.circleci/config.yml index b91f968..07ae488 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,28 +6,10 @@ jobs: steps: - checkout - # this command fails if the requirements.yml is empty - # - run: - # name: Download Ansible dependencies - # command: ansible-galaxy install -p roles -r requirements.yml - # working_directory: ~/project/ansible - - run: - name: Validate Ansible playbook - command: ansible-playbook --syntax-check -i "localhost," -c local wordpress.yml - working_directory: ~/project/ansible - - - run: - name: Validate WordPress template - command: packer validate -syntax-only packer/wordpress.json - - run: name: bootstrap - Set up Terraform command: terraform init -backend=false -input=false working_directory: ~/project/terraform/bootstrap - - run: - name: bootstrap - Validate Terraform - command: terraform validate - working_directory: ~/project/terraform/bootstrap - run: name: mgmt - Create Terraform variables file @@ -37,19 +19,15 @@ jobs: name: mgmt - Set up Terraform command: terraform init -backend=false -input=false working_directory: ~/project/terraform/mgmt - - run: - name: mgmt - Validate Terraform - command: terraform validate - working_directory: ~/project/terraform/mgmt - run: name: env - Set up Terraform command: terraform init "-backend-config=bucket=$TF_ENV_BUCKET" -input=false working_directory: ~/project/terraform/env + - run: - name: env - Validate Terraform - command: terraform validate - working_directory: ~/project/terraform/env + name: Run validations + command: make validate - persist_to_workspace: root: . diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..aeafe57 --- /dev/null +++ b/Makefile @@ -0,0 +1,10 @@ +validate: roles + cd ansible && ansible-playbook --syntax-check -i "localhost," -c local wordpress.yml + packer validate -syntax-only packer/wordpress.json + cd terraform/bootstrap && terraform validate + cd terraform/mgmt && terraform validate + cd terraform/env && terraform validate + +roles: + # this command fails if the requirements.yml is empty + # cd ansible && ansible-galaxy install -p roles -r requirements.yml diff --git a/README.md b/README.md index 6b3a24d..81487e3 100644 --- a/README.md +++ b/README.md @@ -32,8 +32,9 @@ Currently, both the management and environment VPCs will be deployed in the same 1. Install Ansible dependencies. ```sh - ansible-galaxy install -p ansible/roles -r ansible/requirements.yml + make roles ``` + 1. Specify a region ([options](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions)). ```sh From d2b5d3447750e41f74d87a324c5a58b532ce272e Mon Sep 17 00:00:00 2001 From: Aidan Feldman Date: Thu, 18 Jan 2018 03:13:30 -0500 Subject: [PATCH 5/6] add Make target for building the AMI --- .circleci/config.yml | 9 +-------- Makefile | 13 +++++++++++++ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 07ae488..393c785 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,14 +48,7 @@ jobs: - run: name: Build WordPress AMI - command: | - packer build \ - -var db_host=$(terraform output db_host) \ - -var db_name=$(terraform output db_name) \ - -var db_user=$(terraform output db_user) \ - -var "db_pass=${TF_VAR_db_pass}" \ - ../../packer/wordpress.json - working_directory: ~/project/terraform/env + command: make ami # AMI builds are slow, so keeping the environment deploy separate so it can be re-run separately if needed deploy_env: diff --git a/Makefile b/Makefile index aeafe57..d425c42 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,16 @@ +DB_HOST = $(shell cd terraform/env && terraform output db_host) +DB_NAME = $(shell cd terraform/env && terraform output db_name) +DB_USER = $(shell cd terraform/env && terraform output db_user) + +ami: roles + cd terraform/env && \ + packer build \ + -var db_host=$(DB_HOST) \ + -var db_name=$(DB_NAME) \ + -var db_user=$(DB_USER) \ + -var "db_pass=${TF_VAR_db_pass}" \ + ../../packer/wordpress.json + validate: roles cd ansible && ansible-playbook --syntax-check -i "localhost," -c local wordpress.yml packer validate -syntax-only packer/wordpress.json From 8da5d00ec17fff1621f64fa3548dbabb776ee0c3 Mon Sep 17 00:00:00 2001 From: Aidan Feldman Date: Thu, 18 Jan 2018 03:16:51 -0500 Subject: [PATCH 6/6] add instructions on manual environment setup --- README.md | 66 +++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 50 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 81487e3..b58f406 100644 --- a/README.md +++ b/README.md @@ -77,27 +77,61 @@ Currently, both the management and environment VPCs will be deployed in the same ### Application environment 1. [Create an access key for the deployer user.](https://console.aws.amazon.com/iam/home#/users/circleci-deployer?section=security_credentials) -1. Set up CircleCI for the project. - * [Specify environment variables](https://circleci.com/docs/2.0/env-vars/#adding-environment-variables-in-the-app): - * [The required AWS configuration](https://www.terraform.io/docs/providers/aws/index.html#environment-variables) - * `TF_ENV_BUCKET` - get via `terraform output env_backend_bucket` - * `TF_VAR_db_pass` - * `TF_VAR_general_availability_endpoint` - * Generate a deployer key, and add it under the project settings. +1. Follow either the CircleCI or manual steps below. +1. Visit the site, which is the `url` output from Terraform, to complete WordPress setup. - ```sh - ssh-keygen -t rsa -b 4096 -f id_rsa_circleci -C "something@some.gov" -N "" - cat id_rsa_circleci | pbcopy +Note that if the public IP address changes after you set up the site initially, you will need to [change the site URL](https://codex.wordpress.org/Changing_The_Site_URL#Changing_the_Site_URL) in WordPress. - # save as private key in CircleCI +#### CircleCI - rm id_rsa_circleci* - ``` +To set up CircleCI for the project: - * The build will bootstrap the environment. -1. Visit the site, which is the `url` output from Terraform at the end of the CircleCI run, to complete WordPress setup. +* [Specify environment variables](https://circleci.com/docs/2.0/env-vars/#adding-environment-variables-in-the-app): + * [The required AWS configuration](https://www.terraform.io/docs/providers/aws/index.html#environment-variables) + * `TF_ENV_BUCKET` - get via `terraform output env_backend_bucket` + * `TF_VAR_db_pass` + * `TF_VAR_general_availability_endpoint` +* Generate a deployer key, and add it under the project settings. -Note that if the public IP address changes after you set up the site initially, you will need to [change the site URL](https://codex.wordpress.org/Changing_The_Site_URL#Changing_the_Site_URL) in WordPress. + ```sh + ssh-keygen -t rsa -b 4096 -f id_rsa_circleci -C "something@some.gov" -N "" + cat id_rsa_circleci | pbcopy + + # save as private key in CircleCI + + rm id_rsa_circleci* + ``` + +* The build will bootstrap the environment. + +#### Manual + +To set up / deploy Terraform's environment configuration on your local machine: + +1. Set up Terraform. + + ```sh + cd ../env + terraform init "-backend-config=bucket=$(cd ../mgmt && terraform output env_backend_bucket)" + ``` + +1. Bootstrap the environment. + + ```sh + terraform apply -target=aws_route53_record.db + ``` + +1. Create the AMI, making sure to pass your database password. + + ```sh + TF_VAR_db_pass= make ami + ``` + +1. Deploy the rest of the environment. + + ```sh + terraform apply + ``` #### Troubleshooting