diff --git a/CHANGELOG.md b/CHANGELOG.md index 54b59d048..3292441e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,11 +7,19 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## Unreleased -## 4.14.0 - 2020-04094 + +## 4.15.0 - 2020-04-16 + +- Added: support custom docker machine distribution (#216) … +- Bugfix: disabled cache (#212) +- Bugfix: failing curl (#217) … +- Change: Drop supported to manage ec2 keys (#192) + + +## 4.14.0 - 2020-04-04 - Add: Allow traffic from a list of security group IDs (#207) by @fliphess - Bugfix: Fix missing policy for existing cache (#208, #206) - ## 4.13.0 - 2020-03-26 - Add: variables `cache_lifecycle_prefix` and `cache_lifecycle_clear` to increase flexibility of the cache usages. - Add: Parametrize the AWS ARN for policies (#203) @ericamador @@ -296,8 +304,9 @@ Module is available as Terraform 0.11 module, pin module to version 3.x. Please - Update default AMI's to The latest Amazon Linux AMI 2017.09.1 - released on 2018-01-17. - Minor updates in the example -[Unreleased]: https://github.com/npalm/terraform-aws-gitlab-runner/compare/4.14.0...HEAD -[4.13.0]: https://github.com/npalm/terraform-aws-gitlab-runner/compare/4.14.0...4.13.0 +[Unreleased]: https://github.com/npalm/terraform-aws-gitlab-runner/compare/4.15.0...HEAD +[4.15.0]: https://github.com/npalm/terraform-aws-gitlab-runner/compare/4.15.0...4.14.0 +[4.14.0]: https://github.com/npalm/terraform-aws-gitlab-runner/compare/4.14.0...4.13.0 [4.13.0]: https://github.com/npalm/terraform-aws-gitlab-runner/compare/4.13.0...4.12.0 [4.12.0]: https://github.com/npalm/terraform-aws-gitlab-runner/compare/4.12.0...4.11.1 [4.11.1]: https://github.com/npalm/terraform-aws-gitlab-runner/compare/4.11.1...4.11.0 diff --git a/README.md b/README.md index 3f62eb06a..6e3201146 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,10 @@ # Terraform module for GitLab auto scaling runners on AWS spot instances +> "Added support to download docker machine from a different location, e.g. " + +> "Managed ec2 key support dropped": The module will not longer manage an SSH key pair. The module offers two way to access instances. First via the AWS session manager and second by providing an AWS key pair as parameter. + > "Type changes": The types of variable `runners_volumes_tmpfs`, and `runners_services_volumes_tmpfs` are changed to support the Terraform 12 `templatefile` function. Check the [default example](examples/runner-pre-registered/main.tf) for an usages example. ## Terraform versions @@ -257,11 +261,12 @@ terraform destroy | cache\_expiration\_days | Number of days before cache objects expires. | `number` | `1` | no | | cache\_shared | Enables cache sharing between runners, false by default. | `bool` | `false` | no | | cloudwatch\_logging\_retention\_in\_days | Retention for cloudwatch logs. Defaults to unlimited | `number` | `0` | no | -| docker\_machine\_instance\_type | Instance type used for the instances hosting docker-machine. | `string` | `"m5a.large"` | no | +| docker\_machine\_download\_url | Full url pointing to a linux x64 distribution of docker machine. Once set `docker_machine_version` will be ingored. For example the GitLab version, https://gitlab-docker-machine-downloads.s3.amazonaws.com/v0.16.2-gitlab.2/docker-machine. | `string` | `""` | no | +| docker\_machine\_instance\_type | Instance type used for the instances hosting docker-machine. | `string` | `"m5.large"` | no | | docker\_machine\_options | List of additional options for the docker machine config. Each element of this list must be a key=value pair. E.g. '["amazonec2-zone=a"]' | `list(string)` | `[]` | no | | docker\_machine\_role\_json | Docker machine runner instance override policy, expected to be in JSON format. | `string` | `""` | no | | docker\_machine\_spot\_price\_bid | Spot price bid. | `string` | `"0.06"` | no | -| docker\_machine\_version | Version of docker-machine. | `string` | `"0.16.2"` | no | +| docker\_machine\_version | Version of docker-machine. The version will be ingored once `docker_machine_download_url` is set. | `string` | `"0.16.2"` | no | | enable\_cloudwatch\_logging | Boolean used to enable or disable the CloudWatch logging. | `bool` | `true` | no | | enable\_eip | Enable the assignment of an EIP to the gitlab runner instance | `bool` | `false` | no | | enable\_forced\_updates | Enable automatic redeployment of the Runner ASG when the Launch Configs change. | `bool` | `false` | no | @@ -274,7 +279,8 @@ terraform destroy | enable\_schedule | Flag used to enable/disable auto scaling group schedule for the runner instance. | `bool` | `false` | no | | environment | A name that identifies the environment, used as prefix and for tagging. | `string` | n/a | yes | | gitlab\_runner\_registration\_config | Configuration used to register the runner. See the README for an example, or reference the examples in the examples directory of this repo. | `map(string)` | 
{
  "access_level": "",
  "description": "",
  "locked_to_project": "",
  "maximum_timeout": "",
  "registration_token": "",
  "run_untagged": "",
  "tag_list": ""
}
| no | -| gitlab\_runner\_ssh\_cidr\_blocks | List of CIDR blocks to allow SSH Access to the gitlab runner instance. | `list(string)` | 
[
  "0.0.0.0/0"
]
| no | +| gitlab\_runner\_security\_group\_ids | A list of security group ids that are allowed to access the gitlab runner agent | `list(string)` | `[]` | no | +| gitlab\_runner\_ssh\_cidr\_blocks | List of CIDR blocks to allow SSH Access to the gitlab runner instance. | `list(string)` | `[]` | no | | gitlab\_runner\_version | Version of the GitLab runner. | `string` | `"12.8.0"` | no | | instance\_role\_json | Default runner instance override policy, expected to be in JSON format. | `string` | `""` | no | | instance\_type | Instance type used for the GitLab runner. | `string` | `"t3.micro"` | no | @@ -323,8 +329,7 @@ terraform destroy | runners\_volumes\_tmpfs | n/a | 
list(object({
    volume  = string
    options = string
  }))
| `[]` | no | | schedule\_config | Map containing the configuration of the ASG scale-in and scale-up for the runner instance. Will only be used if enable\_schedule is set to true. | `map` | 
{
  "scale_in_count": 0,
  "scale_in_recurrence": "0 18 * * 1-5",
  "scale_out_count": 1,
  "scale_out_recurrence": "0 8 * * 1-5"
}
| no | | secure\_parameter\_store\_runner\_token\_key | The key name used store the Gitlab runner token in Secure Parameter Store | `string` | `"runner-token"` | no | -| ssh\_key\_pair | Set this to use existing AWS key pair | `string` | `""` | no | -| ssh\_public\_key | Public SSH key used for the GitLab runner EC2 instance. | `string` | `""` | no | +| ssh\_key\_pair | Set this to use existing AWS key pair | `string` | n/a | yes | | subnet\_id\_runners | List of subnets used for hosting the gitlab-runners. | `string` | n/a | yes | | subnet\_ids\_gitlab\_runner | Subnet used for hosting the GitLab runner. | `list(string)` | n/a | yes | | tags | Map of tags that will be added to created resources. By default resources will be tagged with name and environment. | `map(string)` | `{}` | no | diff --git a/_docs/README.md b/_docs/README.md index 1de9d494c..9a12c9666 100644 --- a/_docs/README.md +++ b/_docs/README.md @@ -3,6 +3,10 @@ # Terraform module for GitLab auto scaling runners on AWS spot instances +> "Added support to download docker machine from a different location, e.g. https://gitlab.com/gitlab-org/ci-cd/docker-machine" + +> "Managed ec2 key support dropped": The module will not longer manage an SSH key pair. The module offers two way to access instances. First via the AWS session manager and second by providing an AWS key pair as parameter. + > "Type changes": The types of variable `runners_volumes_tmpfs`, and `runners_services_volumes_tmpfs` are changed to support the Terraform 12 `templatefile` function. Check the [default example](examples/runner-pre-registered/main.tf) for an usages example. ## Terraform versions diff --git a/_docs/TF_MODULE.md b/_docs/TF_MODULE.md index bc69485c2..ba58ad440 100644 --- a/_docs/TF_MODULE.md +++ b/_docs/TF_MODULE.md @@ -23,11 +23,12 @@ | cache\_expiration\_days | Number of days before cache objects expires. | `number` | `1` | no | | cache\_shared | Enables cache sharing between runners, false by default. | `bool` | `false` | no | | cloudwatch\_logging\_retention\_in\_days | Retention for cloudwatch logs. Defaults to unlimited | `number` | `0` | no | -| docker\_machine\_instance\_type | Instance type used for the instances hosting docker-machine. | `string` | `"m5a.large"` | no | +| docker\_machine\_download\_url | Full url pointing to a linux x64 distribution of docker machine. Once set `docker_machine_version` will be ingored. For example the GitLab version, https://gitlab-docker-machine-downloads.s3.amazonaws.com/v0.16.2-gitlab.2/docker-machine. | `string` | `""` | no | +| docker\_machine\_instance\_type | Instance type used for the instances hosting docker-machine. | `string` | `"m5.large"` | no | | docker\_machine\_options | List of additional options for the docker machine config. Each element of this list must be a key=value pair. E.g. '["amazonec2-zone=a"]' | `list(string)` | `[]` | no | | docker\_machine\_role\_json | Docker machine runner instance override policy, expected to be in JSON format. | `string` | `""` | no | | docker\_machine\_spot\_price\_bid | Spot price bid. | `string` | `"0.06"` | no | -| docker\_machine\_version | Version of docker-machine. | `string` | `"0.16.2"` | no | +| docker\_machine\_version | Version of docker-machine. The version will be ingored once `docker_machine_download_url` is set. | `string` | `"0.16.2"` | no | | enable\_cloudwatch\_logging | Boolean used to enable or disable the CloudWatch logging. | `bool` | `true` | no | | enable\_eip | Enable the assignment of an EIP to the gitlab runner instance | `bool` | `false` | no | | enable\_forced\_updates | Enable automatic redeployment of the Runner ASG when the Launch Configs change. | `bool` | `false` | no | @@ -40,8 +41,8 @@ | enable\_schedule | Flag used to enable/disable auto scaling group schedule for the runner instance. | `bool` | `false` | no | | environment | A name that identifies the environment, used as prefix and for tagging. | `string` | n/a | yes | | gitlab\_runner\_registration\_config | Configuration used to register the runner. See the README for an example, or reference the examples in the examples directory of this repo. | `map(string)` | 
{
  "access_level": "",
  "description": "",
  "locked_to_project": "",
  "maximum_timeout": "",
  "registration_token": "",
  "run_untagged": "",
  "tag_list": ""
}
| no | -| gitlab\_runner\_ssh\_cidr\_blocks | List of CIDR blocks to allow SSH Access to the gitlab runner instance. | `list(string)` | 
[
]
| no | -| gitlab\_runner\_security\_group\_ids | List of security group IDs to allow Access to the gitlab runner instances. | `list(string)` | 
[
]
` | no | +| gitlab\_runner\_security\_group\_ids | A list of security group ids that are allowed to access the gitlab runner agent | `list(string)` | `[]` | no | +| gitlab\_runner\_ssh\_cidr\_blocks | List of CIDR blocks to allow SSH Access to the gitlab runner instance. | `list(string)` | `[]` | no | | gitlab\_runner\_version | Version of the GitLab runner. | `string` | `"12.8.0"` | no | | instance\_role\_json | Default runner instance override policy, expected to be in JSON format. | `string` | `""` | no | | instance\_type | Instance type used for the GitLab runner. | `string` | `"t3.micro"` | no | @@ -90,8 +91,7 @@ | runners\_volumes\_tmpfs | n/a | 
list(object({
    volume  = string
    options = string
  }))
| `[]` | no | | schedule\_config | Map containing the configuration of the ASG scale-in and scale-up for the runner instance. Will only be used if enable\_schedule is set to true. | `map` | 
{
  "scale_in_count": 0,
  "scale_in_recurrence": "0 18 * * 1-5",
  "scale_out_count": 1,
  "scale_out_recurrence": "0 8 * * 1-5"
}
| no | | secure\_parameter\_store\_runner\_token\_key | The key name used store the Gitlab runner token in Secure Parameter Store | `string` | `"runner-token"` | no | -| ssh\_key\_pair | Set this to use existing AWS key pair | `string` | `""` | no | -| ssh\_public\_key | Public SSH key used for the GitLab runner EC2 instance. | `string` | `""` | no | +| ssh\_key\_pair | Set this to use existing AWS key pair | `string` | n/a | yes | | subnet\_id\_runners | List of subnets used for hosting the gitlab-runners. | `string` | n/a | yes | | subnet\_ids\_gitlab\_runner | Subnet used for hosting the GitLab runner. | `list(string)` | n/a | yes | | tags | Map of tags that will be added to created resources. By default resources will be tagged with name and environment. | `map(string)` | `{}` | no | diff --git a/ci/bin/terraform.sh b/ci/bin/terraform.sh index 5296ce347..aa552f2b7 100755 --- a/ci/bin/terraform.sh +++ b/ci/bin/terraform.sh @@ -3,17 +3,16 @@ TARGET_DIR=/opt PATH=${PATH}:${TARGET_DIR} -TERRAFORM_VERSION=${1:-"0.12.8"} +TERRAFORM_VERSION=${1:-"0.12.24"} OS=${2:-"linux"} TERRAFORM_URL="https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_${OS}_amd64.zip" installTerraform() { echo "Downloading terraform: ${TERRAFORM_URL}" - curl '-#' -fL -o ${TARGET_DIR}/terraform.zip ${TERRAFORM_URL} && \ - unzip -q -d ${TARGET_DIR}/ ${TARGET_DIR}/terraform.zip && \ - - terraform --version + curl '-#' -fL -o ${TARGET_DIR}/terraform.zip ${TERRAFORM_URL} && + unzip -q -d ${TARGET_DIR}/ ${TARGET_DIR}/terraform.zip && + terraform --version } verifyModulesAndPlugins() { @@ -23,7 +22,7 @@ verifyModulesAndPlugins() { formatCheck() { RESULT=$(terraform fmt -recursive -write=false) - if [[ ! -z ${RESULT} ]] ; then + if [[ ! -z ${RESULT} ]]; then echo The following files are formatted incorrectly: $RESULT exit 1 fi @@ -33,4 +32,3 @@ validate() { echo "Validating and checking format of terraform code in $PWD" terraform validate } - diff --git a/examples/runner-default/.terraform-version b/examples/runner-default/.terraform-version index f0481db67..421e954ab 100644 --- a/examples/runner-default/.terraform-version +++ b/examples/runner-default/.terraform-version @@ -1 +1 @@ -0.12.23 +0.12.24 diff --git a/examples/runner-default/README.md b/examples/runner-default/README.md index bec337bab..1145185cc 100644 --- a/examples/runner-default/README.md +++ b/examples/runner-default/README.md @@ -8,6 +8,7 @@ This examples shows: - No SSH keys, you can log into the instance via SSM (Session Manager). - Registration via GitLab token. - Auto scaling using `docker+machine` executor. + - Addtional security groups that are allowed access to the runner agent ![runners-default](https://github.com/npalm/assets/raw/master/images/terraform-aws-gitlab-runner/runner-default.png) @@ -19,7 +20,7 @@ The Terraform version is managed using [tfenv](https://github.com/Zordrak/tfenv) | Name | Version | |------|---------| -| aws | 2.41 | +| aws | 2.56 | | null | 2.1.2 | ## Inputs diff --git a/examples/runner-default/_docs/TF_MODULE.md b/examples/runner-default/_docs/TF_MODULE.md index 4da3c589f..45b6ba63b 100644 --- a/examples/runner-default/_docs/TF_MODULE.md +++ b/examples/runner-default/_docs/TF_MODULE.md @@ -2,7 +2,7 @@ | Name | Version | |------|---------| -| aws | 2.41 | +| aws | 2.56 | | null | 2.1.2 | ## Inputs diff --git a/examples/runner-default/main.tf b/examples/runner-default/main.tf index 218379457..1f13aa7e4 100644 --- a/examples/runner-default/main.tf +++ b/examples/runner-default/main.tf @@ -3,12 +3,13 @@ data "aws_availability_zones" "available" { } data "aws_security_group" "default" { - name = "default" + name = "default" + vpc_id = module.vpc.vpc_id } module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "2.21" + version = "2.33" name = "vpc-${var.environment}" cidr = "10.0.0.0/16" @@ -43,6 +44,7 @@ module "runner" { gitlab_runner_security_group_ids = [data.aws_security_group.default.id] + docker_machine_download_url = "https://gitlab-docker-machine-downloads.s3.amazonaws.com/v0.16.2-gitlab.2/docker-machine" docker_machine_spot_price_bid = "0.06" gitlab_runner_registration_config = { diff --git a/examples/runner-default/providers.tf b/examples/runner-default/providers.tf index 795704a0d..204929b5c 100644 --- a/examples/runner-default/providers.tf +++ b/examples/runner-default/providers.tf @@ -1,6 +1,6 @@ provider "aws" { region = var.aws_region - version = "2.41" + version = "2.56" } provider "local" { diff --git a/examples/runner-docker/.terraform-version b/examples/runner-docker/.terraform-version index f0481db67..421e954ab 100644 --- a/examples/runner-docker/.terraform-version +++ b/examples/runner-docker/.terraform-version @@ -1 +1 @@ -0.12.23 +0.12.24 diff --git a/examples/runner-docker/README.md b/examples/runner-docker/README.md index 6a5a65414..ae2c0491f 100644 --- a/examples/runner-docker/README.md +++ b/examples/runner-docker/README.md @@ -25,10 +25,7 @@ The terraform version is managed using [tfenv](https://github.com/Zordrak/tfenv) | Name | Version | |------|---------| -| aws | 2.41 | -| local | 1.4 | -| null | 2.1.2 | -| tls | 2.1.1 | +| aws | 2.56 | ## Inputs @@ -37,8 +34,6 @@ The terraform version is managed using [tfenv](https://github.com/Zordrak/tfenv) | aws\_region | AWS region. | `string` | `"eu-west-1"` | no | | environment | A name that identifies the environment, will used as prefix and for tagging. | `string` | `"runners-docker"` | no | | gitlab\_url | URL of the gitlab instance to connect to. | `string` | `"https://gitlab.com"` | no | -| private\_ssh\_key\_filename | n/a | `string` | `"generated/id_rsa"` | no | -| public\_ssh\_key\_filename | n/a | `string` | `"generated/id_rsa.pub"` | no | | registration\_token | n/a | `any` | n/a | yes | | runner\_name | Name of the runner, will be used in the runner config.toml | `string` | `"docker"` | no | diff --git a/examples/runner-docker/_docs/TF_MODULE.md b/examples/runner-docker/_docs/TF_MODULE.md index 685653576..633b0b994 100644 --- a/examples/runner-docker/_docs/TF_MODULE.md +++ b/examples/runner-docker/_docs/TF_MODULE.md @@ -2,10 +2,7 @@ | Name | Version | |------|---------| -| aws | 2.41 | -| local | 1.4 | -| null | 2.1.2 | -| tls | 2.1.1 | +| aws | 2.56 | ## Inputs @@ -14,8 +11,6 @@ | aws\_region | AWS region. | `string` | `"eu-west-1"` | no | | environment | A name that identifies the environment, will used as prefix and for tagging. | `string` | `"runners-docker"` | no | | gitlab\_url | URL of the gitlab instance to connect to. | `string` | `"https://gitlab.com"` | no | -| private\_ssh\_key\_filename | n/a | `string` | `"generated/id_rsa"` | no | -| public\_ssh\_key\_filename | n/a | `string` | `"generated/id_rsa.pub"` | no | | registration\_token | n/a | `any` | n/a | yes | | runner\_name | Name of the runner, will be used in the runner config.toml | `string` | `"docker"` | no | diff --git a/examples/runner-docker/main.tf b/examples/runner-docker/main.tf index ae891ac09..2c5169dba 100644 --- a/examples/runner-docker/main.tf +++ b/examples/runner-docker/main.tf @@ -4,7 +4,7 @@ data "aws_availability_zones" "available" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "2.21" + version = "2.33" name = "vpc-${var.environment}" cidr = "10.1.0.0/16" @@ -19,13 +19,20 @@ module "vpc" { } } +module "key_pair" { + source = "../../modules/key-pair" + + environment = var.environment + name = var.runner_name +} + module "runner" { source = "../../" aws_region = var.aws_region environment = var.environment - ssh_public_key = local_file.public_ssh_key.content + ssh_key_pair = module.key_pair.key_pair.key_name runners_use_private_address = false enable_eip = true diff --git a/examples/runner-docker/providers.tf b/examples/runner-docker/providers.tf index 795704a0d..204929b5c 100644 --- a/examples/runner-docker/providers.tf +++ b/examples/runner-docker/providers.tf @@ -1,6 +1,6 @@ provider "aws" { region = var.aws_region - version = "2.41" + version = "2.56" } provider "local" { diff --git a/examples/runner-docker/variables.tf b/examples/runner-docker/variables.tf index 24afa8076..c69522933 100644 --- a/examples/runner-docker/variables.tf +++ b/examples/runner-docker/variables.tf @@ -10,13 +10,13 @@ variable "environment" { type = string } -variable "public_ssh_key_filename" { - default = "generated/id_rsa.pub" -} +# variable "public_ssh_key_filename" { +# default = "generated/id_rsa.pub" +# } -variable "private_ssh_key_filename" { - default = "generated/id_rsa" -} +# variable "private_ssh_key_filename" { +# default = "generated/id_rsa" +# } variable "runner_name" { description = "Name of the runner, will be used in the runner config.toml" diff --git a/examples/runner-pre-registered/.terraform-version b/examples/runner-pre-registered/.terraform-version index f0481db67..421e954ab 100644 --- a/examples/runner-pre-registered/.terraform-version +++ b/examples/runner-pre-registered/.terraform-version @@ -1 +1 @@ -0.12.23 +0.12.24 diff --git a/examples/runner-pre-registered/README.md b/examples/runner-pre-registered/README.md index 3fc1887d3..b4f78bde5 100644 --- a/examples/runner-pre-registered/README.md +++ b/examples/runner-pre-registered/README.md @@ -10,10 +10,7 @@ The terraform version is managed using [tfenv](https://github.com/Zordrak/tfenv) | Name | Version | |------|---------| -| aws | 2.41 | -| local | 1.4 | -| null | 2.1.2 | -| tls | 2.1.1 | +| aws | 2.52 | ## Inputs diff --git a/examples/runner-pre-registered/_docs/TF_MODULE.md b/examples/runner-pre-registered/_docs/TF_MODULE.md index 78d1fdc52..6f66c49f7 100644 --- a/examples/runner-pre-registered/_docs/TF_MODULE.md +++ b/examples/runner-pre-registered/_docs/TF_MODULE.md @@ -2,10 +2,7 @@ | Name | Version | |------|---------| -| aws | 2.41 | -| local | 1.4 | -| null | 2.1.2 | -| tls | 2.1.1 | +| aws | 2.52 | ## Inputs diff --git a/examples/runner-pre-registered/key.tf b/examples/runner-pre-registered/key.tf deleted file mode 100644 index 2dcc51f3f..000000000 --- a/examples/runner-pre-registered/key.tf +++ /dev/null @@ -1,27 +0,0 @@ -resource "tls_private_key" "ssh" { - algorithm = "RSA" -} - -resource "local_file" "public_ssh_key" { - depends_on = [tls_private_key.ssh] - - content = tls_private_key.ssh.public_key_openssh - filename = var.public_ssh_key_filename -} - -resource "local_file" "private_ssh_key" { - depends_on = [tls_private_key.ssh] - - content = tls_private_key.ssh.private_key_pem - filename = var.private_ssh_key_filename -} - -resource "null_resource" "file_permission" { - depends_on = [local_file.private_ssh_key] - - provisioner "local-exec" { - command = format("chmod 600 %s", var.private_ssh_key_filename) - interpreter = ["/bin/bash", "-c"] - } -} - diff --git a/examples/runner-pre-registered/main.tf b/examples/runner-pre-registered/main.tf index b5d5f25d5..6506d6a92 100644 --- a/examples/runner-pre-registered/main.tf +++ b/examples/runner-pre-registered/main.tf @@ -23,13 +23,20 @@ module "vpc" { } } +module "key_pair" { + source = "../../modules/key-pair" + + environment = var.environment + name = var.runner_name +} + module "runner" { source = "../../" aws_region = var.aws_region environment = var.environment - ssh_public_key = local_file.public_ssh_key.content + ssh_key_pair = module.key_pair.key_pair.key_name vpc_id = module.vpc.vpc_id subnet_ids_gitlab_runner = module.vpc.private_subnets diff --git a/examples/runner-pre-registered/providers.tf b/examples/runner-pre-registered/providers.tf index 795704a0d..166b159bb 100644 --- a/examples/runner-pre-registered/providers.tf +++ b/examples/runner-pre-registered/providers.tf @@ -1,6 +1,6 @@ provider "aws" { region = var.aws_region - version = "2.41" + version = "2.52" } provider "local" { diff --git a/examples/runner-public/.terraform-version b/examples/runner-public/.terraform-version index f0481db67..421e954ab 100644 --- a/examples/runner-public/.terraform-version +++ b/examples/runner-public/.terraform-version @@ -1 +1 @@ -0.12.23 +0.12.24 diff --git a/examples/runner-public/README.md b/examples/runner-public/README.md index 975c0553f..30980a71c 100644 --- a/examples/runner-public/README.md +++ b/examples/runner-public/README.md @@ -21,7 +21,7 @@ The Terraform version is managed using [tfenv](https://github.com/Zordrak/tfenv) | Name | Version | |------|---------| -| aws | 2.41 | +| aws | 2.56 | | local | 1.4 | | null | 2.1.2 | | tls | 2.1.1 | diff --git a/examples/runner-public/_docs/TF_MODULE.md b/examples/runner-public/_docs/TF_MODULE.md index d1ff473bb..81c6dcb1e 100644 --- a/examples/runner-public/_docs/TF_MODULE.md +++ b/examples/runner-public/_docs/TF_MODULE.md @@ -2,7 +2,7 @@ | Name | Version | |------|---------| -| aws | 2.41 | +| aws | 2.56 | | local | 1.4 | | null | 2.1.2 | | tls | 2.1.1 | diff --git a/examples/runner-public/main.tf b/examples/runner-public/main.tf index 3f3cfa559..0241d5ab0 100644 --- a/examples/runner-public/main.tf +++ b/examples/runner-public/main.tf @@ -4,7 +4,7 @@ data "aws_availability_zones" "available" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "2.21" + version = "2.33" name = "vpc-${var.environment}" cidr = "10.1.0.0/16" @@ -20,17 +20,24 @@ module "vpc" { } module "cache" { - source = "../../cache" + source = "../../modules/cache" environment = "${var.environment}" } +module "key_pair" { + source = "../../modules/key-pair" + + environment = var.environment + name = var.runner_name +} + module "runner" { source = "../../" aws_region = var.aws_region environment = var.environment - ssh_public_key = local_file.public_ssh_key.content + ssh_key_pair = module.key_pair.key_pair.key_name runners_use_private_address = false vpc_id = module.vpc.vpc_id @@ -77,7 +84,7 @@ module "runner2" { aws_region = var.aws_region environment = "${var.environment}-2" - ssh_public_key = local_file.public_ssh_key.content + ssh_key_pair = module.key_pair.key_pair.key_name runners_use_private_address = false vpc_id = module.vpc.vpc_id diff --git a/examples/runner-public/providers.tf b/examples/runner-public/providers.tf index 795704a0d..204929b5c 100644 --- a/examples/runner-public/providers.tf +++ b/examples/runner-public/providers.tf @@ -1,6 +1,6 @@ provider "aws" { region = var.aws_region - version = "2.41" + version = "2.56" } provider "local" { diff --git a/main.tf b/main.tf index 0d6b0876d..4cbb4fbc4 100644 --- a/main.tf +++ b/main.tf @@ -1,11 +1,5 @@ data "aws_caller_identity" "current" {} -resource "aws_key_pair" "key" { - count = var.ssh_key_pair == "" && var.ssh_public_key != "" ? 1 : 0 - key_name = "${var.environment}-gitlab-runner" - public_key = var.ssh_public_key -} - # Parameter value is managed by the user-data script of the gitlab runner instance resource "aws_ssm_parameter" "runner_registration_token" { name = local.secure_parameter_store_runner_token_key @@ -52,6 +46,7 @@ locals { { gitlab_runner_version = var.gitlab_runner_version docker_machine_version = var.docker_machine_version + docker_machine_download_url = var.docker_machine_download_url runners_config = local.template_runner_config runners_executor = var.runners_executor pre_install = var.userdata_pre_install @@ -206,17 +201,10 @@ data "aws_ami" "runner" { owners = var.ami_owners } -locals { - # Key magic, if public key is provided usthe public key, if key pair is proviced use key pair. Otherwise null - is_ssh_public_key = var.ssh_key_pair == "" && var.ssh_public_key != "" ? aws_key_pair.key[0].key_name : "" - is_key_pair_name = local.is_ssh_public_key != "" ? local.is_ssh_public_key : var.ssh_key_pair - key_pair_name = local.is_key_pair_name != "" ? local.is_key_pair_name : null -} - resource "aws_launch_configuration" "gitlab_runner_instance" { name_prefix = var.runners_name security_groups = [aws_security_group.runner.id] - key_name = local.key_pair_name + key_name = var.ssh_key_pair image_id = data.aws_ami.runner.id user_data = local.template_user_data instance_type = var.instance_type @@ -245,12 +233,12 @@ resource "aws_launch_configuration" "gitlab_runner_instance" { ### Create cache bucket ################################################################################ locals { - bucket_name = var.cache_bucket["create"] ? module.cache.bucket : var.cache_bucket["bucket"] - bucket_policy = var.cache_bucket["create"] ? module.cache.policy_arn : var.cache_bucket["policy"] + bucket_name = var.cache_bucket["create"] ? module.cache.bucket : lookup(var.cache_bucket, "bucket", "") + bucket_policy = var.cache_bucket["create"] ? module.cache.policy_arn : lookup(var.cache_bucket, "policy", "") } module "cache" { - source = "./cache" + source = "./modules/cache" environment = var.environment tags = local.tags @@ -324,7 +312,7 @@ resource "aws_iam_role_policy_attachment" "instance_session_manager_aws_managed" ### Policy for the docker machine instance to access cache ################################################################################ resource "aws_iam_role_policy_attachment" "docker_machine_cache_instance" { - count = var.cache_bucket["create"] || var.cache_bucket["policy"] != "" ? 1 : 0 + count = var.cache_bucket["create"] || lookup(var.cache_bucket, "policy", "") != "" ? 1 : 0 role = aws_iam_role.instance.name policy_arn = local.bucket_policy } diff --git a/cache/README.md b/modules/cache/README.md similarity index 100% rename from cache/README.md rename to modules/cache/README.md diff --git a/cache/_docs/README.md b/modules/cache/_docs/README.md similarity index 100% rename from cache/_docs/README.md rename to modules/cache/_docs/README.md diff --git a/cache/_docs/TF_MODULE.md b/modules/cache/_docs/TF_MODULE.md similarity index 100% rename from cache/_docs/TF_MODULE.md rename to modules/cache/_docs/TF_MODULE.md diff --git a/cache/main.tf b/modules/cache/main.tf similarity index 100% rename from cache/main.tf rename to modules/cache/main.tf diff --git a/cache/outputs.tf b/modules/cache/outputs.tf similarity index 100% rename from cache/outputs.tf rename to modules/cache/outputs.tf diff --git a/cache/policies/cache.json b/modules/cache/policies/cache.json similarity index 100% rename from cache/policies/cache.json rename to modules/cache/policies/cache.json diff --git a/cache/variables.tf b/modules/cache/variables.tf similarity index 100% rename from cache/variables.tf rename to modules/cache/variables.tf diff --git a/cache/versions.tf b/modules/cache/versions.tf similarity index 100% rename from cache/versions.tf rename to modules/cache/versions.tf diff --git a/modules/key-pair/README.md b/modules/key-pair/README.md new file mode 100644 index 000000000..aa3a4e695 --- /dev/null +++ b/modules/key-pair/README.md @@ -0,0 +1,35 @@ +# Module to generate an EC2 key pair. + +A module to generate ec2 key pairs for example modules. + +## Warning + +The should only used for examples + +> > Important Security Notice The private key generated by this resource will be stored unencrypted in your Terraform state file. Use of this resource for production deployments is not recommended. Instead, generate a private key file outside of Terraform and distribute it securely to the system where Terraform will be run. + +## Providers + +| Name | Version | +|------|---------| +| aws | >= 2.46 | +| local | >= 1.4 | +| null | >= 2 | +| tls | >= 2 | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:-----:| +| environment | Name of the environment (aka namespace) to ensure resources are unique. | `string` | n/a | yes | +| name | Name of the key, will be prefixed by the environment name. | `string` | n/a | yes | +| private\_ssh\_key\_filename | Filename (full path) for the private key. | `string` | `"./generated/id_rsa"` | no | +| public\_ssh\_key\_filename | Filename (full path) for the public key. | `string` | `"./generated/id_rsa.pub"` | no | +| rsa\_bits | n/a | `string` | `4048` | no | +| tags | Map of tags that will be added to created resources. By default resources will be tagged with name and environment. | `map(string)` | `{}` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| key\_pair | Generated key pair. | diff --git a/modules/key-pair/_docs/README.md b/modules/key-pair/_docs/README.md new file mode 100644 index 000000000..80663b2cf --- /dev/null +++ b/modules/key-pair/_docs/README.md @@ -0,0 +1,9 @@ +# Module to generate an EC2 key pair. + +A module to generate ec2 key pairs for example modules. + +## Warning +The should only used for examples + +>> Important Security Notice The private key generated by this resource will be stored unencrypted in your Terraform state file. Use of this resource for production deployments is not recommended. Instead, generate a private key file outside of Terraform and distribute it securely to the system where Terraform will be run. + diff --git a/modules/key-pair/_docs/TF_MODULE.md b/modules/key-pair/_docs/TF_MODULE.md new file mode 100644 index 000000000..f55580e5e --- /dev/null +++ b/modules/key-pair/_docs/TF_MODULE.md @@ -0,0 +1,26 @@ +## Providers + +| Name | Version | +|------|---------| +| aws | >= 2.46 | +| local | >= 1.4 | +| null | >= 2 | +| tls | >= 2 | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:-----:| +| environment | Name of the environment (aka namespace) to ensure resources are unique. | `string` | n/a | yes | +| name | Name of the key, will be prefixed by the environment name. | `string` | n/a | yes | +| private\_ssh\_key\_filename | Filename (full path) for the private key. | `string` | `"./generated/id_rsa"` | no | +| public\_ssh\_key\_filename | Filename (full path) for the public key. | `string` | `"./generated/id_rsa.pub"` | no | +| rsa\_bits | n/a | `string` | `4048` | no | +| tags | Map of tags that will be added to created resources. By default resources will be tagged with name and environment. | `map(string)` | `{}` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| key\_pair | Generated key pair. | + diff --git a/examples/runner-docker/key.tf b/modules/key-pair/main.tf similarity index 61% rename from examples/runner-docker/key.tf rename to modules/key-pair/main.tf index 2dcc51f3f..d9aed1fc0 100644 --- a/examples/runner-docker/key.tf +++ b/modules/key-pair/main.tf @@ -1,5 +1,6 @@ resource "tls_private_key" "ssh" { algorithm = "RSA" + rsa_bits = var.rsa_bits } resource "local_file" "public_ssh_key" { @@ -25,3 +26,22 @@ resource "null_resource" "file_permission" { } } +resource "aws_key_pair" "key" { + key_name = var.name != null ? "${var.environment}-${var.name}" : "${var.environment}" + public_key = local_file.public_ssh_key.content + tags = local.tags +} + +locals { + tags = merge( + { + "Name" = format("%s", var.environment) + }, + { + "Environment" = format("%s", var.environment) + }, + var.tags, + ) +} + + diff --git a/modules/key-pair/outputs.tf b/modules/key-pair/outputs.tf new file mode 100644 index 000000000..e6a6c045a --- /dev/null +++ b/modules/key-pair/outputs.tf @@ -0,0 +1,4 @@ +output "key_pair" { + description = "Generated key pair." + value = aws_key_pair.key +} diff --git a/modules/key-pair/variables.tf b/modules/key-pair/variables.tf new file mode 100644 index 000000000..4a19c68a2 --- /dev/null +++ b/modules/key-pair/variables.tf @@ -0,0 +1,34 @@ +variable "rsa_bits" { + description = "" + type = string + default = 4048 +} + +variable "public_ssh_key_filename" { + description = "Filename (full path) for the public key." + type = string + default = "./generated/id_rsa.pub" +} + +variable "private_ssh_key_filename" { + description = "Filename (full path) for the private key." + type = string + default = "./generated/id_rsa" +} + +variable "environment" { + description = "Name of the environment (aka namespace) to ensure resources are unique." + type = string +} + +variable "name" { + description = "Name of the key, will be prefixed by the environment name." + default = null + type = string +} + +variable "tags" { + description = "Map of tags that will be added to created resources. By default resources will be tagged with name and environment." + type = map(string) + default = {} +} diff --git a/modules/key-pair/versions.tf b/modules/key-pair/versions.tf new file mode 100644 index 000000000..88858f37a --- /dev/null +++ b/modules/key-pair/versions.tf @@ -0,0 +1,9 @@ +terraform { + required_version = "~> 0.12" + required_providers { + aws = ">= 2.46" + local = ">= 1.4" + tls = ">= 2" + null = ">= 2" + } +} diff --git a/template/eip.tpl b/template/eip.tpl index 369ee5a40..d389296b9 100644 --- a/template/eip.tpl +++ b/template/eip.tpl @@ -1,6 +1,6 @@ echo 'installing additional software for assigning EIP' -curl -O https://bootstrap.pypa.io/get-pip.py +curl --fail --retry 6 -O https://bootstrap.pypa.io/get-pip.py python get-pip.py --user export PATH=~/.local/bin:$PATH diff --git a/template/gitlab-runner.tpl b/template/gitlab-runner.tpl index 96e45bf1b..141cf6a19 100644 --- a/template/gitlab-runner.tpl +++ b/template/gitlab-runner.tpl @@ -23,12 +23,20 @@ then fi fi -curl -L https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.rpm.sh | bash +curl --fail --retry 6 -L https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.rpm.sh | bash yum install gitlab-runner-${gitlab_runner_version} -y -curl --fail --retry 6 -L https://github.com/docker/machine/releases/download/v${docker_machine_version}/docker-machine-`uname -s`-`uname -m` >/tmp/docker-machine && \ - chmod +x /tmp/docker-machine && \ - cp /tmp/docker-machine /usr/local/bin/docker-machine && \ + +if [[ `echo ${docker_machine_download_url}` == "" ]] +then + curl --fail --retry 6 -L https://github.com/docker/machine/releases/download/v${docker_machine_version}/docker-machine-`uname -s`-`uname -m` >/tmp/docker-machine +else + curl --fail --retry 6 -L ${docker_machine_download_url} >/tmp/docker-machine +fi + +chmod +x /tmp/docker-machine && \ + mv /tmp/docker-machine /usr/local/bin/docker-machine && \ ln -s /usr/local/bin/docker-machine /usr/bin/docker-machine +docker-machine --version # Create a dummy machine so that the cert is generated properly # See: https://gitlab.com/gitlab-org/gitlab-runner/issues/3676 diff --git a/variables.tf b/variables.tf index e24210e32..fadd7a00a 100644 --- a/variables.tf +++ b/variables.tf @@ -56,19 +56,13 @@ variable "runner_instance_spot_price" { variable "ssh_key_pair" { description = "Set this to use existing AWS key pair" type = string - default = "" -} - -variable "ssh_public_key" { - description = "Public SSH key used for the GitLab runner EC2 instance." - type = string - default = "" + default = null } variable "docker_machine_instance_type" { description = "Instance type used for the instances hosting docker-machine." type = string - default = "m5a.large" + default = "m5.large" } variable "docker_machine_spot_price_bid" { @@ -77,8 +71,14 @@ variable "docker_machine_spot_price_bid" { default = "0.06" } +variable "docker_machine_download_url" { + description = "Full url pointing to a linux x64 distribution of docker machine. Once set `docker_machine_version` will be ingored. For example the GitLab version, https://gitlab-docker-machine-downloads.s3.amazonaws.com/v0.16.2-gitlab.2/docker-machine." + type = string + default = "" +} + variable "docker_machine_version" { - description = "Version of docker-machine." + description = "Version of docker-machine. The version will be ingored once `docker_machine_download_url` is set." type = string default = "0.16.2" }