From 558aa608da62839d31317246c619b43a3f09b8f7 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Thu, 12 Dec 2024 16:00:44 -0500 Subject: [PATCH 001/158] start of new version --- .../dev/us-east-1/components/Readme.md | 16 ++++++---- .../components/machine_image/main.tf | 30 ++++++++++++------- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md b/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md index 3f6653d..1f3362c 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md @@ -1,12 +1,18 @@ # Plan ## Phase 1 + 1. create minimal ec2 instance in machine_image + +terraform (later packer) for ubuntu python uvicorn fastapi nginx systemd server with custom git modules + 2. create minimal ec2 ami from instance in machine_image 3. create autoscaling_group of size 1 for image -4. create application_load_balancer -5. create dns_entry -6. create cognito_user_pool for login +4. create application load balancer + +send users back to server via sticky sessions or some id. + +5. create dns_entry +6. create cognito user pool for login 7. create work_queue -8. create lambda_workers on queue +8. create lambda_workers on queue 9. create resource_launchers to create new resources. - diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf index a438646..e108645 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf @@ -1,6 +1,3 @@ -#TASK:write terraform packer for ubuntu python fastapi server with custom git modules - -#To create a Terraform configuration using Packer for an Ubuntu-based FastAPI server with custom Git modules, you'll need to follow a few steps. Below is a concise example of how to structure your Packer template and Terraform files. provider "aws" { region = "us-east-1" @@ -16,14 +13,27 @@ data "aws_ami" "ubuntu" { } } -resource "aws_instance" "fastapi_server_test_instance" { +resource "aws_instance" "swarms_server_test_instance" { count = var.test_server_count - ami = data.aws_ami.ubuntu.id - instance_type = "t2.micro" - - # add in this user data + ami = data.aws_ami.ubuntu.id + instance_type = "t3g.large" + + user_data = <<-EOF +#!/bin/bash +sudo apt update +sudo apt install -y git virtualenv +rm -rf ./src/swarms +if [ ! -d "/opt/swarms/" ]; + then + git clone https://github.com/jmikedupont2/swarms "/opt/swarms/" +fi +cd "/opt/swarms/" || exit 1 # "we need swarms" +export BRANCH=feature/ec2 +git checkout --force $BRANCH +bash -x /opt/swarms/api/install.sh + EOF tags = { - Name = "FastAPI Server" + Name = "Swarms Server" } } - +} From 2138b0450064e6fca2a5e51c00661dac4c3efdb3 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Thu, 12 Dec 2024 16:07:16 -0500 Subject: [PATCH 002/158] now creating --- .../dev/us-east-1/components/machine_image/main.tf | 3 +-- .../dev/us-east-1/components/machine_image/variables.tf | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf index e108645..4ff7a6b 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf @@ -16,7 +16,7 @@ data "aws_ami" "ubuntu" { resource "aws_instance" "swarms_server_test_instance" { count = var.test_server_count ami = data.aws_ami.ubuntu.id - instance_type = "t3g.large" + instance_type = "t3.large" user_data = <<-EOF #!/bin/bash @@ -36,4 +36,3 @@ bash -x /opt/swarms/api/install.sh Name = "Swarms Server" } } -} diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/variables.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/variables.tf index c36c95f..e9e30b2 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/variables.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/variables.tf @@ -1,3 +1,3 @@ variable test_server_count { - value = 1 # how many test servers to run + default = 1 # how many test servers to run } From 2badf08a3ede28ce880657c87643888e4bde1583 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Thu, 12 Dec 2024 17:49:51 -0500 Subject: [PATCH 003/158] machine image --- .../dev/us-east-1/components/Readme.md | 4 + .../components/machine_image/Makefile | 2 +- .../components/machine_image/main.tf | 138 ++++- .../components/machine_image/versions.tf | 10 + .../components/vpc/.terraform.lock.hcl | 20 + .../dev/us-east-1/components/vpc/main.tf | 236 ++++++++ .../dev/us-east-1/components/vpc/outputs.tf | 551 ++++++++++++++++++ .../dev/us-east-1/components/vpc/versions.tf | 10 + 8 files changed, 960 insertions(+), 11 deletions(-) create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/versions.tf create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/.terraform.lock.hcl create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/main.tf create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/outputs.tf create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/versions.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md b/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md index 1f3362c..f21e789 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md @@ -1,6 +1,10 @@ # Plan ## Phase 1 +0. create standard vpc with a private network to host ec2, +this uses a natgw that costs money https://aws.amazon.com/vpc/pricing/ +0.45 so we will put the intances in public for now and use security groups to limit access. + 1. create minimal ec2 instance in machine_image terraform (later packer) for ubuntu python uvicorn fastapi nginx systemd server with custom git modules diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/Makefile b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/Makefile index 307e8a1..6a6bebf 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/Makefile +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/Makefile @@ -1,4 +1,4 @@ doit : tofu init tofu plan - tofu apply + tofu apply -auto-approve diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf index 4ff7a6b..c4f90be 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf @@ -1,11 +1,57 @@ +# outputs +# default_network_acl_id = "acl-032756394b24c5d7a" +# default_route_table_id = "rtb-014dd7a2bcfc284ec" +# default_security_group_id = "sg-0273ddcb04d73df49" +# nat_ids = [] +# nat_public_ips = tolist([]) +# natgw_ids = [] +# private_ipv6_egress_route_ids = [] +# private_nat_gateway_route_ids = [] +# private_route_table_association_ids = [] +# private_route_table_ids = [] +# private_subnet_arns = [] +# private_subnets = [] +# private_subnets_cidr_blocks = tolist([]) +# private_subnets_ipv6_cidr_blocks = tolist([]) +# public_route_table_association_ids = [] +# public_route_table_ids = [] +# public_subnet_arns = [] +# public_subnets = [] +# public_subnets_cidr_blocks = tolist([]) +# public_subnets_ipv6_cidr_blocks = tolist([]) +# this_customer_gateway = {} +# vpc_arn = "arn:aws:ec2:us-east-1:767503528736:vpc/vpc-04f28c9347af48b55" +# vpc_cidr_block = "10.0.0.0/16" +# vpc_enable_dns_hostnames = true +# vpc_enable_dns_support = true +# vpc_flow_log_cloudwatch_iam_role_arn = "" +# vpc_flow_log_destination_arn = "" +# vpc_flow_log_destination_type = "cloud-watch-logs" + +# vpc_instance_tenancy = "default" +# vpc_ipv6_association_id = "" +# vpc_ipv6_cidr_block = "" +# vpc_main_route_table_id = "rtb-014dd7a2bcfc284ec" +# vpc_owner_id = "767503528736" +# vpc_secondary_cidr_blocks = tolist([]) + provider "aws" { region = "us-east-1" } +locals { + name = "swarms" + region = "us-east-1" + ec2_subnet_id = "subnet-057c90cfe7b2e5646" + vpc_id = "vpc-04f28c9347af48b55" + tags = { + project="swarms" + } +} + data "aws_ami" "ubuntu" { most_recent = true - owners = ["099720109477"] # Ubuntu's account ID filter { name = "name" @@ -13,16 +59,55 @@ data "aws_ami" "ubuntu" { } } -resource "aws_instance" "swarms_server_test_instance" { - count = var.test_server_count +module "security_group_instance" { + source = "terraform-aws-modules/security-group/aws" + version = "~> 5.0" + name = "${local.name}-ec2" + description = "Security Group for EC2 Instance" + vpc_id = local.vpc_id + #ingress_rules = ["https-443-tcp"] + ingress_with_cidr_blocks = [ + { + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = "0.0.0.0/0" + # cidr_blocks = "10.10.0.0/16" + } + ] + + #egress_rules = ["https-443-tcp"] + egress_with_cidr_blocks = [ + { + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = "0.0.0.0/0" + } + ] + + tags = local.tags +} + +module "ec2" { + source = "terraform-aws-modules/ec2-instance/aws" + associate_public_ip_address = true # for now + name = local.name ami = data.aws_ami.ubuntu.id instance_type = "t3.large" - + create_iam_instance_profile = true + iam_role_description = "IAM role for EC2 instance" + iam_role_policies = { + AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" + } + vpc_security_group_ids = [module.security_group_instance.security_group_id] + user_data = <<-EOF #!/bin/bash -sudo apt update -sudo apt install -y git virtualenv -rm -rf ./src/swarms +apt update +apt-get install ec2-instance-connect +apt install -y git virtualenv + if [ ! -d "/opt/swarms/" ]; then git clone https://github.com/jmikedupont2/swarms "/opt/swarms/" @@ -32,7 +117,40 @@ export BRANCH=feature/ec2 git checkout --force $BRANCH bash -x /opt/swarms/api/install.sh EOF - tags = { - Name = "Swarms Server" - } + tags = local.tags + create_spot_instance = true + subnet_id = local.ec2_subnet_id + #instance_market_options { market_type = "spot" spot_options { #max_price = 0.0031 + #} +#} } + + +# module "vpc_endpoints" { +# source = "terraform-aws-modules/vpc/aws//modules/vpc-endpoints" +# version = "~> 5.0" + +# vpc_id = module.vpc.vpc_id + +# endpoints = { for service in toset(["ssm", "ssmmessages", "ec2messages"]) : +# replace(service, ".", "_") => +# { +# service = service +# subnet_ids = module.vpc.intra_subnets +# private_dns_enabled = true +# tags = { Name = "${local.name}-${service}" } +# } +# } + +# create_security_group = true +# security_group_name_prefix = "${local.name}-vpc-endpoints-" +# security_group_description = "VPC endpoint security group" +# security_group_rules = { +# ingress_https = { +# description = "HTTPS from subnets" +# cidr_blocks = #module.vpc.intra_subnets_cidr_blocks +# } +# } + +# tags = local.tags +# } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/versions.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/versions.tf new file mode 100644 index 0000000..fd4d116 --- /dev/null +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 4.66" + } + } +} diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/.terraform.lock.hcl b/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/.terraform.lock.hcl new file mode 100644 index 0000000..a688307 --- /dev/null +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/.terraform.lock.hcl @@ -0,0 +1,20 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/hashicorp/aws" { + version = "5.81.0" + constraints = ">= 5.46.0" + hashes = [ + "h1:ird967uf44WBZ0u9rMlOdO4nCKJZMlUknLcD3lz4dWs=", + "zh:0d0c7cc1e16b16cd00ab36de35038162e1871b51a902e9016d08c55c9fc4de35", + "zh:0e4b8c6e46999015292ab4fb9a430ab00107a276f25243552cde66db53c58661", + "zh:30041314cdd4e877d75ee8c9d36aecfca094e276f7a3d8150f929cf5169b2fa5", + "zh:5ebd248ce3e5a7ef9cc2f41499668f182146325e10ea305c70469122f6161a13", + "zh:888a69d371373b418549e03f5922becb2b8074cb463552ecfa65f30455708db0", + "zh:8a21bb7fe60383ff5ca9db8630a287e41fd520d2514c08874a16dc74500fadd7", + "zh:9c4663dcbfe58544642d70ebfdc6c5fa91592ff04164f77c655e32e6024483e2", + "zh:b322873f1209443a8407d5f782d7d917de6a1391239dbd0e7f809ce6507bed76", + "zh:b7c9d5ca14b818b5932ac58a490646a425ebc41b33d149090aa5f48d1ca35c99", + "zh:e76cd202b03749f3082b0cbe849fd2e731cf3f9a6aa994d2d629602c3aede36c", + ] +} diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/main.tf new file mode 100644 index 0000000..5d60b80 --- /dev/null +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/main.tf @@ -0,0 +1,236 @@ +#from https://github.com/terraform-aws-modules/terraform-aws-vpc + +data "aws_availability_zones" "available" {} + +provider "aws" { + region = local.region +} + +locals { + name = "swarms" + region = "us-east-1" + vpc_cidr = "10.0.0.0/16" + azs = slice(data.aws_availability_zones.available.names, 0, 3) + + tags = { + project = local.name +# GithubRepo = "terraform-aws-vpc" +# GithubOrg = "terraform-aws-modules" + } +} + +# resource "vpc" "swarms" { +# source = "terraform-aws-modules/vpc/aws" +# #source = "https://github.com/terraform-aws-modules/terraform-aws-vpc.git" +# name = "swarms" +# cidr = "10.0.0.0/16" +# azs = ["us-east-1a", "us-east-1b", "us-east-1c"] +# private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] +# public_subnets = ["10.0.101.0/24", "10.0.102.0/24", "10.0.103.0/24"] + +# enable_nat_gateway = true +# enable_vpn_gateway = true + +# tags = { +# Terraform = "true" +# Environment = "dev" +# } +# } + +# provider_name = "aws" + + + +################################################################################ +# VPC Module +################################################################################ + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + name = local.name + cidr = local.vpc_cidr + azs = local.azs + private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k)] + public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 4)] + # database_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 8)] + # elasticache_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 12)] + # redshift_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 16)] + # intra_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 20)] + private_subnet_names = ["Private Subnet One", "Private Subnet Two"] + # # public_subnet_names omitted to show default name generation for all three subnets + # database_subnet_names = ["DB Subnet One"] + # elasticache_subnet_names = ["Elasticache Subnet One", "Elasticache Subnet Two"] + # redshift_subnet_names = ["Redshift Subnet One", "Redshift Subnet Two", "Redshift Subnet Three"] + # intra_subnet_names = [] + # create_database_subnet_group = false + # manage_default_network_acl = false + # manage_default_route_table = false + # manage_default_security_group = false + # enable_dns_hostnames = true + # enable_dns_support = true + enable_nat_gateway = false + # single_nat_gateway = false + # customer_gateways = { + # IP1 = { + # bgp_asn = 65112 + # ip_address = "1.2.3.4" + # device_name = "some_name" + # }, + # IP2 = { + # bgp_asn = 65112 + # ip_address = "5.6.7.8" + # } + # } + # enable_vpn_gateway = true + # enable_dhcp_options = true + # dhcp_options_domain_name = "service.consul" + # dhcp_options_domain_name_servers = ["127.0.0.1", "10.10.0.2"] + + # # VPC Flow Logs (Cloudwatch log group and IAM role will be created) + # vpc_flow_log_iam_role_name = "vpc-complete-example-role" + # vpc_flow_log_iam_role_use_name_prefix = false + # enable_flow_log = true + # create_flow_log_cloudwatch_log_group = true + # create_flow_log_cloudwatch_iam_role = true + # flow_log_max_aggregation_interval = 60 + + tags = local.tags +} + +# ################################################################################ +# # VPC Endpoints Module +# ################################################################################ + +# module "vpc_endpoints" { +# source = "../../modules/vpc-endpoints" + +# vpc_id = module.vpc.vpc_id + +# create_security_group = true +# security_group_name_prefix = "${local.name}-vpc-endpoints-" +# security_group_description = "VPC endpoint security group" +# security_group_rules = { +# ingress_https = { +# description = "HTTPS from VPC" +# cidr_blocks = [module.vpc.vpc_cidr_block] +# } +# } + +# endpoints = { +# s3 = { +# service = "s3" +# private_dns_enabled = true +# dns_options = { +# private_dns_only_for_inbound_resolver_endpoint = false +# } +# tags = { Name = "s3-vpc-endpoint" } +# }, +# dynamodb = { +# service = "dynamodb" +# service_type = "Gateway" +# route_table_ids = flatten([module.vpc.intra_route_table_ids, module.vpc.private_route_table_ids, module.vpc.public_route_table_ids]) +# policy = data.aws_iam_policy_document.dynamodb_endpoint_policy.json +# tags = { Name = "dynamodb-vpc-endpoint" } +# }, +# ecs = { +# service = "ecs" +# private_dns_enabled = true +# subnet_ids = module.vpc.private_subnets +# }, +# ecs_telemetry = { +# create = false +# service = "ecs-telemetry" +# private_dns_enabled = true +# subnet_ids = module.vpc.private_subnets +# }, +# ecr_api = { +# service = "ecr.api" +# private_dns_enabled = true +# subnet_ids = module.vpc.private_subnets +# policy = data.aws_iam_policy_document.generic_endpoint_policy.json +# }, +# ecr_dkr = { +# service = "ecr.dkr" +# private_dns_enabled = true +# subnet_ids = module.vpc.private_subnets +# policy = data.aws_iam_policy_document.generic_endpoint_policy.json +# }, +# rds = { +# service = "rds" +# private_dns_enabled = true +# subnet_ids = module.vpc.private_subnets +# security_group_ids = [aws_security_group.rds.id] +# }, +# } + +# tags = merge(local.tags, { +# Project = "Secret" +# Endpoint = "true" +# }) +# } + +# module "vpc_endpoints_nocreate" { +# source = "../../modules/vpc-endpoints" + +# create = false +# } + +# ################################################################################ +# # Supporting Resources +# ################################################################################ + +# data "aws_iam_policy_document" "dynamodb_endpoint_policy" { +# statement { +# effect = "Deny" +# actions = ["dynamodb:*"] +# resources = ["*"] + +# principals { +# type = "*" +# identifiers = ["*"] +# } + +# condition { +# test = "StringNotEquals" +# variable = "aws:sourceVpc" + +# values = [module.vpc.vpc_id] +# } +# } +# } + +# data "aws_iam_policy_document" "generic_endpoint_policy" { +# statement { +# effect = "Deny" +# actions = ["*"] +# resources = ["*"] + +# principals { +# type = "*" +# identifiers = ["*"] +# } + +# condition { +# test = "StringNotEquals" +# variable = "aws:SourceVpc" + +# values = [module.vpc.vpc_id] +# } +# } +# } + +# resource "aws_security_group" "rds" { +# name_prefix = "${local.name}-rds" +# description = "Allow PostgreSQL inbound traffic" +# vpc_id = module.vpc.vpc_id + +# ingress { +# description = "TLS from VPC" +# from_port = 5432 +# to_port = 5432 +# protocol = "tcp" +# cidr_blocks = [module.vpc.vpc_cidr_block] +# } + +# tags = local.tags +# } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/outputs.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/outputs.tf new file mode 100644 index 0000000..655b291 --- /dev/null +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/outputs.tf @@ -0,0 +1,551 @@ +output "vpc_id" { + description = "The ID of the VPC" + value = module.vpc.vpc_id +} + +output "vpc_arn" { + description = "The ARN of the VPC" + value = module.vpc.vpc_arn +} + +output "vpc_cidr_block" { + description = "The CIDR block of the VPC" + value = module.vpc.vpc_cidr_block +} + +output "default_security_group_id" { + description = "The ID of the security group created by default on VPC creation" + value = module.vpc.default_security_group_id +} + +output "default_network_acl_id" { + description = "The ID of the default network ACL" + value = module.vpc.default_network_acl_id +} + +output "default_route_table_id" { + description = "The ID of the default route table" + value = module.vpc.default_route_table_id +} + +output "vpc_instance_tenancy" { + description = "Tenancy of instances spin up within VPC" + value = module.vpc.vpc_instance_tenancy +} + +output "vpc_enable_dns_support" { + description = "Whether or not the VPC has DNS support" + value = module.vpc.vpc_enable_dns_support +} + +output "vpc_enable_dns_hostnames" { + description = "Whether or not the VPC has DNS hostname support" + value = module.vpc.vpc_enable_dns_hostnames +} + +output "vpc_main_route_table_id" { + description = "The ID of the main route table associated with this VPC" + value = module.vpc.vpc_main_route_table_id +} + +output "vpc_ipv6_association_id" { + description = "The association ID for the IPv6 CIDR block" + value = module.vpc.vpc_ipv6_association_id +} + +output "vpc_ipv6_cidr_block" { + description = "The IPv6 CIDR block" + value = module.vpc.vpc_ipv6_cidr_block +} + +output "vpc_secondary_cidr_blocks" { + description = "List of secondary CIDR blocks of the VPC" + value = module.vpc.vpc_secondary_cidr_blocks +} + +output "vpc_owner_id" { + description = "The ID of the AWS account that owns the VPC" + value = module.vpc.vpc_owner_id +} + +output "private_subnets" { + description = "List of IDs of private subnets" + value = module.vpc.private_subnets +} + +output "private_subnet_arns" { + description = "List of ARNs of private subnets" + value = module.vpc.private_subnet_arns +} + +output "private_subnets_cidr_blocks" { + description = "List of cidr_blocks of private subnets" + value = module.vpc.private_subnets_cidr_blocks +} + +output "private_subnets_ipv6_cidr_blocks" { + description = "List of IPv6 cidr_blocks of private subnets in an IPv6 enabled VPC" + value = module.vpc.private_subnets_ipv6_cidr_blocks +} + +output "public_subnets" { + description = "List of IDs of public subnets" + value = module.vpc.public_subnets +} + +output "public_subnet_arns" { + description = "List of ARNs of public subnets" + value = module.vpc.public_subnet_arns +} + +output "public_subnets_cidr_blocks" { + description = "List of cidr_blocks of public subnets" + value = module.vpc.public_subnets_cidr_blocks +} + +output "public_subnets_ipv6_cidr_blocks" { + description = "List of IPv6 cidr_blocks of public subnets in an IPv6 enabled VPC" + value = module.vpc.public_subnets_ipv6_cidr_blocks +} + +# output "outpost_subnets" { +# description = "List of IDs of outpost subnets" +# value = module.vpc.outpost_subnets +# } + +# output "outpost_subnet_arns" { +# description = "List of ARNs of outpost subnets" +# value = module.vpc.outpost_subnet_arns +# } + +# output "outpost_subnets_cidr_blocks" { +# description = "List of cidr_blocks of outpost subnets" +# value = module.vpc.outpost_subnets_cidr_blocks +# } + +# output "outpost_subnets_ipv6_cidr_blocks" { +# description = "List of IPv6 cidr_blocks of outpost subnets in an IPv6 enabled VPC" +# value = module.vpc.outpost_subnets_ipv6_cidr_blocks +# } + +# output "database_subnets" { +# description = "List of IDs of database subnets" +# value = module.vpc.database_subnets +# } + +# output "database_subnet_arns" { +# description = "List of ARNs of database subnets" +# value = module.vpc.database_subnet_arns +# } + +# output "database_subnets_cidr_blocks" { +# description = "List of cidr_blocks of database subnets" +# value = module.vpc.database_subnets_cidr_blocks +# } + +# output "database_subnets_ipv6_cidr_blocks" { +# description = "List of IPv6 cidr_blocks of database subnets in an IPv6 enabled VPC" +# value = module.vpc.database_subnets_ipv6_cidr_blocks +# } + +# output "database_subnet_group" { +# description = "ID of database subnet group" +# value = module.vpc.database_subnet_group +# } + +# output "database_subnet_group_name" { +# description = "Name of database subnet group" +# value = module.vpc.database_subnet_group_name +# } + +# output "redshift_subnets" { +# description = "List of IDs of redshift subnets" +# value = module.vpc.redshift_subnets +# } + +# output "redshift_subnet_arns" { +# description = "List of ARNs of redshift subnets" +# value = module.vpc.redshift_subnet_arns +# } + +# output "redshift_subnets_cidr_blocks" { +# description = "List of cidr_blocks of redshift subnets" +# value = module.vpc.redshift_subnets_cidr_blocks +# } + +# output "redshift_subnets_ipv6_cidr_blocks" { +# description = "List of IPv6 cidr_blocks of redshift subnets in an IPv6 enabled VPC" +# value = module.vpc.redshift_subnets_ipv6_cidr_blocks +# } + +# output "redshift_subnet_group" { +# description = "ID of redshift subnet group" +# value = module.vpc.redshift_subnet_group +# } + +# output "elasticache_subnets" { +# description = "List of IDs of elasticache subnets" +# value = module.vpc.elasticache_subnets +# } + +# output "elasticache_subnet_arns" { +# description = "List of ARNs of elasticache subnets" +# value = module.vpc.elasticache_subnet_arns +# } + +# output "elasticache_subnets_cidr_blocks" { +# description = "List of cidr_blocks of elasticache subnets" +# value = module.vpc.elasticache_subnets_cidr_blocks +# } + +# output "elasticache_subnets_ipv6_cidr_blocks" { +# description = "List of IPv6 cidr_blocks of elasticache subnets in an IPv6 enabled VPC" +# value = module.vpc.elasticache_subnets_ipv6_cidr_blocks +# } + +# output "intra_subnets" { +# description = "List of IDs of intra subnets" +# value = module.vpc.intra_subnets +# } + +# output "intra_subnet_arns" { +# description = "List of ARNs of intra subnets" +# value = module.vpc.intra_subnet_arns +# } + +# output "intra_subnets_cidr_blocks" { +# description = "List of cidr_blocks of intra subnets" +# value = module.vpc.intra_subnets_cidr_blocks +# } + +# output "intra_subnets_ipv6_cidr_blocks" { +# description = "List of IPv6 cidr_blocks of intra subnets in an IPv6 enabled VPC" +# value = module.vpc.intra_subnets_ipv6_cidr_blocks +# } + +# output "elasticache_subnet_group" { +# description = "ID of elasticache subnet group" +# value = module.vpc.elasticache_subnet_group +# } + +# output "elasticache_subnet_group_name" { +# description = "Name of elasticache subnet group" +# value = module.vpc.elasticache_subnet_group_name +# } + +output "public_route_table_ids" { + description = "List of IDs of public route tables" + value = module.vpc.public_route_table_ids +} + +output "private_route_table_ids" { + description = "List of IDs of private route tables" + value = module.vpc.private_route_table_ids +} + +# output "database_route_table_ids" { +# description = "List of IDs of database route tables" +# value = module.vpc.database_route_table_ids +# } + +# output "redshift_route_table_ids" { +# description = "List of IDs of redshift route tables" +# value = module.vpc.redshift_route_table_ids +# } + +# output "elasticache_route_table_ids" { +# description = "List of IDs of elasticache route tables" +# value = module.vpc.elasticache_route_table_ids +# } + +# output "intra_route_table_ids" { +# description = "List of IDs of intra route tables" +# value = module.vpc.intra_route_table_ids +# } + +output "public_internet_gateway_route_id" { + description = "ID of the internet gateway route" + value = module.vpc.public_internet_gateway_route_id +} + +output "public_internet_gateway_ipv6_route_id" { + description = "ID of the IPv6 internet gateway route" + value = module.vpc.public_internet_gateway_ipv6_route_id +} + +# output "database_internet_gateway_route_id" { +# description = "ID of the database internet gateway route" +# value = module.vpc.database_internet_gateway_route_id +# } + +# output "database_nat_gateway_route_ids" { +# description = "List of IDs of the database nat gateway route" +# value = module.vpc.database_nat_gateway_route_ids +# } + +# output "database_ipv6_egress_route_id" { +# description = "ID of the database IPv6 egress route" +# value = module.vpc.database_ipv6_egress_route_id +# } + +output "private_nat_gateway_route_ids" { + description = "List of IDs of the private nat gateway route" + value = module.vpc.private_nat_gateway_route_ids +} + +output "private_ipv6_egress_route_ids" { + description = "List of IDs of the ipv6 egress route" + value = module.vpc.private_ipv6_egress_route_ids +} + +output "private_route_table_association_ids" { + description = "List of IDs of the private route table association" + value = module.vpc.private_route_table_association_ids +} + +# output "database_route_table_association_ids" { +# description = "List of IDs of the database route table association" +# value = module.vpc.database_route_table_association_ids +# } + +# output "redshift_route_table_association_ids" { +# description = "List of IDs of the redshift route table association" +# value = module.vpc.redshift_route_table_association_ids +# } + +# output "redshift_public_route_table_association_ids" { +# description = "List of IDs of the public redshift route table association" +# value = module.vpc.redshift_public_route_table_association_ids +# } + +# output "elasticache_route_table_association_ids" { +# description = "List of IDs of the elasticache route table association" +# value = module.vpc.elasticache_route_table_association_ids +# } + +# output "intra_route_table_association_ids" { +# description = "List of IDs of the intra route table association" +# value = module.vpc.intra_route_table_association_ids +# } + +output "public_route_table_association_ids" { + description = "List of IDs of the public route table association" + value = module.vpc.public_route_table_association_ids +} + +output "dhcp_options_id" { + description = "The ID of the DHCP options" + value = module.vpc.dhcp_options_id +} + +output "nat_ids" { + description = "List of allocation ID of Elastic IPs created for AWS NAT Gateway" + value = module.vpc.nat_ids +} + +output "nat_public_ips" { + description = "List of public Elastic IPs created for AWS NAT Gateway" + value = module.vpc.nat_public_ips +} + +output "natgw_ids" { + description = "List of NAT Gateway IDs" + value = module.vpc.natgw_ids +} + +output "igw_id" { + description = "The ID of the Internet Gateway" + value = module.vpc.igw_id +} + +output "igw_arn" { + description = "The ARN of the Internet Gateway" + value = module.vpc.igw_arn +} + +output "egress_only_internet_gateway_id" { + description = "The ID of the egress only Internet Gateway" + value = module.vpc.egress_only_internet_gateway_id +} + +output "cgw_ids" { + description = "List of IDs of Customer Gateway" + value = module.vpc.cgw_ids +} + +output "cgw_arns" { + description = "List of ARNs of Customer Gateway" + value = module.vpc.cgw_arns +} + +output "this_customer_gateway" { + description = "Map of Customer Gateway attributes" + value = module.vpc.this_customer_gateway +} + +output "vgw_id" { + description = "The ID of the VPN Gateway" + value = module.vpc.vgw_id +} + +output "vgw_arn" { + description = "The ARN of the VPN Gateway" + value = module.vpc.vgw_arn +} + +output "default_vpc_id" { + description = "The ID of the Default VPC" + value = module.vpc.default_vpc_id +} + +output "default_vpc_arn" { + description = "The ARN of the Default VPC" + value = module.vpc.default_vpc_arn +} + +output "default_vpc_cidr_block" { + description = "The CIDR block of the Default VPC" + value = module.vpc.default_vpc_cidr_block +} + +output "default_vpc_default_security_group_id" { + description = "The ID of the security group created by default on Default VPC creation" + value = module.vpc.default_vpc_default_security_group_id +} + +output "default_vpc_default_network_acl_id" { + description = "The ID of the default network ACL of the Default VPC" + value = module.vpc.default_vpc_default_network_acl_id +} + +output "default_vpc_default_route_table_id" { + description = "The ID of the default route table of the Default VPC" + value = module.vpc.default_vpc_default_route_table_id +} + +output "default_vpc_instance_tenancy" { + description = "Tenancy of instances spin up within Default VPC" + value = module.vpc.default_vpc_instance_tenancy +} + +output "default_vpc_enable_dns_support" { + description = "Whether or not the Default VPC has DNS support" + value = module.vpc.default_vpc_enable_dns_support +} + +output "default_vpc_enable_dns_hostnames" { + description = "Whether or not the Default VPC has DNS hostname support" + value = module.vpc.default_vpc_enable_dns_hostnames +} + +output "default_vpc_main_route_table_id" { + description = "The ID of the main route table associated with the Default VPC" + value = module.vpc.default_vpc_main_route_table_id +} + +output "public_network_acl_id" { + description = "ID of the public network ACL" + value = module.vpc.public_network_acl_id +} + +output "public_network_acl_arn" { + description = "ARN of the public network ACL" + value = module.vpc.public_network_acl_arn +} + +output "private_network_acl_id" { + description = "ID of the private network ACL" + value = module.vpc.private_network_acl_id +} + +output "private_network_acl_arn" { + description = "ARN of the private network ACL" + value = module.vpc.private_network_acl_arn +} + +# output "outpost_network_acl_id" { +# description = "ID of the outpost network ACL" +# value = module.vpc.outpost_network_acl_id +# } + +# output "outpost_network_acl_arn" { +# description = "ARN of the outpost network ACL" +# value = module.vpc.outpost_network_acl_arn +# } + +# output "intra_network_acl_id" { +# description = "ID of the intra network ACL" +# value = module.vpc.intra_network_acl_id +# } + +# output "intra_network_acl_arn" { +# description = "ARN of the intra network ACL" +# value = module.vpc.intra_network_acl_arn +# } + +# output "database_network_acl_id" { +# description = "ID of the database network ACL" +# value = module.vpc.database_network_acl_id +# } + +# output "database_network_acl_arn" { +# description = "ARN of the database network ACL" +# value = module.vpc.database_network_acl_arn +# } + +# output "redshift_network_acl_id" { +# description = "ID of the redshift network ACL" +# value = module.vpc.redshift_network_acl_id +# } + +# output "redshift_network_acl_arn" { +# description = "ARN of the redshift network ACL" +# value = module.vpc.redshift_network_acl_arn +# } + +# output "elasticache_network_acl_id" { +# description = "ID of the elasticache network ACL" +# value = module.vpc.elasticache_network_acl_id +# } + +# output "elasticache_network_acl_arn" { +# description = "ARN of the elasticache network ACL" +# value = module.vpc.elasticache_network_acl_arn +# } + +# VPC flow log +output "vpc_flow_log_id" { + description = "The ID of the Flow Log resource" + value = module.vpc.vpc_flow_log_id +} + +output "vpc_flow_log_destination_arn" { + description = "The ARN of the destination for VPC Flow Logs" + value = module.vpc.vpc_flow_log_destination_arn +} + +output "vpc_flow_log_destination_type" { + description = "The type of the destination for VPC Flow Logs" + value = module.vpc.vpc_flow_log_destination_type +} + +output "vpc_flow_log_cloudwatch_iam_role_arn" { + description = "The ARN of the IAM role used when pushing logs to Cloudwatch log group" + value = module.vpc.vpc_flow_log_cloudwatch_iam_role_arn +} + +# VPC endpoints +# output "vpc_endpoints" { +# description = "Array containing the full resource object and attributes for all endpoints created" +# value = module.vpc_endpoints.endpoints +# } + +# output "vpc_endpoints_security_group_arn" { +# description = "Amazon Resource Name (ARN) of the security group" +# value = module.vpc_endpoints.security_group_arn +# } + +# output "vpc_endpoints_security_group_id" { +# description = "ID of the security group" +# value = module.vpc_endpoints.security_group_id +# } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/versions.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/versions.tf new file mode 100644 index 0000000..e1ed610 --- /dev/null +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.46" + } + } +} From 51a09062b9ba131e086fe2dbddfd0569faab4ad9 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Thu, 12 Dec 2024 18:18:40 -0500 Subject: [PATCH 004/158] update --- .../dev/us-east-1/components/machine_image/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf index c4f90be..a1d8aec 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf @@ -104,9 +104,9 @@ module "ec2" { user_data = <<-EOF #!/bin/bash +export HOME=/root apt update -apt-get install ec2-instance-connect -apt install -y git virtualenv +apt-get install -y ec2-instance-connect git virtualenv if [ ! -d "/opt/swarms/" ]; then From fdac521729e4b8c0ada66150cb3209be00a899e9 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Thu, 12 Dec 2024 20:49:29 -0500 Subject: [PATCH 005/158] now running --- .../dev/us-east-1/components/Readme.md | 2 +- .../components/machine_image/Readme.md | 5 ++++ .../components/machine_image/main.tf | 28 +++++++++++++++++-- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md b/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md index f21e789..b9e7c04 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md @@ -2,7 +2,7 @@ ## Phase 1 0. create standard vpc with a private network to host ec2, -this uses a natgw that costs money https://aws.amazon.com/vpc/pricing/ +this uses a natgw that costs money bhttps://aws.amazon.com/vpc/pricing/ 0.45 so we will put the intances in public for now and use security groups to limit access. 1. create minimal ec2 instance in machine_image diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/Readme.md b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/Readme.md index 2ee3737..7051cde 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/Readme.md +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/Readme.md @@ -1 +1,6 @@ machine_image + + +`tofu apply -destroy -target module.ec2.aws_spot_instance_request.this[0] -auto-approve` + +aws ec2 describe-images --owners 099720109477 > images.json diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf index a1d8aec..48bd41b 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf @@ -55,7 +55,14 @@ data "aws_ami" "ubuntu" { owners = ["099720109477"] # Ubuntu's account ID filter { name = "name" - values = ["ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"] + values = [ + #"ubuntu/images/hvm-ssd/ubuntu*24*amd64-server*" + "ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*", + ] + + #"ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-arm64-server-20240823", + #"ubuntu-minimal/images/hvm-ssd-gp3/ubuntu-noble-24.04-arm64-minimal-20240824", + #"ubuntu-minimal/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-minimal-20240813", } } @@ -72,8 +79,13 @@ module "security_group_instance" { to_port = 443 protocol = "tcp" cidr_blocks = "0.0.0.0/0" - # cidr_blocks = "10.10.0.0/16" - } + }, + { + from_port = 80 + to_port = 80 + protocol = "tcp" + cidr_blocks = "0.0.0.0/0" + } ] #egress_rules = ["https-443-tcp"] @@ -102,6 +114,16 @@ module "ec2" { } vpc_security_group_ids = [module.security_group_instance.security_group_id] + root_block_device = [ + { + #encrypted = true + #volume_type = "gp3" + #throughput = 200 + volume_size = 50 +# tags = local.tags + } + ] + user_data = <<-EOF #!/bin/bash export HOME=/root From 5ed453cd15c7a867c060e81c1622fe943922ab37 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Fri, 13 Dec 2024 09:54:54 -0500 Subject: [PATCH 006/158] using ami --- .../components/machine_image/main.tf | 40 +++++++++++++++---- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf index 48bd41b..128bd4a 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf @@ -41,6 +41,7 @@ provider "aws" { } locals { + ami = "ami-0e2c8caa4b6378d8c" name = "swarms" region = "us-east-1" ec2_subnet_id = "subnet-057c90cfe7b2e5646" @@ -59,7 +60,32 @@ data "aws_ami" "ubuntu" { #"ubuntu/images/hvm-ssd/ubuntu*24*amd64-server*" "ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*", ] - + +# from https://us-east-1.console.aws.amazon.com/ec2/home?region=us-east-1#AMICatalog:https://us-east-1.console.aws.amazon.com/ec2/home?region=us-east-1#AMICatalog: +# ubuntu (2 filtered, 8 unfiltered) +# Free tier eligible +# Ubuntu Server 24.04 LTS (HVM), SSD Volume Type +# ami-0e2c8caa4b6378d8c (64-bit (x86)) / ami-0932ffb346ea84d48 (64-bit (Arm)) +# Platform: ubuntu +# Root device type: ebs +# Virtualization: hvm +# ENA enabled: Yes +# Select +# 64-bit (x86) +# 64-bit (Arm) +# Ubuntu + + +# ami-005fc0f236362e99f (64-bit (x86)) / ami-07ee04759daf109de (64-bit (Arm)) +# Ubuntu Server 22.04 LTS (HVM),EBS General Purpose (SSD) Volume Type. +# Platform: ubuntu +# Root device type: ebs +# Virtualization: hvm +# ENA enabled: Yes +# Select +# 64-bit (x86) +# 64-bit (Arm) + #"ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-arm64-server-20240823", #"ubuntu-minimal/images/hvm-ssd-gp3/ubuntu-noble-24.04-arm64-minimal-20240824", #"ubuntu-minimal/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-minimal-20240813", @@ -105,7 +131,7 @@ module "ec2" { source = "terraform-aws-modules/ec2-instance/aws" associate_public_ip_address = true # for now name = local.name - ami = data.aws_ami.ubuntu.id + ami = local.ami # data.aws_ami.ubuntu.id instance_type = "t3.large" create_iam_instance_profile = true iam_role_description = "IAM role for EC2 instance" @@ -116,11 +142,11 @@ module "ec2" { root_block_device = [ { - #encrypted = true - #volume_type = "gp3" - #throughput = 200 - volume_size = 50 -# tags = local.tags + + # best practice is encrypted at rest + encrypted = true + volume_size = 30 + volume_type = "gp3" } ] From f29b5df959a681539f641bb37a21fe827d78477f Mon Sep 17 00:00:00 2001 From: mike dupont Date: Fri, 13 Dec 2024 12:14:54 -0500 Subject: [PATCH 007/158] bugfix egress --- .../components/machine_image/main.tf | 96 +++++++++---------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf index 128bd4a..8b46068 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf @@ -51,46 +51,46 @@ locals { } } -data "aws_ami" "ubuntu" { - most_recent = true - owners = ["099720109477"] # Ubuntu's account ID - filter { - name = "name" - values = [ - #"ubuntu/images/hvm-ssd/ubuntu*24*amd64-server*" - "ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*", - ] - -# from https://us-east-1.console.aws.amazon.com/ec2/home?region=us-east-1#AMICatalog:https://us-east-1.console.aws.amazon.com/ec2/home?region=us-east-1#AMICatalog: -# ubuntu (2 filtered, 8 unfiltered) -# Free tier eligible -# Ubuntu Server 24.04 LTS (HVM), SSD Volume Type -# ami-0e2c8caa4b6378d8c (64-bit (x86)) / ami-0932ffb346ea84d48 (64-bit (Arm)) -# Platform: ubuntu -# Root device type: ebs -# Virtualization: hvm -# ENA enabled: Yes -# Select -# 64-bit (x86) -# 64-bit (Arm) -# Ubuntu - - -# ami-005fc0f236362e99f (64-bit (x86)) / ami-07ee04759daf109de (64-bit (Arm)) -# Ubuntu Server 22.04 LTS (HVM),EBS General Purpose (SSD) Volume Type. -# Platform: ubuntu -# Root device type: ebs -# Virtualization: hvm -# ENA enabled: Yes -# Select -# 64-bit (x86) -# 64-bit (Arm) +# data "aws_ami" "ubuntu" { +# most_recent = true +# owners = ["099720109477"] # Ubuntu's account ID +# filter { +# name = "name" +# values = [ +# #"ubuntu/images/hvm-ssd/ubuntu*24*amd64-server*" +# "ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*", +# ] + +# # from https://us-east-1.console.aws.amazon.com/ec2/home?region=us-east-1#AMICatalog:https://us-east-1.console.aws.amazon.com/ec2/home?region=us-east-1#AMICatalog: +# # ubuntu (2 filtered, 8 unfiltered) +# # Free tier eligible +# # Ubuntu Server 24.04 LTS (HVM), SSD Volume Type +# # ami-0e2c8caa4b6378d8c (64-bit (x86)) / ami-0932ffb346ea84d48 (64-bit (Arm)) +# # Platform: ubuntu +# # Root device type: ebs +# # Virtualization: hvm +# # ENA enabled: Yes +# # Select +# # 64-bit (x86) +# # 64-bit (Arm) +# # Ubuntu + + +# # ami-005fc0f236362e99f (64-bit (x86)) / ami-07ee04759daf109de (64-bit (Arm)) +# # Ubuntu Server 22.04 LTS (HVM),EBS General Purpose (SSD) Volume Type. +# # Platform: ubuntu +# # Root device type: ebs +# # Virtualization: hvm +# # ENA enabled: Yes +# # Select +# # 64-bit (x86) +# # 64-bit (Arm) - #"ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-arm64-server-20240823", - #"ubuntu-minimal/images/hvm-ssd-gp3/ubuntu-noble-24.04-arm64-minimal-20240824", - #"ubuntu-minimal/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-minimal-20240813", - } -} +# #"ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-arm64-server-20240823", +# #"ubuntu-minimal/images/hvm-ssd-gp3/ubuntu-noble-24.04-arm64-minimal-20240824", +# #"ubuntu-minimal/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-minimal-20240813", +# } +# } module "security_group_instance" { source = "terraform-aws-modules/security-group/aws" @@ -114,15 +114,15 @@ module "security_group_instance" { } ] - #egress_rules = ["https-443-tcp"] - egress_with_cidr_blocks = [ - { - from_port = 443 - to_port = 443 - protocol = "tcp" - cidr_blocks = "0.0.0.0/0" - } - ] + egress_rules = ["all-all"] + # egress_with_cidr_blocks = [ + # { + # from_port = 0 + # to_port = 0 + # protocol = "tcp" + # cidr_blocks = "0.0.0.0/0" + # } + # ] tags = local.tags } From f381197f3d80ca9e42b2749cb6b0e3a8ed84828d Mon Sep 17 00:00:00 2001 From: mike dupont Date: Fri, 13 Dec 2024 14:14:28 -0500 Subject: [PATCH 008/158] adding updates and proposed changes --- .../components/machine_image/Readme.md | 3 + .../components/machine_image/main.tf | 126 +-------- .../components/machine_image/main.tf.asg | 253 ++++++++++++++++++ .../machine_image/main.tf.consts.org | 194 ++++++++++++++ 4 files changed, 454 insertions(+), 122 deletions(-) create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf.asg create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf.consts.org diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/Readme.md b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/Readme.md index 7051cde..69b85c3 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/Readme.md +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/Readme.md @@ -4,3 +4,6 @@ machine_image `tofu apply -destroy -target module.ec2.aws_spot_instance_request.this[0] -auto-approve` aws ec2 describe-images --owners 099720109477 > images.json +* + +tofu state rm "module.ec2.aws_spot_instance_request.this[0]" diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf index 8b46068..2dbea5b 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf @@ -1,41 +1,3 @@ - -# outputs -# default_network_acl_id = "acl-032756394b24c5d7a" -# default_route_table_id = "rtb-014dd7a2bcfc284ec" -# default_security_group_id = "sg-0273ddcb04d73df49" -# nat_ids = [] -# nat_public_ips = tolist([]) -# natgw_ids = [] -# private_ipv6_egress_route_ids = [] -# private_nat_gateway_route_ids = [] -# private_route_table_association_ids = [] -# private_route_table_ids = [] -# private_subnet_arns = [] -# private_subnets = [] -# private_subnets_cidr_blocks = tolist([]) -# private_subnets_ipv6_cidr_blocks = tolist([]) -# public_route_table_association_ids = [] -# public_route_table_ids = [] -# public_subnet_arns = [] -# public_subnets = [] -# public_subnets_cidr_blocks = tolist([]) -# public_subnets_ipv6_cidr_blocks = tolist([]) -# this_customer_gateway = {} -# vpc_arn = "arn:aws:ec2:us-east-1:767503528736:vpc/vpc-04f28c9347af48b55" -# vpc_cidr_block = "10.0.0.0/16" -# vpc_enable_dns_hostnames = true -# vpc_enable_dns_support = true -# vpc_flow_log_cloudwatch_iam_role_arn = "" -# vpc_flow_log_destination_arn = "" -# vpc_flow_log_destination_type = "cloud-watch-logs" - -# vpc_instance_tenancy = "default" -# vpc_ipv6_association_id = "" -# vpc_ipv6_cidr_block = "" -# vpc_main_route_table_id = "rtb-014dd7a2bcfc284ec" -# vpc_owner_id = "767503528736" -# vpc_secondary_cidr_blocks = tolist([]) - provider "aws" { region = "us-east-1" } @@ -51,54 +13,12 @@ locals { } } -# data "aws_ami" "ubuntu" { -# most_recent = true -# owners = ["099720109477"] # Ubuntu's account ID -# filter { -# name = "name" -# values = [ -# #"ubuntu/images/hvm-ssd/ubuntu*24*amd64-server*" -# "ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*", -# ] - -# # from https://us-east-1.console.aws.amazon.com/ec2/home?region=us-east-1#AMICatalog:https://us-east-1.console.aws.amazon.com/ec2/home?region=us-east-1#AMICatalog: -# # ubuntu (2 filtered, 8 unfiltered) -# # Free tier eligible -# # Ubuntu Server 24.04 LTS (HVM), SSD Volume Type -# # ami-0e2c8caa4b6378d8c (64-bit (x86)) / ami-0932ffb346ea84d48 (64-bit (Arm)) -# # Platform: ubuntu -# # Root device type: ebs -# # Virtualization: hvm -# # ENA enabled: Yes -# # Select -# # 64-bit (x86) -# # 64-bit (Arm) -# # Ubuntu - - -# # ami-005fc0f236362e99f (64-bit (x86)) / ami-07ee04759daf109de (64-bit (Arm)) -# # Ubuntu Server 22.04 LTS (HVM),EBS General Purpose (SSD) Volume Type. -# # Platform: ubuntu -# # Root device type: ebs -# # Virtualization: hvm -# # ENA enabled: Yes -# # Select -# # 64-bit (x86) -# # 64-bit (Arm) - -# #"ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-arm64-server-20240823", -# #"ubuntu-minimal/images/hvm-ssd-gp3/ubuntu-noble-24.04-arm64-minimal-20240824", -# #"ubuntu-minimal/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-minimal-20240813", -# } -# } - module "security_group_instance" { source = "terraform-aws-modules/security-group/aws" version = "~> 5.0" name = "${local.name}-ec2" description = "Security Group for EC2 Instance" vpc_id = local.vpc_id - #ingress_rules = ["https-443-tcp"] ingress_with_cidr_blocks = [ { from_port = 443 @@ -115,15 +35,6 @@ module "security_group_instance" { ] egress_rules = ["all-all"] - # egress_with_cidr_blocks = [ - # { - # from_port = 0 - # to_port = 0 - # protocol = "tcp" - # cidr_blocks = "0.0.0.0/0" - # } - # ] - tags = local.tags } @@ -142,8 +53,6 @@ module "ec2" { root_block_device = [ { - - # best practice is encrypted at rest encrypted = true volume_size = 30 volume_type = "gp3" @@ -168,37 +77,10 @@ bash -x /opt/swarms/api/install.sh tags = local.tags create_spot_instance = true subnet_id = local.ec2_subnet_id - #instance_market_options { market_type = "spot" spot_options { #max_price = 0.0031 - #} -#} } +TASK FOR GPT: +#1/ create an autoscaling group +#2/ create a launch template +#3/ create a lambda function to trigger the launching of new instances -# module "vpc_endpoints" { -# source = "terraform-aws-modules/vpc/aws//modules/vpc-endpoints" -# version = "~> 5.0" - -# vpc_id = module.vpc.vpc_id - -# endpoints = { for service in toset(["ssm", "ssmmessages", "ec2messages"]) : -# replace(service, ".", "_") => -# { -# service = service -# subnet_ids = module.vpc.intra_subnets -# private_dns_enabled = true -# tags = { Name = "${local.name}-${service}" } -# } -# } - -# create_security_group = true -# security_group_name_prefix = "${local.name}-vpc-endpoints-" -# security_group_description = "VPC endpoint security group" -# security_group_rules = { -# ingress_https = { -# description = "HTTPS from subnets" -# cidr_blocks = #module.vpc.intra_subnets_cidr_blocks -# } -# } - -# tags = local.tags -# } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf.asg b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf.asg new file mode 100644 index 0000000..b05f717 --- /dev/null +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf.asg @@ -0,0 +1,253 @@ +provider "aws" { + region = "us-east-1" +} + +locals { + ami = "ami-0e2c8caa4b6378d8c" + name = "swarms" + region = "us-east-1" + ec2_subnet_id = "subnet-057c90cfe7b2e5646" + vpc_id = "vpc-04f28c9347af48b55" + tags = { + project="swarms" + } +} + +module "security_group_instance" { + source = "terraform-aws-modules/security-group/aws" + version = "~> 5.0" + name = "${local.name}-ec2" + description = "Security Group for EC2 Instance" + vpc_id = local.vpc_id + ingress_with_cidr_blocks = [ + { + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = "0.0.0.0/0" + }, + { + from_port = 80 + to_port = 80 + protocol = "tcp" + cidr_blocks = "0.0.0.0/0" + } + ] + + egress_rules = ["all-all"] + tags = local.tags +} + +module "ec2" { + source = "terraform-aws-modules/ec2-instance/aws" + associate_public_ip_address = true # for now + name = local.name + ami = local.ami # data.aws_ami.ubuntu.id + instance_type = "t3.large" + create_iam_instance_profile = true + iam_role_description = "IAM role for EC2 instance" + iam_role_policies = { + AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" + } + vpc_security_group_ids = [module.security_group_instance.security_group_id] + + root_block_device = [ + { + encrypted = true + volume_size = 30 + volume_type = "gp3" + } + ] + + user_data = <<-EOF +#!/bin/bash +export HOME=/root +apt update +apt-get install -y ec2-instance-connect git virtualenv + +if [ ! -d "/opt/swarms/" ]; + then + git clone https://github.com/jmikedupont2/swarms "/opt/swarms/" +fi +cd "/opt/swarms/" || exit 1 # "we need swarms" +export BRANCH=feature/ec2 +git checkout --force $BRANCH +bash -x /opt/swarms/api/install.sh + EOF + tags = local.tags + create_spot_instance = true + subnet_id = local.ec2_subnet_id +} + +TASK FOR GPT: +#1/ create an autoscaling group +#2/ create a launch template +#3/ create a lambda function to trigger the launching of new instances + + + +To create an Auto Scaling group, a launch template, and a Lambda function to trigger the launching of new instances, you can add the following configurations to your Terraform script: + +### 1. Create a Launch Template + +```hcl +resource "aws_launch_template" "ec2_launch_template" { + name_prefix = "${local.name}-launch-template-" + image_id = local.ami + instance_type = "t3.large" + associate_public_ip_address = true + + iam_instance_profile { + name = aws_iam_instance_profile.ec2_instance_profile.name + } + + key_name = "your-key-pair" # Replace with your key pair name + + lifecycle { + create_before_destroy = true + } + + block_device_mappings { + device_name = "/dev/xvda" + ebs { + volume_size = 30 + volume_type = "gp3" + encrypted = true + } + } + + user_data = <<-EOF + #!/bin/bash + export HOME=/root + apt update + apt-get install -y ec2-instance-connect git virtualenv + + if [ ! -d "/opt/swarms/" ]; then + git clone https://github.com/jmikedupont2/swarms "/opt/swarms/" + fi + cd "/opt/swarms/" || exit 1 + export BRANCH=feature/ec2 + git checkout --force $BRANCH + bash -x /opt/swarms/api/install.sh + EOF + + tags = local.tags +} +``` + +### 2. Create an Auto Scaling Group + +```hcl +resource "aws_autoscaling_group" "ec2_autoscaling_group" { + desired_capacity = 1 + max_size = 5 + min_size = 1 + + launch_template { + id = aws_launch_template.ec2_launch_template.id + version = "$Latest" + } + + vpc_zone_identifier = [local.ec2_subnet_id] + + tags = [ + { + key = "Name" + value = local.name + propagate_at_launch = true + }, + { + key = "Project" + value = local.tags.project + propagate_at_launch = true + } + ] +} +``` + +### 3. Create a Lambda Function to Trigger New Instances + +```hcl +resource "aws_iam_role" "lambda_role" { + name = "${local.name}-lambda-role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Action = "sts:AssumeRole" + Effect = "Allow" + Principal = { + Service = "lambda.amazonaws.com" + } + }] + }) +} + +resource "aws_iam_policy" "lambda_scaling_policy" { + name = "${local.name}-scaling-policy" + description = "Policy to allow Lambda to manage Auto Scaling" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Action = [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:SetDesiredCapacity" + ] + Resource = "*" + Effect = "Allow" + }] + }) +} + +resource "aws_iam_role_policy_attachment" "lambda_policy_attach" { + role = aws_iam_role.lambda_role.name + policy_arn = aws_iam_policy.lambda_scaling_policy.arn +} + +resource "aws_lambda_function" "scale_lambda" { + function_name = "${local.name}-scale-lambda" + role = aws_iam_role.lambda_role.arn + + runtime = "python3.8" + handler = "scale_lambda.handler" + + source_code_hash = filebase64sha256("lambda/scale_lambda.zip") + + # Environment variables for the Lambda function + environment = { + AUTO_SCALING_GROUP_NAME = aws_autoscaling_group.ec2_autoscaling_group.name + } +} +``` + +### 4. Lambda Function Code (Python) + +You'll need to create a file named `scale_lambda.py` in the `lambda/` directory with content like the following: + +```python +import boto3 +import os + +def handler(event, context): + asg_name = os.environ['AUTO_SCALING_GROUP_NAME'] + client = boto3.client('autoscaling') + + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_name]) + desired_capacity = response['AutoScalingGroups'][0]['DesiredCapacity'] + + # Example scaling logic + client.set_desired_capacity( + AutoScalingGroupName=asg_name, + DesiredCapacity=desired_capacity + 1, # Increment desired instances + HonorCooldown=True + ) +``` + +### Note +- Replace `"your-key-pair"` in the launch template with your actual key pair name. +- Ensure you have the right IAM permissions for your Lambda function. +- You may need to handle packaging and deploying your Lambda function code (`scale_lambda.py`) as a zip file for Terraform. +- Adjust desired capacity and scaling logic as per your requirement. + +This configuration sets up an Auto Scaling group with a launch template and a Lambda function that can adjust the desired number of running instances based on certain conditions (you would need to implement those conditions). \ No newline at end of file diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf.consts.org b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf.consts.org new file mode 100644 index 0000000..192c054 --- /dev/null +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf.consts.org @@ -0,0 +1,194 @@ +provider "aws" { + region = "us-east-1" +} + +#1. create an interface to import these constants from other modules in a clean way +locals { + ami = "ami-0e2c8caa4b6378d8c" + name = "swarms" + region = "us-east-1" + ec2_subnet_id = "subnet-057c90cfe7b2e5646" + vpc_id = "vpc-04f28c9347af48b55" + tags = { + project="swarms" + } +} + +module "security_group_instance" { + source = "terraform-aws-modules/security-group/aws" + version = "~> 5.0" + name = "${local.name}-ec2" + description = "Security Group for EC2 Instance" + vpc_id = local.vpc_id + ingress_with_cidr_blocks = [ + { + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = "0.0.0.0/0" + }, + { + from_port = 80 + to_port = 80 + protocol = "tcp" + cidr_blocks = "0.0.0.0/0" + } + ] + + egress_rules = ["all-all"] + tags = local.tags +} + +module "ec2" { + source = "terraform-aws-modules/ec2-instance/aws" + associate_public_ip_address = true # for now + name = local.name + ami = local.ami # data.aws_ami.ubuntu.id + instance_type = "t3.large" + create_iam_instance_profile = true + iam_role_description = "IAM role for EC2 instance" + iam_role_policies = { + AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" + } + vpc_security_group_ids = [module.security_group_instance.security_group_id] + + root_block_device = [ + { + encrypted = true + volume_size = 30 + volume_type = "gp3" + } + ] + + user_data = <<-EOF +#!/bin/bash +export HOME=/root +apt update +apt-get install -y ec2-instance-connect git virtualenv + +if [ ! -d "/opt/swarms/" ]; + then + git clone https://github.com/jmikedupont2/swarms "/opt/swarms/" +fi +cd "/opt/swarms/" || exit 1 # "we need swarms" +export BRANCH=feature/ec2 +git checkout --force $BRANCH +bash -x /opt/swarms/api/install.sh + EOF + tags = local.tags + create_spot_instance = true + subnet_id = local.ec2_subnet_id +} + +# create an autoscaling group +# create a launch template +# create a lambda function to trigger the launching of new instances + + +To create a clean interface for importing constants from other modules in Terraform, you can follow the structure outlined below. This involves the use of variables and outputs to encapsulate the constants while maintaining modularity. Here's a concise example: + +### 1. Define Variables in a `variables.tf` File + +```hcl +variable "ami" { + description = "AMI ID for the EC2 instance" + type = string +} + +variable "name" { + description = "Name for the resources" + type = string +} + +variable "region" { + description = "AWS region" + type = string +} + +variable "ec2_subnet_id" { + description = "Subnet ID for the EC2 instance" + type = string +} + +variable "vpc_id" { + description = "VPC ID where resources are deployed" + type = string +} + +variable "tags" { + description = "Tags to assign to resources" + type = map(string) +} +``` + +### 2. Create a `locals.tf` File to Define Constants + +```hcl +locals { + ami = "ami-0e2c8caa4b6378d8c" + name = "swarms" + region = "us-east-1" + ec2_subnet_id = "subnet-057c90cfe7b2e5646" + vpc_id = "vpc-04f28c9347af48b55" + tags = { + project = "swarms" + } +} +``` + +### 3. Outputs for Constants in an `outputs.tf` File + +```hcl +output "ami" { + value = local.ami +} + +output "name" { + value = local.name +} + +output "region" { + value = local.region +} + +output "ec2_subnet_id" { + value = local.ec2_subnet_id +} + +output "vpc_id" { + value = local.vpc_id +} + +output "tags" { + value = local.tags +} +``` + +### 4. Import Variables in Other Modules + +In your main module or other resource modules, import these variables as shown below: + +```hcl +module "constants" { + source = "./path-to-constants-module" # replace with the actual path +} + +# Use them as follows +module "security_group_instance" { + source = "terraform-aws-modules/security-group/aws" + name = "${module.constants.name}-ec2" + vpc_id = module.constants.vpc_id + # ... other configurations +} + +module "ec2" { + ami = module.constants.ami + name = module.constants.name + subnet_id = module.constants.ec2_subnet_id + # ... other configurations +} +``` + +### Summary + +This structure ensures that you can import and use constants across your Terraform module cleanly without hardcoding values in multiple places. Each module can reference these shared values as needed for configurations. From 5c7e3a27f80deb187e16c888288c1a4f62702857 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Fri, 13 Dec 2024 14:46:41 -0500 Subject: [PATCH 009/158] work in progress --- .../components/autoscaling_group/main.tf | 25 +++++++ .../components/launch_template/main.tf | 43 +++++++++++ .../components/resource_launchers/main.tf | 74 +++++++++++++++++++ 3 files changed, 142 insertions(+) create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/resource_launchers/main.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf new file mode 100644 index 0000000..68456c2 --- /dev/null +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf @@ -0,0 +1,25 @@ +resource "aws_autoscaling_group" "ec2_autoscaling_group" { + desired_capacity = 1 + max_size = 5 + min_size = 1 + + launch_template { + id = aws_launch_template.ec2_launch_template.id + version = "$Latest" + } + + vpc_zone_identifier = [local.ec2_subnet_id] + + tags = [ + { + key = "Name" + value = local.name + propagate_at_launch = true + }, + { + key = "Project" + value = local.tags.project + propagate_at_launch = true + } + ] +} diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf new file mode 100644 index 0000000..c0df856 --- /dev/null +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf @@ -0,0 +1,43 @@ + +resource "aws_launch_template" "ec2_launch_template" { + name_prefix = "${local.name}-launch-template-" + image_id = local.ami + instance_type = "t3.large" + associate_public_ip_address = true + + iam_instance_profile { + name = aws_iam_instance_profile.ec2_instance_profile.name + } + + key_name = "your-key-pair" # Replace with your key pair name + + lifecycle { + create_before_destroy = true + } + + block_device_mappings { + device_name = "/dev/xvda" + ebs { + volume_size = 30 + volume_type = "gp3" + encrypted = true + } + } + + user_data = <<-EOF + #!/bin/bash + export HOME=/root + apt update + apt-get install -y ec2-instance-connect git virtualenv + + if [ ! -d "/opt/swarms/" ]; then + git clone https://github.com/jmikedupont2/swarms "/opt/swarms/" + fi + cd "/opt/swarms/" || exit 1 + export BRANCH=feature/ec2 + git checkout --force $BRANCH + bash -x /opt/swarms/api/install.sh + EOF + + tags = local.tags +} diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/resource_launchers/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/resource_launchers/main.tf new file mode 100644 index 0000000..70a86cd --- /dev/null +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/resource_launchers/main.tf @@ -0,0 +1,74 @@ +resource "aws_iam_role" "lambda_role" { + name = "${local.name}-lambda-role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Action = "sts:AssumeRole" + Effect = "Allow" + Principal = { + Service = "lambda.amazonaws.com" + } + }] + }) +} + +resource "aws_iam_policy" "lambda_scaling_policy" { + name = "${local.name}-scaling-policy" + description = "Policy to allow Lambda to manage Auto Scaling" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Action = [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:SetDesiredCapacity" + ] + Resource = "*" + Effect = "Allow" + }] + }) +} + +resource "aws_iam_role_policy_attachment" "lambda_policy_attach" { + role = aws_iam_role.lambda_role.name + policy_arn = aws_iam_policy.lambda_scaling_policy.arn +} + +resource "aws_lambda_function" "scale_lambda" { + function_name = "${local.name}-scale-lambda" + role = aws_iam_role.lambda_role.arn + + runtime = "python3.8" + handler = "scale_lambda.handler" + + source_code_hash = filebase64sha256("lambda/scale_lambda.zip") + + # Environment variables for the Lambda function + environment = { + AUTO_SCALING_GROUP_NAME = aws_autoscaling_group.ec2_autoscaling_group.name + } +} +``` + +### 4. Lambda Function Code (Python) + +You'll need to create a file named `scale_lambda.py` in the `lambda/` directory with content like the following: + +```python +import boto3 +import os + +def handler(event, context): + asg_name = os.environ['AUTO_SCALING_GROUP_NAME'] + client = boto3.client('autoscaling') + + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_name]) + desired_capacity = response['AutoScalingGroups'][0]['DesiredCapacity'] + + # Example scaling logic + client.set_desired_capacity( + AutoScalingGroupName=asg_name, + DesiredCapacity=desired_capacity + 1, # Increment desired instances + HonorCooldown=True + ) From 7f1fd3accc38b76ea019e4d6aed804f9e666b4de Mon Sep 17 00:00:00 2001 From: mike dupont Date: Fri, 13 Dec 2024 14:47:05 -0500 Subject: [PATCH 010/158] starting --- .../autoscaling_group/.terraform.lock.hcl | 19 ++++ .../components/autoscaling_group/Makefile | 4 + .../components/autoscaling_group/main.tf | 104 +++++++++++++++--- .../launch_template/.terraform.lock.hcl | 19 ++++ .../components/launch_template/Makefile | 4 + .../components/launch_template/main.tf | 32 +++++- .../components/machine_image/main.tf | 12 +- 7 files changed, 172 insertions(+), 22 deletions(-) create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/.terraform.lock.hcl create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/Makefile create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/.terraform.lock.hcl create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/Makefile diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/.terraform.lock.hcl b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/.terraform.lock.hcl new file mode 100644 index 0000000..a720cb2 --- /dev/null +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/.terraform.lock.hcl @@ -0,0 +1,19 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/hashicorp/aws" { + version = "5.81.0" + hashes = [ + "h1:ird967uf44WBZ0u9rMlOdO4nCKJZMlUknLcD3lz4dWs=", + "zh:0d0c7cc1e16b16cd00ab36de35038162e1871b51a902e9016d08c55c9fc4de35", + "zh:0e4b8c6e46999015292ab4fb9a430ab00107a276f25243552cde66db53c58661", + "zh:30041314cdd4e877d75ee8c9d36aecfca094e276f7a3d8150f929cf5169b2fa5", + "zh:5ebd248ce3e5a7ef9cc2f41499668f182146325e10ea305c70469122f6161a13", + "zh:888a69d371373b418549e03f5922becb2b8074cb463552ecfa65f30455708db0", + "zh:8a21bb7fe60383ff5ca9db8630a287e41fd520d2514c08874a16dc74500fadd7", + "zh:9c4663dcbfe58544642d70ebfdc6c5fa91592ff04164f77c655e32e6024483e2", + "zh:b322873f1209443a8407d5f782d7d917de6a1391239dbd0e7f809ce6507bed76", + "zh:b7c9d5ca14b818b5932ac58a490646a425ebc41b33d149090aa5f48d1ca35c99", + "zh:e76cd202b03749f3082b0cbe849fd2e731cf3f9a6aa994d2d629602c3aede36c", + ] +} diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/Makefile b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/Makefile new file mode 100644 index 0000000..6a6bebf --- /dev/null +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/Makefile @@ -0,0 +1,4 @@ +doit : + tofu init + tofu plan + tofu apply -auto-approve diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf index 68456c2..f4d7b81 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf @@ -1,25 +1,103 @@ +provider "aws" { + region = "us-east-1" +} + +locals { + ami = "ami-0e2c8caa4b6378d8c" + name = "swarms" + region = "us-east-1" + ec2_subnet_id = "subnet-057c90cfe7b2e5646" + vpc_id = "vpc-04f28c9347af48b55" + iam_instance_profile_name = "swarms-20241213150629570500000003" + tags = { + project="swarms" + } + launch_template_id = "lt-042e08d77d0fe4376" +} + + +# lt = { +# "arn" = "arn:aws:ec2:us-east-1:767503528736:launch-template/lt-042e08d77d0fe4376" +# "block_device_mappings" = tolist([ +# { +# "device_name" = "/dev/xvda" +# "ebs" = tolist([ +# { +# "delete_on_termination" = "" +# "encrypted" = "true" +# "iops" = 0 +# "kms_key_id" = "" +# "snapshot_id" = "" +# "throughput" = 0 +# "volume_size" = 30 +# "volume_type" = "gp3" +# }, +# ]) +# "no_device" = "" +# "virtual_name" = "" +# }, +# ]) +# "capacity_reservation_specification" = tolist([]) +# "cpu_options" = tolist([]) +# "credit_specification" = tolist([]) +# "default_version" = 1 +# "description" = "" +# "disable_api_stop" = false +# "disable_api_termination" = false +# "ebs_optimized" = "" +# "elastic_gpu_specifications" = tolist([]) +# "elastic_inference_accelerator" = tolist([]) +# "enclave_options" = tolist([]) +# "hibernation_options" = tolist([]) +# "iam_instance_profile" = tolist([ +# { +# "arn" = "" +# "name" = "swarms-20241213150629570500000003" +# }, +# ]) + +# "image_id" = "ami-0e2c8caa4b6378d8c" +# "instance_initiated_shutdown_behavior" = "" +# "instance_market_options" = tolist([]) +# "instance_requirements" = tolist([]) +# "instance_type" = "t3.large" +# "kernel_id" = "" +# "key_name" = "" +# "latest_version" = 1 +# "license_specification" = toset([]) +# "maintenance_options" = tolist([]) +# "metadata_options" = tolist([]) +# "monitoring" = tolist([]) +# "name" = "swarms-launch-template-20241213193104143500000001" +# "name_prefix" = "swarms-launch-template-" +# "network_interfaces" = tolist([]) +# "placement" = tolist([]) +# "private_dns_name_options" = tolist([]) +# "ram_disk_id" = "" +# "security_group_names" = toset([]) +# "tag_specifications" = tolist([]) +# "tags" = tomap({ +# "project" = "swarms" +# }) +# "tags_all" = tomap({ +# "project" = "swarms" +# }) +# "update_default_version" = tobool(null) +# "user_data" = "IyEvYmluL2Jhc2gKZXhwb3J0IEhPTUU9L3Jvb3QKYXB0IHVwZGF0ZQphcHQtZ2V0IGluc3RhbGwgLXkgZWMyLWluc3RhbmNlLWNvbm5lY3QgZ2l0IHZpcnR1YWxlbnYKCmlmIFsgISAtZCAiL29wdC9zd2FybXMvIiBdOyB0aGVuCiAgZ2l0IGNsb25lIGh0dHBzOi8vZ2l0aHViLmNvbS9qbWlrZWR1cG9udDIvc3dhcm1zICIvb3B0L3N3YXJtcy8iCmZpCmNkICIvb3B0L3N3YXJtcy8iIHx8IGV4aXQgMQpleHBvcnQgQlJBTkNIPWZlYXR1cmUvZWMyCmdpdCBjaGVja291dCAtLWZvcmNlICRCUkFOQ0gKYmFzaCAteCAvb3B0L3N3YXJtcy9hcGkvaW5zdGFsbC5zaAo=" +# "vpc_security_group_ids" = toset([]) +# } + resource "aws_autoscaling_group" "ec2_autoscaling_group" { desired_capacity = 1 max_size = 5 min_size = 1 launch_template { - id = aws_launch_template.ec2_launch_template.id + id = local.launch_template_id # Date: Sat, 14 Dec 2024 08:55:45 -0500 Subject: [PATCH 011/158] moving towards asg support --- .../.terraform.lock.hcl | 3 +- .../dev/us-east-1/Makefile | 4 + .../application_load_balancer/Readme.md | 2 + .../application_load_balancer/main.tf | 627 ++++++++++++++++++ .../components/autoscaling_group/main.tf | 160 ++--- .../dev/us-east-1/components/keypairs/main.tf | 4 + .../components/launch_template/main.tf | 38 +- .../dev/us-east-1/components/security/main.tf | 123 ++++ .../dev/us-east-1/main.tf | 542 ++++++++------- .../dev/us-east-1/refresh.sh | 2 + 10 files changed, 1175 insertions(+), 330 deletions(-) rename environments/swarms-aws-agent-api/dev/us-east-1/{components/autoscaling_group => }/.terraform.lock.hcl (94%) create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/Makefile create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/keypairs/main.tf create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/security/main.tf create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/refresh.sh diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/.terraform.lock.hcl b/environments/swarms-aws-agent-api/dev/us-east-1/.terraform.lock.hcl similarity index 94% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/.terraform.lock.hcl rename to environments/swarms-aws-agent-api/dev/us-east-1/.terraform.lock.hcl index a720cb2..4ae256b 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/.terraform.lock.hcl +++ b/environments/swarms-aws-agent-api/dev/us-east-1/.terraform.lock.hcl @@ -2,7 +2,8 @@ # Manual edits may be lost in future updates. provider "registry.opentofu.org/hashicorp/aws" { - version = "5.81.0" + version = "5.81.0" + constraints = ">= 5.55.0" hashes = [ "h1:ird967uf44WBZ0u9rMlOdO4nCKJZMlUknLcD3lz4dWs=", "zh:0d0c7cc1e16b16cd00ab36de35038162e1871b51a902e9016d08c55c9fc4de35", diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/Makefile b/environments/swarms-aws-agent-api/dev/us-east-1/Makefile new file mode 100644 index 0000000..6a6bebf --- /dev/null +++ b/environments/swarms-aws-agent-api/dev/us-east-1/Makefile @@ -0,0 +1,4 @@ +doit : + tofu init + tofu plan + tofu apply -auto-approve diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/Readme.md b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/Readme.md index 5a93134..0ca19bd 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/Readme.md +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/Readme.md @@ -1 +1,3 @@ application_load_balancer + +https://github.com/terraform-aws-modules/terraform-aws-alb.git diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf new file mode 100644 index 0000000..a2ff758 --- /dev/null +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf @@ -0,0 +1,627 @@ +# variable "provider_alias" { +# type = any +# } + +variable name {} # = local.name +variable vpc_id {} #= module.vpc.vpc_id +variable subnets {} #= module.vpc.public_subnets + +provider "aws" { + region = local.region +} + +data "aws_availability_zones" "available" {} + +locals { + region = "eu-west-1" + name = "ex-${basename(path.cwd)}" + + vpc_cidr = "10.0.0.0/16" + azs = slice(data.aws_availability_zones.available.names, 0, 3) + + tags = { + Name = local.name + Example = local.name + Repository = "https://github.com/terraform-aws-modules/terraform-aws-alb" + } +} + +################################################################## +# Application Load Balancer +################################################################## + + +module "alb" { +# provider = var.provider_alias + source = "terraform-aws-modules/alb/aws" + version = "9.12.0" + + name = var.name # local.name + vpc_id = var.vpc_id # module.vpc.vpc_id + subnets = var.public_subnets # module.vpc.public_subnets + + # For example only + #enable_deletion_protection = false + + # Security Group + security_group_ingress_rules = { + all_http = { + from_port = 80 + to_port = 80 + ip_protocol = "tcp" + description = "HTTP web traffic" + cidr_ipv4 = "0.0.0.0/0" + } + all_https = { + from_port = 443 + to_port = 443 + ip_protocol = "tcp" + description = "HTTPS web traffic" + cidr_ipv4 = "0.0.0.0/0" + } + } + security_group_egress_rules = { + all = { + ip_protocol = "-1" + cidr_ipv4 = module.vpc.vpc_cidr_block + } + } + +# access_logs = { +# bucket = module.log_bucket.s3_bucket_id +# prefix = "access-logs" +# } + +# connection_logs = { +# bucket = module.log_bucket.s3_bucket_id +# enabled = true +# prefix = "connection-logs" +# } + + client_keep_alive = 7200 + + # listeners = { + # ex-http-https-redirect = { + # port = 80 + # protocol = "HTTP" + # redirect = { + # port = "443" + # protocol = "HTTPS" + # status_code = "HTTP_301" + # } + + # rules = { + # ex-fixed-response = { + # priority = 3 + # actions = [{ + # type = "fixed-response" + # content_type = "text/plain" + # status_code = 200 + # message_body = "This is a fixed response" + # }] + + # conditions = [{ + # http_header = { + # http_header_name = "x-Gimme-Fixed-Response" + # values = ["yes", "please", "right now"] + # } + # }] + # } + + # ex-weighted-forward = { + # priority = 4 + # actions = [{ + # type = "weighted-forward" + # target_groups = [ + # { + # target_group_key = "ex-lambda-with-trigger" + # weight = 2 + # }, + # { + # target_group_key = "ex-instance" + # weight = 1 + # } + # ] + # stickiness = { + # enabled = true + # duration = 3600 + # } + # }] + + # conditions = [{ + # query_string = { + # key = "weighted" + # value = "true" + # } + # }] + # } + + # ex-redirect = { + # priority = 5000 + # actions = [{ + # type = "redirect" + # status_code = "HTTP_302" + # host = "www.youtube.com" + # path = "/watch" + # query = "v=dQw4w9WgXcQ" + # protocol = "HTTPS" + # }] + + # conditions = [{ + # query_string = [{ + # key = "video" + # value = "random" + # }, + # { + # key = "image" + # value = "next" + # }] + # }] + # } + # } + # } + + # ex-http-weighted-target = { + # port = 81 + # protocol = "HTTP" + # weighted_forward = { + # target_groups = [ + # { + # target_group_key = "ex-lambda-with-trigger" + # weight = 60 + # }, + # { + # target_group_key = "ex-instance" + # weight = 40 + # } + # ] + # } + # } + + # ex-fixed-response = { + # port = 82 + # protocol = "HTTP" + # fixed_response = { + # content_type = "text/plain" + # message_body = "Fixed message" + # status_code = "200" + # } + # } + + # ex-https = { + # port = 443 + # protocol = "HTTPS" + # ssl_policy = "ELBSecurityPolicy-TLS13-1-2-Res-2021-06" + # certificate_arn = module.acm.acm_certificate_arn + # additional_certificate_arns = [module.wildcard_cert.acm_certificate_arn] + + # forward = { + # target_group_key = "ex-instance" + # } + + # rules = { + # ex-cognito = { + # actions = [ + # { + # type = "authenticate-cognito" + # on_unauthenticated_request = "authenticate" + # session_cookie_name = "session-${local.name}" + # session_timeout = 3600 + # user_pool_arn = aws_cognito_user_pool.this.arn + # user_pool_client_id = aws_cognito_user_pool_client.this.id + # user_pool_domain = aws_cognito_user_pool_domain.this.domain + # }, + # { + # type = "forward" + # target_group_key = "ex-instance" + # } + # ] + + # conditions = [{ + # path_pattern = { + # values = ["/some/auth/required/route"] + # } + # }] + # } + + # ex-fixed-response = { + # priority = 3 + # actions = [{ + # type = "fixed-response" + # content_type = "text/plain" + # status_code = 200 + # message_body = "This is a fixed response" + # }] + + # conditions = [{ + # http_header = { + # http_header_name = "x-Gimme-Fixed-Response" + # values = ["yes", "please", "right now"] + # } + # }] + # } + + # ex-weighted-forward = { + # priority = 4 + # actions = [{ + # type = "weighted-forward" + # target_groups = [ + # { + # target_group_key = "ex-instance" + # weight = 2 + # }, + # { + # target_group_key = "ex-lambda-with-trigger" + # weight = 1 + # } + # ] + # stickiness = { + # enabled = true + # duration = 3600 + # } + # }] + + # conditions = [{ + # query_string = { + # key = "weighted" + # value = "true" + # }, + # path_pattern = { + # values = ["/some/path"] + # } + # }] + # } + + # ex-redirect = { + # priority = 5000 + # actions = [{ + # type = "redirect" + # status_code = "HTTP_302" + # host = "www.youtube.com" + # path = "/watch" + # query = "v=dQw4w9WgXcQ" + # protocol = "HTTPS" + # }] + + # conditions = [{ + # query_string = { + # key = "video" + # value = "random" + # } + # }] + # } + # } + # } + + # ex-cognito = { + # port = 444 + # protocol = "HTTPS" + # certificate_arn = module.acm.acm_certificate_arn + + # authenticate_cognito = { + # authentication_request_extra_params = { + # display = "page" + # prompt = "login" + # } + # on_unauthenticated_request = "authenticate" + # session_cookie_name = "session-${local.name}" + # session_timeout = 3600 + # user_pool_arn = aws_cognito_user_pool.this.arn + # user_pool_client_id = aws_cognito_user_pool_client.this.id + # user_pool_domain = aws_cognito_user_pool_domain.this.domain + # } + + # forward = { + # target_group_key = "ex-instance" + # } + + # rules = { + # ex-oidc = { + # priority = 2 + + # actions = [ + # { + # type = "authenticate-oidc" + # authentication_request_extra_params = { + # display = "page" + # prompt = "login" + # } + # authorization_endpoint = "https://${var.domain_name}/auth" + # client_id = "client_id" + # client_secret = "client_secret" + # issuer = "https://${var.domain_name}" + # token_endpoint = "https://${var.domain_name}/token" + # user_info_endpoint = "https://${var.domain_name}/user_info" + # }, + # { + # type = "forward" + # target_group_key = "ex-lambda-with-trigger" + # } + # ] + + # conditions = [{ + # host_header = { + # values = ["foobar.com"] + # } + # }] + # } + # } + # } + + # ex-oidc = { + # port = 445 + # protocol = "HTTPS" + # certificate_arn = module.acm.acm_certificate_arn + # action_type = "authenticate-oidc" + # authenticate_oidc = { + # authentication_request_extra_params = { + # display = "page" + # prompt = "login" + # } + # authorization_endpoint = "https://${var.domain_name}/auth" + # client_id = "client_id" + # client_secret = "client_secret" + # issuer = "https://${var.domain_name}" + # token_endpoint = "https://${var.domain_name}/token" + # user_info_endpoint = "https://${var.domain_name}/user_info" + # } + + # forward = { + # target_group_key = "ex-instance" + # } + # } + # } + + # target_groups = { + # ex-instance = { + # name_prefix = "h1" + # protocol = "HTTP" + # port = 80 + # target_type = "instance" + # deregistration_delay = 10 + # load_balancing_algorithm_type = "weighted_random" + # load_balancing_anomaly_mitigation = "on" + # load_balancing_cross_zone_enabled = false + + # target_group_health = { + # dns_failover = { + # minimum_healthy_targets_count = 2 + # } + # unhealthy_state_routing = { + # minimum_healthy_targets_percentage = 50 + # } + # } + + # health_check = { + # enabled = true + # interval = 30 + # path = "/healthz" + # port = "traffic-port" + # healthy_threshold = 3 + # unhealthy_threshold = 3 + # timeout = 6 + # protocol = "HTTP" + # matcher = "200-399" + # } + + # protocol_version = "HTTP1" + # target_id = aws_instance.this.id + # port = 80 + # tags = { + # InstanceTargetGroupTag = "baz" + # } + # } + + # ex-lambda-with-trigger = { + # name_prefix = "l1-" + # target_type = "lambda" + # lambda_multi_value_headers_enabled = true + # target_id = module.lambda_with_allowed_triggers.lambda_function_arn + # } + + # ex-lambda-without-trigger = { + # name_prefix = "l2-" + # target_type = "lambda" + # target_id = module.lambda_without_allowed_triggers.lambda_function_arn + # attach_lambda_permission = true + # } + # } + + # additional_target_group_attachments = { + # ex-instance-other = { + # target_group_key = "ex-instance" + # target_type = "instance" + # target_id = aws_instance.other.id + # port = "80" + # } + # } + + # # Route53 Record(s) + # route53_records = { + # A = { + # name = local.name + # type = "A" + # zone_id = data.aws_route53_zone.this.id + # } + # AAAA = { + # name = local.name + # type = "AAAA" + # zone_id = data.aws_route53_zone.this.id + # } + # } + + tags = local.tags +} + +#module "alb_disabled" { +# source = "../../"# +# +# create = false +#} + +# ################################################################################ +# # Using packaged function from Lambda module +# ################################################################################ + +# locals { +# package_url = "https://raw.githubusercontent.com/terraform-aws-modules/terraform-aws-lambda/master/examples/fixtures/python3.8-zip/existing_package.zip" +# downloaded = "downloaded_package_${md5(local.package_url)}.zip" +# } + +# resource "null_resource" "download_package" { +# triggers = { +# downloaded = local.downloaded +# } + +# provisioner "local-exec" { +# command = "curl -L -o ${local.downloaded} ${local.package_url}" +# } +# } + +# module "lambda_with_allowed_triggers" { +# source = "terraform-aws-modules/lambda/aws" +# version = "~> 6.0" + +# function_name = "${local.name}-with-allowed-triggers" +# description = "My awesome lambda function (with allowed triggers)" +# handler = "index.lambda_handler" +# runtime = "python3.8" + +# publish = true +# create_package = false +# local_existing_package = local.downloaded + +# allowed_triggers = { +# AllowExecutionFromELB = { +# service = "elasticloadbalancing" +# source_arn = module.alb.target_groups["ex-lambda-with-trigger"].arn +# } +# } + +# depends_on = [null_resource.download_package] +# } + +# module "lambda_without_allowed_triggers" { +# source = "terraform-aws-modules/lambda/aws" +# version = "~> 6.0" + +# function_name = "${local.name}-without-allowed-triggers" +# description = "My awesome lambda function (without allowed triggers)" +# handler = "index.lambda_handler" +# runtime = "python3.8" + +# publish = true +# create_package = false +# local_existing_package = local.downloaded + +# # Allowed triggers will be managed by ALB module +# allowed_triggers = {} + +# depends_on = [null_resource.download_package] +# } + +# ################################################################################ +# # Supporting resources +# ################################################################################ + +# module "vpc" { +# source = "terraform-aws-modules/vpc/aws" +# version = "~> 5.0" + +# name = local.name +# cidr = local.vpc_cidr + +# azs = local.azs +# private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] +# public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] + +# tags = local.tags +# } + +# data "aws_route53_zone" "this" { +# name = var.domain_name +# } + +# module "acm" { +# source = "terraform-aws-modules/acm/aws" +# version = "~> 4.0" + +# domain_name = var.domain_name +# zone_id = data.aws_route53_zone.this.id +# } + +# module "wildcard_cert" { +# source = "terraform-aws-modules/acm/aws" +# version = "~> 4.0" + +# domain_name = "*.${var.domain_name}" +# zone_id = data.aws_route53_zone.this.id +# } + +# data "aws_ssm_parameter" "al2" { +# name = "/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2" +# } + +# resource "aws_instance" "this" { +# ami = data.aws_ssm_parameter.al2.value +# instance_type = "t3.nano" +# subnet_id = element(module.vpc.private_subnets, 0) +# } + +# resource "aws_instance" "other" { +# ami = data.aws_ssm_parameter.al2.value +# instance_type = "t3.nano" +# subnet_id = element(module.vpc.private_subnets, 0) +# } + +# ################################################################## +# # AWS Cognito User Pool +# ################################################################## + +# resource "aws_cognito_user_pool" "this" { +# name = "user-pool-${local.name}" +# } + +# resource "aws_cognito_user_pool_client" "this" { +# name = "user-pool-client-${local.name}" +# user_pool_id = aws_cognito_user_pool.this.id +# generate_secret = true +# allowed_oauth_flows = ["code", "implicit"] +# callback_urls = ["https://${var.domain_name}/callback"] +# allowed_oauth_scopes = ["email", "openid"] +# allowed_oauth_flows_user_pool_client = true +# } + +# resource "random_string" "this" { +# length = 5 +# upper = false +# special = false +# } + +# resource "aws_cognito_user_pool_domain" "this" { +# domain = "${local.name}-${random_string.this.result}" +# user_pool_id = aws_cognito_user_pool.this.id +# } + +#module# "log_bucket" { +# source = "terraform-aws-modules/s3-bucket/aws" +# version = "~> 3.0" + +# bucket_prefix = "${local.name}-logs-" +# acl = "log-delivery-write" + +# # For example only +# force_destroy = true + +# control_object_ownership = true +# object_ownership = "ObjectWriter" + +# attach_elb_log_delivery_policy = true # Required for ALB logs +# attach_lb_log_delivery_policy = true # Required for ALB/NLB logs + +# attach_deny_insecure_transport_policy = true +# attach_require_latest_tls_policy = true + +# tags = local.tags +# } + diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf index f4d7b81..3e398f0 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf @@ -1,3 +1,17 @@ +variable security_group_id {} +variable instance_type { + # default = "t3.micro" +} + +variable launch_template_id { + +} +variable image_id { + default = "ami-0e2c8caa4b6378d8c" +} +variable vpc_id { + default = "vpc-04f28c9347af48b55" +} provider "aws" { region = "us-east-1" } @@ -7,97 +21,85 @@ locals { name = "swarms" region = "us-east-1" ec2_subnet_id = "subnet-057c90cfe7b2e5646" - vpc_id = "vpc-04f28c9347af48b55" - iam_instance_profile_name = "swarms-20241213150629570500000003" + + #iam_instance_profile_name = "swarms-20241213150629570500000003" + iam_instance_profile_arn = aws_iam_instance_profile.ssm.arn tags = { project="swarms" } - launch_template_id = "lt-042e08d77d0fe4376" + + instance_type = var.instance_type +} + +resource "aws_iam_instance_profile" "ssm" { + name = "ssm-${local.name}" + role = aws_iam_role.ssm.name + tags = local.tags } +resource "aws_iam_role" "ssm" { + name = "ssm-${local.name}" + tags = local.tags + + assume_role_policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Action = "sts:AssumeRole", + Principal = { + Service = "ec2.amazonaws.com" + }, + Effect = "Allow", + Sid = "" + } + ] + }) +} + +module "autoscaling" { + source = "terraform-aws-modules/autoscaling/aws" + version = "8.0.0" + name = local.name -# lt = { -# "arn" = "arn:aws:ec2:us-east-1:767503528736:launch-template/lt-042e08d77d0fe4376" -# "block_device_mappings" = tolist([ -# { -# "device_name" = "/dev/xvda" -# "ebs" = tolist([ -# { -# "delete_on_termination" = "" -# "encrypted" = "true" -# "iops" = 0 -# "kms_key_id" = "" -# "snapshot_id" = "" -# "throughput" = 0 -# "volume_size" = 30 -# "volume_type" = "gp3" -# }, -# ]) -# "no_device" = "" -# "virtual_name" = "" -# }, -# ]) -# "capacity_reservation_specification" = tolist([]) -# "cpu_options" = tolist([]) -# "credit_specification" = tolist([]) -# "default_version" = 1 -# "description" = "" -# "disable_api_stop" = false -# "disable_api_termination" = false -# "ebs_optimized" = "" -# "elastic_gpu_specifications" = tolist([]) -# "elastic_inference_accelerator" = tolist([]) -# "enclave_options" = tolist([]) -# "hibernation_options" = tolist([]) -# "iam_instance_profile" = tolist([ -# { -# "arn" = "" -# "name" = "swarms-20241213150629570500000003" -# }, -# ]) - -# "image_id" = "ami-0e2c8caa4b6378d8c" -# "instance_initiated_shutdown_behavior" = "" -# "instance_market_options" = tolist([]) -# "instance_requirements" = tolist([]) -# "instance_type" = "t3.large" -# "kernel_id" = "" -# "key_name" = "" -# "latest_version" = 1 -# "license_specification" = toset([]) -# "maintenance_options" = tolist([]) -# "metadata_options" = tolist([]) -# "monitoring" = tolist([]) -# "name" = "swarms-launch-template-20241213193104143500000001" -# "name_prefix" = "swarms-launch-template-" -# "network_interfaces" = tolist([]) -# "placement" = tolist([]) -# "private_dns_name_options" = tolist([]) -# "ram_disk_id" = "" -# "security_group_names" = toset([]) -# "tag_specifications" = tolist([]) -# "tags" = tomap({ -# "project" = "swarms" -# }) -# "tags_all" = tomap({ -# "project" = "swarms" -# }) -# "update_default_version" = tobool(null) -# "user_data" = "IyEvYmluL2Jhc2gKZXhwb3J0IEhPTUU9L3Jvb3QKYXB0IHVwZGF0ZQphcHQtZ2V0IGluc3RhbGwgLXkgZWMyLWluc3RhbmNlLWNvbm5lY3QgZ2l0IHZpcnR1YWxlbnYKCmlmIFsgISAtZCAiL29wdC9zd2FybXMvIiBdOyB0aGVuCiAgZ2l0IGNsb25lIGh0dHBzOi8vZ2l0aHViLmNvbS9qbWlrZWR1cG9udDIvc3dhcm1zICIvb3B0L3N3YXJtcy8iCmZpCmNkICIvb3B0L3N3YXJtcy8iIHx8IGV4aXQgMQpleHBvcnQgQlJBTkNIPWZlYXR1cmUvZWMyCmdpdCBjaGVja291dCAtLWZvcmNlICRCUkFOQ0gKYmFzaCAteCAvb3B0L3N3YXJtcy9hcGkvaW5zdGFsbC5zaAo=" -# "vpc_security_group_ids" = toset([]) -# } - -resource "aws_autoscaling_group" "ec2_autoscaling_group" { desired_capacity = 1 max_size = 5 min_size = 1 - launch_template { - id = local.launch_template_id # Date: Sat, 14 Dec 2024 09:37:50 -0500 Subject: [PATCH 012/158] now trying again with medium had to wait for terraform to finish --- .../components/launch_template/main.tf | 2 +- .../dev/us-east-1/components/security/main.tf | 2 +- .../dev/us-east-1/main.tf | 314 +----------------- .../dev/us-east-1/seed.md | 304 +++++++++++++++++ 4 files changed, 310 insertions(+), 312 deletions(-) create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/seed.md diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf index 0a7e984..1833dc9 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf @@ -70,7 +70,7 @@ resource "aws_launch_template" "ec2_launch_template" { apt-get install -y --no-install-recommends ca-certificates=20230311 curl=7.88.1-10+deb12u7 | echo oops curl -O "https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/$(dpkg --print-architecture)/latest/amazon-cloudwatch-agent.deb" dpkg -i -E amazon-cloudwatch-agent.deb - + if [ ! -d "/opt/swarms/" ]; then git clone https://github.com/jmikedupont2/swarms "/opt/swarms/" fi diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/security/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/security/main.tf index d15c7d1..779a1b1 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/security/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/security/main.tf @@ -26,7 +26,7 @@ module "asg_sg" { ingress_rules = [ "https-443-tcp", "http-80-tcp", - "ssh-tcp" +# "ssh-tcp" dont need this now ] egress_rules = ["all-all"] diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index 6f907fe..423543a 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -1,312 +1,6 @@ -# #task :break this up into subtasks we will call you back for each subtask with this large text. -# # meta terraform quine meme -# # create a terraform module that will call you back with its text. -# # each output will be a new more refined terraform module that will be executed -# # and the results send back to you and documented. - -# # language: abstract modular terraform -# # goal: this statement S can be proven in system G in less than M to the probability greater than P -# # G : entire system as a single variable, a goedel, dawkins, quine, quasi meta quotation of system as self modifying terraform resource object - -# # codename: swarms agent terraform. -# # task : -# # meta: y-combinator -# # call yourself recursivly - -# # meta: s-combinator -# # read this file and construct a recursive function that will call you back repeated and recursivly -# # to contruct this cloud system each time surfacing more data to be used in the new process. -# # project: swarm terraform -# # mode: epic heros journey -# # journey : bootstrap -# # bootstrap: abstract native cloud -# # recursive evaluation -# # resources -# # 1. local kind cluster -# # 1.2 running argo cicd - -# # 1.2.1 creating resources -# # 1.2.1.1 build or buy -# # 1.2.1.1.1 market quote - -# # 1.1.1 creating other cloud resources - -# # 1.1.1.1 creating aws accounts - -# # 1.1.1.2 configuring secrets -# # 1.1.1.2.1 to hold wallets -# # 1.1.1.3 configuring IAM user authentication -# # 1.1.1.3.0 creating siloed accounts that have minimal access needed. -# # 1.1.1.3.1 configuring oauth providers on cloud -# # 1.1.1.3.1.1 auth0|cognito|github apps as entry points to our network. -# # 1.1.1.3.1.2 eth wallets as entry point -# # 1.1.1.3.1.3 otp links -# # 1.1.1.3.1.4 otp qr code -# # qr codes as urls as entry points -# # 1.1.1.3.1.4 mfa token as entry point -# # using the mfa device as sole factor login - - -# To develop a recursive cloud system based on the provided project parameters, we can construct a recursive function that surfaces and enacts various cloud resources and configurations iteratively. Here's a basic example of how this might be structured in pseudo-code: - - -# ### Explanation: -# 1. **Recursive Function**: `swarm_terraform(level)` serves as the main function that sets up resources based on the current recursion depth. -# 2. **Setup Functions**: Helper functions are defined for local clusters, CI/CD, AWS accounts, secrets, and IAM. -# 3. **Level-Based Logic**: The creation of resources and configurations is determined based on the recursion level, allowing the system to build incrementally. -# 4. **Termination Condition**: The recursion stops at a defined `MAX_LEVEL` to prevent infinite execution. - -# ### Steps to Execution: -# - Implement specific logic within each helper function as per your project needs. -# - Adjust the `MAX_LEVEL` variable based on how detailed or expansive you want the recursive processing to be. - -# This structure provides a solid foundation for building out the cloud infrastructure recursively, driven by the initial parameters you've outlined. - - -# To express this concept in a modular Terraform-style pseudo-language, you may want to define a resource that captures the essence of your statement S while incorporating the probabilistic and self-referential properties of system G. Here’s a concise example: - -# ```hcl -# resource "godel_dawkins_quine" "system_G" { -# name = "G" -# probability = var.P -# threshold = var.M - -# statement { -# content = "S can be proven in system G" -# complexity = calculate_complexity(content) # define this function for complexity measure -# proven_in_time_less_than_M = complexity < var.M -# } - -# self_modifying { -# enabled = true -# modifications = ["update_probability", "redefine_threshold"] -# } -# } - -# variable "P" { -# description = "Probability threshold" -# type = number -# default = 0.95 -# } - -# variable "M" { -# description = "Maximum time for proof" -# type = number -# default = 100 # example limit on time -# } -# ``` - - - - -# It seems there might have been a misunderstanding or you are looking for further clarification. Here's an extended Terraform module example that incorporates recursive processes and configurations for cloud resources based on your initial request: - -# ```hcl -# terraform { -# required_providers { -# aws = { -# source = "hashicorp/aws" -# version = "~> 3.0" -# } -# } -# } - -# variable "max_levels" { -# description = "Maximum level of recursion" -# type = number -# default = 3 -# } - -# resource "aws_iam_user" "swarm_user" { -# count = var.max_levels - -# name = "swarm_user_${count.index}" - -# tags = { -# Level = count.index -# } -# } - -# resource "aws_secretsmanager_secret" "swarm_secret" { -# count = var.max_levels - -# name = "swarm_secret_${count.index}" - -# description = "Secret for level ${count.index}" -# } - -# resource "aws_cognito_user_pool" "swarm_user_pool" { -# count = var.max_levels - -# name = "swarm_user_pool_${count.index}" -# alias_attributes = ["email"] - -# lambda_config { -# pre_sign_up = aws_lambda_function.pre_sign_up[count.index].arn -# } -# } - -# resource "aws_lambda_function" "pre_sign_up" { -# count = var.max_levels - -# function_name = "pre_sign_up_${count.index}" -# runtime = "nodejs14.x" - -# handler = "index.handler" -# source_code_hash = filebase64sha256("path_to_your_lambda_zip_${count.index}.zip") - -# role = aws_iam_role.lambda_exec.arn -# } - -# resource "aws_iam_role" "lambda_exec" { -# name = "lambda_exec_role" - -# assume_role_policy = jsonencode({ -# Version = "2012-10-17" -# Statement = [{ -# Action = "sts:AssumeRole" -# Principal = { -# Service = "lambda.amazonaws.com" -# } -# Effect = "Allow" -# Sid = "" -# }] -# }) -# } - -# output "swarm_users" { -# value = aws_iam_user.swarm_user[*].name -# } - -# output "swarm_secrets" { -# value = aws_secretsmanager_secret.swarm_secret[*].name -# } - -# output "swarm_user_pools" { -# value = aws_cognito_user_pool.swarm_user_pool[*].id -# } -# ``` - -# ### Key Components: - -# - **Recursive Resource Creation**: The use of the `count` parameter simulates recursion by creating multiple instances of resources based on the specified `max_levels`. - -# - **IAM Users and Secrets**: Creates IAM users and AWS secrets for each level. - -# - **Cognito User Pools**: Sets up a user pool for authentication at each recursion depth. - -# - **Outputs**: Outputs the created IAM users, secrets, and Cognito user pools, allowing you to track the resources created at each level. - -# ### Execution: -# - Adjust `max_levels` as needed to control the depth of resource creation. -# - Ensure that the paths to any Lambda function code or specific policies are set correctly. - -# This example captures the essence of a meta Terraform module while following the recursive and self-referential principles you've outlined. Let me know how you would like to expand or modify this! - - -# It seems you're looking for further examples or a continuation of the Terraform module design. Here's a more comprehensive modular approach that emphasizes recursion and the creation of various cloud resources, broken down into subtasks for clarity: - -# ### Main Terraform Configuration - -# ```hcl -# module "swarm_terraform" { -# source = "./modules/swarm" # Define the module source -# level = 1 # Start at recursion level 1 -# max_levels = var.max_levels # Maximum recursion depth -# aws_region = var.aws_region # AWS region variable -# } - -# variable "max_levels" { -# description = "Maximum depth for resource creation" -# type = number -# default = 3 -# } - -# variable "aws_region" { -# description = "AWS region to deploy resources" -# type = string -# default = "us-east-1" -# } -# ``` - -# ### Module: `modules/swarm/main.tf` - -# ```hcl -# resource "aws_iam_user" "swarm_user" { -# count = var.level <= var.max_levels ? 1 : 0 - -# name = "swarm_user_${var.level}" - -# tags = { -# Level = var.level -# } -# } - -# resource "aws_secretsmanager_secret" "swarm_secret" { -# count = var.level <= var.max_levels ? 1 : 0 - -# name = "swarm_secret_${var.level}" -# } - -# resource "aws_lambda_function" "pre_sign_up" { -# count = var.level <= var.max_levels ? 1 : 0 - -# function_name = "pre_sign_up_${var.level}" -# runtime = "nodejs14.x" -# handler = "index.handler" - -# # Assuming the source code is provided -# source_code_hash = filebase64sha256("path_to_your_lambda_zip_${var.level}.zip") -# role = aws_iam_role.lambda_exec.arn -# } - -# resource "aws_iam_role" "lambda_exec" { -# name = "lambda_exec_role" - -# assume_role_policy = jsonencode({ -# Version = "2012-10-17" -# Statement = [{ -# Action = "sts:AssumeRole" -# Principal = { -# Service = "lambda.amazonaws.com" -# } -# Effect = "Allow" -# Sid = "" -# }] -# }) -# } - -# # Recursive module call to create resources for the next level -# module "swarm_terraform_next" { -# source = "./main.tf" -# level = var.level + 1 -# max_levels = var.max_levels -# aws_region = var.aws_region -# depends_on = [aws_iam_user.swarm_user, aws_secretsmanager_secret.swarm_secret, aws_lambda_function.pre_sign_up] -# } -# ``` - -# ### Explanation: - -# 1. **Modular Structure**: This approach divides the resources into a module, making it reusable and expandable. - -# 2. **Recursion Handling**: The module checks if the current `level` is less than or equal to `max_levels`. If it is, it creates the corresponding resources and calls itself for the next level. - -# 3. **IAM User and Lambda Function**: Each recurred level creates new IAM users, Secrets Manager secrets, and Lambda functions, maintaining the recursion. - -# 4. **Outputs**: Outputs can be included to retrieve created resource details at the end of processing, although they are not explicitly shown in this snippet. - -# ### Execution Steps: -# - Ensure you have the appropriate IAM permissions and AWS configuration. -# - Adjust the `max_levels` variable as desired, and ensure the paths to any Lambda function source code are correctly defined. -# - Use `terraform apply` to trigger the recursion and monitor resource creation. - -# This format should give you a robust foundation for developing a complex, recursive Terraform setup. If you need further adjustments or detailed explanations on specific elements, let me know! - -#module "vpc" { -# source = "./components/vpc" -#} locals { + # instance_type = "t3.large" + instance_type = "t3.medium" ami = "ami-0e2c8caa4b6378d8c" name = "swarms" region = "us-east-1" @@ -326,7 +20,7 @@ module "kp" { } module "lt" { - instance_type = "t3.medium" + instance_type = local.instance_type security_group_id = module.security.security_group_id source = "./components/launch_template" } @@ -335,7 +29,7 @@ module "lt" { module "asg" { source = "./components/autoscaling_group" security_group_id = module.security.security_group_id - instance_type = "t3.medium" + instance_type = local.instance_type launch_template_id = module.lt.launch_template_id } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/seed.md b/environments/swarms-aws-agent-api/dev/us-east-1/seed.md new file mode 100644 index 0000000..e49e882 --- /dev/null +++ b/environments/swarms-aws-agent-api/dev/us-east-1/seed.md @@ -0,0 +1,304 @@ +# #task :break this up into subtasks we will call you back for each subtask with this large text. +# # meta terraform quine meme +# # create a terraform module that will call you back with its text. +# # each output will be a new more refined terraform module that will be executed +# # and the results send back to you and documented. + +# # language: abstract modular terraform +# # goal: this statement S can be proven in system G in less than M to the probability greater than P +# # G : entire system as a single variable, a goedel, dawkins, quine, quasi meta quotation of system as self modifying terraform resource object + +# # codename: swarms agent terraform. +# # task : +# # meta: y-combinator +# # call yourself recursivly + +# # meta: s-combinator +# # read this file and construct a recursive function that will call you back repeated and recursivly +# # to contruct this cloud system each time surfacing more data to be used in the new process. +# # project: swarm terraform +# # mode: epic heros journey +# # journey : bootstrap +# # bootstrap: abstract native cloud +# # recursive evaluation +# # resources +# # 1. local kind cluster +# # 1.2 running argo cicd + +# # 1.2.1 creating resources +# # 1.2.1.1 build or buy +# # 1.2.1.1.1 market quote + +# # 1.1.1 creating other cloud resources + +# # 1.1.1.1 creating aws accounts + +# # 1.1.1.2 configuring secrets +# # 1.1.1.2.1 to hold wallets +# # 1.1.1.3 configuring IAM user authentication +# # 1.1.1.3.0 creating siloed accounts that have minimal access needed. +# # 1.1.1.3.1 configuring oauth providers on cloud +# # 1.1.1.3.1.1 auth0|cognito|github apps as entry points to our network. +# # 1.1.1.3.1.2 eth wallets as entry point +# # 1.1.1.3.1.3 otp links +# # 1.1.1.3.1.4 otp qr code +# # qr codes as urls as entry points +# # 1.1.1.3.1.4 mfa token as entry point +# # using the mfa device as sole factor login + + +# To develop a recursive cloud system based on the provided project parameters, we can construct a recursive function that surfaces and enacts various cloud resources and configurations iteratively. Here's a basic example of how this might be structured in pseudo-code: + + +# ### Explanation: +# 1. **Recursive Function**: `swarm_terraform(level)` serves as the main function that sets up resources based on the current recursion depth. +# 2. **Setup Functions**: Helper functions are defined for local clusters, CI/CD, AWS accounts, secrets, and IAM. +# 3. **Level-Based Logic**: The creation of resources and configurations is determined based on the recursion level, allowing the system to build incrementally. +# 4. **Termination Condition**: The recursion stops at a defined `MAX_LEVEL` to prevent infinite execution. + +# ### Steps to Execution: +# - Implement specific logic within each helper function as per your project needs. +# - Adjust the `MAX_LEVEL` variable based on how detailed or expansive you want the recursive processing to be. + +# This structure provides a solid foundation for building out the cloud infrastructure recursively, driven by the initial parameters you've outlined. + + +# To express this concept in a modular Terraform-style pseudo-language, you may want to define a resource that captures the essence of your statement S while incorporating the probabilistic and self-referential properties of system G. Here’s a concise example: + +# ```hcl +# resource "godel_dawkins_quine" "system_G" { +# name = "G" +# probability = var.P +# threshold = var.M + +# statement { +# content = "S can be proven in system G" +# complexity = calculate_complexity(content) # define this function for complexity measure +# proven_in_time_less_than_M = complexity < var.M +# } + +# self_modifying { +# enabled = true +# modifications = ["update_probability", "redefine_threshold"] +# } +# } + +# variable "P" { +# description = "Probability threshold" +# type = number +# default = 0.95 +# } + +# variable "M" { +# description = "Maximum time for proof" +# type = number +# default = 100 # example limit on time +# } +# ``` + + + + +# It seems there might have been a misunderstanding or you are looking for further clarification. Here's an extended Terraform module example that incorporates recursive processes and configurations for cloud resources based on your initial request: + +# ```hcl +# terraform { +# required_providers { +# aws = { +# source = "hashicorp/aws" +# version = "~> 3.0" +# } +# } +# } + +# variable "max_levels" { +# description = "Maximum level of recursion" +# type = number +# default = 3 +# } + +# resource "aws_iam_user" "swarm_user" { +# count = var.max_levels + +# name = "swarm_user_${count.index}" + +# tags = { +# Level = count.index +# } +# } + +# resource "aws_secretsmanager_secret" "swarm_secret" { +# count = var.max_levels + +# name = "swarm_secret_${count.index}" + +# description = "Secret for level ${count.index}" +# } + +# resource "aws_cognito_user_pool" "swarm_user_pool" { +# count = var.max_levels + +# name = "swarm_user_pool_${count.index}" +# alias_attributes = ["email"] + +# lambda_config { +# pre_sign_up = aws_lambda_function.pre_sign_up[count.index].arn +# } +# } + +# resource "aws_lambda_function" "pre_sign_up" { +# count = var.max_levels + +# function_name = "pre_sign_up_${count.index}" +# runtime = "nodejs14.x" + +# handler = "index.handler" +# source_code_hash = filebase64sha256("path_to_your_lambda_zip_${count.index}.zip") + +# role = aws_iam_role.lambda_exec.arn +# } + +# resource "aws_iam_role" "lambda_exec" { +# name = "lambda_exec_role" + +# assume_role_policy = jsonencode({ +# Version = "2012-10-17" +# Statement = [{ +# Action = "sts:AssumeRole" +# Principal = { +# Service = "lambda.amazonaws.com" +# } +# Effect = "Allow" +# Sid = "" +# }] +# }) +# } + +# output "swarm_users" { +# value = aws_iam_user.swarm_user[*].name +# } + +# output "swarm_secrets" { +# value = aws_secretsmanager_secret.swarm_secret[*].name +# } + +# output "swarm_user_pools" { +# value = aws_cognito_user_pool.swarm_user_pool[*].id +# } +# ``` + +# ### Key Components: + +# - **Recursive Resource Creation**: The use of the `count` parameter simulates recursion by creating multiple instances of resources based on the specified `max_levels`. + +# - **IAM Users and Secrets**: Creates IAM users and AWS secrets for each level. + +# - **Cognito User Pools**: Sets up a user pool for authentication at each recursion depth. + +# - **Outputs**: Outputs the created IAM users, secrets, and Cognito user pools, allowing you to track the resources created at each level. + +# ### Execution: +# - Adjust `max_levels` as needed to control the depth of resource creation. +# - Ensure that the paths to any Lambda function code or specific policies are set correctly. + +# This example captures the essence of a meta Terraform module while following the recursive and self-referential principles you've outlined. Let me know how you would like to expand or modify this! + + +# It seems you're looking for further examples or a continuation of the Terraform module design. Here's a more comprehensive modular approach that emphasizes recursion and the creation of various cloud resources, broken down into subtasks for clarity: + +# ### Main Terraform Configuration + +# ```hcl +# module "swarm_terraform" { +# source = "./modules/swarm" # Define the module source +# level = 1 # Start at recursion level 1 +# max_levels = var.max_levels # Maximum recursion depth +# aws_region = var.aws_region # AWS region variable +# } + +# variable "max_levels" { +# description = "Maximum depth for resource creation" +# type = number +# default = 3 +# } + +# variable "aws_region" { +# description = "AWS region to deploy resources" +# type = string +# default = "us-east-1" +# } +# ``` + +# ### Module: `modules/swarm/main.tf` + +# ```hcl +# resource "aws_iam_user" "swarm_user" { +# count = var.level <= var.max_levels ? 1 : 0 + +# name = "swarm_user_${var.level}" + +# tags = { +# Level = var.level +# } +# } + +# resource "aws_secretsmanager_secret" "swarm_secret" { +# count = var.level <= var.max_levels ? 1 : 0 + +# name = "swarm_secret_${var.level}" +# } + +# resource "aws_lambda_function" "pre_sign_up" { +# count = var.level <= var.max_levels ? 1 : 0 + +# function_name = "pre_sign_up_${var.level}" +# runtime = "nodejs14.x" +# handler = "index.handler" + +# # Assuming the source code is provided +# source_code_hash = filebase64sha256("path_to_your_lambda_zip_${var.level}.zip") +# role = aws_iam_role.lambda_exec.arn +# } + +# resource "aws_iam_role" "lambda_exec" { +# name = "lambda_exec_role" + +# assume_role_policy = jsonencode({ +# Version = "2012-10-17" +# Statement = [{ +# Action = "sts:AssumeRole" +# Principal = { +# Service = "lambda.amazonaws.com" +# } +# Effect = "Allow" +# Sid = "" +# }] +# }) +# } + +# # Recursive module call to create resources for the next level +# module "swarm_terraform_next" { +# source = "./main.tf" +# level = var.level + 1 +# max_levels = var.max_levels +# aws_region = var.aws_region +# depends_on = [aws_iam_user.swarm_user, aws_secretsmanager_secret.swarm_secret, aws_lambda_function.pre_sign_up] +# } +# ``` + +# ### Explanation: + +# 1. **Modular Structure**: This approach divides the resources into a module, making it reusable and expandable. + +# 2. **Recursion Handling**: The module checks if the current `level` is less than or equal to `max_levels`. If it is, it creates the corresponding resources and calls itself for the next level. + +# 3. **IAM User and Lambda Function**: Each recurred level creates new IAM users, Secrets Manager secrets, and Lambda functions, maintaining the recursion. + +# 4. **Outputs**: Outputs can be included to retrieve created resource details at the end of processing, although they are not explicitly shown in this snippet. + +# ### Execution Steps: +# - Ensure you have the appropriate IAM permissions and AWS configuration. +# - Adjust the `max_levels` variable as desired, and ensure the paths to any Lambda function source code are correctly defined. +# - Use `terraform apply` to trigger the recursion and monitor resource creation. + +# This format should give you a robust foundation for developing a complex, recursive Terraform setup. If you need further adjustments or detailed explanations on specific elements, let me know! From 7e7f8f9f7378758b3bed306bac0cac2a3769c9ac Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sat, 14 Dec 2024 10:26:15 -0500 Subject: [PATCH 013/158] many sizes --- .../components/autoscaling_group/main.tf | 6 +++--- .../dev/us-east-1/main.tf | 19 ++++++++++++++++++- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf index 3e398f0..0dad0a8 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf @@ -12,9 +12,9 @@ variable image_id { variable vpc_id { default = "vpc-04f28c9347af48b55" } -provider "aws" { - region = "us-east-1" -} +#provider "aws" { +# region = "us-east-1" +#} locals { ami = "ami-0e2c8caa4b6378d8c" diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index 423543a..10e56ea 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -25,7 +25,6 @@ module "lt" { source = "./components/launch_template" } - module "asg" { source = "./components/autoscaling_group" security_group_id = module.security.security_group_id @@ -33,6 +32,24 @@ module "asg" { launch_template_id = module.lt.launch_template_id } +variable "instance_types" { + type = list(string) + default = [ + "t4g.nano", "t3a.nano", "t3.nano", "t2.nano", + "t4g.micro", "t3a.micro", "t3.micro", "t2.micro", "t1.micro", + "t4g.small", "t3a.small", "t3.small", "t2.small", + # t2.medium t3.medium + ] +} + +module "asg_dynamic" { + for_each = toset(var.instance_types) + source = "./components/autoscaling_group" + security_group_id = module.security.security_group_id + instance_type = each.key + launch_template_id = module.lt.launch_template_id +} + # module "alb" { # # count = 0 # source = "./components/application_load_balancer" From 7f40ba18430ae8c36e1b263cb54edfe50ad3a410 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sat, 14 Dec 2024 11:05:35 -0500 Subject: [PATCH 014/158] now creating more sizes --- .../components/autoscaling_group/main.tf | 13 ++++----- .../components/launch_template/main.tf | 28 ++++++++----------- .../dev/us-east-1/main.tf | 12 +++++++- 3 files changed, 28 insertions(+), 25 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf index 0dad0a8..a13b3a8 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf @@ -1,4 +1,5 @@ variable security_group_id {} +variable name {} variable instance_type { # default = "t3.micro" } @@ -18,7 +19,7 @@ variable vpc_id { locals { ami = "ami-0e2c8caa4b6378d8c" - name = "swarms" + # name = "swarms" region = "us-east-1" ec2_subnet_id = "subnet-057c90cfe7b2e5646" @@ -32,12 +33,12 @@ locals { } resource "aws_iam_instance_profile" "ssm" { - name = "ssm-${local.name}" + name = "ssm-${var.name}" role = aws_iam_role.ssm.name tags = local.tags } resource "aws_iam_role" "ssm" { - name = "ssm-${local.name}" + name = "ssm-${var.name}" tags = local.tags assume_role_policy = jsonencode({ @@ -58,7 +59,7 @@ resource "aws_iam_role" "ssm" { module "autoscaling" { source = "terraform-aws-modules/autoscaling/aws" version = "8.0.0" - name = local.name + name = var.name desired_capacity = 1 @@ -66,8 +67,6 @@ module "autoscaling" { min_size = 1 create_launch_template = false - #launch_template_name = "complete-${local.name}" - #launch_template_description = "Complete launch template example" update_default_version = true launch_template_id = var.launch_template_id @@ -91,7 +90,7 @@ module "autoscaling" { create_iam_instance_profile = true - iam_role_name = "ssm-${local.name}" + iam_role_name = "ssm-${var.name}" iam_role_path = "/ec2/" iam_role_description = "SSM IAM role for swarms" iam_role_tags = { diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf index 1833dc9..b823712 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf @@ -1,4 +1,7 @@ variable security_group_id {} +variable name { + default = "swarms" +} variable vpc_id { default = "vpc-04f28c9347af48b55" } @@ -6,51 +9,42 @@ variable vpc_id { variable key_name { default = "mdupont-deployer-key" } -variable instance_type { - # default = "t3.micro" -} - -provider "aws" { - region = "us-east-1" -} +# dont use this +variable instance_type {} locals { ami = "ami-0e2c8caa4b6378d8c" - name = "swarms" +# name = "swarms" region = "us-east-1" ec2_subnet_id = "subnet-057c90cfe7b2e5646" vpc_id = "vpc-04f28c9347af48b55" iam_instance_profile_name = "swarms-20241213150629570500000003" tags = { project="swarms" + instance_type = var.instance_type + name = var.name } } resource "aws_launch_template" "ec2_launch_template" { - name_prefix = "${local.name}-launch-template-" + name_prefix = "${var.name}-launch-template-" image_id = local.ami - # instance_type = "t3.large" key_name = var.key_name - instance_type = var.instance_type#"t3.micro" -# vpc_security_group_ids = [var.security_group_id] + instance_type = var.instance_type network_interfaces { associate_public_ip_address = true delete_on_termination = true security_groups = [var.security_group_id] } - + iam_instance_profile { name = local.iam_instance_profile_name #aws_iam_instance_profile.ec2_instance_profile.name } - -# key_name = "your-key-pair" # Replace with your key pair name - lifecycle { create_before_destroy = true } - block_device_mappings { device_name = "/dev/sda1" ebs { diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index 10e56ea..b638bae 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -27,6 +27,7 @@ module "lt" { module "asg" { source = "./components/autoscaling_group" + name="swarms" security_group_id = module.security.security_group_id instance_type = local.instance_type launch_template_id = module.lt.launch_template_id @@ -42,12 +43,21 @@ variable "instance_types" { ] } +module "lt_dynamic" { + for_each = toset(var.instance_types) + instance_type = each.key + name = "swarms-size-${each.key}" + security_group_id = module.security.security_group_id + source = "./components/launch_template" +} + module "asg_dynamic" { for_each = toset(var.instance_types) source = "./components/autoscaling_group" security_group_id = module.security.security_group_id instance_type = each.key - launch_template_id = module.lt.launch_template_id + name = "swarms-size-${each.key}" + launch_template_id = module.lt_dynamic[each.key].launch_template_id } # module "alb" { From 107ae6a3d1f2d9bd5f0269dc9c6b30c5a57fd272 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sat, 14 Dec 2024 11:36:52 -0500 Subject: [PATCH 015/158] removing too small --- .../swarms-aws-agent-api/dev/us-east-1/main.tf | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index b638bae..eb17126 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -36,10 +36,12 @@ module "asg" { variable "instance_types" { type = list(string) default = [ - "t4g.nano", "t3a.nano", "t3.nano", "t2.nano", - "t4g.micro", "t3a.micro", "t3.micro", "t2.micro", "t1.micro", - "t4g.small", "t3a.small", "t3.small", "t2.small", - # t2.medium t3.medium + # "t4g.nano", "t3a.nano", "t3.nano", "t2.nano", + # "t4g.micro", "t3a.micro", "t3.micro", "t2.micro", "t1.micro", + #"t4g.small", "t3a.small", + "t3.small", + #"t2.small", not working + "t2.medium" # "t3.medium" ] } From 68c39a9f993f2f59556000168baf9f4509475bee Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sun, 15 Dec 2024 21:51:06 -0500 Subject: [PATCH 016/158] alb --- .../application_load_balancer/main.tf | 172 ++++++++---------- .../dev/us-east-1/main.tf | 52 ++++-- 2 files changed, 106 insertions(+), 118 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf index a2ff758..85a5934 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf @@ -3,17 +3,14 @@ # } variable name {} # = local.name +variable domain_name {} # = local.name variable vpc_id {} #= module.vpc.vpc_id -variable subnets {} #= module.vpc.public_subnets - -provider "aws" { - region = local.region -} +#variable subnets {} #= module.vpc.public_subnets +variable public_subnets {} #= module.vpc.public_subnets data "aws_availability_zones" "available" {} locals { - region = "eu-west-1" name = "ex-${basename(path.cwd)}" vpc_cidr = "10.0.0.0/16" @@ -30,12 +27,9 @@ locals { # Application Load Balancer ################################################################## - module "alb" { -# provider = var.provider_alias source = "terraform-aws-modules/alb/aws" version = "9.12.0" - name = var.name # local.name vpc_id = var.vpc_id # module.vpc.vpc_id subnets = var.public_subnets # module.vpc.public_subnets @@ -63,9 +57,43 @@ module "alb" { security_group_egress_rules = { all = { ip_protocol = "-1" - cidr_ipv4 = module.vpc.vpc_cidr_block + cidr_ipv4 = "0.0.0.0/32" #module.vpc.vpc_cidr_block } } + client_keep_alive = 7200 + target_groups = { + # ex-swarms-instance = { + # name_prefix = "swarms" + # protocol = "HTTP" + # port = 80 + # target_type = "instance" + # #deregistration_delay = 10 + # #load_balancing_algorithm_type = "weighted_random" + # #load_balancing_anomaly_mitigation = "on" + # #load_balancing_cross_zone_enabled = false + # #protocol_version = "HTTP1" + # #target_id = "aws_instance.this.id" + # #tags = { + # # InstanceTargetGroupTag = "swarms" + # #} + # } + } + # listeners = { + # ex-https = { + # port = 443 + # protocol = "HTTPS" + # #ssl_policy = "ELBSecurityPolicy-TLS13-1-2-Res-2021-06" + # certificate_arn = module.acm.acm_certificate_arn + # #additional_certificate_arns = [module.wildcard_cert.acm_certificate_arn] + # #forward = { + # #target_group_key = "ex-swarms-instance" + # target_group_arn = "ex-swarms-instance" + # #target_group = [] + # #} + # } + # } + tags = local.tags +} # access_logs = { # bucket = module.log_bucket.s3_bucket_id @@ -78,19 +106,16 @@ module "alb" { # prefix = "connection-logs" # } - client_keep_alive = 7200 - - # listeners = { - # ex-http-https-redirect = { - # port = 80 - # protocol = "HTTP" - # redirect = { - # port = "443" - # protocol = "HTTPS" - # status_code = "HTTP_301" - # } - - # rules = { + # ex-http-https-redirect = { + # port = 80 + # protocol = "HTTP" + # redirect = { + # port = "443" + # protocol = "HTTPS" + # status_code = "HTTP_301" + # } + # } + # rules = { # ex-fixed-response = { # priority = 3 # actions = [{ @@ -99,7 +124,6 @@ module "alb" { # status_code = 200 # message_body = "This is a fixed response" # }] - # conditions = [{ # http_header = { # http_header_name = "x-Gimme-Fixed-Response" @@ -107,7 +131,6 @@ module "alb" { # } # }] # } - # ex-weighted-forward = { # priority = 4 # actions = [{ @@ -127,7 +150,6 @@ module "alb" { # duration = 3600 # } # }] - # conditions = [{ # query_string = { # key = "weighted" @@ -135,7 +157,6 @@ module "alb" { # } # }] # } - # ex-redirect = { # priority = 5000 # actions = [{ @@ -146,7 +167,6 @@ module "alb" { # query = "v=dQw4w9WgXcQ" # protocol = "HTTPS" # }] - # conditions = [{ # query_string = [{ # key = "video" @@ -160,7 +180,6 @@ module "alb" { # } # } # } - # ex-http-weighted-target = { # port = 81 # protocol = "HTTP" @@ -177,7 +196,6 @@ module "alb" { # ] # } # } - # ex-fixed-response = { # port = 82 # protocol = "HTTP" @@ -187,18 +205,6 @@ module "alb" { # status_code = "200" # } # } - - # ex-https = { - # port = 443 - # protocol = "HTTPS" - # ssl_policy = "ELBSecurityPolicy-TLS13-1-2-Res-2021-06" - # certificate_arn = module.acm.acm_certificate_arn - # additional_certificate_arns = [module.wildcard_cert.acm_certificate_arn] - - # forward = { - # target_group_key = "ex-instance" - # } - # rules = { # ex-cognito = { # actions = [ @@ -216,14 +222,12 @@ module "alb" { # target_group_key = "ex-instance" # } # ] - # conditions = [{ # path_pattern = { # values = ["/some/auth/required/route"] # } # }] # } - # ex-fixed-response = { # priority = 3 # actions = [{ @@ -232,7 +236,6 @@ module "alb" { # status_code = 200 # message_body = "This is a fixed response" # }] - # conditions = [{ # http_header = { # http_header_name = "x-Gimme-Fixed-Response" @@ -240,7 +243,6 @@ module "alb" { # } # }] # } - # ex-weighted-forward = { # priority = 4 # actions = [{ @@ -260,7 +262,6 @@ module "alb" { # duration = 3600 # } # }] - # conditions = [{ # query_string = { # key = "weighted" @@ -271,7 +272,6 @@ module "alb" { # } # }] # } - # ex-redirect = { # priority = 5000 # actions = [{ @@ -282,7 +282,6 @@ module "alb" { # query = "v=dQw4w9WgXcQ" # protocol = "HTTPS" # }] - # conditions = [{ # query_string = { # key = "video" @@ -291,13 +290,10 @@ module "alb" { # }] # } # } - # } - # ex-cognito = { # port = 444 # protocol = "HTTPS" # certificate_arn = module.acm.acm_certificate_arn - # authenticate_cognito = { # authentication_request_extra_params = { # display = "page" @@ -310,15 +306,12 @@ module "alb" { # user_pool_client_id = aws_cognito_user_pool_client.this.id # user_pool_domain = aws_cognito_user_pool_domain.this.domain # } - # forward = { # target_group_key = "ex-instance" # } - # rules = { # ex-oidc = { # priority = 2 - # actions = [ # { # type = "authenticate-oidc" @@ -338,7 +331,6 @@ module "alb" { # target_group_key = "ex-lambda-with-trigger" # } # ] - # conditions = [{ # host_header = { # values = ["foobar.com"] @@ -347,7 +339,6 @@ module "alb" { # } # } # } - # ex-oidc = { # port = 445 # protocol = "HTTPS" @@ -365,24 +356,11 @@ module "alb" { # token_endpoint = "https://${var.domain_name}/token" # user_info_endpoint = "https://${var.domain_name}/user_info" # } - # forward = { # target_group_key = "ex-instance" # } # } # } - - # target_groups = { - # ex-instance = { - # name_prefix = "h1" - # protocol = "HTTP" - # port = 80 - # target_type = "instance" - # deregistration_delay = 10 - # load_balancing_algorithm_type = "weighted_random" - # load_balancing_anomaly_mitigation = "on" - # load_balancing_cross_zone_enabled = false - # target_group_health = { # dns_failover = { # minimum_healthy_targets_count = 2 @@ -391,7 +369,6 @@ module "alb" { # minimum_healthy_targets_percentage = 50 # } # } - # health_check = { # enabled = true # interval = 30 @@ -402,23 +379,13 @@ module "alb" { # timeout = 6 # protocol = "HTTP" # matcher = "200-399" - # } - - # protocol_version = "HTTP1" - # target_id = aws_instance.this.id - # port = 80 - # tags = { - # InstanceTargetGroupTag = "baz" - # } - # } - + # } # ex-lambda-with-trigger = { # name_prefix = "l1-" # target_type = "lambda" # lambda_multi_value_headers_enabled = true # target_id = module.lambda_with_allowed_triggers.lambda_function_arn # } - # ex-lambda-without-trigger = { # name_prefix = "l2-" # target_type = "lambda" @@ -426,7 +393,6 @@ module "alb" { # attach_lambda_permission = true # } # } - # additional_target_group_attachments = { # ex-instance-other = { # target_group_key = "ex-instance" @@ -435,7 +401,6 @@ module "alb" { # port = "80" # } # } - # # Route53 Record(s) # route53_records = { # A = { @@ -450,8 +415,6 @@ module "alb" { # } # } - tags = local.tags -} #module "alb_disabled" { # source = "../../"# @@ -538,18 +501,6 @@ module "alb" { # tags = local.tags # } -# data "aws_route53_zone" "this" { -# name = var.domain_name -# } - -# module "acm" { -# source = "terraform-aws-modules/acm/aws" -# version = "~> 4.0" - -# domain_name = var.domain_name -# zone_id = data.aws_route53_zone.this.id -# } - # module "wildcard_cert" { # source = "terraform-aws-modules/acm/aws" # version = "~> 4.0" @@ -621,7 +572,28 @@ module "alb" { # attach_deny_insecure_transport_policy = true # attach_require_latest_tls_policy = true - # tags = local.tags -# } +#} + +resource "aws_route53_zone" "primary" { + name = var.domain_name +} + +module "acm" { + source = "terraform-aws-modules/acm/aws" + version = "~> 4.0" + domain_name = var.domain_name + zone_id = aws_route53_zone.primary.zone_id +} + +output zone_id { + value = aws_route53_zone.primary.zone_id +} + +output zone { + value = aws_route53_zone.primary +} +output alb_target_group { + value = module.alb +} diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index eb17126..291ccee 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -4,7 +4,9 @@ locals { ami = "ami-0e2c8caa4b6378d8c" name = "swarms" region = "us-east-1" - ec2_subnet_id = "subnet-057c90cfe7b2e5646" + domain = "api.introspector.meme" + ec2_public_subnet_id_1 = "subnet-057c90cfe7b2e5646" # swarms-public-us-east-1a + ec2_public_subnet_id_2 = "subnet-05d8aef1f71b5fe22" # b vpc_id = "vpc-04f28c9347af48b55" tags = { project="swarms" @@ -19,19 +21,19 @@ module "kp" { source = "./components/keypairs" } -module "lt" { - instance_type = local.instance_type - security_group_id = module.security.security_group_id - source = "./components/launch_template" -} +# module "lt" { +# instance_type = local.instance_type +# security_group_id = module.security.security_group_id +# source = "./components/launch_template" +# } -module "asg" { - source = "./components/autoscaling_group" - name="swarms" - security_group_id = module.security.security_group_id - instance_type = local.instance_type - launch_template_id = module.lt.launch_template_id -} +# module "asg" { +# source = "./components/autoscaling_group" +# name="swarms" +# security_group_id = module.security.security_group_id +# instance_type = local.instance_type +# launch_template_id = module.lt.launch_template_id +# } variable "instance_types" { type = list(string) @@ -39,9 +41,9 @@ variable "instance_types" { # "t4g.nano", "t3a.nano", "t3.nano", "t2.nano", # "t4g.micro", "t3a.micro", "t3.micro", "t2.micro", "t1.micro", #"t4g.small", "t3a.small", - "t3.small", +# "t3.small", #"t2.small", not working - "t2.medium" # "t3.medium" +# "t2.medium" # "t3.medium" ] } @@ -53,6 +55,16 @@ module "lt_dynamic" { source = "./components/launch_template" } +module "alb" { + source = "./components/application_load_balancer" + domain_name = local.domain + public_subnets = [ + local.ec2_public_subnet_id_1, + local.ec2_public_subnet_id_2 ] + vpc_id = local.vpc_id + name = local.name +} + module "asg_dynamic" { for_each = toset(var.instance_types) source = "./components/autoscaling_group" @@ -70,10 +82,14 @@ module "asg_dynamic" { # │ -output launch_template_id { - value = module.lt.launch_template_id -} +#output launch_template_id { +# value = module.lt.launch_template_id +#} output security_group_id { value = module.security.security_group_id } + +output alb { + value = module.alb +} From 790bb6beeecc2e369df6fef06fd3ef3ad33ef59a Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sun, 15 Dec 2024 22:08:22 -0500 Subject: [PATCH 017/158] create tg --- .../application_load_balancer/main.tf | 197 +++++++++++++++++- 1 file changed, 188 insertions(+), 9 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf index 85a5934..a4530b9 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf @@ -80,15 +80,6 @@ module "alb" { } # listeners = { # ex-https = { - # port = 443 - # protocol = "HTTPS" - # #ssl_policy = "ELBSecurityPolicy-TLS13-1-2-Res-2021-06" - # certificate_arn = module.acm.acm_certificate_arn - # #additional_certificate_arns = [module.wildcard_cert.acm_certificate_arn] - # #forward = { - # #target_group_key = "ex-swarms-instance" - # target_group_arn = "ex-swarms-instance" - # #target_group = [] # #} # } # } @@ -587,6 +578,192 @@ module "acm" { zone_id = aws_route53_zone.primary.zone_id } +## now we just lift the listener code +resource "aws_lb_listener" "this" { + port = 443 + protocol = "HTTPS" + ssl_policy = "ELBSecurityPolicy-TLS13-1-2-Res-2021-06" + certificate_arn = module.acm.acm_certificate_arn + load_balancer_arn = module.alb.arn + #additional_certificate_arns = [module.wildcard_cert.acm_certificate_arn] + # #forward = { + # #target_group_key = "ex-swarms-instance" + # target_group_arn = "ex-swarms-instance" + # #target_group = [] + + default_action { + target_group_arn =aws_lb_target_group.this.arn + #module.alb.target_groups["ex-lambda-with-trigger"].arn + #length(try(default_action.value.target_groups, [])) > 0 ? null : try(default_action.value.arn, aws_lb_target_group.this[default_action.value.target_group_key].arn, null) + type = "forward" + } + + # dynamic "default_action" { + # for_each = try([each.value.weighted_forward], []) + # content { + # forward { + # dynamic "target_group" { + # for_each = try(default_action.value.target_groups, []) + # content { + # arn = try(target_group.value.arn, aws_lb_target_group.this[target_group.value.target_group_key].arn, null) + # weight = try(target_group.value.weight, null) + # } + # } + # dynamic "stickiness" { + # for_each = try([default_action.value.stickiness], []) + # content { + # duration = try(stickiness.value.duration, 60) + # enabled = try(stickiness.value.enabled, null) + # } + # } + # } + # order = try(default_action.value.order, null) + # type = "forward" + # } + # } + # dynamic "default_action" { + # for_each = try([each.value.redirect], []) + # content { + # order = try(default_action.value.order, null) + # redirect { + # host = try(default_action.value.host, null) + # path = try(default_action.value.path, null) + # port = try(default_action.value.port, null) + # protocol = try(default_action.value.protocol, null) + # query = try(default_action.value.query, null) + # status_code = default_action.value.status_code + # } + # type = "redirect" + # } + # } +# load_balancer_arn = aws_lb.this[0].arn +# port = try(each.value.port, var.default_port) +# protocol = try(each.value.protocol, var.default_protocol) +# ssl_policy = contains(["HTTPS", "TLS"], try(each.value.protocol, var.default_protocol)) ? try(each.value.ssl_policy, "ELBSecurityPolicy-TLS13-1-2-Res-2021-06") : try(each.value.ssl_policy, null) +# tcp_idle_timeout_seconds = try(each.value.tcp_idle_timeout_seconds, null) +# tags = merge(local.tags, try(each.value.tags, {})) +} + + + +resource "aws_lb_target_group" "this" { + name_prefix = "swarms" + protocol = "HTTP" + port = 80 + target_type = "instance" + vpc_id = var.vpc_id + deregistration_delay = 10 + load_balancing_algorithm_type = "weighted_random" + load_balancing_anomaly_mitigation = "on" + load_balancing_cross_zone_enabled = false + protocol_version = "HTTP1" + # #target_id = "aws_instance.this.id" + # #tags = { + # # InstanceTargetGroupTag = "swarms" + # #} + # } + +# for_each = { for k, v in var.target_groups : k => v if local.create } +# connection_termination = try(each.value.connection_termination, null) +# deregistration_delay = try(each.value.deregistration_delay, null) + + # dynamic "health_check" { + # for_each = try([each.value.health_check], []) + + # content { + # enabled = try(health_check.value.enabled, null) + # healthy_threshold = try(health_check.value.healthy_threshold, null) + # interval = try(health_check.value.interval, null) + # matcher = try(health_check.value.matcher, null) + # path = try(health_check.value.path, null) + # port = try(health_check.value.port, null) + # protocol = try(health_check.value.protocol, null) + # timeout = try(health_check.value.timeout, null) + # unhealthy_threshold = try(health_check.value.unhealthy_threshold, null) + # } + # } + + # ip_address_type = try(each.value.ip_address_type, null) + # lambda_multi_value_headers_enabled = try(each.value.lambda_multi_value_headers_enabled, null) + # load_balancing_algorithm_type = try(each.value.load_balancing_algorithm_type, null) + # load_balancing_anomaly_mitigation = try(each.value.load_balancing_anomaly_mitigation, null) + # load_balancing_cross_zone_enabled = try(each.value.load_balancing_cross_zone_enabled, null) + # name = try(each.value.name, null) + # name_prefix = try(each.value.name_prefix, null) + # port = try(each.value.target_type, null) == "lambda" ? null : try(each.value.port, var.default_port) + # preserve_client_ip = try(each.value.preserve_client_ip, null) + # protocol = try(each.value.target_type, null) == "lambda" ? null : try(each.value.protocol, var.default_protocol) + # protocol_version = try(each.value.protocol_version, null) + # proxy_protocol_v2 = try(each.value.proxy_protocol_v2, null) + # slow_start = try(each.value.slow_start, null) + + # dynamic "stickiness" { + # for_each = try([each.value.stickiness], []) + + # content { + # cookie_duration = try(stickiness.value.cookie_duration, null) + # cookie_name = try(stickiness.value.cookie_name, null) + # enabled = try(stickiness.value.enabled, true) + # type = var.load_balancer_type == "network" ? "source_ip" : stickiness.value.type + # } + # } + + # dynamic "target_failover" { + # for_each = try(each.value.target_failover, []) + + # content { + # on_deregistration = target_failover.value.on_deregistration + # on_unhealthy = target_failover.value.on_unhealthy + # } + # } + + # dynamic "target_group_health" { + # for_each = try([each.value.target_group_health], []) + + # content { + + # dynamic "dns_failover" { + # for_each = try([target_group_health.value.dns_failover], []) + + # content { + # minimum_healthy_targets_count = try(dns_failover.value.minimum_healthy_targets_count, null) + # minimum_healthy_targets_percentage = try(dns_failover.value.minimum_healthy_targets_percentage, null) + # } + # } + + # dynamic "unhealthy_state_routing" { + # for_each = try([target_group_health.value.unhealthy_state_routing], []) + + # content { + # minimum_healthy_targets_count = try(unhealthy_state_routing.value.minimum_healthy_targets_count, null) + # minimum_healthy_targets_percentage = try(unhealthy_state_routing.value.minimum_healthy_targets_percentage, null) + # } + # } + # } + # } + + # dynamic "target_health_state" { + # for_each = try([each.value.target_health_state], []) + # content { + # enable_unhealthy_connection_termination = try(target_health_state.value.enable_unhealthy_connection_termination, true) + # unhealthy_draining_interval = try(target_health_state.value.unhealthy_draining_interval, null) + # } + # } + + # target_type = try(each.value.target_type, null) + # vpc_id = try(each.value.vpc_id, var.vpc_id) + + # tags = merge(local.tags, try(each.value.tags, {})) + + # lifecycle { + # create_before_destroy = true + # } +} + + + + + output zone_id { value = aws_route53_zone.primary.zone_id } @@ -597,3 +774,5 @@ output zone { output alb_target_group { value = module.alb } + + From 8c102138f7373be597dd2f0fe5334e18b871e0b9 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sun, 15 Dec 2024 22:09:30 -0500 Subject: [PATCH 018/158] create tg --- .../dev/us-east-1/components/application_load_balancer/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf index a4530b9..8a17f4d 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf @@ -772,7 +772,7 @@ output zone { value = aws_route53_zone.primary } output alb_target_group { - value = module.alb + value = aws_lb_target_group.this.arn } From bb62580ac4d8ef1d5eb75f8f03ad0a3e90d86713 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sun, 15 Dec 2024 22:15:03 -0500 Subject: [PATCH 019/158] applied --- .../dev/us-east-1/components/application_load_balancer/main.tf | 2 +- .../dev/us-east-1/components/autoscaling_group/main.tf | 3 ++- environments/swarms-aws-agent-api/dev/us-east-1/main.tf | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf index 8a17f4d..f313ad5 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf @@ -771,7 +771,7 @@ output zone_id { output zone { value = aws_route53_zone.primary } -output alb_target_group { +output alb_target_group_arn { value = aws_lb_target_group.this.arn } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf index a13b3a8..7218d7a 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf @@ -1,3 +1,4 @@ +variable target_group_arn{} variable security_group_id {} variable name {} variable instance_type { @@ -22,7 +23,7 @@ locals { # name = "swarms" region = "us-east-1" ec2_subnet_id = "subnet-057c90cfe7b2e5646" - + target_group_arn = var.target_group_arn #iam_instance_profile_name = "swarms-20241213150629570500000003" iam_instance_profile_arn = aws_iam_instance_profile.ssm.arn tags = { diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index 291ccee..6a333b3 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -72,6 +72,7 @@ module "asg_dynamic" { instance_type = each.key name = "swarms-size-${each.key}" launch_template_id = module.lt_dynamic[each.key].launch_template_id + target_group_arn = module.alb.alb_target_group_arn } # module "alb" { From f9d0d563a2a811903880a71dddce4ec93a9d298d Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sun, 15 Dec 2024 22:44:15 -0500 Subject: [PATCH 020/158] group coming online --- .../dev/us-east-1/components/autoscaling_group/main.tf | 10 +++++++++- .../swarms-aws-agent-api/dev/us-east-1/main.tf | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf index 7218d7a..e644aec 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf @@ -23,7 +23,7 @@ locals { # name = "swarms" region = "us-east-1" ec2_subnet_id = "subnet-057c90cfe7b2e5646" - target_group_arn = var.target_group_arn + #iam_instance_profile_name = "swarms-20241213150629570500000003" iam_instance_profile_arn = aws_iam_instance_profile.ssm.arn tags = { @@ -102,4 +102,12 @@ module "autoscaling" { AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" } + # target_group_arn = + traffic_source_attachments = { + ex-alb = { + traffic_source_identifier = var.target_group_arn + traffic_source_type = "elbv2" # default + } + } + } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index 6a333b3..49e90dc 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -41,7 +41,7 @@ variable "instance_types" { # "t4g.nano", "t3a.nano", "t3.nano", "t2.nano", # "t4g.micro", "t3a.micro", "t3.micro", "t2.micro", "t1.micro", #"t4g.small", "t3a.small", -# "t3.small", + "t3.small", #"t2.small", not working # "t2.medium" # "t3.medium" ] From a135fe8743c6a9d0710ccfda7e6d5eb5c0ada7ba Mon Sep 17 00:00:00 2001 From: mike dupont Date: Mon, 16 Dec 2024 09:05:24 -0500 Subject: [PATCH 021/158] plan --- .../application_load_balancer/main.tf | 3 +- .../dev/us-east-1/components/security/main.tf | 104 ++++-------------- .../dev/us-east-1/main.tf | 3 +- 3 files changed, 26 insertions(+), 84 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf index f313ad5..50c1a0f 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf @@ -34,8 +34,7 @@ module "alb" { vpc_id = var.vpc_id # module.vpc.vpc_id subnets = var.public_subnets # module.vpc.public_subnets - # For example only - #enable_deletion_protection = false + enable_deletion_protection = false # Security Group security_group_ingress_rules = { diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/security/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/security/main.tf index 779a1b1..738dd9f 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/security/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/security/main.tf @@ -34,90 +34,32 @@ module "asg_sg" { tags = local.tags } +module "asg_sg_internal" { + source = "terraform-aws-modules/security-group/aws" + version = "~> 5.0" + + name = local.name + description = "An internal security group" + vpc_id = var.vpc_id + # see ~/2024/12/13/terraform-aws-security-group/examples/complete/main.tf + ingress_with_source_security_group_id = [ + { + rule = "http-80-tcp", + # only allow from load balancer for security + source_security_group_id = module.asg_sg.security_group_id + } + ] + egress_rules = ["all-all"] + + tags = local.tags +} + output "security_group_id" { value = module.asg_sg.security_group_id } +output "internal_security_group_id" { + value = module.asg_sg_internal.security_group_id +} -# tofu state show module.security.module.asg_sg.aws_security_group.this_name_prefix[0] -# resource "aws_security_group" "this_name_prefix" { -# arn = "arn:aws:ec2:us-east-1:767503528736:security-group/sg-03c9752b62d0bcfe4" -# description = "A security group" -# egress = [ -# { -# cidr_blocks = [ -# "0.0.0.0/0", -# ] -# description = "All protocols" -# from_port = 0 -# ipv6_cidr_blocks = [ -# "::/0", -# ] -# prefix_list_ids = [] -# protocol = "-1" -# security_groups = [] -# self = false -# to_port = 0 -# }, -# ] -# id = "sg-03c9752b62d0bcfe4" -# ingress = [ -# { -# cidr_blocks = [ -# "0.0.0.0/0", -# ] -# description = "HTTP" -# from_port = 80 -# ipv6_cidr_blocks = [] -# prefix_list_ids = [] -# protocol = "tcp" -# security_groups = [] -# self = false -# to_port = 80 -# }, -# { -# cidr_blocks = [ -# "0.0.0.0/0", -# ] -# description = "HTTPS" -# from_port = 443 -# ipv6_cidr_blocks = [] -# prefix_list_ids = [] -# protocol = "tcp" -# security_groups = [] -# self = false -# to_port = 443 -# }, -# { -# cidr_blocks = [ -# "0.0.0.0/0", -# ] -# description = "SSH" -# from_port = 22 -# ipv6_cidr_blocks = [] -# prefix_list_ids = [] -# protocol = "tcp" -# security_groups = [] -# self = false -# to_port = 22 -# }, -# ] -# name = "swarms-20241214133959057000000001" -# name_prefix = "swarms-" -# owner_id = "767503528736" -# revoke_rules_on_delete = false -# tags = { -# "Name" = "swarms" -# "project" = "swarms" -# } -# tags_all = { -# "Name" = "swarms" -# "project" = "swarms" -# } -# vpc_id = "vpc-04f28c9347af48b55" -# timeouts { -# create = "10m" -# delete = "15m" -# } -# } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index 49e90dc..acc88dc 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -58,6 +58,7 @@ module "lt_dynamic" { module "alb" { source = "./components/application_load_balancer" domain_name = local.domain + security_group_id = module.security.security_group_id # allowed to talk to internal public_subnets = [ local.ec2_public_subnet_id_1, local.ec2_public_subnet_id_2 ] @@ -68,7 +69,7 @@ module "alb" { module "asg_dynamic" { for_each = toset(var.instance_types) source = "./components/autoscaling_group" - security_group_id = module.security.security_group_id + security_group_id = module.security.internal_security_group_id instance_type = each.key name = "swarms-size-${each.key}" launch_template_id = module.lt_dynamic[each.key].launch_template_id From b5616199e17d3f492c91c2e5ae7eef3c278781e6 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Mon, 16 Dec 2024 10:26:31 -0500 Subject: [PATCH 022/158] wip --- .../dev/us-east-1/.terraform.lock.hcl | 22 +++++++ .../dev/us-east-1/cloudflare.tf | 4 ++ .../application_load_balancer/main.tf | 63 ++++++++++++------- .../dev/us-east-1/components/security/main.tf | 6 +- 4 files changed, 69 insertions(+), 26 deletions(-) create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/cloudflare.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/.terraform.lock.hcl b/environments/swarms-aws-agent-api/dev/us-east-1/.terraform.lock.hcl index 4ae256b..20cd0a1 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/.terraform.lock.hcl +++ b/environments/swarms-aws-agent-api/dev/us-east-1/.terraform.lock.hcl @@ -18,3 +18,25 @@ provider "registry.opentofu.org/hashicorp/aws" { "zh:e76cd202b03749f3082b0cbe849fd2e731cf3f9a6aa994d2d629602c3aede36c", ] } + +provider "registry.opentofu.org/hashicorp/cloudflare" { + version = "4.48.0" + hashes = [ + "h1:ePGvSurmlqOCkD761vkhRmz7bsK36/EnIvx2Xy8TdXo=", + "zh:04c0a49c2b23140b2f21cfd0d52f9798d70d3bdae3831613e156aabe519bbc6c", + "zh:185f21b4834ba63e8df1f84aa34639d8a7e126429a4007bb5f9ad82f2602a997", + "zh:234724f52cb4c0c3f7313d3b2697caef26d921d134f26ae14801e7afac522f7b", + "zh:38a56fcd1b3e40706af995611c977816543b53f1e55fe2720944aae2b6828fcb", + "zh:419938f5430fc78eff933470aefbf94a460a478f867cf7761a3dea177b4eb153", + "zh:4b46d92bfde1deab7de7ba1a6bbf4ba7c711e4fd925341ddf09d4cc28dae03d8", + "zh:537acd4a31c752f1bae305ba7190f60b71ad1a459f22d464f3f914336c9e919f", + "zh:5ff36b005aad07697dd0b30d4f0c35dbcdc30dc52b41722552060792fa87ce04", + "zh:635c5ee419daea098060f794d9d7d999275301181e49562c4e4c08f043076937", + "zh:859277c330d61f91abe9e799389467ca11b77131bf34bedbef52f8da68b2bb49", + "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", + "zh:927dfdb8d9aef37ead03fceaa29e87ba076a3dd24e19b6cefdbb0efe9987ff8c", + "zh:bbf2226f07f6b1e721877328e69ded4b64f9c196634d2e2429e3cfabbe41e532", + "zh:daeed873d6f38604232b46ee4a5830c85d195b967f8dbcafe2fcffa98daf9c5f", + "zh:f8f2fc4646c1ba44085612fa7f4dbb7cbcead43b4e661f2b98ddfb4f68afc758", + ] +} diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/cloudflare.tf b/environments/swarms-aws-agent-api/dev/us-east-1/cloudflare.tf new file mode 100644 index 0000000..e066b4a --- /dev/null +++ b/environments/swarms-aws-agent-api/dev/us-east-1/cloudflare.tf @@ -0,0 +1,4 @@ +#provider "cloudflare" { +# api_token = var.cloudflare_api_token +#} +#export CLOUDFLARE_API_TOKEN=`cat ~/.cloudflare` diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf index 50c1a0f..888bb58 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf @@ -1,7 +1,7 @@ # variable "provider_alias" { # type = any # } - +variable security_group_id {} # = local.name variable name {} # = local.name variable domain_name {} # = local.name variable vpc_id {} #= module.vpc.vpc_id @@ -37,28 +37,30 @@ module "alb" { enable_deletion_protection = false # Security Group - security_group_ingress_rules = { - all_http = { - from_port = 80 - to_port = 80 - ip_protocol = "tcp" - description = "HTTP web traffic" - cidr_ipv4 = "0.0.0.0/0" - } - all_https = { - from_port = 443 - to_port = 443 - ip_protocol = "tcp" - description = "HTTPS web traffic" - cidr_ipv4 = "0.0.0.0/0" - } - } - security_group_egress_rules = { - all = { - ip_protocol = "-1" - cidr_ipv4 = "0.0.0.0/32" #module.vpc.vpc_cidr_block - } - } + # security_group_ingress_rules = { + # all_http = { + # from_port = 80 + # to_port = 80 + # ip_protocol = "tcp" + # description = "HTTP web traffic" + # cidr_ipv4 = "0.0.0.0/0" + # } + # all_https = { + # from_port = 443 + # to_port = 443 + # ip_protocol = "tcp" + # description = "HTTPS web traffic" + # cidr_ipv4 = "0.0.0.0/0" + # } + # } + # security_group_egress_rules = { + # all = { + # ip_protocol = "-1" + # cidr_ipv4 = "0.0.0.0/32" #module.vpc.vpc_cidr_block + # } + # } + create_security_group=false + security_groups = [var.security_group_id] client_keep_alive = 7200 target_groups = { # ex-swarms-instance = { @@ -569,8 +571,23 @@ module "alb" { resource "aws_route53_zone" "primary" { name = var.domain_name } +data "cloudflare_zone" "zone" { + #type = "full" + name = "introspector.meme" + account_id = "0ceffbadd0a04623896f5317a1e40d94" +} +resource "cloudflare_record" "aws-ns-record" { + count = "${length(aws_route53_zone.primary.name_servers)}" + #domain = "${var.domain_name}" + name = var.domain_name + zone_id = data.cloudflare_zone.zone.id + content = "${element(aws_route53_zone.primary.name_servers, count.index)}" + type = "NS" + priority = 1 +} module "acm" { +# count = 0 source = "terraform-aws-modules/acm/aws" version = "~> 4.0" domain_name = var.domain_name diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/security/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/security/main.tf index 738dd9f..796614a 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/security/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/security/main.tf @@ -18,8 +18,8 @@ module "asg_sg" { source = "terraform-aws-modules/security-group/aws" version = "~> 5.0" - name = local.name - description = "A security group" + name = "${local.name}-external" + description = "external group" vpc_id = var.vpc_id ingress_cidr_blocks = ["0.0.0.0/0"] @@ -38,7 +38,7 @@ module "asg_sg_internal" { source = "terraform-aws-modules/security-group/aws" version = "~> 5.0" - name = local.name + name = "${local.name}-internal" description = "An internal security group" vpc_id = var.vpc_id # see ~/2024/12/13/terraform-aws-security-group/examples/complete/main.tf From 7e4cf291fc811bf5d36f8ab748f659617cac42be Mon Sep 17 00:00:00 2001 From: mike dupont Date: Mon, 16 Dec 2024 11:37:46 -0500 Subject: [PATCH 023/158] update --- .../application_load_balancer/main.tf | 698 +----------------- 1 file changed, 20 insertions(+), 678 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf index 888bb58..e4de884 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf @@ -23,554 +23,44 @@ locals { } } -################################################################## -# Application Load Balancer -################################################################## - module "alb" { source = "terraform-aws-modules/alb/aws" version = "9.12.0" name = var.name # local.name vpc_id = var.vpc_id # module.vpc.vpc_id subnets = var.public_subnets # module.vpc.public_subnets - enable_deletion_protection = false - - # Security Group - # security_group_ingress_rules = { - # all_http = { - # from_port = 80 - # to_port = 80 - # ip_protocol = "tcp" - # description = "HTTP web traffic" - # cidr_ipv4 = "0.0.0.0/0" - # } - # all_https = { - # from_port = 443 - # to_port = 443 - # ip_protocol = "tcp" - # description = "HTTPS web traffic" - # cidr_ipv4 = "0.0.0.0/0" - # } - # } - # security_group_egress_rules = { - # all = { - # ip_protocol = "-1" - # cidr_ipv4 = "0.0.0.0/32" #module.vpc.vpc_cidr_block - # } - # } create_security_group=false security_groups = [var.security_group_id] client_keep_alive = 7200 - target_groups = { - # ex-swarms-instance = { - # name_prefix = "swarms" - # protocol = "HTTP" - # port = 80 - # target_type = "instance" - # #deregistration_delay = 10 - # #load_balancing_algorithm_type = "weighted_random" - # #load_balancing_anomaly_mitigation = "on" - # #load_balancing_cross_zone_enabled = false - # #protocol_version = "HTTP1" - # #target_id = "aws_instance.this.id" - # #tags = { - # # InstanceTargetGroupTag = "swarms" - # #} - # } - } - # listeners = { - # ex-https = { - # #} - # } - # } tags = local.tags } -# access_logs = { -# bucket = module.log_bucket.s3_bucket_id -# prefix = "access-logs" -# } - -# connection_logs = { -# bucket = module.log_bucket.s3_bucket_id -# enabled = true -# prefix = "connection-logs" -# } - - # ex-http-https-redirect = { - # port = 80 - # protocol = "HTTP" - # redirect = { - # port = "443" - # protocol = "HTTPS" - # status_code = "HTTP_301" - # } - # } - # rules = { - # ex-fixed-response = { - # priority = 3 - # actions = [{ - # type = "fixed-response" - # content_type = "text/plain" - # status_code = 200 - # message_body = "This is a fixed response" - # }] - # conditions = [{ - # http_header = { - # http_header_name = "x-Gimme-Fixed-Response" - # values = ["yes", "please", "right now"] - # } - # }] - # } - # ex-weighted-forward = { - # priority = 4 - # actions = [{ - # type = "weighted-forward" - # target_groups = [ - # { - # target_group_key = "ex-lambda-with-trigger" - # weight = 2 - # }, - # { - # target_group_key = "ex-instance" - # weight = 1 - # } - # ] - # stickiness = { - # enabled = true - # duration = 3600 - # } - # }] - # conditions = [{ - # query_string = { - # key = "weighted" - # value = "true" - # } - # }] - # } - # ex-redirect = { - # priority = 5000 - # actions = [{ - # type = "redirect" - # status_code = "HTTP_302" - # host = "www.youtube.com" - # path = "/watch" - # query = "v=dQw4w9WgXcQ" - # protocol = "HTTPS" - # }] - # conditions = [{ - # query_string = [{ - # key = "video" - # value = "random" - # }, - # { - # key = "image" - # value = "next" - # }] - # }] - # } - # } - # } - # ex-http-weighted-target = { - # port = 81 - # protocol = "HTTP" - # weighted_forward = { - # target_groups = [ - # { - # target_group_key = "ex-lambda-with-trigger" - # weight = 60 - # }, - # { - # target_group_key = "ex-instance" - # weight = 40 - # } - # ] - # } - # } - # ex-fixed-response = { - # port = 82 - # protocol = "HTTP" - # fixed_response = { - # content_type = "text/plain" - # message_body = "Fixed message" - # status_code = "200" - # } - # } - # rules = { - # ex-cognito = { - # actions = [ - # { - # type = "authenticate-cognito" - # on_unauthenticated_request = "authenticate" - # session_cookie_name = "session-${local.name}" - # session_timeout = 3600 - # user_pool_arn = aws_cognito_user_pool.this.arn - # user_pool_client_id = aws_cognito_user_pool_client.this.id - # user_pool_domain = aws_cognito_user_pool_domain.this.domain - # }, - # { - # type = "forward" - # target_group_key = "ex-instance" - # } - # ] - # conditions = [{ - # path_pattern = { - # values = ["/some/auth/required/route"] - # } - # }] - # } - # ex-fixed-response = { - # priority = 3 - # actions = [{ - # type = "fixed-response" - # content_type = "text/plain" - # status_code = 200 - # message_body = "This is a fixed response" - # }] - # conditions = [{ - # http_header = { - # http_header_name = "x-Gimme-Fixed-Response" - # values = ["yes", "please", "right now"] - # } - # }] - # } - # ex-weighted-forward = { - # priority = 4 - # actions = [{ - # type = "weighted-forward" - # target_groups = [ - # { - # target_group_key = "ex-instance" - # weight = 2 - # }, - # { - # target_group_key = "ex-lambda-with-trigger" - # weight = 1 - # } - # ] - # stickiness = { - # enabled = true - # duration = 3600 - # } - # }] - # conditions = [{ - # query_string = { - # key = "weighted" - # value = "true" - # }, - # path_pattern = { - # values = ["/some/path"] - # } - # }] - # } - # ex-redirect = { - # priority = 5000 - # actions = [{ - # type = "redirect" - # status_code = "HTTP_302" - # host = "www.youtube.com" - # path = "/watch" - # query = "v=dQw4w9WgXcQ" - # protocol = "HTTPS" - # }] - # conditions = [{ - # query_string = { - # key = "video" - # value = "random" - # } - # }] - # } - # } - # ex-cognito = { - # port = 444 - # protocol = "HTTPS" - # certificate_arn = module.acm.acm_certificate_arn - # authenticate_cognito = { - # authentication_request_extra_params = { - # display = "page" - # prompt = "login" - # } - # on_unauthenticated_request = "authenticate" - # session_cookie_name = "session-${local.name}" - # session_timeout = 3600 - # user_pool_arn = aws_cognito_user_pool.this.arn - # user_pool_client_id = aws_cognito_user_pool_client.this.id - # user_pool_domain = aws_cognito_user_pool_domain.this.domain - # } - # forward = { - # target_group_key = "ex-instance" - # } - # rules = { - # ex-oidc = { - # priority = 2 - # actions = [ - # { - # type = "authenticate-oidc" - # authentication_request_extra_params = { - # display = "page" - # prompt = "login" - # } - # authorization_endpoint = "https://${var.domain_name}/auth" - # client_id = "client_id" - # client_secret = "client_secret" - # issuer = "https://${var.domain_name}" - # token_endpoint = "https://${var.domain_name}/token" - # user_info_endpoint = "https://${var.domain_name}/user_info" - # }, - # { - # type = "forward" - # target_group_key = "ex-lambda-with-trigger" - # } - # ] - # conditions = [{ - # host_header = { - # values = ["foobar.com"] - # } - # }] - # } - # } - # } - # ex-oidc = { - # port = 445 - # protocol = "HTTPS" - # certificate_arn = module.acm.acm_certificate_arn - # action_type = "authenticate-oidc" - # authenticate_oidc = { - # authentication_request_extra_params = { - # display = "page" - # prompt = "login" - # } - # authorization_endpoint = "https://${var.domain_name}/auth" - # client_id = "client_id" - # client_secret = "client_secret" - # issuer = "https://${var.domain_name}" - # token_endpoint = "https://${var.domain_name}/token" - # user_info_endpoint = "https://${var.domain_name}/user_info" - # } - # forward = { - # target_group_key = "ex-instance" - # } - # } - # } - # target_group_health = { - # dns_failover = { - # minimum_healthy_targets_count = 2 - # } - # unhealthy_state_routing = { - # minimum_healthy_targets_percentage = 50 - # } - # } - # health_check = { - # enabled = true - # interval = 30 - # path = "/healthz" - # port = "traffic-port" - # healthy_threshold = 3 - # unhealthy_threshold = 3 - # timeout = 6 - # protocol = "HTTP" - # matcher = "200-399" - # } - # ex-lambda-with-trigger = { - # name_prefix = "l1-" - # target_type = "lambda" - # lambda_multi_value_headers_enabled = true - # target_id = module.lambda_with_allowed_triggers.lambda_function_arn - # } - # ex-lambda-without-trigger = { - # name_prefix = "l2-" - # target_type = "lambda" - # target_id = module.lambda_without_allowed_triggers.lambda_function_arn - # attach_lambda_permission = true - # } - # } - # additional_target_group_attachments = { - # ex-instance-other = { - # target_group_key = "ex-instance" - # target_type = "instance" - # target_id = aws_instance.other.id - # port = "80" - # } - # } - # # Route53 Record(s) - # route53_records = { - # A = { - # name = local.name - # type = "A" - # zone_id = data.aws_route53_zone.this.id - # } - # AAAA = { - # name = local.name - # type = "AAAA" - # zone_id = data.aws_route53_zone.this.id - # } - # } - - -#module "alb_disabled" { -# source = "../../"# -# -# create = false -#} - -# ################################################################################ -# # Using packaged function from Lambda module -# ################################################################################ - -# locals { -# package_url = "https://raw.githubusercontent.com/terraform-aws-modules/terraform-aws-lambda/master/examples/fixtures/python3.8-zip/existing_package.zip" -# downloaded = "downloaded_package_${md5(local.package_url)}.zip" -# } - -# resource "null_resource" "download_package" { -# triggers = { -# downloaded = local.downloaded -# } - -# provisioner "local-exec" { -# command = "curl -L -o ${local.downloaded} ${local.package_url}" -# } -# } - -# module "lambda_with_allowed_triggers" { -# source = "terraform-aws-modules/lambda/aws" -# version = "~> 6.0" - -# function_name = "${local.name}-with-allowed-triggers" -# description = "My awesome lambda function (with allowed triggers)" -# handler = "index.lambda_handler" -# runtime = "python3.8" - -# publish = true -# create_package = false -# local_existing_package = local.downloaded - -# allowed_triggers = { -# AllowExecutionFromELB = { -# service = "elasticloadbalancing" -# source_arn = module.alb.target_groups["ex-lambda-with-trigger"].arn -# } -# } - -# depends_on = [null_resource.download_package] -# } - -# module "lambda_without_allowed_triggers" { -# source = "terraform-aws-modules/lambda/aws" -# version = "~> 6.0" - -# function_name = "${local.name}-without-allowed-triggers" -# description = "My awesome lambda function (without allowed triggers)" -# handler = "index.lambda_handler" -# runtime = "python3.8" - -# publish = true -# create_package = false -# local_existing_package = local.downloaded - -# # Allowed triggers will be managed by ALB module -# allowed_triggers = {} - -# depends_on = [null_resource.download_package] -# } - -# ################################################################################ -# # Supporting resources -# ################################################################################ - -# module "vpc" { -# source = "terraform-aws-modules/vpc/aws" -# version = "~> 5.0" - -# name = local.name -# cidr = local.vpc_cidr - -# azs = local.azs -# private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] -# public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] - -# tags = local.tags -# } - -# module "wildcard_cert" { -# source = "terraform-aws-modules/acm/aws" -# version = "~> 4.0" - -# domain_name = "*.${var.domain_name}" -# zone_id = data.aws_route53_zone.this.id -# } - -# data "aws_ssm_parameter" "al2" { -# name = "/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2" -# } - -# resource "aws_instance" "this" { -# ami = data.aws_ssm_parameter.al2.value -# instance_type = "t3.nano" -# subnet_id = element(module.vpc.private_subnets, 0) -# } - -# resource "aws_instance" "other" { -# ami = data.aws_ssm_parameter.al2.value -# instance_type = "t3.nano" -# subnet_id = element(module.vpc.private_subnets, 0) -# } - -# ################################################################## -# # AWS Cognito User Pool -# ################################################################## - -# resource "aws_cognito_user_pool" "this" { -# name = "user-pool-${local.name}" -# } - -# resource "aws_cognito_user_pool_client" "this" { -# name = "user-pool-client-${local.name}" -# user_pool_id = aws_cognito_user_pool.this.id -# generate_secret = true -# allowed_oauth_flows = ["code", "implicit"] -# callback_urls = ["https://${var.domain_name}/callback"] -# allowed_oauth_scopes = ["email", "openid"] -# allowed_oauth_flows_user_pool_client = true -# } - -# resource "random_string" "this" { -# length = 5 -# upper = false -# special = false -# } - -# resource "aws_cognito_user_pool_domain" "this" { -# domain = "${local.name}-${random_string.this.result}" -# user_pool_id = aws_cognito_user_pool.this.id -# } +resource "aws_route53_zone" "primary" { + name = var.domain_name +} -#module# "log_bucket" { -# source = "terraform-aws-modules/s3-bucket/aws" -# version = "~> 3.0" -# bucket_prefix = "${local.name}-logs-" -# acl = "log-delivery-write" +# name = var.domain_name -# # For example only -# force_destroy = true +resource "aws_route53_record" "api-cname" { + zone_id = aws_route53_zone.primary.zone_id + name = "us-east-1.${var.domain_name}" + type = "CNAME" + ttl = 5 -# control_object_ownership = true -# object_ownership = "ObjectWriter" + weighted_routing_policy { + weight = 10 + } -# attach_elb_log_delivery_policy = true # Required for ALB logs -# attach_lb_log_delivery_policy = true # Required for ALB/NLB logs + set_identifier = "dev" + records = [ + module.alb.dns_name + ] +} -# attach_deny_insecure_transport_policy = true -# attach_require_latest_tls_policy = true -# tags = local.tags -#} -resource "aws_route53_zone" "primary" { - name = var.domain_name -} data "cloudflare_zone" "zone" { #type = "full" name = "introspector.meme" @@ -592,6 +82,9 @@ module "acm" { version = "~> 4.0" domain_name = var.domain_name zone_id = aws_route53_zone.primary.zone_id + subject_alternative_names = [ + "*.${var.domain_name}" + ] } ## now we just lift the listener code @@ -613,55 +106,9 @@ resource "aws_lb_listener" "this" { #length(try(default_action.value.target_groups, [])) > 0 ? null : try(default_action.value.arn, aws_lb_target_group.this[default_action.value.target_group_key].arn, null) type = "forward" } - - # dynamic "default_action" { - # for_each = try([each.value.weighted_forward], []) - # content { - # forward { - # dynamic "target_group" { - # for_each = try(default_action.value.target_groups, []) - # content { - # arn = try(target_group.value.arn, aws_lb_target_group.this[target_group.value.target_group_key].arn, null) - # weight = try(target_group.value.weight, null) - # } - # } - # dynamic "stickiness" { - # for_each = try([default_action.value.stickiness], []) - # content { - # duration = try(stickiness.value.duration, 60) - # enabled = try(stickiness.value.enabled, null) - # } - # } - # } - # order = try(default_action.value.order, null) - # type = "forward" - # } - # } - # dynamic "default_action" { - # for_each = try([each.value.redirect], []) - # content { - # order = try(default_action.value.order, null) - # redirect { - # host = try(default_action.value.host, null) - # path = try(default_action.value.path, null) - # port = try(default_action.value.port, null) - # protocol = try(default_action.value.protocol, null) - # query = try(default_action.value.query, null) - # status_code = default_action.value.status_code - # } - # type = "redirect" - # } - # } -# load_balancer_arn = aws_lb.this[0].arn -# port = try(each.value.port, var.default_port) -# protocol = try(each.value.protocol, var.default_protocol) -# ssl_policy = contains(["HTTPS", "TLS"], try(each.value.protocol, var.default_protocol)) ? try(each.value.ssl_policy, "ELBSecurityPolicy-TLS13-1-2-Res-2021-06") : try(each.value.ssl_policy, null) -# tcp_idle_timeout_seconds = try(each.value.tcp_idle_timeout_seconds, null) -# tags = merge(local.tags, try(each.value.tags, {})) } - resource "aws_lb_target_group" "this" { name_prefix = "swarms" protocol = "HTTP" @@ -673,113 +120,8 @@ resource "aws_lb_target_group" "this" { load_balancing_anomaly_mitigation = "on" load_balancing_cross_zone_enabled = false protocol_version = "HTTP1" - # #target_id = "aws_instance.this.id" - # #tags = { - # # InstanceTargetGroupTag = "swarms" - # #} - # } - -# for_each = { for k, v in var.target_groups : k => v if local.create } -# connection_termination = try(each.value.connection_termination, null) -# deregistration_delay = try(each.value.deregistration_delay, null) - - # dynamic "health_check" { - # for_each = try([each.value.health_check], []) - - # content { - # enabled = try(health_check.value.enabled, null) - # healthy_threshold = try(health_check.value.healthy_threshold, null) - # interval = try(health_check.value.interval, null) - # matcher = try(health_check.value.matcher, null) - # path = try(health_check.value.path, null) - # port = try(health_check.value.port, null) - # protocol = try(health_check.value.protocol, null) - # timeout = try(health_check.value.timeout, null) - # unhealthy_threshold = try(health_check.value.unhealthy_threshold, null) - # } - # } - - # ip_address_type = try(each.value.ip_address_type, null) - # lambda_multi_value_headers_enabled = try(each.value.lambda_multi_value_headers_enabled, null) - # load_balancing_algorithm_type = try(each.value.load_balancing_algorithm_type, null) - # load_balancing_anomaly_mitigation = try(each.value.load_balancing_anomaly_mitigation, null) - # load_balancing_cross_zone_enabled = try(each.value.load_balancing_cross_zone_enabled, null) - # name = try(each.value.name, null) - # name_prefix = try(each.value.name_prefix, null) - # port = try(each.value.target_type, null) == "lambda" ? null : try(each.value.port, var.default_port) - # preserve_client_ip = try(each.value.preserve_client_ip, null) - # protocol = try(each.value.target_type, null) == "lambda" ? null : try(each.value.protocol, var.default_protocol) - # protocol_version = try(each.value.protocol_version, null) - # proxy_protocol_v2 = try(each.value.proxy_protocol_v2, null) - # slow_start = try(each.value.slow_start, null) - - # dynamic "stickiness" { - # for_each = try([each.value.stickiness], []) - - # content { - # cookie_duration = try(stickiness.value.cookie_duration, null) - # cookie_name = try(stickiness.value.cookie_name, null) - # enabled = try(stickiness.value.enabled, true) - # type = var.load_balancer_type == "network" ? "source_ip" : stickiness.value.type - # } - # } - - # dynamic "target_failover" { - # for_each = try(each.value.target_failover, []) - - # content { - # on_deregistration = target_failover.value.on_deregistration - # on_unhealthy = target_failover.value.on_unhealthy - # } - # } - - # dynamic "target_group_health" { - # for_each = try([each.value.target_group_health], []) - - # content { - - # dynamic "dns_failover" { - # for_each = try([target_group_health.value.dns_failover], []) - - # content { - # minimum_healthy_targets_count = try(dns_failover.value.minimum_healthy_targets_count, null) - # minimum_healthy_targets_percentage = try(dns_failover.value.minimum_healthy_targets_percentage, null) - # } - # } - - # dynamic "unhealthy_state_routing" { - # for_each = try([target_group_health.value.unhealthy_state_routing], []) - - # content { - # minimum_healthy_targets_count = try(unhealthy_state_routing.value.minimum_healthy_targets_count, null) - # minimum_healthy_targets_percentage = try(unhealthy_state_routing.value.minimum_healthy_targets_percentage, null) - # } - # } - # } - # } - - # dynamic "target_health_state" { - # for_each = try([each.value.target_health_state], []) - # content { - # enable_unhealthy_connection_termination = try(target_health_state.value.enable_unhealthy_connection_termination, true) - # unhealthy_draining_interval = try(target_health_state.value.unhealthy_draining_interval, null) - # } - # } - - # target_type = try(each.value.target_type, null) - # vpc_id = try(each.value.vpc_id, var.vpc_id) - - # tags = merge(local.tags, try(each.value.tags, {})) - - # lifecycle { - # create_before_destroy = true - # } } - - - - output zone_id { value = aws_route53_zone.primary.zone_id } From 1cf229557fc04a9996f3d40bd20ffac6b6a4e988 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Mon, 16 Dec 2024 12:28:59 -0500 Subject: [PATCH 024/158] health check --- .../application_load_balancer/main.tf | 47 +++++++++++++------ .../components/autoscaling_group/main.tf | 6 +-- .../dev/us-east-1/main.tf | 4 +- 3 files changed, 38 insertions(+), 19 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf index e4de884..6c06194 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf @@ -26,7 +26,7 @@ locals { module "alb" { source = "terraform-aws-modules/alb/aws" version = "9.12.0" - name = var.name # local.name + name = "${var.name}-api" # local.name vpc_id = var.vpc_id # module.vpc.vpc_id subnets = var.public_subnets # module.vpc.public_subnets enable_deletion_protection = false @@ -40,32 +40,27 @@ resource "aws_route53_zone" "primary" { name = var.domain_name } - -# name = var.domain_name - resource "aws_route53_record" "api-cname" { zone_id = aws_route53_zone.primary.zone_id name = "us-east-1.${var.domain_name}" type = "CNAME" ttl = 5 - weighted_routing_policy { - weight = 10 - } - - set_identifier = "dev" +# weighted_routing_policy { +# weight = 10 +# } + #set_identifier = "dev" records = [ module.alb.dns_name ] } - - data "cloudflare_zone" "zone" { #type = "full" name = "introspector.meme" account_id = "0ceffbadd0a04623896f5317a1e40d94" } + resource "cloudflare_record" "aws-ns-record" { count = "${length(aws_route53_zone.primary.name_servers)}" #domain = "${var.domain_name}" @@ -116,10 +111,28 @@ resource "aws_lb_target_group" "this" { target_type = "instance" vpc_id = var.vpc_id deregistration_delay = 10 - load_balancing_algorithm_type = "weighted_random" - load_balancing_anomaly_mitigation = "on" - load_balancing_cross_zone_enabled = false + #load_balancing_algorithm_type = "weighted_random" + #load_balancing_anomaly_mitigation = "on" + #load_balancing_cross_zone_enabled = false protocol_version = "HTTP1" + # + health_check { + path = "/v1/docs" # the docs api + enabled = true + healthy_threshold = 10 + interval = 130 + port = "traffic-port" + protocol = "HTTP" + timeout = 120 + unhealthy_threshold = 10 + } + +# stickiness { +# cookie_duration = 86400 +# enabled = true +# type = "lb_cookie" +# } + } output zone_id { @@ -133,4 +146,10 @@ output alb_target_group_arn { value = aws_lb_target_group.this.arn } +output dns { + value = module.alb.dns_name +} +output cname { + value = aws_route53_record.api-cname.fqdn +} diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf index e644aec..bb78a6e 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf @@ -1,5 +1,5 @@ variable target_group_arn{} -variable security_group_id {} +#variable security_group_id {} variable name {} variable instance_type { # default = "t3.micro" @@ -62,7 +62,7 @@ module "autoscaling" { version = "8.0.0" name = var.name - + health_check_type = "EC2" desired_capacity = 1 max_size = 5 min_size = 1 @@ -83,7 +83,7 @@ module "autoscaling" { device_index = 0 delete_on_termination = true description = "interface1" - security_groups = [var.security_group_id] +# security_groups = [var.security_group_id] } ] instance_type = var.instance_type diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index acc88dc..744dcc3 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -51,7 +51,7 @@ module "lt_dynamic" { for_each = toset(var.instance_types) instance_type = each.key name = "swarms-size-${each.key}" - security_group_id = module.security.security_group_id + security_group_id = module.security.internal_security_group_id source = "./components/launch_template" } @@ -69,7 +69,7 @@ module "alb" { module "asg_dynamic" { for_each = toset(var.instance_types) source = "./components/autoscaling_group" - security_group_id = module.security.internal_security_group_id +# security_group_id = module.security.internal_security_group_id instance_type = each.key name = "swarms-size-${each.key}" launch_template_id = module.lt_dynamic[each.key].launch_template_id From c6fd42cd730a1f68b233e7e54b15f009e0c4cf2e Mon Sep 17 00:00:00 2001 From: mike dupont Date: Mon, 16 Dec 2024 12:43:25 -0500 Subject: [PATCH 025/158] update --- .../swarms-aws-agent-api/dev/us-east-1/Readme.md | 9 ++++++++- .../dev/us-east-1/components/Readme.md | 15 +++++++++------ .../swarms-aws-agent-api/dev/us-east-1/main.tf | 7 ++++--- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/Readme.md b/environments/swarms-aws-agent-api/dev/us-east-1/Readme.md index 4f57e60..144ec90 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/Readme.md +++ b/environments/swarms-aws-agent-api/dev/us-east-1/Readme.md @@ -1,3 +1,10 @@ +# Done + 1. create vpc +2. create iam roles and users +3. create asg and alb and route53 and acm + +# Todo + +1. create cognito pool 2. create ami -4. create iam roles and users diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md b/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md index b9e7c04..92f42ec 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md @@ -2,21 +2,24 @@ ## Phase 1 0. create standard vpc with a private network to host ec2, -this uses a natgw that costs money bhttps://aws.amazon.com/vpc/pricing/ -0.45 so we will put the intances in public for now and use security groups to limit access. +this uses a natgw that costs money https://aws.amazon.com/vpc/pricing/ +so we will put the intances in public for now and use security groups to limit access. 1. create minimal ec2 instance in machine_image -terraform (later packer) for ubuntu python uvicorn fastapi nginx systemd server with custom git modules +terraform for ubuntu python uvicorn fastapi nginx systemd server with custom git modules 2. create minimal ec2 ami from instance in machine_image 3. create autoscaling_group of size 1 for image 4. create application load balancer +5. create dns_entry -send users back to server via sticky sessions or some id. -5. create dns_entry -6. create cognito user pool for login +# todo + +1. alb sticky sessions :send users back to server via sticky sessions or some id. +2. create cognito user pool for login 7. create work_queue 8. create lambda_workers on queue 9. create resource_launchers to create new resources. +10. packer ami for ubuntu python uvicorn fastapi nginx systemd server with custom git modules diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index 744dcc3..629d942 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -1,6 +1,6 @@ locals { # instance_type = "t3.large" - instance_type = "t3.medium" + # instance_type = "t3.medium" ami = "ami-0e2c8caa4b6378d8c" name = "swarms" region = "us-east-1" @@ -41,9 +41,10 @@ variable "instance_types" { # "t4g.nano", "t3a.nano", "t3.nano", "t2.nano", # "t4g.micro", "t3a.micro", "t3.micro", "t2.micro", "t1.micro", #"t4g.small", "t3a.small", - "t3.small", + #"t3.small", #"t2.small", not working -# "t2.medium" # "t3.medium" + # "t2.medium" # + "t3.medium" ] } From 049c6ef373d827a1b8da68acd117073a8e384c0a Mon Sep 17 00:00:00 2001 From: mike dupont Date: Mon, 16 Dec 2024 13:47:43 -0500 Subject: [PATCH 026/158] start of swarms deploy --- accounts/swarms/.terraform.lock.hcl | 42 +++++++++++++++++++++++++++++ accounts/swarms/main.tf | 15 +++++++++++ 2 files changed, 57 insertions(+) create mode 100644 accounts/swarms/.terraform.lock.hcl create mode 100644 accounts/swarms/main.tf diff --git a/accounts/swarms/.terraform.lock.hcl b/accounts/swarms/.terraform.lock.hcl new file mode 100644 index 0000000..cb584db --- /dev/null +++ b/accounts/swarms/.terraform.lock.hcl @@ -0,0 +1,42 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/hashicorp/aws" { + version = "5.81.0" + constraints = ">= 3.29.0, >= 4.40.0, >= 5.55.0, >= 5.73.0" + hashes = [ + "h1:ird967uf44WBZ0u9rMlOdO4nCKJZMlUknLcD3lz4dWs=", + "zh:0d0c7cc1e16b16cd00ab36de35038162e1871b51a902e9016d08c55c9fc4de35", + "zh:0e4b8c6e46999015292ab4fb9a430ab00107a276f25243552cde66db53c58661", + "zh:30041314cdd4e877d75ee8c9d36aecfca094e276f7a3d8150f929cf5169b2fa5", + "zh:5ebd248ce3e5a7ef9cc2f41499668f182146325e10ea305c70469122f6161a13", + "zh:888a69d371373b418549e03f5922becb2b8074cb463552ecfa65f30455708db0", + "zh:8a21bb7fe60383ff5ca9db8630a287e41fd520d2514c08874a16dc74500fadd7", + "zh:9c4663dcbfe58544642d70ebfdc6c5fa91592ff04164f77c655e32e6024483e2", + "zh:b322873f1209443a8407d5f782d7d917de6a1391239dbd0e7f809ce6507bed76", + "zh:b7c9d5ca14b818b5932ac58a490646a425ebc41b33d149090aa5f48d1ca35c99", + "zh:e76cd202b03749f3082b0cbe849fd2e731cf3f9a6aa994d2d629602c3aede36c", + ] +} + +provider "registry.opentofu.org/hashicorp/cloudflare" { + version = "4.48.0" + hashes = [ + "h1:ePGvSurmlqOCkD761vkhRmz7bsK36/EnIvx2Xy8TdXo=", + "zh:04c0a49c2b23140b2f21cfd0d52f9798d70d3bdae3831613e156aabe519bbc6c", + "zh:185f21b4834ba63e8df1f84aa34639d8a7e126429a4007bb5f9ad82f2602a997", + "zh:234724f52cb4c0c3f7313d3b2697caef26d921d134f26ae14801e7afac522f7b", + "zh:38a56fcd1b3e40706af995611c977816543b53f1e55fe2720944aae2b6828fcb", + "zh:419938f5430fc78eff933470aefbf94a460a478f867cf7761a3dea177b4eb153", + "zh:4b46d92bfde1deab7de7ba1a6bbf4ba7c711e4fd925341ddf09d4cc28dae03d8", + "zh:537acd4a31c752f1bae305ba7190f60b71ad1a459f22d464f3f914336c9e919f", + "zh:5ff36b005aad07697dd0b30d4f0c35dbcdc30dc52b41722552060792fa87ce04", + "zh:635c5ee419daea098060f794d9d7d999275301181e49562c4e4c08f043076937", + "zh:859277c330d61f91abe9e799389467ca11b77131bf34bedbef52f8da68b2bb49", + "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", + "zh:927dfdb8d9aef37ead03fceaa29e87ba076a3dd24e19b6cefdbb0efe9987ff8c", + "zh:bbf2226f07f6b1e721877328e69ded4b64f9c196634d2e2429e3cfabbe41e532", + "zh:daeed873d6f38604232b46ee4a5830c85d195b967f8dbcafe2fcffa98daf9c5f", + "zh:f8f2fc4646c1ba44085612fa7f4dbb7cbcead43b4e661f2b98ddfb4f68afc758", + ] +} diff --git a/accounts/swarms/main.tf b/accounts/swarms/main.tf new file mode 100644 index 0000000..8067eb2 --- /dev/null +++ b/accounts/swarms/main.tf @@ -0,0 +1,15 @@ +output dns { + value = "api.swarms.ai" +} + +output profile { + value = "swarms" +} + +output account { + value = "916723593639" +} + +module "swarms_api" { + source = "../../environments/swarms-aws-agent-api/dev/us-east-1" +} From 05ed0bd8b7580f71aa857ccb230aaddd96286ec2 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Mon, 16 Dec 2024 14:12:05 -0500 Subject: [PATCH 027/158] start of swarms --- accounts/swarms/.terraform.lock.hcl | 22 -------------- accounts/swarms/main.tf | 15 +++++++++- .../application_load_balancer/cloudflare.tf | 18 +++++++++++ .../application_load_balancer/main.tf | 15 ---------- .../components/machine_image/main.tf | 3 -- .../components/machine_image/versions.tf | 10 ------- .../dev/us-east-1/components/vpc/main.tf | 3 -- .../dev/us-east-1/components/vpc/outputs.tf | 7 +++++ .../dev/us-east-1/components/vpc/versions.tf | 10 ------- .../dev/us-east-1/main.tf | 30 +++++++++++++++---- 10 files changed, 63 insertions(+), 70 deletions(-) create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/cloudflare.tf diff --git a/accounts/swarms/.terraform.lock.hcl b/accounts/swarms/.terraform.lock.hcl index cb584db..f639596 100644 --- a/accounts/swarms/.terraform.lock.hcl +++ b/accounts/swarms/.terraform.lock.hcl @@ -18,25 +18,3 @@ provider "registry.opentofu.org/hashicorp/aws" { "zh:e76cd202b03749f3082b0cbe849fd2e731cf3f9a6aa994d2d629602c3aede36c", ] } - -provider "registry.opentofu.org/hashicorp/cloudflare" { - version = "4.48.0" - hashes = [ - "h1:ePGvSurmlqOCkD761vkhRmz7bsK36/EnIvx2Xy8TdXo=", - "zh:04c0a49c2b23140b2f21cfd0d52f9798d70d3bdae3831613e156aabe519bbc6c", - "zh:185f21b4834ba63e8df1f84aa34639d8a7e126429a4007bb5f9ad82f2602a997", - "zh:234724f52cb4c0c3f7313d3b2697caef26d921d134f26ae14801e7afac522f7b", - "zh:38a56fcd1b3e40706af995611c977816543b53f1e55fe2720944aae2b6828fcb", - "zh:419938f5430fc78eff933470aefbf94a460a478f867cf7761a3dea177b4eb153", - "zh:4b46d92bfde1deab7de7ba1a6bbf4ba7c711e4fd925341ddf09d4cc28dae03d8", - "zh:537acd4a31c752f1bae305ba7190f60b71ad1a459f22d464f3f914336c9e919f", - "zh:5ff36b005aad07697dd0b30d4f0c35dbcdc30dc52b41722552060792fa87ce04", - "zh:635c5ee419daea098060f794d9d7d999275301181e49562c4e4c08f043076937", - "zh:859277c330d61f91abe9e799389467ca11b77131bf34bedbef52f8da68b2bb49", - "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", - "zh:927dfdb8d9aef37ead03fceaa29e87ba076a3dd24e19b6cefdbb0efe9987ff8c", - "zh:bbf2226f07f6b1e721877328e69ded4b64f9c196634d2e2429e3cfabbe41e532", - "zh:daeed873d6f38604232b46ee4a5830c85d195b967f8dbcafe2fcffa98daf9c5f", - "zh:f8f2fc4646c1ba44085612fa7f4dbb7cbcead43b4e661f2b98ddfb4f68afc758", - ] -} diff --git a/accounts/swarms/main.tf b/accounts/swarms/main.tf index 8067eb2..ff3317e 100644 --- a/accounts/swarms/main.tf +++ b/accounts/swarms/main.tf @@ -1,5 +1,12 @@ +locals { + dns = "api.swarms.ai" + +} +provider aws { + region = "us-east-2" +} output dns { - value = "api.swarms.ai" + value = local.dns } output profile { @@ -10,6 +17,12 @@ output account { value = "916723593639" } +output region { + value = "us-east-2" +} + module "swarms_api" { source = "../../environments/swarms-aws-agent-api/dev/us-east-1" + domain = local.dns } + diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/cloudflare.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/cloudflare.tf new file mode 100644 index 0000000..be2ab3b --- /dev/null +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/cloudflare.tf @@ -0,0 +1,18 @@ +#variable "dns_name" {} +#variable "account_id" {} +# data "cloudflare_zone" "zone" { +# count =0 +# name = var.dns_name +# account_id = var.account_id +# } + +# resource "cloudflare_record" "aws-ns-record" { +# count = 0 +# # count = "${length(aws_route53_zone.primary.name_servers)}" +# #domain = "${var.domain_name}" +# name = var.domain_name +# # zone_id = data.cloudflare_zone[0].zone.id +# content = "${element(aws_route53_zone.primary.name_servers, count.index)}" +# type = "NS" +# priority = 1 +# } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf index 6c06194..896e006 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf @@ -55,21 +55,6 @@ resource "aws_route53_record" "api-cname" { ] } -data "cloudflare_zone" "zone" { - #type = "full" - name = "introspector.meme" - account_id = "0ceffbadd0a04623896f5317a1e40d94" -} - -resource "cloudflare_record" "aws-ns-record" { - count = "${length(aws_route53_zone.primary.name_servers)}" - #domain = "${var.domain_name}" - name = var.domain_name - zone_id = data.cloudflare_zone.zone.id - content = "${element(aws_route53_zone.primary.name_servers, count.index)}" - type = "NS" - priority = 1 -} module "acm" { # count = 0 diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf index ec203e8..36c5cbd 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf @@ -1,6 +1,3 @@ -provider "aws" { - region = "us-east-1" -} locals { ami = "ami-0e2c8caa4b6378d8c" diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/versions.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/versions.tf index fd4d116..e69de29 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/versions.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/versions.tf @@ -1,10 +0,0 @@ -terraform { - required_version = ">= 1.0" - - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 4.66" - } - } -} diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/main.tf index 5d60b80..2448aae 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/main.tf @@ -2,9 +2,6 @@ data "aws_availability_zones" "available" {} -provider "aws" { - region = local.region -} locals { name = "swarms" diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/outputs.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/outputs.tf index 655b291..5cb68e1 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/outputs.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/outputs.tf @@ -549,3 +549,10 @@ output "vpc_flow_log_cloudwatch_iam_role_arn" { # description = "ID of the security group" # value = module.vpc_endpoints.security_group_id # } +output ec2_public_subnet_id_2 { + value = 1 +} + +output ec2_public_subnet_id_1 { + value = 1 +} diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/versions.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/versions.tf index e1ed610..e69de29 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/versions.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/versions.tf @@ -1,10 +0,0 @@ -terraform { - required_version = ">= 1.0" - - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 5.46" - } - } -} diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index 629d942..3814095 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -1,18 +1,36 @@ +variable domain {} locals { # instance_type = "t3.large" # instance_type = "t3.medium" - ami = "ami-0e2c8caa4b6378d8c" + #ami = "ami-0e2c8caa4b6378d8c" + ami_name = "ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*" name = "swarms" - region = "us-east-1" - domain = "api.introspector.meme" - ec2_public_subnet_id_1 = "subnet-057c90cfe7b2e5646" # swarms-public-us-east-1a - ec2_public_subnet_id_2 = "subnet-05d8aef1f71b5fe22" # b - vpc_id = "vpc-04f28c9347af48b55" + region = "us-east-2" + domain = var.domain tags = { project="swarms" } } +data "aws_ami" "ami" { + most_recent = true + name_regex = "^${local.ami_name}" +} + +locals { + ami_id = data.aws_ami.ami.id +} + +module "vpc" { + source = "./components/vpc" +} + +locals { + ec2_public_subnet_id_1 = module.vpc.ec2_public_subnet_id_1 + ec2_public_subnet_id_2 = module.vpc.ec2_public_subnet_id_2 + vpc_id = module.vpc.vpc_id +} + module "security" { source = "./components/security" } From 248221ca533a1890173f7148556033b0513ea73f Mon Sep 17 00:00:00 2001 From: mike dupont Date: Mon, 16 Dec 2024 19:17:12 -0500 Subject: [PATCH 028/158] now applying --- Makefile | 2 + accounts/swarms/main.tf | 25 +- accounts/swarms/main.txt | 1769 +++++++++++++++++ .../dev/us-east-1/alb/main.tf | 0 .../dev/us-east-1/asg/main.tf | 0 .../application_load_balancer/https/main.tf | 32 + .../application_load_balancer/main.tf | 106 +- .../application_load_balancer/notes.prg | 479 +++++ .../application_load_balancer/route53/main.tf | 36 + .../target_group/main.tf | 34 + .../components/autoscaling_group/main.tf | 63 +- .../components/launch_template/main.tf | 29 +- .../components/machine_image/main.tf | 4 - .../dev/us-east-1/components/security/main.tf | 26 +- .../dev/us-east-1/components/vpc/main.tf | 7 +- .../dev/us-east-1/components/vpc/outputs.tf | 4 +- .../dev/us-east-1/main.tf | 85 +- 17 files changed, 2477 insertions(+), 224 deletions(-) create mode 100644 Makefile create mode 100644 accounts/swarms/main.txt create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/alb/main.tf create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/asg/main.tf create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/notes.prg create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/target_group/main.tf diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..de928aa --- /dev/null +++ b/Makefile @@ -0,0 +1,2 @@ +test: + find -name \*.tf -exec grep -r provider {} \; -print diff --git a/accounts/swarms/main.tf b/accounts/swarms/main.tf index ff3317e..2468b5b 100644 --- a/accounts/swarms/main.tf +++ b/accounts/swarms/main.tf @@ -1,9 +1,14 @@ locals { - dns = "api.swarms.ai" - + #ami_name = "ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*" + ami_name = "ubuntu-minimal/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-minimal-*" + dns = "api.swarms.ai" + account = "916723593639" + region = "us-east-2" } + provider aws { region = "us-east-2" + profile = "swarms" } output dns { value = local.dns @@ -21,8 +26,24 @@ output region { value = "us-east-2" } +#SLOW + data "aws_ami" "ami" { + most_recent = true + name_regex = "^${local.ami_name}" + } + module "swarms_api" { source = "../../environments/swarms-aws-agent-api/dev/us-east-1" domain = local.dns + ami_id = data.aws_ami.ami.id + #"ami-0ad5d6c7069ce56ac" + #ami_id = "ami-0ad5d6c7069ce56ac" + + name = "swarms" + tags = {project="swarms"} + } +output api { + value = module.swarms_api +} diff --git a/accounts/swarms/main.txt b/accounts/swarms/main.txt new file mode 100644 index 0000000..65f2ed4 --- /dev/null +++ b/accounts/swarms/main.txt @@ -0,0 +1,1769 @@ +mdupont@mdupont-G470:~/2024/12/swarms-terraform/accounts/swarms$ tofu apply +module.swarms_api.module.vpc.data.aws_availability_zones.available: Reading... +module.swarms_api.module.alb.module.alb.data.aws_partition.current: Reading... +module.swarms_api.module.asg_dynamic["t3.medium"].module.autoscaling.data.aws_partition.current: Reading... +module.swarms_api.module.kp.aws_key_pair.mdupont-deployer: Refreshing state... [id=mdupont-deployer-key] +module.swarms_api.data.aws_ami.ami: Reading... +module.swarms_api.module.alb.module.acm.aws_route53_record.validation[0]: Refreshing state... [id=Z05433953QQ7Q4A095TXN__2d35674ee842f81e6c1d4b1ff81bf202.api.swarms.ai._CNAME] +module.swarms_api.module.alb.aws_route53_zone.primary: Refreshing state... [id=Z05433953QQ7Q4A095TXN] +module.swarms_api.module.vpc.module.vpc.aws_vpc.this[0]: Refreshing state... [id=vpc-0474ec5cd8a0dd94b] +module.swarms_api.module.alb.module.alb.data.aws_partition.current: Read complete after 0s [id=aws] +module.swarms_api.module.asg_dynamic["t3.medium"].module.autoscaling.data.aws_partition.current: Read complete after 0s [id=aws] +module.swarms_api.module.alb.module.acm.aws_acm_certificate.this[0]: Refreshing state... [id=arn:aws:acm:us-east-2:767503528736:certificate/e4c7475f-9b0a-471f-b02c-134759b27668] +module.swarms_api.module.alb.data.aws_availability_zones.available: Reading... +module.swarms_api.module.asg_dynamic["t3.medium"].aws_iam_role.ssm: Refreshing state... [id=ssm-swarms-size-t3.medium] +module.swarms_api.module.asg_dynamic["t3.medium"].module.autoscaling.data.aws_iam_policy_document.assume_role_policy[0]: Reading... +module.swarms_api.module.asg_dynamic["t3.medium"].module.autoscaling.data.aws_iam_policy_document.assume_role_policy[0]: Read complete after 0s [id=1256122602] +module.swarms_api.module.asg_dynamic["t3.medium"].module.autoscaling.aws_iam_role.this[0]: Refreshing state... [id=ssm-swarms-size-t3.medium-20241216190828467900000002] +module.swarms_api.module.alb.data.aws_availability_zones.available: Read complete after 0s [id=us-east-2] +module.swarms_api.module.vpc.data.aws_availability_zones.available: Read complete after 0s [id=us-east-2] +module.swarms_api.module.asg_dynamic["t3.medium"].aws_iam_instance_profile.ssm: Refreshing state... [id=ssm-swarms-size-t3.medium] +module.swarms_api.module.asg_dynamic["t3.medium"].module.autoscaling.aws_iam_instance_profile.this[0]: Refreshing state... [id=ssm-swarms-size-t3.medium-20241216190828732900000006] +module.swarms_api.module.asg_dynamic["t3.medium"].module.autoscaling.aws_iam_role_policy_attachment.this["AmazonSSMManagedInstanceCore"]: Refreshing state... [id=ssm-swarms-size-t3.medium-20241216190828467900000002-20241216190828805000000007] +module.swarms_api.module.vpc.module.vpc.aws_default_route_table.default[0]: Refreshing state... [id=rtb-0834ea0a1cb0b3e20] +module.swarms_api.module.vpc.module.vpc.aws_default_security_group.this[0]: Refreshing state... [id=sg-09e45cb7d4cd6e40f] +module.swarms_api.module.vpc.module.vpc.aws_internet_gateway.this[0]: Refreshing state... [id=igw-094056be43e3ff84b] +module.swarms_api.module.vpc.module.vpc.aws_default_network_acl.this[0]: Refreshing state... [id=acl-0757b0776826bfbf8] +module.swarms_api.module.vpc.module.vpc.aws_route_table.public[0]: Refreshing state... [id=rtb-01ab4dcce11853b1d] +module.swarms_api.module.vpc.module.vpc.aws_subnet.public[1]: Refreshing state... [id=subnet-0bc4a9913a3ce2359] +module.swarms_api.module.vpc.module.vpc.aws_subnet.public[2]: Refreshing state... [id=subnet-02678c830b74a5b5d] +module.swarms_api.module.vpc.module.vpc.aws_subnet.public[0]: Refreshing state... [id=subnet-0c0af7a5f74ac54cc] +module.swarms_api.module.vpc.module.vpc.aws_route_table.private[0]: Refreshing state... [id=rtb-078545b8b8efb0295] +module.swarms_api.module.vpc.module.vpc.aws_route_table.private[2]: Refreshing state... [id=rtb-062a89232fc0355b2] +module.swarms_api.module.vpc.module.vpc.aws_route_table.private[1]: Refreshing state... [id=rtb-0513c8c9e488570c3] +module.swarms_api.module.vpc.module.vpc.aws_subnet.private[0]: Refreshing state... [id=subnet-03f08e37b2201be9c] +module.swarms_api.module.vpc.module.vpc.aws_subnet.private[1]: Refreshing state... [id=subnet-0e6c2fef4377b5590] +module.swarms_api.module.vpc.module.vpc.aws_subnet.private[2]: Refreshing state... [id=subnet-0bbfe21680e03572b] +module.swarms_api.module.vpc.module.vpc.aws_route.public_internet_gateway[0]: Refreshing state... [id=r-rtb-01ab4dcce11853b1d1080289494] +module.swarms_api.module.alb.aws_lb_target_group.this: Refreshing state... [id=arn:aws:elasticloadbalancing:us-east-2:767503528736:targetgroup/swarms2024121619084133320000000c/001ec925380d4bcd] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.public[2]: Refreshing state... [id=rtbassoc-037ae6ac8e6a6692b] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.public[0]: Refreshing state... [id=rtbassoc-07c9642c84441a2c1] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.public[1]: Refreshing state... [id=rtbassoc-08546098c2d85fd6f] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.private[0]: Refreshing state... [id=rtbassoc-02e71f6004f433e3e] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.private[2]: Refreshing state... [id=rtbassoc-082b7f543579d5f38] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.private[1]: Refreshing state... [id=rtbassoc-05cfcd90f6d653fe1] +module.swarms_api.data.aws_ami.ami: Still reading... [10s elapsed] +module.swarms_api.data.aws_ami.ami: Read complete after 13s [id=ami-0ad5d6c7069ce56ac] + +OpenTofu used the selected providers to generate the following execution plan. Resource actions are indicated with +the following symbols: + + create + - destroy + +OpenTofu will perform the following actions: + + # module.swarms_api.module.lt_dynamic["t3.medium"].aws_launch_template.ec2_launch_template will be created + + resource "aws_launch_template" "ec2_launch_template" { + + arn = (known after apply) + + default_version = (known after apply) + + id = (known after apply) + + image_id = + + instance_type = "t3.medium" + + key_name = "mdupont-deployer-key" + + latest_version = (known after apply) + + name = (known after apply) + + name_prefix = "swarms-size-t3.medium-launch-template-" + + tags = { + + "instance_type" = "t3.medium" + + "name" = "swarms-size-t3.medium" + + "project" = "swarms" + } + + tags_all = { + + "instance_type" = "t3.medium" + + "name" = "swarms-size-t3.medium" + + "project" = "swarms" + } + + user_data = "IyEvYmluL2Jhc2gKZXhwb3J0IEhPTUU9L3Jvb3QKYXB0IHVwZGF0ZQphcHQtZ2V0IGluc3RhbGwgLXkgZWMyLWluc3RhbmNlLWNvbm5lY3QgZ2l0IHZpcnR1YWxlbnYKc25hcCBpbnN0YWxsIGFtYXpvbi1zc20tYWdlbnQgLS1jbGFzc2ljIHx8IGVjaG8gb29wczEKc25hcCBzdGFydCBhbWF6b24tc3NtLWFnZW50IHx8IGVjaG8gb29wczIKYXB0LWdldCBpbnN0YWxsIC15IC0tbm8taW5zdGFsbC1yZWNvbW1lbmRzIGNhLWNlcnRpZmljYXRlcz0yMDIzMDMxMSBjdXJsPTcuODguMS0xMCtkZWIxMnU3IHwgIGVjaG8gb29wcwpjdXJsIC1PICJodHRwczovL3MzLmFtYXpvbmF3cy5jb20vYW1hem9uY2xvdWR3YXRjaC1hZ2VudC91YnVudHUvJChkcGtnIC0tcHJpbnQtYXJjaGl0ZWN0dXJlKS9sYXRlc3QvYW1hem9uLWNsb3Vkd2F0Y2gtYWdlbnQuZGViIgpkcGtnIC1pIC1FIGFtYXpvbi1jbG91ZHdhdGNoLWFnZW50LmRlYgogCmlmIFsgISAtZCAiL29wdC9zd2FybXMvIiBdOyB0aGVuCiAgZ2l0IGNsb25lIGh0dHBzOi8vZ2l0aHViLmNvbS9qbWlrZWR1cG9udDIvc3dhcm1zICIvb3B0L3N3YXJtcy8iCmZpCmNkICIvb3B0L3N3YXJtcy8iIHx8IGV4aXQgMQpleHBvcnQgQlJBTkNIPWZlYXR1cmUvZWMyCmdpdCBzdGFzaApnaXQgY2hlY2tvdXQgLS1mb3JjZSAkQlJBTkNICmJhc2ggLXggL29wdC9zd2FybXMvYXBpL2luc3RhbGwuc2gK" + + + block_device_mappings { + + device_name = "/dev/sda1" + + + ebs { + + encrypted = "true" + + iops = (known after apply) + + throughput = (known after apply) + + volume_size = 30 + + volume_type = "gp3" + } + } + + + iam_instance_profile { + + name = "swarms-20241213150629570500000003" + } + + + network_interfaces { + + associate_public_ip_address = "true" + + delete_on_termination = "true" + + security_groups = (known after apply) + } + } + + # module.swarms_api.module.alb.module.acm.aws_acm_certificate.this[0] will be destroyed + # (because aws_acm_certificate.this is not in configuration) + - resource "aws_acm_certificate" "this" { + - arn = "arn:aws:acm:us-east-2:767503528736:certificate/e4c7475f-9b0a-471f-b02c-134759b27668" -> null + - domain_name = "api.swarms.ai" -> null + - domain_validation_options = [ + - { + - domain_name = "*.api.swarms.ai" + - resource_record_name = "_2d35674ee842f81e6c1d4b1ff81bf202.api.swarms.ai." + - resource_record_type = "CNAME" + - resource_record_value = "_14001873a994d4ba61251844ed588611.zfyfvmchrl.acm-validations.aws." + }, + - { + - domain_name = "api.swarms.ai" + - resource_record_name = "_2d35674ee842f81e6c1d4b1ff81bf202.api.swarms.ai." + - resource_record_type = "CNAME" + - resource_record_value = "_14001873a994d4ba61251844ed588611.zfyfvmchrl.acm-validations.aws." + }, + ] -> null + - id = "arn:aws:acm:us-east-2:767503528736:certificate/e4c7475f-9b0a-471f-b02c-134759b27668" -> null + - key_algorithm = "RSA_2048" -> null + - pending_renewal = false -> null + - renewal_eligibility = "INELIGIBLE" -> null + - renewal_summary = [] -> null + - status = "PENDING_VALIDATION" -> null + - subject_alternative_names = [ + - "*.api.swarms.ai", + - "api.swarms.ai", + ] -> null + - tags = {} -> null + - tags_all = {} -> null + - type = "AMAZON_ISSUED" -> null + - validation_emails = [] -> null + - validation_method = "DNS" -> null + + - options { + - certificate_transparency_logging_preference = "ENABLED" -> null + } + } + + # module.swarms_api.module.alb.module.acm.aws_route53_record.validation[0] will be destroyed + # (because aws_route53_record.validation is not in configuration) + - resource "aws_route53_record" "validation" { + - allow_overwrite = true -> null + - fqdn = "_2d35674ee842f81e6c1d4b1ff81bf202.api.swarms.ai" -> null + - id = "Z05433953QQ7Q4A095TXN__2d35674ee842f81e6c1d4b1ff81bf202.api.swarms.ai._CNAME" -> null + - multivalue_answer_routing_policy = false -> null + - name = "_2d35674ee842f81e6c1d4b1ff81bf202.api.swarms.ai" -> null + - records = [ + - "_14001873a994d4ba61251844ed588611.zfyfvmchrl.acm-validations.aws.", + ] -> null + - ttl = 60 -> null + - type = "CNAME" -> null + - zone_id = "Z05433953QQ7Q4A095TXN" -> null + } + + # module.swarms_api.module.alb.module.alb.aws_lb.this[0] will be created + + resource "aws_lb" "this" { + + arn = (known after apply) + + arn_suffix = (known after apply) + + client_keep_alive = 7200 + + desync_mitigation_mode = "defensive" + + dns_name = (known after apply) + + drop_invalid_header_fields = true + + enable_deletion_protection = false + + enable_http2 = true + + enable_tls_version_and_cipher_suite_headers = false + + enable_waf_fail_open = false + + enable_xff_client_port = false + + enforce_security_group_inbound_rules_on_private_link_traffic = (known after apply) + + id = (known after apply) + + idle_timeout = 60 + + internal = (known after apply) + + ip_address_type = (known after apply) + + load_balancer_type = "application" + + name = "swarms-api" + + name_prefix = (known after apply) + + preserve_host_header = false + + security_groups = (known after apply) + + subnets = [ + + "1", + ] + + tags = { + + "Example" = "ex-swarms" + + "Name" = "ex-swarms" + + "Repository" = "https://github.com/terraform-aws-modules/terraform-aws-alb" + + "terraform-aws-modules" = "alb" + } + + tags_all = { + + "Example" = "ex-swarms" + + "Name" = "ex-swarms" + + "Repository" = "https://github.com/terraform-aws-modules/terraform-aws-alb" + + "terraform-aws-modules" = "alb" + } + + vpc_id = (known after apply) + + xff_header_processing_mode = "append" + + zone_id = (known after apply) + + + timeouts {} + } + + # module.swarms_api.module.asg_dynamic["t3.medium"].module.autoscaling.aws_autoscaling_group.this[0] will be created + + resource "aws_autoscaling_group" "this" { + + arn = (known after apply) + + availability_zones = (known after apply) + + default_cooldown = (known after apply) + + desired_capacity = 1 + + force_delete = false + + force_delete_warm_pool = false + + health_check_grace_period = 300 + + health_check_type = "EC2" + + id = (known after apply) + + ignore_failed_scaling_activities = false + + load_balancers = (known after apply) + + max_size = 5 + + metrics_granularity = "1Minute" + + min_size = 1 + + name = (known after apply) + + name_prefix = "swarms-size-t3.medium-" + + predicted_capacity = (known after apply) + + protect_from_scale_in = false + + service_linked_role_arn = (known after apply) + + target_group_arns = (known after apply) + + termination_policies = [] + + vpc_zone_identifier = [ + + "subnet-057c90cfe7b2e5646", + ] + + wait_for_capacity_timeout = "10m" + + warm_pool_size = (known after apply) + + + launch_template { + + id = (known after apply) + + name = (known after apply) + + version = "$Latest" + } + + + tag { + + key = "Name" + + propagate_at_launch = true + + value = "swarms-size-t3.medium" + } + + + timeouts {} + } + + # module.swarms_api.module.asg_dynamic["t3.medium"].module.autoscaling.aws_autoscaling_traffic_source_attachment.this["ex-alb"] will be created + + resource "aws_autoscaling_traffic_source_attachment" "this" { + + autoscaling_group_name = (known after apply) + + id = (known after apply) + + + traffic_source { + + identifier = "arn:aws:elasticloadbalancing:us-east-2:767503528736:targetgroup/swarms2024121619084133320000000c/001ec925380d4bcd" + + type = "elbv2" + } + } + + # module.swarms_api.module.security.module.asg_sg.aws_security_group.this_name_prefix[0] will be created + + resource "aws_security_group" "this_name_prefix" { + + arn = (known after apply) + + description = "external group" + + egress = (known after apply) + + id = (known after apply) + + ingress = (known after apply) + + name = (known after apply) + + name_prefix = "swarms-external-" + + owner_id = (known after apply) + + revoke_rules_on_delete = false + + tags = { + + "Name" = "swarms-external" + + "project" = "swarms" + } + + tags_all = { + + "Name" = "swarms-external" + + "project" = "swarms" + } + + vpc_id = "vpc-04f28c9347af48b55" + + + timeouts { + + create = "10m" + + delete = "15m" + } + } + + # module.swarms_api.module.security.module.asg_sg.aws_security_group_rule.egress_rules[0] will be created + + resource "aws_security_group_rule" "egress_rules" { + + cidr_blocks = [ + + "0.0.0.0/0", + ] + + description = "All protocols" + + from_port = -1 + + id = (known after apply) + + ipv6_cidr_blocks = [ + + "::/0", + ] + + prefix_list_ids = [] + + protocol = "-1" + + security_group_id = (known after apply) + + security_group_rule_id = (known after apply) + + self = false + + source_security_group_id = (known after apply) + + to_port = -1 + + type = "egress" + } + + # module.swarms_api.module.security.module.asg_sg.aws_security_group_rule.ingress_rules[0] will be created + + resource "aws_security_group_rule" "ingress_rules" { + + cidr_blocks = [ + + "0.0.0.0/0", + ] + + description = "HTTPS" + + from_port = 443 + + id = (known after apply) + + ipv6_cidr_blocks = [] + + prefix_list_ids = [] + + protocol = "tcp" + + security_group_id = (known after apply) + + security_group_rule_id = (known after apply) + + self = false + + source_security_group_id = (known after apply) + + to_port = 443 + + type = "ingress" + } + + # module.swarms_api.module.security.module.asg_sg.aws_security_group_rule.ingress_rules[1] will be created + + resource "aws_security_group_rule" "ingress_rules" { + + cidr_blocks = [ + + "0.0.0.0/0", + ] + + description = "HTTP" + + from_port = 80 + + id = (known after apply) + + ipv6_cidr_blocks = [] + + prefix_list_ids = [] + + protocol = "tcp" + + security_group_id = (known after apply) + + security_group_rule_id = (known after apply) + + self = false + + source_security_group_id = (known after apply) + + to_port = 80 + + type = "ingress" + } + + # module.swarms_api.module.security.module.asg_sg_internal.aws_security_group.this_name_prefix[0] will be created + + resource "aws_security_group" "this_name_prefix" { + + arn = (known after apply) + + description = "An internal security group" + + egress = (known after apply) + + id = (known after apply) + + ingress = (known after apply) + + name = (known after apply) + + name_prefix = "swarms-internal-" + + owner_id = (known after apply) + + revoke_rules_on_delete = false + + tags = { + + "Name" = "swarms-internal" + + "project" = "swarms" + } + + tags_all = { + + "Name" = "swarms-internal" + + "project" = "swarms" + } + + vpc_id = "vpc-04f28c9347af48b55" + + + timeouts { + + create = "10m" + + delete = "15m" + } + } + + # module.swarms_api.module.security.module.asg_sg_internal.aws_security_group_rule.egress_rules[0] will be created + + resource "aws_security_group_rule" "egress_rules" { + + cidr_blocks = [ + + "0.0.0.0/0", + ] + + description = "All protocols" + + from_port = -1 + + id = (known after apply) + + ipv6_cidr_blocks = [ + + "::/0", + ] + + prefix_list_ids = [] + + protocol = "-1" + + security_group_id = (known after apply) + + security_group_rule_id = (known after apply) + + self = false + + source_security_group_id = (known after apply) + + to_port = -1 + + type = "egress" + } + + # module.swarms_api.module.security.module.asg_sg_internal.aws_security_group_rule.ingress_with_source_security_group_id[0] will be created + + resource "aws_security_group_rule" "ingress_with_source_security_group_id" { + + description = "Ingress Rule" + + from_port = 80 + + id = (known after apply) + + prefix_list_ids = [] + + protocol = "tcp" + + security_group_id = (known after apply) + + security_group_rule_id = (known after apply) + + self = false + + source_security_group_id = (known after apply) + + to_port = 80 + + type = "ingress" + } + + + + +mdupont@mdupont-G470:~/2024/12/swarms-terraform/accounts/swarms$ tofu apply -auto-approve +module.swarms_api.module.alb.module.route53.data.aws_route53_zone.primary: Reading... +module.swarms_api.module.alb.module.alb.data.aws_partition.current: Reading... +module.swarms_api.module.vpc.data.aws_availability_zones.available: Reading... +module.swarms_api.module.alb.data.aws_availability_zones.available: Reading... +module.swarms_api.module.kp.aws_key_pair.mdupont-deployer: Refreshing state... [id=mdupont-deployer-key] +module.swarms_api.module.vpc.module.vpc.aws_vpc.this[0]: Refreshing state... [id=vpc-0b4cedd083227068d] +module.swarms_api.module.alb.module.alb.data.aws_partition.current: Read complete after 0s [id=aws] +module.swarms_api.module.alb.data.aws_availability_zones.available: Read complete after 0s [id=us-east-2] +module.swarms_api.module.vpc.data.aws_availability_zones.available: Read complete after 0s [id=us-east-2] +module.swarms_api.module.alb.module.route53.data.aws_route53_zone.primary: Read complete after 0s [id=Z04162952OP7P14Z97UWY] +module.swarms_api.module.vpc.module.vpc.aws_default_route_table.default[0]: Refreshing state... [id=rtb-053dc8c3d37f2f58f] +module.swarms_api.module.vpc.module.vpc.aws_default_security_group.this[0]: Refreshing state... [id=sg-0a42c3ef338285431] +module.swarms_api.module.vpc.module.vpc.aws_default_network_acl.this[0]: Refreshing state... [id=acl-07d9084294e0492bd] +module.swarms_api.module.vpc.module.vpc.aws_route_table.private[1]: Refreshing state... [id=rtb-0bbc3caa1cdc40cd0] +module.swarms_api.module.vpc.module.vpc.aws_route_table.private[0]: Refreshing state... [id=rtb-033eec802f743baca] +module.swarms_api.module.vpc.module.vpc.aws_route_table.private[2]: Refreshing state... [id=rtb-081f2bb265d2703ee] +module.swarms_api.module.vpc.module.vpc.aws_subnet.public[2]: Refreshing state... [id=subnet-027f4abd09a6543f0] +module.swarms_api.module.vpc.module.vpc.aws_route_table.public[0]: Refreshing state... [id=rtb-0445a2ea1c510657b] +module.swarms_api.module.vpc.module.vpc.aws_subnet.public[0]: Refreshing state... [id=subnet-04b3bdd4b0dc877f0] +module.swarms_api.module.vpc.module.vpc.aws_subnet.public[1]: Refreshing state... [id=subnet-051fecdee9028af79] +module.swarms_api.module.vpc.module.vpc.aws_subnet.private[1]: Refreshing state... [id=subnet-0de17633cfb166d29] +module.swarms_api.module.vpc.module.vpc.aws_subnet.private[0]: Refreshing state... [id=subnet-0ea44af2f97e12b1a] +module.swarms_api.module.vpc.module.vpc.aws_subnet.private[2]: Refreshing state... [id=subnet-0e926a2b5ae3f5acc] +module.swarms_api.module.vpc.module.vpc.aws_internet_gateway.this[0]: Refreshing state... [id=igw-022da11de86f41ae0] +module.swarms_api.module.security.module.asg_sg_internal.aws_security_group.this_name_prefix[0]: Refreshing state... [id=sg-0516329c97694b300] +module.swarms_api.module.security.module.asg_sg.aws_security_group.this_name_prefix[0]: Refreshing state... [id=sg-09e0227357b33ab1e] +module.swarms_api.module.alb.module.tg.aws_lb_target_group.this: Refreshing state... [id=arn:aws:elasticloadbalancing:us-east-2:916723593639:targetgroup/swarms2024121620041957850000000a/e915743119e96771] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.public[1]: Refreshing state... [id=rtbassoc-000d8799150d904e6] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.public[2]: Refreshing state... [id=rtbassoc-0b047c79fe554852d] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.public[0]: Refreshing state... [id=rtbassoc-033c4fa608f79a594] +module.swarms_api.module.vpc.module.vpc.aws_route.public_internet_gateway[0]: Refreshing state... [id=r-rtb-0445a2ea1c510657b1080289494] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.private[0]: Refreshing state... [id=rtbassoc-06f87a2b8106090d9] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.private[2]: Refreshing state... [id=rtbassoc-0657562cf0177710a] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.private[1]: Refreshing state... [id=rtbassoc-085a04c2858a3164e] +module.swarms_api.module.security.module.asg_sg_internal.aws_security_group_rule.egress_rules[0]: Refreshing state... [id=sgrule-1938904650] +module.swarms_api.module.security.module.asg_sg.aws_security_group_rule.ingress_rules[0]: Refreshing state... [id=sgrule-2188377888] +module.swarms_api.module.security.module.asg_sg.aws_security_group_rule.ingress_rules[1]: Refreshing state... [id=sgrule-1557830365] +module.swarms_api.module.security.module.asg_sg.aws_security_group_rule.egress_rules[0]: Refreshing state... [id=sgrule-2310235202] +module.swarms_api.module.security.module.asg_sg_internal.aws_security_group_rule.ingress_with_source_security_group_id[0]: Refreshing state... [id=sgrule-513653582] +module.swarms_api.module.lt_dynamic["t3.medium"].aws_launch_template.ec2_launch_template: Refreshing state... [id=lt-0ef5358bfbb143f2d] + +OpenTofu used the selected providers to generate the following execution plan. Resource actions are indicated with +the following symbols: + + create + +OpenTofu will perform the following actions: + + # module.swarms_api.module.alb.module.alb.aws_lb.this[0] will be created + + resource "aws_lb" "this" { + + arn = (known after apply) + + arn_suffix = (known after apply) + + client_keep_alive = 7200 + + desync_mitigation_mode = "defensive" + + dns_name = (known after apply) + + drop_invalid_header_fields = true + + enable_deletion_protection = false + + enable_http2 = true + + enable_tls_version_and_cipher_suite_headers = false + + enable_waf_fail_open = false + + enable_xff_client_port = false + + enforce_security_group_inbound_rules_on_private_link_traffic = (known after apply) + + id = (known after apply) + + idle_timeout = 60 + + internal = (known after apply) + + ip_address_type = (known after apply) + + load_balancer_type = "application" + + name = "swarms-api" + + name_prefix = (known after apply) + + preserve_host_header = false + + security_groups = [ + + "sg-09e0227357b33ab1e", + ] + + subnets = [ + + "1", + ] + + tags = { + + "Example" = "ex-swarms" + + "Name" = "ex-swarms" + + "Repository" = "https://github.com/terraform-aws-modules/terraform-aws-alb" + + "terraform-aws-modules" = "alb" + } + + tags_all = { + + "Example" = "ex-swarms" + + "Name" = "ex-swarms" + + "Repository" = "https://github.com/terraform-aws-modules/terraform-aws-alb" + + "terraform-aws-modules" = "alb" + } + + vpc_id = (known after apply) + + xff_header_processing_mode = "append" + + zone_id = (known after apply) + + + timeouts {} + } + + # module.swarms_api.module.alb.module.route53.aws_route53_record.api-cname will be created + + resource "aws_route53_record" "api-cname" { + + allow_overwrite = (known after apply) + + fqdn = (known after apply) + + id = (known after apply) + + name = "api.swarms.ai" + + records = (known after apply) + + ttl = 5 + + type = "CNAME" + + zone_id = "Z04162952OP7P14Z97UWY" + } + +Plan: 2 to add, 0 to change, 0 to destroy. +module.swarms_api.module.alb.module.alb.aws_lb.this[0]: Creating... +╷ +│ Error: creating ELBv2 application Load Balancer (swarms-api): operation error Elastic Load Balancing v2: CreateLoadBalancer, https response error StatusCode: 400, RequestID: 33d4b4ca-900e-4148-989a-932978bd7bfc, InvalidSubnet: The subnet ID '1' is not valid +│ +│ with module.swarms_api.module.alb.module.alb.aws_lb.this[0], +│ on .terraform/modules/swarms_api.alb.alb/main.tf line 12, in resource "aws_lb" "this": +│ 12: resource "aws_lb" "this" { +│ +╵ +mdupont@mdupont-G470:~/2024/12/swarms-terraform/accounts/swarms$ tofu apply -auto-approve +╷ +│ Error: Reference to undeclared module +│ +│ on ../../environments/swarms-aws-agent-api/dev/us-east-1/main.tf line 107, in output "alb": +│ 107: value = module.alb +│ +│ No module call named "alb" is declared in module.swarms_api. +╵ +mdupont@mdupont-G470:~/2024/12/swarms-terraform/accounts/swarms$ tofu apply -auto-approve +╷ +│ Error: Reference to undeclared module +│ +│ on ../../environments/swarms-aws-agent-api/dev/us-east-1/main.tf line 107, in output "alb": +│ 107: value = module.alb +│ +│ No module call named "alb" is declared in module.swarms_api. +╵ +mdupont@mdupont-G470:~/2024/12/swarms-terraform/accounts/swarms$ +mdupont@mdupont-G470:~/2024/12/swarms-terraform/accounts/swarms$ tofu apply -auto-approve +module.swarms_api.module.vpc.data.aws_availability_zones.available: Reading... +module.swarms_api.module.kp.aws_key_pair.mdupont-deployer: Refreshing state... [id=mdupont-deployer-key] +module.swarms_api.module.vpc.module.vpc.aws_vpc.this[0]: Refreshing state... [id=vpc-0b4cedd083227068d] +module.swarms_api.module.alb.module.tg.aws_lb_target_group.this: Refreshing state... [id=arn:aws:elasticloadbalancing:us-east-2:916723593639:targetgroup/swarms2024121620041957850000000a/e915743119e96771] +module.swarms_api.module.vpc.data.aws_availability_zones.available: Read complete after 0s [id=us-east-2] +module.swarms_api.module.vpc.module.vpc.aws_default_security_group.this[0]: Refreshing state... [id=sg-0a42c3ef338285431] +module.swarms_api.module.vpc.module.vpc.aws_default_route_table.default[0]: Refreshing state... [id=rtb-053dc8c3d37f2f58f] +module.swarms_api.module.vpc.module.vpc.aws_route_table.public[0]: Refreshing state... [id=rtb-0445a2ea1c510657b] +module.swarms_api.module.vpc.module.vpc.aws_default_network_acl.this[0]: Refreshing state... [id=acl-07d9084294e0492bd] +module.swarms_api.module.vpc.module.vpc.aws_subnet.public[1]: Refreshing state... [id=subnet-051fecdee9028af79] +module.swarms_api.module.vpc.module.vpc.aws_subnet.public[2]: Refreshing state... [id=subnet-027f4abd09a6543f0] +module.swarms_api.module.vpc.module.vpc.aws_subnet.public[0]: Refreshing state... [id=subnet-04b3bdd4b0dc877f0] +module.swarms_api.module.vpc.module.vpc.aws_route_table.private[0]: Refreshing state... [id=rtb-033eec802f743baca] +module.swarms_api.module.vpc.module.vpc.aws_route_table.private[2]: Refreshing state... [id=rtb-081f2bb265d2703ee] +module.swarms_api.module.vpc.module.vpc.aws_route_table.private[1]: Refreshing state... [id=rtb-0bbc3caa1cdc40cd0] +module.swarms_api.module.vpc.module.vpc.aws_internet_gateway.this[0]: Refreshing state... [id=igw-022da11de86f41ae0] +module.swarms_api.module.vpc.module.vpc.aws_subnet.private[0]: Refreshing state... [id=subnet-0ea44af2f97e12b1a] +module.swarms_api.module.vpc.module.vpc.aws_subnet.private[1]: Refreshing state... [id=subnet-0de17633cfb166d29] +module.swarms_api.module.vpc.module.vpc.aws_subnet.private[2]: Refreshing state... [id=subnet-0e926a2b5ae3f5acc] +module.swarms_api.module.security.module.asg_sg_internal.aws_security_group.this_name_prefix[0]: Refreshing state... [id=sg-0516329c97694b300] +module.swarms_api.module.security.module.asg_sg.aws_security_group.this_name_prefix[0]: Refreshing state... [id=sg-09e0227357b33ab1e] +module.swarms_api.module.vpc.module.vpc.aws_route.public_internet_gateway[0]: Refreshing state... [id=r-rtb-0445a2ea1c510657b1080289494] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.public[2]: Refreshing state... [id=rtbassoc-0b047c79fe554852d] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.public[1]: Refreshing state... [id=rtbassoc-000d8799150d904e6] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.public[0]: Refreshing state... [id=rtbassoc-033c4fa608f79a594] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.private[2]: Refreshing state... [id=rtbassoc-0657562cf0177710a] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.private[0]: Refreshing state... [id=rtbassoc-06f87a2b8106090d9] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.private[1]: Refreshing state... [id=rtbassoc-085a04c2858a3164e] +module.swarms_api.module.security.module.asg_sg_internal.aws_security_group_rule.egress_rules[0]: Refreshing state... [id=sgrule-1938904650] +module.swarms_api.module.security.module.asg_sg.aws_security_group_rule.ingress_rules[1]: Refreshing state... [id=sgrule-1557830365] +module.swarms_api.module.security.module.asg_sg.aws_security_group_rule.ingress_rules[0]: Refreshing state... [id=sgrule-2188377888] +module.swarms_api.module.security.module.asg_sg.aws_security_group_rule.egress_rules[0]: Refreshing state... [id=sgrule-2310235202] +module.swarms_api.module.security.module.asg_sg_internal.aws_security_group_rule.ingress_with_source_security_group_id[0]: Refreshing state... [id=sgrule-513653582] +module.swarms_api.module.lt_dynamic["t3.medium"].aws_launch_template.ec2_launch_template: Refreshing state... [id=lt-0ef5358bfbb143f2d] + +OpenTofu used the selected providers to generate the following execution plan. Resource actions are indicated with +the following symbols: + - destroy + +OpenTofu will perform the following actions: + + # module.swarms_api.module.alb.module.tg.aws_lb_target_group.this will be destroyed + # (because aws_lb_target_group.this is not in configuration) + - resource "aws_lb_target_group" "this" { + - arn = "arn:aws:elasticloadbalancing:us-east-2:916723593639:targetgroup/swarms2024121620041957850000000a/e915743119e96771" -> null + - arn_suffix = "targetgroup/swarms2024121620041957850000000a/e915743119e96771" -> null + - deregistration_delay = "10" -> null + - id = "arn:aws:elasticloadbalancing:us-east-2:916723593639:targetgroup/swarms2024121620041957850000000a/e915743119e96771" -> null + - ip_address_type = "ipv4" -> null + - lambda_multi_value_headers_enabled = false -> null + - load_balancer_arns = [] -> null + - load_balancing_algorithm_type = "round_robin" -> null + - load_balancing_anomaly_mitigation = "off" -> null + - load_balancing_cross_zone_enabled = "use_load_balancer_configuration" -> null + - name = "swarms2024121620041957850000000a" -> null + - name_prefix = "swarms" -> null + - port = 80 -> null + - protocol = "HTTP" -> null + - protocol_version = "HTTP1" -> null + - proxy_protocol_v2 = false -> null + - slow_start = 0 -> null + - tags = {} -> null + - tags_all = {} -> null + - target_type = "instance" -> null + - vpc_id = "vpc-0b4cedd083227068d" -> null + + - health_check { + - enabled = true -> null + - healthy_threshold = 10 -> null + - interval = 130 -> null + - matcher = "200" -> null + - path = "/v1/docs" -> null + - port = "traffic-port" -> null + - protocol = "HTTP" -> null + - timeout = 120 -> null + - unhealthy_threshold = 10 -> null + } + + - stickiness { + - cookie_duration = 86400 -> null + - enabled = false -> null + - type = "lb_cookie" -> null + } + + - target_failover {} + + - target_group_health { + - dns_failover { + - minimum_healthy_targets_count = "1" -> null + - minimum_healthy_targets_percentage = "off" -> null + } + - unhealthy_state_routing { + - minimum_healthy_targets_count = 1 -> null + - minimum_healthy_targets_percentage = "off" -> null + } + } + + - target_health_state {} + } + +Plan: 0 to add, 0 to change, 1 to destroy. +module.swarms_api.module.alb.module.tg.aws_lb_target_group.this: Destroying... [id=arn:aws:elasticloadbalancing:us-east-2:916723593639:targetgroup/swarms2024121620041957850000000a/e915743119e96771] +module.swarms_api.module.alb.module.tg.aws_lb_target_group.this: Destruction complete after 1s + +Apply complete! Resources: 0 added, 0 changed, 1 destroyed. + +Outputs: + +account = "916723593639" +dns = "api.swarms.ai" +profile = "swarms" +region = "us-east-2" +mdupont@mdupont-G470:~/2024/12/swarms-terraform/accounts/swarms$ tofu output +account = "916723593639" +dns = "api.swarms.ai" +profile = "swarms" +region = "us-east-2" +mdupont@mdupont-G470:~/2024/12/swarms-terraform/accounts/swarms$ tofu output +account = "916723593639" +dns = "api.swarms.ai" +profile = "swarms" +region = "us-east-2" +mdupont@mdupont-G470:~/2024/12/swarms-terraform/accounts/swarms$ tofu apply +module.swarms_api.module.vpc.data.aws_availability_zones.available: Reading... +module.swarms_api.module.kp.aws_key_pair.mdupont-deployer: Refreshing state... [id=mdupont-deployer-key] +module.swarms_api.module.vpc.module.vpc.aws_vpc.this[0]: Refreshing state... [id=vpc-0b4cedd083227068d] +module.swarms_api.module.vpc.data.aws_availability_zones.available: Read complete after 0s [id=us-east-2] +module.swarms_api.module.vpc.module.vpc.aws_default_route_table.default[0]: Refreshing state... [id=rtb-053dc8c3d37f2f58f] +module.swarms_api.module.vpc.module.vpc.aws_default_security_group.this[0]: Refreshing state... [id=sg-0a42c3ef338285431] +module.swarms_api.module.vpc.module.vpc.aws_route_table.public[0]: Refreshing state... [id=rtb-0445a2ea1c510657b] +module.swarms_api.module.vpc.module.vpc.aws_default_network_acl.this[0]: Refreshing state... [id=acl-07d9084294e0492bd] +module.swarms_api.module.vpc.module.vpc.aws_subnet.private[2]: Refreshing state... [id=subnet-0e926a2b5ae3f5acc] +module.swarms_api.module.vpc.module.vpc.aws_subnet.public[2]: Refreshing state... [id=subnet-027f4abd09a6543f0] +module.swarms_api.module.vpc.module.vpc.aws_subnet.private[0]: Refreshing state... [id=subnet-0ea44af2f97e12b1a] +module.swarms_api.module.vpc.module.vpc.aws_subnet.public[0]: Refreshing state... [id=subnet-04b3bdd4b0dc877f0] +module.swarms_api.module.vpc.module.vpc.aws_subnet.public[1]: Refreshing state... [id=subnet-051fecdee9028af79] +module.swarms_api.module.vpc.module.vpc.aws_subnet.private[1]: Refreshing state... [id=subnet-0de17633cfb166d29] +module.swarms_api.module.vpc.module.vpc.aws_internet_gateway.this[0]: Refreshing state... [id=igw-022da11de86f41ae0] +module.swarms_api.module.vpc.module.vpc.aws_route_table.private[0]: Refreshing state... [id=rtb-033eec802f743baca] +module.swarms_api.module.vpc.module.vpc.aws_route_table.private[1]: Refreshing state... [id=rtb-0bbc3caa1cdc40cd0] +module.swarms_api.module.vpc.module.vpc.aws_route_table.private[2]: Refreshing state... [id=rtb-081f2bb265d2703ee] +module.swarms_api.module.security.module.asg_sg_internal.aws_security_group.this_name_prefix[0]: Refreshing state... [id=sg-0516329c97694b300] +module.swarms_api.module.security.module.asg_sg.aws_security_group.this_name_prefix[0]: Refreshing state... [id=sg-09e0227357b33ab1e] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.public[1]: Refreshing state... [id=rtbassoc-000d8799150d904e6] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.public[2]: Refreshing state... [id=rtbassoc-0b047c79fe554852d] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.public[0]: Refreshing state... [id=rtbassoc-033c4fa608f79a594] +module.swarms_api.module.vpc.module.vpc.aws_route.public_internet_gateway[0]: Refreshing state... [id=r-rtb-0445a2ea1c510657b1080289494] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.private[0]: Refreshing state... [id=rtbassoc-06f87a2b8106090d9] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.private[2]: Refreshing state... [id=rtbassoc-0657562cf0177710a] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.private[1]: Refreshing state... [id=rtbassoc-085a04c2858a3164e] +module.swarms_api.module.security.module.asg_sg_internal.aws_security_group_rule.egress_rules[0]: Refreshing state... [id=sgrule-1938904650] +module.swarms_api.module.lt_dynamic["t3.medium"].aws_launch_template.ec2_launch_template: Refreshing state... [id=lt-0ef5358bfbb143f2d] +module.swarms_api.module.security.module.asg_sg.aws_security_group_rule.ingress_rules[0]: Refreshing state... [id=sgrule-2188377888] +module.swarms_api.module.security.module.asg_sg.aws_security_group_rule.ingress_rules[1]: Refreshing state... [id=sgrule-1557830365] +module.swarms_api.module.security.module.asg_sg.aws_security_group_rule.egress_rules[0]: Refreshing state... [id=sgrule-2310235202] +module.swarms_api.module.security.module.asg_sg_internal.aws_security_group_rule.ingress_with_source_security_group_id[0]: Refreshing state... [id=sgrule-513653582] + +Changes to Outputs: + + api = { + + security_group_id = "sg-09e0227357b33ab1e" + } + +You can apply this plan to save these new output values to the OpenTofu state, without changing any real +infrastructure. + +Do you want to perform these actions? + OpenTofu will perform the actions described above. + Only 'yes' will be accepted to approve. + + Enter a value: yes + + +Apply complete! Resources: 0 added, 0 changed, 0 destroyed. + +Outputs: + +account = "916723593639" +api = { + "security_group_id" = "sg-09e0227357b33ab1e" +} +dns = "api.swarms.ai" +profile = "swarms" +region = "us-east-2" +mdupont@mdupont-G470:~/2024/12/swarms-terraform/accounts/swarms$ tofu apply +module.swarms_api.module.vpc.data.aws_availability_zones.available: Reading... +module.swarms_api.module.kp.aws_key_pair.mdupont-deployer: Refreshing state... [id=mdupont-deployer-key] +module.swarms_api.module.vpc.module.vpc.aws_vpc.this[0]: Refreshing state... [id=vpc-0b4cedd083227068d] +module.swarms_api.module.vpc.data.aws_availability_zones.available: Read complete after 0s [id=us-east-2] +module.swarms_api.module.vpc.module.vpc.aws_default_route_table.default[0]: Refreshing state... [id=rtb-053dc8c3d37f2f58f] +module.swarms_api.module.vpc.module.vpc.aws_default_network_acl.this[0]: Refreshing state... [id=acl-07d9084294e0492bd] +module.swarms_api.module.vpc.module.vpc.aws_default_security_group.this[0]: Refreshing state... [id=sg-0a42c3ef338285431] +module.swarms_api.module.vpc.module.vpc.aws_subnet.public[2]: Refreshing state... [id=subnet-027f4abd09a6543f0] +module.swarms_api.module.vpc.module.vpc.aws_subnet.public[0]: Refreshing state... [id=subnet-04b3bdd4b0dc877f0] +module.swarms_api.module.vpc.module.vpc.aws_internet_gateway.this[0]: Refreshing state... [id=igw-022da11de86f41ae0] +module.swarms_api.module.vpc.module.vpc.aws_subnet.public[1]: Refreshing state... [id=subnet-051fecdee9028af79] +module.swarms_api.module.vpc.module.vpc.aws_route_table.public[0]: Refreshing state... [id=rtb-0445a2ea1c510657b] +module.swarms_api.module.vpc.module.vpc.aws_route_table.private[1]: Refreshing state... [id=rtb-0bbc3caa1cdc40cd0] +module.swarms_api.module.vpc.module.vpc.aws_route_table.private[0]: Refreshing state... [id=rtb-033eec802f743baca] +module.swarms_api.module.vpc.module.vpc.aws_route_table.private[2]: Refreshing state... [id=rtb-081f2bb265d2703ee] +module.swarms_api.module.vpc.module.vpc.aws_subnet.private[0]: Refreshing state... [id=subnet-0ea44af2f97e12b1a] +module.swarms_api.module.vpc.module.vpc.aws_subnet.private[2]: Refreshing state... [id=subnet-0e926a2b5ae3f5acc] +module.swarms_api.module.vpc.module.vpc.aws_subnet.private[1]: Refreshing state... [id=subnet-0de17633cfb166d29] +module.swarms_api.module.security.module.asg_sg.aws_security_group.this_name_prefix[0]: Refreshing state... [id=sg-09e0227357b33ab1e] +module.swarms_api.module.security.module.asg_sg_internal.aws_security_group.this_name_prefix[0]: Refreshing state... [id=sg-0516329c97694b300] +module.swarms_api.module.vpc.module.vpc.aws_route.public_internet_gateway[0]: Refreshing state... [id=r-rtb-0445a2ea1c510657b1080289494] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.public[0]: Refreshing state... [id=rtbassoc-033c4fa608f79a594] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.public[2]: Refreshing state... [id=rtbassoc-0b047c79fe554852d] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.public[1]: Refreshing state... [id=rtbassoc-000d8799150d904e6] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.private[1]: Refreshing state... [id=rtbassoc-085a04c2858a3164e] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.private[0]: Refreshing state... [id=rtbassoc-06f87a2b8106090d9] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.private[2]: Refreshing state... [id=rtbassoc-0657562cf0177710a] +module.swarms_api.module.security.module.asg_sg_internal.aws_security_group_rule.egress_rules[0]: Refreshing state... [id=sgrule-1938904650] +module.swarms_api.module.security.module.asg_sg.aws_security_group_rule.ingress_rules[1]: Refreshing state... [id=sgrule-1557830365] +module.swarms_api.module.security.module.asg_sg.aws_security_group_rule.ingress_rules[0]: Refreshing state... [id=sgrule-2188377888] +module.swarms_api.module.security.module.asg_sg.aws_security_group_rule.egress_rules[0]: Refreshing state... [id=sgrule-2310235202] +module.swarms_api.module.security.module.asg_sg_internal.aws_security_group_rule.ingress_with_source_security_group_id[0]: Refreshing state... [id=sgrule-513653582] +module.swarms_api.module.lt_dynamic["t3.medium"].aws_launch_template.ec2_launch_template: Refreshing state... [id=lt-0ef5358bfbb143f2d] + +No changes. Your infrastructure matches the configuration. + +OpenTofu has compared your real infrastructure against your configuration and found no differences, so no changes are +needed. + +Apply complete! Resources: 0 added, 0 changed, 0 destroyed. + +Outputs: + +account = "916723593639" +api = { + "security_group_id" = "sg-09e0227357b33ab1e" +} +dns = "api.swarms.ai" +profile = "swarms" +region = "us-east-2" +mdupont@mdupont-G470:~/2024/12/swarms-terraform/accounts/swarms$ tofu apply +module.swarms_api.module.vpc.data.aws_availability_zones.available: Reading... +module.swarms_api.module.kp.aws_key_pair.mdupont-deployer: Refreshing state... [id=mdupont-deployer-key] +module.swarms_api.module.vpc.module.vpc.aws_vpc.this[0]: Refreshing state... [id=vpc-0b4cedd083227068d] +module.swarms_api.module.vpc.data.aws_availability_zones.available: Read complete after 0s [id=us-east-2] +module.swarms_api.module.vpc.module.vpc.aws_default_route_table.default[0]: Refreshing state... [id=rtb-053dc8c3d37f2f58f] +module.swarms_api.module.vpc.module.vpc.aws_default_security_group.this[0]: Refreshing state... [id=sg-0a42c3ef338285431] +module.swarms_api.module.vpc.module.vpc.aws_default_network_acl.this[0]: Refreshing state... [id=acl-07d9084294e0492bd] +module.swarms_api.module.vpc.module.vpc.aws_subnet.private[2]: Refreshing state... [id=subnet-0e926a2b5ae3f5acc] +module.swarms_api.module.vpc.module.vpc.aws_subnet.private[1]: Refreshing state... [id=subnet-0de17633cfb166d29] +module.swarms_api.module.vpc.module.vpc.aws_subnet.private[0]: Refreshing state... [id=subnet-0ea44af2f97e12b1a] +module.swarms_api.module.vpc.module.vpc.aws_subnet.public[0]: Refreshing state... [id=subnet-04b3bdd4b0dc877f0] +module.swarms_api.module.vpc.module.vpc.aws_subnet.public[1]: Refreshing state... [id=subnet-051fecdee9028af79] +module.swarms_api.module.vpc.module.vpc.aws_subnet.public[2]: Refreshing state... [id=subnet-027f4abd09a6543f0] +module.swarms_api.module.vpc.module.vpc.aws_route_table.private[2]: Refreshing state... [id=rtb-081f2bb265d2703ee] +module.swarms_api.module.vpc.module.vpc.aws_route_table.private[0]: Refreshing state... [id=rtb-033eec802f743baca] +module.swarms_api.module.vpc.module.vpc.aws_route_table.private[1]: Refreshing state... [id=rtb-0bbc3caa1cdc40cd0] +module.swarms_api.module.vpc.module.vpc.aws_internet_gateway.this[0]: Refreshing state... [id=igw-022da11de86f41ae0] +module.swarms_api.module.vpc.module.vpc.aws_route_table.public[0]: Refreshing state... [id=rtb-0445a2ea1c510657b] +module.swarms_api.module.security.module.asg_sg_internal.aws_security_group.this_name_prefix[0]: Refreshing state... [id=sg-0516329c97694b300] +module.swarms_api.module.security.module.asg_sg.aws_security_group.this_name_prefix[0]: Refreshing state... [id=sg-09e0227357b33ab1e] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.private[0]: Refreshing state... [id=rtbassoc-06f87a2b8106090d9] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.private[2]: Refreshing state... [id=rtbassoc-0657562cf0177710a] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.private[1]: Refreshing state... [id=rtbassoc-085a04c2858a3164e] +module.swarms_api.module.vpc.module.vpc.aws_route.public_internet_gateway[0]: Refreshing state... [id=r-rtb-0445a2ea1c510657b1080289494] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.public[0]: Refreshing state... [id=rtbassoc-033c4fa608f79a594] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.public[1]: Refreshing state... [id=rtbassoc-000d8799150d904e6] +module.swarms_api.module.vpc.module.vpc.aws_route_table_association.public[2]: Refreshing state... [id=rtbassoc-0b047c79fe554852d] +module.swarms_api.module.security.module.asg_sg_internal.aws_security_group_rule.egress_rules[0]: Refreshing state... [id=sgrule-1938904650] +module.swarms_api.module.security.module.asg_sg.aws_security_group_rule.egress_rules[0]: Refreshing state... [id=sgrule-2310235202] +module.swarms_api.module.security.module.asg_sg_internal.aws_security_group_rule.ingress_with_source_security_group_id[0]: Refreshing state... [id=sgrule-513653582] +module.swarms_api.module.security.module.asg_sg.aws_security_group_rule.ingress_rules[0]: Refreshing state... [id=sgrule-2188377888] +module.swarms_api.module.lt_dynamic["t3.medium"].aws_launch_template.ec2_launch_template: Refreshing state... [id=lt-0ef5358bfbb143f2d] +module.swarms_api.module.security.module.asg_sg.aws_security_group_rule.ingress_rules[1]: Refreshing state... [id=sgrule-1557830365] + +Changes to Outputs: + ~ api = { + + vpc = { + + cgw_arns = [] + + cgw_ids = [] + + default_network_acl_id = "acl-07d9084294e0492bd" + + default_route_table_id = "rtb-053dc8c3d37f2f58f" + + default_security_group_id = "sg-0a42c3ef338285431" + + default_vpc_arn = null + + default_vpc_cidr_block = null + + default_vpc_default_network_acl_id = null + + default_vpc_default_route_table_id = null + + default_vpc_default_security_group_id = null + + default_vpc_enable_dns_hostnames = null + + default_vpc_enable_dns_support = null + + default_vpc_id = null + + default_vpc_instance_tenancy = null + + default_vpc_main_route_table_id = null + + dhcp_options_id = null + + ec2_public_subnet_id_1 = 1 + + ec2_public_subnet_id_2 = 1 + + egress_only_internet_gateway_id = null + + igw_arn = "arn:aws:ec2:us-east-2:916723593639:internet-gateway/igw-022da11de86f41ae0" + + igw_id = "igw-022da11de86f41ae0" + + nat_ids = [] + + nat_public_ips = [] + + natgw_ids = [] + + private_ipv6_egress_route_ids = [] + + private_nat_gateway_route_ids = [] + + private_network_acl_arn = null + + private_network_acl_id = null + + private_route_table_association_ids = [ + + "rtbassoc-06f87a2b8106090d9", + + "rtbassoc-085a04c2858a3164e", + + "rtbassoc-0657562cf0177710a", + ] + + private_route_table_ids = [ + + "rtb-033eec802f743baca", + + "rtb-0bbc3caa1cdc40cd0", + + "rtb-081f2bb265d2703ee", + ] + + private_subnet_arns = [ + + "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-0ea44af2f97e12b1a", + + "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-0de17633cfb166d29", + + "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-0e926a2b5ae3f5acc", + ] + + private_subnets = [ + + "subnet-0ea44af2f97e12b1a", + + "subnet-0de17633cfb166d29", + + "subnet-0e926a2b5ae3f5acc", + ] + + private_subnets_cidr_blocks = [ + + "10.0.0.0/24", + + "10.0.1.0/24", + + "10.0.2.0/24", + ] + + private_subnets_ipv6_cidr_blocks = [] + + public_internet_gateway_ipv6_route_id = null + + public_internet_gateway_route_id = "r-rtb-0445a2ea1c510657b1080289494" + + public_network_acl_arn = null + + public_network_acl_id = null + + public_route_table_association_ids = [ + + "rtbassoc-033c4fa608f79a594", + + "rtbassoc-000d8799150d904e6", + + "rtbassoc-0b047c79fe554852d", + ] + + public_route_table_ids = [ + + "rtb-0445a2ea1c510657b", + ] + + public_subnet_arns = [ + + "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-04b3bdd4b0dc877f0", + + "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-051fecdee9028af79", + + "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-027f4abd09a6543f0", + ] + + public_subnets = [ + + "subnet-04b3bdd4b0dc877f0", + + "subnet-051fecdee9028af79", + + "subnet-027f4abd09a6543f0", + ] + + public_subnets_cidr_blocks = [ + + "10.0.4.0/24", + + "10.0.5.0/24", + + "10.0.6.0/24", + ] + + public_subnets_ipv6_cidr_blocks = [] + + this_customer_gateway = {} + + vgw_arn = null + + vgw_id = null + + vpc = { + + azs = [ + + "us-east-2a", + + "us-east-2b", + + "us-east-2c", + ] + + cgw_arns = [] + + cgw_ids = [] + + database_internet_gateway_route_id = null + + database_ipv6_egress_route_id = null + + database_nat_gateway_route_ids = [] + + database_network_acl_arn = null + + database_network_acl_id = null + + database_route_table_association_ids = [] + + database_route_table_ids = [ + + "rtb-033eec802f743baca", + + "rtb-0bbc3caa1cdc40cd0", + + "rtb-081f2bb265d2703ee", + ] + + database_subnet_arns = [] + + database_subnet_group = null + + database_subnet_group_name = null + + database_subnet_objects = [] + + database_subnets = [] + + database_subnets_cidr_blocks = [] + + database_subnets_ipv6_cidr_blocks = [] + + default_network_acl_id = "acl-07d9084294e0492bd" + + default_route_table_id = "rtb-053dc8c3d37f2f58f" + + default_security_group_id = "sg-0a42c3ef338285431" + + default_vpc_arn = null + + default_vpc_cidr_block = null + + default_vpc_default_network_acl_id = null + + default_vpc_default_route_table_id = null + + default_vpc_default_security_group_id = null + + default_vpc_enable_dns_hostnames = null + + default_vpc_enable_dns_support = null + + default_vpc_id = null + + default_vpc_instance_tenancy = null + + default_vpc_main_route_table_id = null + + dhcp_options_id = null + + egress_only_internet_gateway_id = null + + elasticache_network_acl_arn = null + + elasticache_network_acl_id = null + + elasticache_route_table_association_ids = [] + + elasticache_route_table_ids = [ + + "rtb-033eec802f743baca", + + "rtb-0bbc3caa1cdc40cd0", + + "rtb-081f2bb265d2703ee", + ] + + elasticache_subnet_arns = [] + + elasticache_subnet_group = null + + elasticache_subnet_group_name = null + + elasticache_subnet_objects = [] + + elasticache_subnets = [] + + elasticache_subnets_cidr_blocks = [] + + elasticache_subnets_ipv6_cidr_blocks = [] + + igw_arn = "arn:aws:ec2:us-east-2:916723593639:internet-gateway/igw-022da11de86f41ae0" + + igw_id = "igw-022da11de86f41ae0" + + intra_network_acl_arn = null + + intra_network_acl_id = null + + intra_route_table_association_ids = [] + + intra_route_table_ids = [] + + intra_subnet_arns = [] + + intra_subnet_objects = [] + + intra_subnets = [] + + intra_subnets_cidr_blocks = [] + + intra_subnets_ipv6_cidr_blocks = [] + + name = "swarms" + + nat_ids = [] + + nat_public_ips = [] + + natgw_ids = [] + + natgw_interface_ids = [] + + outpost_network_acl_arn = null + + outpost_network_acl_id = null + + outpost_subnet_arns = [] + + outpost_subnet_objects = [] + + outpost_subnets = [] + + outpost_subnets_cidr_blocks = [] + + outpost_subnets_ipv6_cidr_blocks = [] + + private_ipv6_egress_route_ids = [] + + private_nat_gateway_route_ids = [] + + private_network_acl_arn = null + + private_network_acl_id = null + + private_route_table_association_ids = [ + + "rtbassoc-06f87a2b8106090d9", + + "rtbassoc-085a04c2858a3164e", + + "rtbassoc-0657562cf0177710a", + ] + + private_route_table_ids = [ + + "rtb-033eec802f743baca", + + "rtb-0bbc3caa1cdc40cd0", + + "rtb-081f2bb265d2703ee", + ] + + private_subnet_arns = [ + + "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-0ea44af2f97e12b1a", + + "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-0de17633cfb166d29", + + "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-0e926a2b5ae3f5acc", + ] + + private_subnet_objects = [ + + { + + arn = "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-0ea44af2f97e12b1a" + + assign_ipv6_address_on_creation = false + + availability_zone = "us-east-2a" + + availability_zone_id = "use2-az1" + + cidr_block = "10.0.0.0/24" + + customer_owned_ipv4_pool = "" + + enable_dns64 = false + + enable_lni_at_device_index = 0 + + enable_resource_name_dns_a_record_on_launch = false + + enable_resource_name_dns_aaaa_record_on_launch = false + + id = "subnet-0ea44af2f97e12b1a" + + ipv6_cidr_block = "" + + ipv6_cidr_block_association_id = "" + + ipv6_native = false + + map_customer_owned_ip_on_launch = false + + map_public_ip_on_launch = false + + outpost_arn = "" + + owner_id = "916723593639" + + private_dns_hostname_type_on_launch = "ip-name" + + tags = { + + Name = "Private Subnet One" + + project = "swarms" + } + + tags_all = { + + Name = "Private Subnet One" + + project = "swarms" + } + + timeouts = null + + vpc_id = "vpc-0b4cedd083227068d" + }, + + { + + arn = "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-0de17633cfb166d29" + + assign_ipv6_address_on_creation = false + + availability_zone = "us-east-2b" + + availability_zone_id = "use2-az2" + + cidr_block = "10.0.1.0/24" + + customer_owned_ipv4_pool = "" + + enable_dns64 = false + + enable_lni_at_device_index = 0 + + enable_resource_name_dns_a_record_on_launch = false + + enable_resource_name_dns_aaaa_record_on_launch = false + + id = "subnet-0de17633cfb166d29" + + ipv6_cidr_block = "" + + ipv6_cidr_block_association_id = "" + + ipv6_native = false + + map_customer_owned_ip_on_launch = false + + map_public_ip_on_launch = false + + outpost_arn = "" + + owner_id = "916723593639" + + private_dns_hostname_type_on_launch = "ip-name" + + tags = { + + Name = "Private Subnet Two" + + project = "swarms" + } + + tags_all = { + + Name = "Private Subnet Two" + + project = "swarms" + } + + timeouts = null + + vpc_id = "vpc-0b4cedd083227068d" + }, + + { + + arn = "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-0e926a2b5ae3f5acc" + + assign_ipv6_address_on_creation = false + + availability_zone = "us-east-2c" + + availability_zone_id = "use2-az3" + + cidr_block = "10.0.2.0/24" + + customer_owned_ipv4_pool = "" + + enable_dns64 = false + + enable_lni_at_device_index = 0 + + enable_resource_name_dns_a_record_on_launch = false + + enable_resource_name_dns_aaaa_record_on_launch = false + + id = "subnet-0e926a2b5ae3f5acc" + + ipv6_cidr_block = "" + + ipv6_cidr_block_association_id = "" + + ipv6_native = false + + map_customer_owned_ip_on_launch = false + + map_public_ip_on_launch = false + + outpost_arn = "" + + owner_id = "916723593639" + + private_dns_hostname_type_on_launch = "ip-name" + + tags = { + + Name = "swarms-private-us-east-2c" + + project = "swarms" + } + + tags_all = { + + Name = "swarms-private-us-east-2c" + + project = "swarms" + } + + timeouts = null + + vpc_id = "vpc-0b4cedd083227068d" + }, + ] + + private_subnets = [ + + "subnet-0ea44af2f97e12b1a", + + "subnet-0de17633cfb166d29", + + "subnet-0e926a2b5ae3f5acc", + ] + + private_subnets_cidr_blocks = [ + + "10.0.0.0/24", + + "10.0.1.0/24", + + "10.0.2.0/24", + ] + + private_subnets_ipv6_cidr_blocks = [] + + public_internet_gateway_ipv6_route_id = null + + public_internet_gateway_route_id = "r-rtb-0445a2ea1c510657b1080289494" + + public_network_acl_arn = null + + public_network_acl_id = null + + public_route_table_association_ids = [ + + "rtbassoc-033c4fa608f79a594", + + "rtbassoc-000d8799150d904e6", + + "rtbassoc-0b047c79fe554852d", + ] + + public_route_table_ids = [ + + "rtb-0445a2ea1c510657b", + ] + + public_subnet_arns = [ + + "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-04b3bdd4b0dc877f0", + + "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-051fecdee9028af79", + + "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-027f4abd09a6543f0", + ] + + public_subnet_objects = [ + + { + + arn = "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-04b3bdd4b0dc877f0" + + assign_ipv6_address_on_creation = false + + availability_zone = "us-east-2a" + + availability_zone_id = "use2-az1" + + cidr_block = "10.0.4.0/24" + + customer_owned_ipv4_pool = "" + + enable_dns64 = false + + enable_lni_at_device_index = 0 + + enable_resource_name_dns_a_record_on_launch = false + + enable_resource_name_dns_aaaa_record_on_launch = false + + id = "subnet-04b3bdd4b0dc877f0" + + ipv6_cidr_block = "" + + ipv6_cidr_block_association_id = "" + + ipv6_native = false + + map_customer_owned_ip_on_launch = false + + map_public_ip_on_launch = false + + outpost_arn = "" + + owner_id = "916723593639" + + private_dns_hostname_type_on_launch = "ip-name" + + tags = { + + Name = "swarms-public-us-east-2a" + + project = "swarms" + } + + tags_all = { + + Name = "swarms-public-us-east-2a" + + project = "swarms" + } + + timeouts = null + + vpc_id = "vpc-0b4cedd083227068d" + }, + + { + + arn = "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-051fecdee9028af79" + + assign_ipv6_address_on_creation = false + + availability_zone = "us-east-2b" + + availability_zone_id = "use2-az2" + + cidr_block = "10.0.5.0/24" + + customer_owned_ipv4_pool = "" + + enable_dns64 = false + + enable_lni_at_device_index = 0 + + enable_resource_name_dns_a_record_on_launch = false + + enable_resource_name_dns_aaaa_record_on_launch = false + + id = "subnet-051fecdee9028af79" + + ipv6_cidr_block = "" + + ipv6_cidr_block_association_id = "" + + ipv6_native = false + + map_customer_owned_ip_on_launch = false + + map_public_ip_on_launch = false + + outpost_arn = "" + + owner_id = "916723593639" + + private_dns_hostname_type_on_launch = "ip-name" + + tags = { + + Name = "swarms-public-us-east-2b" + + project = "swarms" + } + + tags_all = { + + Name = "swarms-public-us-east-2b" + + project = "swarms" + } + + timeouts = null + + vpc_id = "vpc-0b4cedd083227068d" + }, + + { + + arn = "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-027f4abd09a6543f0" + + assign_ipv6_address_on_creation = false + + availability_zone = "us-east-2c" + + availability_zone_id = "use2-az3" + + cidr_block = "10.0.6.0/24" + + customer_owned_ipv4_pool = "" + + enable_dns64 = false + + enable_lni_at_device_index = 0 + + enable_resource_name_dns_a_record_on_launch = false + + enable_resource_name_dns_aaaa_record_on_launch = false + + id = "subnet-027f4abd09a6543f0" + + ipv6_cidr_block = "" + + ipv6_cidr_block_association_id = "" + + ipv6_native = false + + map_customer_owned_ip_on_launch = false + + map_public_ip_on_launch = false + + outpost_arn = "" + + owner_id = "916723593639" + + private_dns_hostname_type_on_launch = "ip-name" + + tags = { + + Name = "swarms-public-us-east-2c" + + project = "swarms" + } + + tags_all = { + + Name = "swarms-public-us-east-2c" + + project = "swarms" + } + + timeouts = null + + vpc_id = "vpc-0b4cedd083227068d" + }, + ] + + public_subnets = [ + + "subnet-04b3bdd4b0dc877f0", + + "subnet-051fecdee9028af79", + + "subnet-027f4abd09a6543f0", + ] + + public_subnets_cidr_blocks = [ + + "10.0.4.0/24", + + "10.0.5.0/24", + + "10.0.6.0/24", + ] + + public_subnets_ipv6_cidr_blocks = [] + + redshift_network_acl_arn = null + + redshift_network_acl_id = null + + redshift_public_route_table_association_ids = [] + + redshift_route_table_association_ids = [] + + redshift_route_table_ids = [ + + "rtb-033eec802f743baca", + + "rtb-0bbc3caa1cdc40cd0", + + "rtb-081f2bb265d2703ee", + ] + + redshift_subnet_arns = [] + + redshift_subnet_group = null + + redshift_subnet_objects = [] + + redshift_subnets = [] + + redshift_subnets_cidr_blocks = [] + + redshift_subnets_ipv6_cidr_blocks = [] + + this_customer_gateway = {} + + vgw_arn = null + + vgw_id = null + + vpc_arn = "arn:aws:ec2:us-east-2:916723593639:vpc/vpc-0b4cedd083227068d" + + vpc_cidr_block = "10.0.0.0/16" + + vpc_enable_dns_hostnames = true + + vpc_enable_dns_support = true + + vpc_flow_log_cloudwatch_iam_role_arn = "" + + vpc_flow_log_deliver_cross_account_role = null + + vpc_flow_log_destination_arn = "" + + vpc_flow_log_destination_type = "cloud-watch-logs" + + vpc_flow_log_id = null + + vpc_id = "vpc-0b4cedd083227068d" + + vpc_instance_tenancy = "default" + + vpc_ipv6_association_id = "" + + vpc_ipv6_cidr_block = "" + + vpc_main_route_table_id = "rtb-053dc8c3d37f2f58f" + + vpc_owner_id = "916723593639" + + vpc_secondary_cidr_blocks = [] + } + + vpc_arn = "arn:aws:ec2:us-east-2:916723593639:vpc/vpc-0b4cedd083227068d" + + vpc_cidr_block = "10.0.0.0/16" + + vpc_enable_dns_hostnames = true + + vpc_enable_dns_support = true + + vpc_flow_log_cloudwatch_iam_role_arn = "" + + vpc_flow_log_destination_arn = "" + + vpc_flow_log_destination_type = "cloud-watch-logs" + + vpc_flow_log_id = null + + vpc_id = "vpc-0b4cedd083227068d" + + vpc_instance_tenancy = "default" + + vpc_ipv6_association_id = "" + + vpc_ipv6_cidr_block = "" + + vpc_main_route_table_id = "rtb-053dc8c3d37f2f58f" + + vpc_owner_id = "916723593639" + + vpc_secondary_cidr_blocks = [] + } + # (1 unchanged attribute hidden) + } + +You can apply this plan to save these new output values to the OpenTofu state, without changing any real +infrastructure. + +Do you want to perform these actions? + OpenTofu will perform the actions described above. + Only 'yes' will be accepted to approve. + + Enter a value: yes + + +Apply complete! Resources: 0 added, 0 changed, 0 destroyed. + +Outputs: + +account = "916723593639" +api = { + "security_group_id" = "sg-09e0227357b33ab1e" + "vpc" = { + "cgw_arns" = [] + "cgw_ids" = [] + "default_network_acl_id" = "acl-07d9084294e0492bd" + "default_route_table_id" = "rtb-053dc8c3d37f2f58f" + "default_security_group_id" = "sg-0a42c3ef338285431" + "default_vpc_arn" = null + "default_vpc_cidr_block" = null + "default_vpc_default_network_acl_id" = null + "default_vpc_default_route_table_id" = null + "default_vpc_default_security_group_id" = null + "default_vpc_enable_dns_hostnames" = null + "default_vpc_enable_dns_support" = null + "default_vpc_id" = null + "default_vpc_instance_tenancy" = null + "default_vpc_main_route_table_id" = null + "dhcp_options_id" = null + "ec2_public_subnet_id_1" = 1 + "ec2_public_subnet_id_2" = 1 + "egress_only_internet_gateway_id" = null + "igw_arn" = "arn:aws:ec2:us-east-2:916723593639:internet-gateway/igw-022da11de86f41ae0" + "igw_id" = "igw-022da11de86f41ae0" + "nat_ids" = [] + "nat_public_ips" = tolist([]) + "natgw_ids" = [] + "private_ipv6_egress_route_ids" = [] + "private_nat_gateway_route_ids" = [] + "private_network_acl_arn" = null + "private_network_acl_id" = null + "private_route_table_association_ids" = [ + "rtbassoc-06f87a2b8106090d9", + "rtbassoc-085a04c2858a3164e", + "rtbassoc-0657562cf0177710a", + ] + "private_route_table_ids" = [ + "rtb-033eec802f743baca", + "rtb-0bbc3caa1cdc40cd0", + "rtb-081f2bb265d2703ee", + ] + "private_subnet_arns" = [ + "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-0ea44af2f97e12b1a", + "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-0de17633cfb166d29", + "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-0e926a2b5ae3f5acc", + ] + "private_subnets" = [ + "subnet-0ea44af2f97e12b1a", + "subnet-0de17633cfb166d29", + "subnet-0e926a2b5ae3f5acc", + ] + "private_subnets_cidr_blocks" = tolist([ + "10.0.0.0/24", + "10.0.1.0/24", + "10.0.2.0/24", + ]) + "private_subnets_ipv6_cidr_blocks" = tolist([]) + "public_internet_gateway_ipv6_route_id" = null + "public_internet_gateway_route_id" = "r-rtb-0445a2ea1c510657b1080289494" + "public_network_acl_arn" = null + "public_network_acl_id" = null + "public_route_table_association_ids" = [ + "rtbassoc-033c4fa608f79a594", + "rtbassoc-000d8799150d904e6", + "rtbassoc-0b047c79fe554852d", + ] + "public_route_table_ids" = [ + "rtb-0445a2ea1c510657b", + ] + "public_subnet_arns" = [ + "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-04b3bdd4b0dc877f0", + "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-051fecdee9028af79", + "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-027f4abd09a6543f0", + ] + "public_subnets" = [ + "subnet-04b3bdd4b0dc877f0", + "subnet-051fecdee9028af79", + "subnet-027f4abd09a6543f0", + ] + "public_subnets_cidr_blocks" = tolist([ + "10.0.4.0/24", + "10.0.5.0/24", + "10.0.6.0/24", + ]) + "public_subnets_ipv6_cidr_blocks" = tolist([]) + "this_customer_gateway" = {} + "vgw_arn" = null + "vgw_id" = null + "vpc" = { + "azs" = tolist([ + "us-east-2a", + "us-east-2b", + "us-east-2c", + ]) + "cgw_arns" = [] + "cgw_ids" = [] + "database_internet_gateway_route_id" = null + "database_ipv6_egress_route_id" = null + "database_nat_gateway_route_ids" = [] + "database_network_acl_arn" = null + "database_network_acl_id" = null + "database_route_table_association_ids" = [] + "database_route_table_ids" = tolist([ + "rtb-033eec802f743baca", + "rtb-0bbc3caa1cdc40cd0", + "rtb-081f2bb265d2703ee", + ]) + "database_subnet_arns" = [] + "database_subnet_group" = null + "database_subnet_group_name" = null + "database_subnet_objects" = [] + "database_subnets" = [] + "database_subnets_cidr_blocks" = tolist([]) + "database_subnets_ipv6_cidr_blocks" = tolist([]) + "default_network_acl_id" = "acl-07d9084294e0492bd" + "default_route_table_id" = "rtb-053dc8c3d37f2f58f" + "default_security_group_id" = "sg-0a42c3ef338285431" + "default_vpc_arn" = null + "default_vpc_cidr_block" = null + "default_vpc_default_network_acl_id" = null + "default_vpc_default_route_table_id" = null + "default_vpc_default_security_group_id" = null + "default_vpc_enable_dns_hostnames" = null + "default_vpc_enable_dns_support" = null + "default_vpc_id" = null + "default_vpc_instance_tenancy" = null + "default_vpc_main_route_table_id" = null + "dhcp_options_id" = null + "egress_only_internet_gateway_id" = null + "elasticache_network_acl_arn" = null + "elasticache_network_acl_id" = null + "elasticache_route_table_association_ids" = [] + "elasticache_route_table_ids" = [ + "rtb-033eec802f743baca", + "rtb-0bbc3caa1cdc40cd0", + "rtb-081f2bb265d2703ee", + ] + "elasticache_subnet_arns" = [] + "elasticache_subnet_group" = null + "elasticache_subnet_group_name" = null + "elasticache_subnet_objects" = [] + "elasticache_subnets" = [] + "elasticache_subnets_cidr_blocks" = tolist([]) + "elasticache_subnets_ipv6_cidr_blocks" = tolist([]) + "igw_arn" = "arn:aws:ec2:us-east-2:916723593639:internet-gateway/igw-022da11de86f41ae0" + "igw_id" = "igw-022da11de86f41ae0" + "intra_network_acl_arn" = null + "intra_network_acl_id" = null + "intra_route_table_association_ids" = [] + "intra_route_table_ids" = [] + "intra_subnet_arns" = [] + "intra_subnet_objects" = [] + "intra_subnets" = [] + "intra_subnets_cidr_blocks" = tolist([]) + "intra_subnets_ipv6_cidr_blocks" = tolist([]) + "name" = "swarms" + "nat_ids" = [] + "nat_public_ips" = tolist([]) + "natgw_ids" = [] + "natgw_interface_ids" = [] + "outpost_network_acl_arn" = null + "outpost_network_acl_id" = null + "outpost_subnet_arns" = [] + "outpost_subnet_objects" = [] + "outpost_subnets" = [] + "outpost_subnets_cidr_blocks" = tolist([]) + "outpost_subnets_ipv6_cidr_blocks" = tolist([]) + "private_ipv6_egress_route_ids" = [] + "private_nat_gateway_route_ids" = [] + "private_network_acl_arn" = null + "private_network_acl_id" = null + "private_route_table_association_ids" = [ + "rtbassoc-06f87a2b8106090d9", + "rtbassoc-085a04c2858a3164e", + "rtbassoc-0657562cf0177710a", + ] + "private_route_table_ids" = [ + "rtb-033eec802f743baca", + "rtb-0bbc3caa1cdc40cd0", + "rtb-081f2bb265d2703ee", + ] + "private_subnet_arns" = [ + "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-0ea44af2f97e12b1a", + "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-0de17633cfb166d29", + "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-0e926a2b5ae3f5acc", + ] + "private_subnet_objects" = [ + { + "arn" = "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-0ea44af2f97e12b1a" + "assign_ipv6_address_on_creation" = false + "availability_zone" = "us-east-2a" + "availability_zone_id" = "use2-az1" + "cidr_block" = "10.0.0.0/24" + "customer_owned_ipv4_pool" = "" + "enable_dns64" = false + "enable_lni_at_device_index" = 0 + "enable_resource_name_dns_a_record_on_launch" = false + "enable_resource_name_dns_aaaa_record_on_launch" = false + "id" = "subnet-0ea44af2f97e12b1a" + "ipv6_cidr_block" = "" + "ipv6_cidr_block_association_id" = "" + "ipv6_native" = false + "map_customer_owned_ip_on_launch" = false + "map_public_ip_on_launch" = false + "outpost_arn" = "" + "owner_id" = "916723593639" + "private_dns_hostname_type_on_launch" = "ip-name" + "tags" = tomap({ + "Name" = "Private Subnet One" + "project" = "swarms" + }) + "tags_all" = tomap({ + "Name" = "Private Subnet One" + "project" = "swarms" + }) + "timeouts" = null /* object */ + "vpc_id" = "vpc-0b4cedd083227068d" + }, + { + "arn" = "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-0de17633cfb166d29" + "assign_ipv6_address_on_creation" = false + "availability_zone" = "us-east-2b" + "availability_zone_id" = "use2-az2" + "cidr_block" = "10.0.1.0/24" + "customer_owned_ipv4_pool" = "" + "enable_dns64" = false + "enable_lni_at_device_index" = 0 + "enable_resource_name_dns_a_record_on_launch" = false + "enable_resource_name_dns_aaaa_record_on_launch" = false + "id" = "subnet-0de17633cfb166d29" + "ipv6_cidr_block" = "" + "ipv6_cidr_block_association_id" = "" + "ipv6_native" = false + "map_customer_owned_ip_on_launch" = false + "map_public_ip_on_launch" = false + "outpost_arn" = "" + "owner_id" = "916723593639" + "private_dns_hostname_type_on_launch" = "ip-name" + "tags" = tomap({ + "Name" = "Private Subnet Two" + "project" = "swarms" + }) + "tags_all" = tomap({ + "Name" = "Private Subnet Two" + "project" = "swarms" + }) + "timeouts" = null /* object */ + "vpc_id" = "vpc-0b4cedd083227068d" + }, + { + "arn" = "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-0e926a2b5ae3f5acc" + "assign_ipv6_address_on_creation" = false + "availability_zone" = "us-east-2c" + "availability_zone_id" = "use2-az3" + "cidr_block" = "10.0.2.0/24" + "customer_owned_ipv4_pool" = "" + "enable_dns64" = false + "enable_lni_at_device_index" = 0 + "enable_resource_name_dns_a_record_on_launch" = false + "enable_resource_name_dns_aaaa_record_on_launch" = false + "id" = "subnet-0e926a2b5ae3f5acc" + "ipv6_cidr_block" = "" + "ipv6_cidr_block_association_id" = "" + "ipv6_native" = false + "map_customer_owned_ip_on_launch" = false + "map_public_ip_on_launch" = false + "outpost_arn" = "" + "owner_id" = "916723593639" + "private_dns_hostname_type_on_launch" = "ip-name" + "tags" = tomap({ + "Name" = "swarms-private-us-east-2c" + "project" = "swarms" + }) + "tags_all" = tomap({ + "Name" = "swarms-private-us-east-2c" + "project" = "swarms" + }) + "timeouts" = null /* object */ + "vpc_id" = "vpc-0b4cedd083227068d" + }, + ] + "private_subnets" = [ + "subnet-0ea44af2f97e12b1a", + "subnet-0de17633cfb166d29", + "subnet-0e926a2b5ae3f5acc", + ] + "private_subnets_cidr_blocks" = tolist([ + "10.0.0.0/24", + "10.0.1.0/24", + "10.0.2.0/24", + ]) + "private_subnets_ipv6_cidr_blocks" = tolist([]) + "public_internet_gateway_ipv6_route_id" = null + "public_internet_gateway_route_id" = "r-rtb-0445a2ea1c510657b1080289494" + "public_network_acl_arn" = null + "public_network_acl_id" = null + "public_route_table_association_ids" = [ + "rtbassoc-033c4fa608f79a594", + "rtbassoc-000d8799150d904e6", + "rtbassoc-0b047c79fe554852d", + ] + "public_route_table_ids" = [ + "rtb-0445a2ea1c510657b", + ] + "public_subnet_arns" = [ + "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-04b3bdd4b0dc877f0", + "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-051fecdee9028af79", + "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-027f4abd09a6543f0", + ] + "public_subnet_objects" = [ + { + "arn" = "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-04b3bdd4b0dc877f0" + "assign_ipv6_address_on_creation" = false + "availability_zone" = "us-east-2a" + "availability_zone_id" = "use2-az1" + "cidr_block" = "10.0.4.0/24" + "customer_owned_ipv4_pool" = "" + "enable_dns64" = false + "enable_lni_at_device_index" = 0 + "enable_resource_name_dns_a_record_on_launch" = false + "enable_resource_name_dns_aaaa_record_on_launch" = false + "id" = "subnet-04b3bdd4b0dc877f0" + "ipv6_cidr_block" = "" + "ipv6_cidr_block_association_id" = "" + "ipv6_native" = false + "map_customer_owned_ip_on_launch" = false + "map_public_ip_on_launch" = false + "outpost_arn" = "" + "owner_id" = "916723593639" + "private_dns_hostname_type_on_launch" = "ip-name" + "tags" = tomap({ + "Name" = "swarms-public-us-east-2a" + "project" = "swarms" + }) + "tags_all" = tomap({ + "Name" = "swarms-public-us-east-2a" + "project" = "swarms" + }) + "timeouts" = null /* object */ + "vpc_id" = "vpc-0b4cedd083227068d" + }, + { + "arn" = "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-051fecdee9028af79" + "assign_ipv6_address_on_creation" = false + "availability_zone" = "us-east-2b" + "availability_zone_id" = "use2-az2" + "cidr_block" = "10.0.5.0/24" + "customer_owned_ipv4_pool" = "" + "enable_dns64" = false + "enable_lni_at_device_index" = 0 + "enable_resource_name_dns_a_record_on_launch" = false + "enable_resource_name_dns_aaaa_record_on_launch" = false + "id" = "subnet-051fecdee9028af79" + "ipv6_cidr_block" = "" + "ipv6_cidr_block_association_id" = "" + "ipv6_native" = false + "map_customer_owned_ip_on_launch" = false + "map_public_ip_on_launch" = false + "outpost_arn" = "" + "owner_id" = "916723593639" + "private_dns_hostname_type_on_launch" = "ip-name" + "tags" = tomap({ + "Name" = "swarms-public-us-east-2b" + "project" = "swarms" + }) + "tags_all" = tomap({ + "Name" = "swarms-public-us-east-2b" + "project" = "swarms" + }) + "timeouts" = null /* object */ + "vpc_id" = "vpc-0b4cedd083227068d" + }, + { + "arn" = "arn:aws:ec2:us-east-2:916723593639:subnet/subnet-027f4abd09a6543f0" + "assign_ipv6_address_on_creation" = false + "availability_zone" = "us-east-2c" + "availability_zone_id" = "use2-az3" + "cidr_block" = "10.0.6.0/24" + "customer_owned_ipv4_pool" = "" + "enable_dns64" = false + "enable_lni_at_device_index" = 0 + "enable_resource_name_dns_a_record_on_launch" = false + "enable_resource_name_dns_aaaa_record_on_launch" = false + "id" = "subnet-027f4abd09a6543f0" + "ipv6_cidr_block" = "" + "ipv6_cidr_block_association_id" = "" + "ipv6_native" = false + "map_customer_owned_ip_on_launch" = false + "map_public_ip_on_launch" = false + "outpost_arn" = "" + "owner_id" = "916723593639" + "private_dns_hostname_type_on_launch" = "ip-name" + "tags" = tomap({ + "Name" = "swarms-public-us-east-2c" + "project" = "swarms" + }) + "tags_all" = tomap({ + "Name" = "swarms-public-us-east-2c" + "project" = "swarms" + }) + "timeouts" = null /* object */ + "vpc_id" = "vpc-0b4cedd083227068d" + }, + ] + "public_subnets" = [ + "subnet-04b3bdd4b0dc877f0", + "subnet-051fecdee9028af79", + "subnet-027f4abd09a6543f0", + ] + "public_subnets_cidr_blocks" = tolist([ + "10.0.4.0/24", + "10.0.5.0/24", + "10.0.6.0/24", + ]) + "public_subnets_ipv6_cidr_blocks" = tolist([]) + "redshift_network_acl_arn" = null + "redshift_network_acl_id" = null + "redshift_public_route_table_association_ids" = [] + "redshift_route_table_association_ids" = [] + "redshift_route_table_ids" = tolist([ + "rtb-033eec802f743baca", + "rtb-0bbc3caa1cdc40cd0", + "rtb-081f2bb265d2703ee", + ]) + "redshift_subnet_arns" = [] + "redshift_subnet_group" = null + "redshift_subnet_objects" = [] + "redshift_subnets" = [] + "redshift_subnets_cidr_blocks" = tolist([]) + "redshift_subnets_ipv6_cidr_blocks" = tolist([]) + "this_customer_gateway" = {} + "vgw_arn" = null + "vgw_id" = null + "vpc_arn" = "arn:aws:ec2:us-east-2:916723593639:vpc/vpc-0b4cedd083227068d" + "vpc_cidr_block" = "10.0.0.0/16" + "vpc_enable_dns_hostnames" = true + "vpc_enable_dns_support" = true + "vpc_flow_log_cloudwatch_iam_role_arn" = "" + "vpc_flow_log_deliver_cross_account_role" = null + "vpc_flow_log_destination_arn" = "" + "vpc_flow_log_destination_type" = "cloud-watch-logs" + "vpc_flow_log_id" = null + "vpc_id" = "vpc-0b4cedd083227068d" + "vpc_instance_tenancy" = "default" + "vpc_ipv6_association_id" = "" + "vpc_ipv6_cidr_block" = "" + "vpc_main_route_table_id" = "rtb-053dc8c3d37f2f58f" + "vpc_owner_id" = "916723593639" + "vpc_secondary_cidr_blocks" = tolist([]) + } + "vpc_arn" = "arn:aws:ec2:us-east-2:916723593639:vpc/vpc-0b4cedd083227068d" + "vpc_cidr_block" = "10.0.0.0/16" + "vpc_enable_dns_hostnames" = true + "vpc_enable_dns_support" = true + "vpc_flow_log_cloudwatch_iam_role_arn" = "" + "vpc_flow_log_destination_arn" = "" + "vpc_flow_log_destination_type" = "cloud-watch-logs" + "vpc_flow_log_id" = null + "vpc_id" = "vpc-0b4cedd083227068d" + "vpc_instance_tenancy" = "default" + "vpc_ipv6_association_id" = "" + "vpc_ipv6_cidr_block" = "" + "vpc_main_route_table_id" = "rtb-053dc8c3d37f2f58f" + "vpc_owner_id" = "916723593639" + "vpc_secondary_cidr_blocks" = tolist([]) + } +} +dns = "api.swarms.ai" +profile = "swarms" +region = "us-east-2" +mdupont@mdupont-G470:~/2024/12/swarms-terraform/accounts/swarms$ diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/alb/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/alb/main.tf new file mode 100644 index 0000000..e69de29 diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/asg/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/asg/main.tf new file mode 100644 index 0000000..e69de29 diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf new file mode 100644 index 0000000..171f170 --- /dev/null +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf @@ -0,0 +1,32 @@ +module "acm" { +# count = 0 + source = "terraform-aws-modules/acm/aws" + version = "~> 4.0" + domain_name = var.domain_name + zone_id = aws_route53_zone.primary.zone_id + subject_alternative_names = [ + "*.${var.domain_name}" + ] +} + +## now we just lift the listener code +resource "aws_lb_listener" "this" { + port = 443 + protocol = "HTTPS" + ssl_policy = "ELBSecurityPolicy-TLS13-1-2-Res-2021-06" + certificate_arn = module.acm.acm_certificate_arn + load_balancer_arn = module.alb.arn + #additional_certificate_arns = [module.wildcard_cert.acm_certificate_arn] + # #forward = { + # #target_group_key = "ex-swarms-instance" + # target_group_arn = "ex-swarms-instance" + # #target_group = [] + + default_action { + target_group_arn =aws_lb_target_group.this.arn + #module.alb.target_groups["ex-lambda-with-trigger"].arn + #length(try(default_action.value.target_groups, [])) > 0 ? null : try(default_action.value.arn, aws_lb_target_group.this[default_action.value.target_group_key].arn, null) + type = "forward" + } +} + diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf index 896e006..b5c33cf 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf @@ -1,11 +1,7 @@ -# variable "provider_alias" { -# type = any -# } +variable vpc_id {} variable security_group_id {} # = local.name variable name {} # = local.name variable domain_name {} # = local.name -variable vpc_id {} #= module.vpc.vpc_id -#variable subnets {} #= module.vpc.public_subnets variable public_subnets {} #= module.vpc.public_subnets data "aws_availability_zones" "available" {} @@ -36,105 +32,27 @@ module "alb" { tags = local.tags } -resource "aws_route53_zone" "primary" { - name = var.domain_name -} -resource "aws_route53_record" "api-cname" { - zone_id = aws_route53_zone.primary.zone_id - name = "us-east-1.${var.domain_name}" - type = "CNAME" - ttl = 5 -# weighted_routing_policy { -# weight = 10 -# } - #set_identifier = "dev" - records = [ - module.alb.dns_name - ] -} -module "acm" { -# count = 0 - source = "terraform-aws-modules/acm/aws" - version = "~> 4.0" - domain_name = var.domain_name - zone_id = aws_route53_zone.primary.zone_id - subject_alternative_names = [ - "*.${var.domain_name}" - ] +output dns { + value = module.alb.dns_name } -## now we just lift the listener code -resource "aws_lb_listener" "this" { - port = 443 - protocol = "HTTPS" - ssl_policy = "ELBSecurityPolicy-TLS13-1-2-Res-2021-06" - certificate_arn = module.acm.acm_certificate_arn - load_balancer_arn = module.alb.arn - #additional_certificate_arns = [module.wildcard_cert.acm_certificate_arn] - # #forward = { - # #target_group_key = "ex-swarms-instance" - # target_group_arn = "ex-swarms-instance" - # #target_group = [] - - default_action { - target_group_arn =aws_lb_target_group.this.arn - #module.alb.target_groups["ex-lambda-with-trigger"].arn - #length(try(default_action.value.target_groups, [])) > 0 ? null : try(default_action.value.arn, aws_lb_target_group.this[default_action.value.target_group_key].arn, null) - type = "forward" - } +module "route53" { + source = "./route53/" + alb_dns_name = module.alb.dns_name + alb_dns_zone = module.alb.zone_id + domain_name = var.domain_name } - -resource "aws_lb_target_group" "this" { - name_prefix = "swarms" - protocol = "HTTP" - port = 80 - target_type = "instance" - vpc_id = var.vpc_id - deregistration_delay = 10 - #load_balancing_algorithm_type = "weighted_random" - #load_balancing_anomaly_mitigation = "on" - #load_balancing_cross_zone_enabled = false - protocol_version = "HTTP1" - # - health_check { - path = "/v1/docs" # the docs api - enabled = true - healthy_threshold = 10 - interval = 130 - port = "traffic-port" - protocol = "HTTP" - timeout = 120 - unhealthy_threshold = 10 - } - -# stickiness { -# cookie_duration = 86400 -# enabled = true -# type = "lb_cookie" -# } - +module "tg" { + source = "./target_group/" + vpc_id = var.vpc_id # module.vpc.vpc_id } -output zone_id { - value = aws_route53_zone.primary.zone_id -} -output zone { - value = aws_route53_zone.primary -} output alb_target_group_arn { - value = aws_lb_target_group.this.arn -} - -output dns { - value = module.alb.dns_name -} - -output cname { - value = aws_route53_record.api-cname.fqdn + value = module.tg.alb_target_group_arn } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/notes.prg b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/notes.prg new file mode 100644 index 0000000..df1c054 --- /dev/null +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/notes.prg @@ -0,0 +1,479 @@ +# access_logs = { +# bucket = module.log_bucket.s3_bucket_id +# prefix = "access-logs" +# } + +# connection_logs = { +# bucket = module.log_bucket.s3_bucket_id +# enabled = true +# prefix = "connection-logs" +# } + + # ex-http-https-redirect = { + # port = 80 + # protocol = "HTTP" + # redirect = { + # port = "443" + # protocol = "HTTPS" + # status_code = "HTTP_301" + # } + # } + # rules = { + # ex-fixed-response = { + # priority = 3 + # actions = [{ + # type = "fixed-response" + # content_type = "text/plain" + # status_code = 200 + # message_body = "This is a fixed response" + # }] + # conditions = [{ + # http_header = { + # http_header_name = "x-Gimme-Fixed-Response" + # values = ["yes", "please", "right now"] + # } + # }] + # } + # ex-weighted-forward = { + # priority = 4 + # actions = [{ + # type = "weighted-forward" + # target_groups = [ + # { + # target_group_key = "ex-lambda-with-trigger" + # weight = 2 + # }, + # { + # target_group_key = "ex-instance" + # weight = 1 + # } + # ] + # stickiness = { + # enabled = true + # duration = 3600 + # } + # }] + # conditions = [{ + # query_string = { + # key = "weighted" + # value = "true" + # } + # }] + # } + # ex-redirect = { + # priority = 5000 + # actions = [{ + # type = "redirect" + # status_code = "HTTP_302" + # host = "www.youtube.com" + # path = "/watch" + # query = "v=dQw4w9WgXcQ" + # protocol = "HTTPS" + # }] + # conditions = [{ + # query_string = [{ + # key = "video" + # value = "random" + # }, + # { + # key = "image" + # value = "next" + # }] + # }] + # } + # } + # } + # ex-http-weighted-target = { + # port = 81 + # protocol = "HTTP" + # weighted_forward = { + # target_groups = [ + # { + # target_group_key = "ex-lambda-with-trigger" + # weight = 60 + # }, + # { + # target_group_key = "ex-instance" + # weight = 40 + # } + # ] + # } + # } + # ex-fixed-response = { + # port = 82 + # protocol = "HTTP" + # fixed_response = { + # content_type = "text/plain" + # message_body = "Fixed message" + # status_code = "200" + # } + # } + # rules = { + # ex-cognito = { + # actions = [ + # { + # type = "authenticate-cognito" + # on_unauthenticated_request = "authenticate" + # session_cookie_name = "session-${local.name}" + # session_timeout = 3600 + # user_pool_arn = aws_cognito_user_pool.this.arn + # user_pool_client_id = aws_cognito_user_pool_client.this.id + # user_pool_domain = aws_cognito_user_pool_domain.this.domain + # }, + # { + # type = "forward" + # target_group_key = "ex-instance" + # } + # ] + # conditions = [{ + # path_pattern = { + # values = ["/some/auth/required/route"] + # } + # }] + # } + # ex-fixed-response = { + # priority = 3 + # actions = [{ + # type = "fixed-response" + # content_type = "text/plain" + # status_code = 200 + # message_body = "This is a fixed response" + # }] + # conditions = [{ + # http_header = { + # http_header_name = "x-Gimme-Fixed-Response" + # values = ["yes", "please", "right now"] + # } + # }] + # } + # ex-weighted-forward = { + # priority = 4 + # actions = [{ + # type = "weighted-forward" + # target_groups = [ + # { + # target_group_key = "ex-instance" + # weight = 2 + # }, + # { + # target_group_key = "ex-lambda-with-trigger" + # weight = 1 + # } + # ] + # stickiness = { + # enabled = true + # duration = 3600 + # } + # }] + # conditions = [{ + # query_string = { + # key = "weighted" + # value = "true" + # }, + # path_pattern = { + # values = ["/some/path"] + # } + # }] + # } + # ex-redirect = { + # priority = 5000 + # actions = [{ + # type = "redirect" + # status_code = "HTTP_302" + # host = "www.youtube.com" + # path = "/watch" + # query = "v=dQw4w9WgXcQ" + # protocol = "HTTPS" + # }] + # conditions = [{ + # query_string = { + # key = "video" + # value = "random" + # } + # }] + # } + # } + # ex-cognito = { + # port = 444 + # protocol = "HTTPS" + # certificate_arn = module.acm.acm_certificate_arn + # authenticate_cognito = { + # authentication_request_extra_params = { + # display = "page" + # prompt = "login" + # } + # on_unauthenticated_request = "authenticate" + # session_cookie_name = "session-${local.name}" + # session_timeout = 3600 + # user_pool_arn = aws_cognito_user_pool.this.arn + # user_pool_client_id = aws_cognito_user_pool_client.this.id + # user_pool_domain = aws_cognito_user_pool_domain.this.domain + # } + # forward = { + # target_group_key = "ex-instance" + # } + # rules = { + # ex-oidc = { + # priority = 2 + # actions = [ + # { + # type = "authenticate-oidc" + # authentication_request_extra_params = { + # display = "page" + # prompt = "login" + # } + # authorization_endpoint = "https://${var.domain_name}/auth" + # client_id = "client_id" + # client_secret = "client_secret" + # issuer = "https://${var.domain_name}" + # token_endpoint = "https://${var.domain_name}/token" + # user_info_endpoint = "https://${var.domain_name}/user_info" + # }, + # { + # type = "forward" + # target_group_key = "ex-lambda-with-trigger" + # } + # ] + # conditions = [{ + # host_header = { + # values = ["foobar.com"] + # } + # }] + # } + # } + # } + # ex-oidc = { + # port = 445 + # protocol = "HTTPS" + # certificate_arn = module.acm.acm_certificate_arn + # action_type = "authenticate-oidc" + # authenticate_oidc = { + # authentication_request_extra_params = { + # display = "page" + # prompt = "login" + # } + # authorization_endpoint = "https://${var.domain_name}/auth" + # client_id = "client_id" + # client_secret = "client_secret" + # issuer = "https://${var.domain_name}" + # token_endpoint = "https://${var.domain_name}/token" + # user_info_endpoint = "https://${var.domain_name}/user_info" + # } + # forward = { + # target_group_key = "ex-instance" + # } + # } + # } + # target_group_health = { + # dns_failover = { + # minimum_healthy_targets_count = 2 + # } + # unhealthy_state_routing = { + # minimum_healthy_targets_percentage = 50 + # } + # } + # health_check = { + # enabled = true + # interval = 30 + # path = "/healthz" + # port = "traffic-port" + # healthy_threshold = 3 + # unhealthy_threshold = 3 + # timeout = 6 + # protocol = "HTTP" + # matcher = "200-399" + # } + # ex-lambda-with-trigger = { + # name_prefix = "l1-" + # target_type = "lambda" + # lambda_multi_value_headers_enabled = true + # target_id = module.lambda_with_allowed_triggers.lambda_function_arn + # } + # ex-lambda-without-trigger = { + # name_prefix = "l2-" + # target_type = "lambda" + # target_id = module.lambda_without_allowed_triggers.lambda_function_arn + # attach_lambda_permission = true + # } + # } + # additional_target_group_attachments = { + # ex-instance-other = { + # target_group_key = "ex-instance" + # target_type = "instance" + # target_id = aws_instance.other.id + # port = "80" + # } + # } + # # Route53 Record(s) + # route53_records = { + # A = { + # name = local.name + # type = "A" + # zone_id = data.aws_route53_zone.this.id + # } + # AAAA = { + # name = local.name + # type = "AAAA" + # zone_id = data.aws_route53_zone.this.id + # } + # } + + +#module "alb_disabled" { +# source = "../../"# +# +# create = false +#} + +# ################################################################################ +# # Using packaged function from Lambda module +# ################################################################################ + +# locals { +# package_url = "https://raw.githubusercontent.com/terraform-aws-modules/terraform-aws-lambda/master/examples/fixtures/python3.8-zip/existing_package.zip" +# downloaded = "downloaded_package_${md5(local.package_url)}.zip" +# } + +# resource "null_resource" "download_package" { +# triggers = { +# downloaded = local.downloaded +# } + +# provisioner "local-exec" { +# command = "curl -L -o ${local.downloaded} ${local.package_url}" +# } +# } + +# module "lambda_with_allowed_triggers" { +# source = "terraform-aws-modules/lambda/aws" +# version = "~> 6.0" + +# function_name = "${local.name}-with-allowed-triggers" +# description = "My awesome lambda function (with allowed triggers)" +# handler = "index.lambda_handler" +# runtime = "python3.8" + +# publish = true +# create_package = false +# local_existing_package = local.downloaded + +# allowed_triggers = { +# AllowExecutionFromELB = { +# service = "elasticloadbalancing" +# source_arn = module.alb.target_groups["ex-lambda-with-trigger"].arn +# } +# } + +# depends_on = [null_resource.download_package] +# } + +# module "lambda_without_allowed_triggers" { +# source = "terraform-aws-modules/lambda/aws" +# version = "~> 6.0" + +# function_name = "${local.name}-without-allowed-triggers" +# description = "My awesome lambda function (without allowed triggers)" +# handler = "index.lambda_handler" +# runtime = "python3.8" + +# publish = true +# create_package = false +# local_existing_package = local.downloaded + +# # Allowed triggers will be managed by ALB module +# allowed_triggers = {} + +# depends_on = [null_resource.download_package] +# } + +# ################################################################################ +# # Supporting resources +# ################################################################################ + +# module "vpc" { +# source = "terraform-aws-modules/vpc/aws" +# version = "~> 5.0" + +# name = local.name +# cidr = local.vpc_cidr + +# azs = local.azs +# private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] +# public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] + +# tags = local.tags +# } + +# module "wildcard_cert" { +# source = "terraform-aws-modules/acm/aws" +# version = "~> 4.0" + +# domain_name = "*.${var.domain_name}" +# zone_id = data.aws_route53_zone.this.id +# } + +# data "aws_ssm_parameter" "al2" { +# name = "/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2" +# } + +# resource "aws_instance" "this" { +# ami = data.aws_ssm_parameter.al2.value +# instance_type = "t3.nano" +# subnet_id = element(module.vpc.private_subnets, 0) +# } + +# resource "aws_instance" "other" { +# ami = data.aws_ssm_parameter.al2.value +# instance_type = "t3.nano" +# subnet_id = element(module.vpc.private_subnets, 0) +# } + +# ################################################################## +# # AWS Cognito User Pool +# ################################################################## + +# resource "aws_cognito_user_pool" "this" { +# name = "user-pool-${local.name}" +# } + +# resource "aws_cognito_user_pool_client" "this" { +# name = "user-pool-client-${local.name}" +# user_pool_id = aws_cognito_user_pool.this.id +# generate_secret = true +# allowed_oauth_flows = ["code", "implicit"] +# callback_urls = ["https://${var.domain_name}/callback"] +# allowed_oauth_scopes = ["email", "openid"] +# allowed_oauth_flows_user_pool_client = true +# } + +# resource "random_string" "this" { +# length = 5 +# upper = false +# special = false +# } + +# resource "aws_cognito_user_pool_domain" "this" { +# domain = "${local.name}-${random_string.this.result}" +# user_pool_id = aws_cognito_user_pool.this.id +# } + +#module# "log_bucket" { +# source = "terraform-aws-modules/s3-bucket/aws" +# version = "~> 3.0" + +# bucket_prefix = "${local.name}-logs-" +# acl = "log-delivery-write" + +# # For example only +# force_destroy = true + +# control_object_ownership = true +# object_ownership = "ObjectWriter" + +# attach_elb_log_delivery_policy = true # Required for ALB logs +# attach_lb_log_delivery_policy = true # Required for ALB/NLB logs + +# attach_deny_insecure_transport_policy = true +# attach_require_latest_tls_policy = true +# tags = local.tags +#} diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf new file mode 100644 index 0000000..b29c4e2 --- /dev/null +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf @@ -0,0 +1,36 @@ +variable domain_name {} # = local.name +variable alb_dns_name {} +variable alb_dns_zone {} +data "aws_route53_zone" "primary" { + name = var.domain_name +} + +resource "aws_route53_record" "api-cname" { + zone_id = data.aws_route53_zone.primary.zone_id + name = var.domain_name + # type = "CNAME" + type = "A" +# ttl = 5 + +# weighted_routing_policy { +# weight = 10 +# } + #set_identifier = "dev" + alias { + name = var.alb_dns_name + zone_id = var.alb_dns_zone + evaluate_target_health = true + + # + } +} + +output cname { + value = aws_route53_record.api-cname.fqdn +} +output zone { + value = data.aws_route53_zone.primary +} +output zone_id { + value = data.aws_route53_zone.primary.zone_id +} diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/target_group/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/target_group/main.tf new file mode 100644 index 0000000..017efed --- /dev/null +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/target_group/main.tf @@ -0,0 +1,34 @@ +variable vpc_id {} +resource "aws_lb_target_group" "this" { + name_prefix = "swarms" + protocol = "HTTP" + port = 80 + target_type = "instance" + vpc_id = var.vpc_id + deregistration_delay = 10 + #load_balancing_algorithm_type = "weighted_random" + #load_balancing_anomaly_mitigation = "on" + #load_balancing_cross_zone_enabled = false + protocol_version = "HTTP1" + # + health_check { + path = "/v1/docs" # the docs api + enabled = true + healthy_threshold = 10 + interval = 130 + port = "traffic-port" + protocol = "HTTP" + timeout = 120 + unhealthy_threshold = 10 + } + +# stickiness { +# cookie_duration = 86400 +# enabled = true +# type = "lb_cookie" +# } + +} +output alb_target_group_arn { + value = aws_lb_target_group.this.arn +} diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf index bb78a6e..e5e31ef 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf @@ -1,62 +1,18 @@ +variable aws_iam_instance_profile_ssm_arn {} variable target_group_arn{} -#variable security_group_id {} variable name {} -variable instance_type { - # default = "t3.micro" -} - -variable launch_template_id { - -} -variable image_id { - default = "ami-0e2c8caa4b6378d8c" -} -variable vpc_id { - default = "vpc-04f28c9347af48b55" -} -#provider "aws" { -# region = "us-east-1" -#} +variable instance_type {} +variable launch_template_id {} +variable image_id {} +variable vpc_id {} +variable tags {} +variable ec2_subnet_id {} locals { - ami = "ami-0e2c8caa4b6378d8c" - # name = "swarms" - region = "us-east-1" - ec2_subnet_id = "subnet-057c90cfe7b2e5646" - - #iam_instance_profile_name = "swarms-20241213150629570500000003" - iam_instance_profile_arn = aws_iam_instance_profile.ssm.arn - tags = { - project="swarms" - } - + iam_instance_profile_arn = var.aws_iam_instance_profile_ssm_arn instance_type = var.instance_type } -resource "aws_iam_instance_profile" "ssm" { - name = "ssm-${var.name}" - role = aws_iam_role.ssm.name - tags = local.tags -} -resource "aws_iam_role" "ssm" { - name = "ssm-${var.name}" - tags = local.tags - - assume_role_policy = jsonencode({ - Version = "2012-10-17", - Statement = [ - { - Action = "sts:AssumeRole", - Principal = { - Service = "ec2.amazonaws.com" - }, - Effect = "Allow", - Sid = "" - } - ] - }) -} - module "autoscaling" { source = "terraform-aws-modules/autoscaling/aws" version = "8.0.0" @@ -73,7 +29,7 @@ module "autoscaling" { launch_template_id = var.launch_template_id launch_template_version = "$Latest" - vpc_zone_identifier = [local.ec2_subnet_id] + vpc_zone_identifier = [var.ec2_subnet_id] instance_market_options = { market_type = "spot" @@ -88,7 +44,6 @@ module "autoscaling" { ] instance_type = var.instance_type image_id = var.image_id - create_iam_instance_profile = true iam_role_name = "ssm-${var.name}" diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf index b823712..09801f7 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf @@ -1,36 +1,26 @@ +variable iam_instance_profile_name {} variable security_group_id {} -variable name { - default = "swarms" -} -variable vpc_id { - default = "vpc-04f28c9347af48b55" -} - +variable name {} +variable vpc_id {} +variable ami_id {} +variable tags {} variable key_name { default = "mdupont-deployer-key" } + # dont use this variable instance_type {} locals { - ami = "ami-0e2c8caa4b6378d8c" -# name = "swarms" - region = "us-east-1" - ec2_subnet_id = "subnet-057c90cfe7b2e5646" - vpc_id = "vpc-04f28c9347af48b55" - iam_instance_profile_name = "swarms-20241213150629570500000003" tags = { project="swarms" instance_type = var.instance_type name = var.name } } - - - resource "aws_launch_template" "ec2_launch_template" { name_prefix = "${var.name}-launch-template-" - image_id = local.ami + image_id = var.ami_id key_name = var.key_name instance_type = var.instance_type network_interfaces { @@ -40,7 +30,8 @@ resource "aws_launch_template" "ec2_launch_template" { } iam_instance_profile { - name = local.iam_instance_profile_name #aws_iam_instance_profile.ec2_instance_profile.name + # iam_instance_profile_arn = aws_iam_instance_profile.ssm.arn + name = var.iam_instance_profile_name #aws_iam_instance_profile.ec2_instance_profile.name } lifecycle { create_before_destroy = true @@ -75,7 +66,7 @@ resource "aws_launch_template" "ec2_launch_template" { bash -x /opt/swarms/api/install.sh EOF ) - tags = local.tags + tags = var.tags } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf index 36c5cbd..2b7fbaa 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf @@ -1,10 +1,6 @@ locals { - ami = "ami-0e2c8caa4b6378d8c" name = "swarms" - region = "us-east-1" - ec2_subnet_id = "subnet-057c90cfe7b2e5646" - vpc_id = "vpc-04f28c9347af48b55" tags = { project="swarms" } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/security/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/security/main.tf index 796614a..5572aba 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/security/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/security/main.tf @@ -1,24 +1,12 @@ -variable vpc_id { - default = "vpc-04f28c9347af48b55" -} - -locals { - ami = "ami-0e2c8caa4b6378d8c" - name = "swarms" - region = "us-east-1" - ec2_subnet_id = "subnet-057c90cfe7b2e5646" - - tags = { - project="swarms" - } - -} +variable vpc_id { } +variable tags { } +variable name { } module "asg_sg" { source = "terraform-aws-modules/security-group/aws" version = "~> 5.0" - name = "${local.name}-external" + name = "${var.name}-external" description = "external group" vpc_id = var.vpc_id @@ -31,14 +19,14 @@ module "asg_sg" { egress_rules = ["all-all"] - tags = local.tags + tags = var.tags } module "asg_sg_internal" { source = "terraform-aws-modules/security-group/aws" version = "~> 5.0" - name = "${local.name}-internal" + name = "${var.name}-internal" description = "An internal security group" vpc_id = var.vpc_id # see ~/2024/12/13/terraform-aws-security-group/examples/complete/main.tf @@ -51,7 +39,7 @@ module "asg_sg_internal" { ] egress_rules = ["all-all"] - tags = local.tags + tags = var.tags } output "security_group_id" { diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/main.tf index 2448aae..303c8fc 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/main.tf @@ -2,10 +2,9 @@ data "aws_availability_zones" "available" {} - locals { name = "swarms" - region = "us-east-1" + vpc_cidr = "10.0.0.0/16" azs = slice(data.aws_availability_zones.available.names, 0, 3) @@ -21,7 +20,6 @@ locals { # #source = "https://github.com/terraform-aws-modules/terraform-aws-vpc.git" # name = "swarms" # cidr = "10.0.0.0/16" -# azs = ["us-east-1a", "us-east-1b", "us-east-1c"] # private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] # public_subnets = ["10.0.101.0/24", "10.0.102.0/24", "10.0.103.0/24"] @@ -231,3 +229,6 @@ module "vpc" { # tags = local.tags # } +output "vpc" { +value = module.vpc +} diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/outputs.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/outputs.tf index 5cb68e1..32381e3 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/outputs.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/outputs.tf @@ -550,9 +550,9 @@ output "vpc_flow_log_cloudwatch_iam_role_arn" { # value = module.vpc_endpoints.security_group_id # } output ec2_public_subnet_id_2 { - value = 1 + value = module.vpc.public_subnets[1] } output ec2_public_subnet_id_1 { - value = 1 + value = module.vpc.public_subnets[0] } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index 3814095..c811e52 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -1,8 +1,6 @@ -variable domain {} locals { # instance_type = "t3.large" # instance_type = "t3.medium" - #ami = "ami-0e2c8caa4b6378d8c" ami_name = "ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*" name = "swarms" region = "us-east-2" @@ -11,16 +9,19 @@ locals { project="swarms" } } +variable domain {} +variable ami_id {} +variable tags {} +variable name {} -data "aws_ami" "ami" { - most_recent = true - name_regex = "^${local.ami_name}" -} locals { - ami_id = data.aws_ami.ami.id +# ami_id = data.aws_ami.ami.id + ami_id = var.ami_id + } +# SLOW module "vpc" { source = "./components/vpc" } @@ -33,6 +34,9 @@ locals { module "security" { source = "./components/security" + vpc_id = local.vpc_id + tags = local.tags + name = local.name } module "kp" { @@ -66,14 +70,52 @@ variable "instance_types" { ] } +resource "aws_iam_role" "ssm" { + name = "ssm-${local.name}" + tags = var.tags + + assume_role_policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Action = "sts:AssumeRole", + Principal = { + Service = "ec2.amazonaws.com" + }, + Effect = "Allow", + Sid = "" + } + ] + }) +} + +resource "aws_iam_instance_profile" "ssm" { + name = "ssm-${var.name}" + role = aws_iam_role.ssm.name + tags = var.tags +} + module "lt_dynamic" { + vpc_id = local.vpc_id for_each = toset(var.instance_types) instance_type = each.key name = "swarms-size-${each.key}" security_group_id = module.security.internal_security_group_id + ami_id = var.ami_id + tags= local.tags source = "./components/launch_template" + iam_instance_profile_name = aws_iam_instance_profile.ssm.name +} + +output security_group_id { + value = module.security.security_group_id +} + +output vpc { + value = module.vpc } + module "alb" { source = "./components/application_load_balancer" domain_name = local.domain @@ -84,9 +126,18 @@ module "alb" { vpc_id = local.vpc_id name = local.name } +output alb { + value = module.alb +} + module "asg_dynamic" { + tags = local.tags + vpc_id = local.vpc_id + image_id = local.ami_id + ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 for_each = toset(var.instance_types) + aws_iam_instance_profile_ssm_arn = aws_iam_instance_profile.ssm.arn source = "./components/autoscaling_group" # security_group_id = module.security.internal_security_group_id instance_type = each.key @@ -94,23 +145,3 @@ module "asg_dynamic" { launch_template_id = module.lt_dynamic[each.key].launch_template_id target_group_arn = module.alb.alb_target_group_arn } - -# module "alb" { -# # count = 0 -# source = "./components/application_load_balancer" -# vpc_id = local.vpc_id -# } -# │ - - -#output launch_template_id { -# value = module.lt.launch_template_id -#} - -output security_group_id { - value = module.security.security_group_id -} - -output alb { - value = module.alb -} From 1063aed4b6e1d9d479b224d2a598cb20689b8bc4 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Mon, 16 Dec 2024 19:39:53 -0500 Subject: [PATCH 029/158] now applying --- .../application_load_balancer/https/main.tf | 12 ++++++++---- .../components/application_load_balancer/main.tf | 12 +++++++++++- .../application_load_balancer/route53/main.tf | 2 +- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf index 171f170..1d97095 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf @@ -1,9 +1,14 @@ +variable alb_arn{} +variable domain_name{} +variable zone_id{} +variable aws_lb_target_group_arn{} + module "acm" { # count = 0 source = "terraform-aws-modules/acm/aws" version = "~> 4.0" domain_name = var.domain_name - zone_id = aws_route53_zone.primary.zone_id + zone_id = var.zone_id subject_alternative_names = [ "*.${var.domain_name}" ] @@ -15,7 +20,7 @@ resource "aws_lb_listener" "this" { protocol = "HTTPS" ssl_policy = "ELBSecurityPolicy-TLS13-1-2-Res-2021-06" certificate_arn = module.acm.acm_certificate_arn - load_balancer_arn = module.alb.arn + load_balancer_arn = var.alb_arn #additional_certificate_arns = [module.wildcard_cert.acm_certificate_arn] # #forward = { # #target_group_key = "ex-swarms-instance" @@ -23,10 +28,9 @@ resource "aws_lb_listener" "this" { # #target_group = [] default_action { - target_group_arn =aws_lb_target_group.this.arn + target_group_arn =var.aws_lb_target_group_arn #module.alb.target_groups["ex-lambda-with-trigger"].arn #length(try(default_action.value.target_groups, [])) > 0 ? null : try(default_action.value.arn, aws_lb_target_group.this[default_action.value.target_group_key].arn, null) type = "forward" } } - diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf index b5c33cf..5762415 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf @@ -1,7 +1,7 @@ variable vpc_id {} variable security_group_id {} # = local.name variable name {} # = local.name -variable domain_name {} # = local.name +variable domain_name {} variable public_subnets {} #= module.vpc.public_subnets data "aws_availability_zones" "available" {} @@ -52,6 +52,16 @@ module "tg" { vpc_id = var.vpc_id # module.vpc.vpc_id } +module "https" { + source = "./https/" + # vpc_id = var.vpc_id # module.vpc.vpc_id + zone_id = module.route53.primary_zone_id + domain_name = var.domain_name + alb_arn = module.alb.arn + aws_lb_target_group_arn = module.tg.alb_target_group_arn + #aws_lb_target_group.this.arn +} + output alb_target_group_arn { value = module.tg.alb_target_group_arn diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf index b29c4e2..697bffa 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf @@ -31,6 +31,6 @@ output cname { output zone { value = data.aws_route53_zone.primary } -output zone_id { +output primary_zone_id { value = data.aws_route53_zone.primary.zone_id } From 06f9a86606ea184d716e64fdd9b7854f1dbc49d7 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Tue, 17 Dec 2024 06:45:45 -0500 Subject: [PATCH 030/158] adding readme --- accounts/swarms/Readme.md | 60 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 accounts/swarms/Readme.md diff --git a/accounts/swarms/Readme.md b/accounts/swarms/Readme.md new file mode 100644 index 0000000..db2cd84 --- /dev/null +++ b/accounts/swarms/Readme.md @@ -0,0 +1,60 @@ + +# credentials + +set up ~/.aws/credentials +``` +[swarms] +aws_access_key_id =${your key} +aws_secret_access_key=${your SECRET} +``` + +# install opentofu or terraform +# install aws cli +# install aws ssm plugin + +# create openai secret token + +TODO: +`aws ssm set-parameter --name "swarms_openai_key"` + +# tofu init +# tofu plan +# tofu apply +point the dns api.swarms.ai at the dns servers in godaddy + +`tofu state show module.swarms_api.module.alb.module.route53.data.aws_route53_zone.primary` + +```terraform +# module.swarms_api.module.alb.module.route53.data.aws_route53_zone.primary: +data "aws_route53_zone" "primary" { + arn = "arn:aws:route53:::hostedzone/Z04162952OP7P14Z97UWY" + caller_reference = "937599df-113d-4b02-8c75-4a20f8e6293e" + id = "Z04162952OP7P14Z97UWY" + name = "api.swarms.ai" + name_servers = [ + "ns-864.awsdns-44.net", + "ns-1595.awsdns-07.co.uk", + "ns-1331.awsdns-38.org", + "ns-463.awsdns-57.com", + ] + primary_name_server = "ns-864.awsdns-44.net" + private_zone = false + resource_record_set_count = 3 + tags = {} + zone_id = "Z04162952OP7P14Z97UWY" +} +``` +so we need 4 records + +1. NS api -> "ns-864.awsdns-44.net" +2. NS api -> "ns-1595.awsdns-07.co.uk" +3. NS api -> "ns-1331.awsdns-38.org" +4. NS api -> "ns-463.awsdns-57.com" + +see youtube or +https://youtu.be/3BI6_gq-lSU +https://dev.to/diegop0s/managing-your-godaddy-domain-with-route53-5f2p + +# tofu apply + +`tofu apply` From 790758e269b1a202d64974ea2515019469570c75 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Tue, 17 Dec 2024 10:02:51 -0500 Subject: [PATCH 031/158] starting cognito --- accounts/mdupont/Makefile | 3 +- accounts/mdupont/custom_style.css | 1 + accounts/mdupont/data.tf | 21 +- accounts/mdupont/logo.png | Bin 0 -> 172 bytes accounts/mdupont/main.tf | 15 + accounts/mdupont/output.tf | 15 +- accounts/mdupont/variables.tf | 27 +- .../components/cognito_user_pool/README.md | 264 +++++++++++++++ .../components/cognito_user_pool/main.tf | 300 ++++++++++++++++++ .../components/cognito_user_pool/provider.tf | 4 + .../components/cognito_user_pool/variables.tf | 4 + 11 files changed, 625 insertions(+), 29 deletions(-) create mode 100644 accounts/mdupont/custom_style.css create mode 100644 accounts/mdupont/logo.png create mode 100644 accounts/mdupont/main.tf create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/README.md create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/main.tf create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/provider.tf create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/variables.tf diff --git a/accounts/mdupont/Makefile b/accounts/mdupont/Makefile index 9d2b026..47e8dbb 100644 --- a/accounts/mdupont/Makefile +++ b/accounts/mdupont/Makefile @@ -1,3 +1,4 @@ make : - #tofu init + tofu init tofu plan + tofu apply diff --git a/accounts/mdupont/custom_style.css b/accounts/mdupont/custom_style.css new file mode 100644 index 0000000..7fb0c8b --- /dev/null +++ b/accounts/mdupont/custom_style.css @@ -0,0 +1 @@ +.label-customizable {font-weight: 400;} diff --git a/accounts/mdupont/data.tf b/accounts/mdupont/data.tf index d4e6bbb..fbb2afa 100644 --- a/accounts/mdupont/data.tf +++ b/accounts/mdupont/data.tf @@ -7,21 +7,20 @@ data "aws_iam_user" "example_user" { user_name = var.iam_user } -data "aws_dynamodb_table" "terraform_dynamo_table" { - name = var.table_name -} +#data "aws_dynamodb_table" "terraform_dynamo_table" { +# name = var.table_name +#} # Data resource for AWS call identity data "aws_caller_identity" "current" {} -data "aws_s3_bucket" "terraform_logging" { - bucket = "${var.project_name}-tf-state-log-${var.aws_region}" -} - -data "aws_s3_bucket" "terraform_state" { - bucket = "${var.project_name}-tf-state-${var.aws_region}" -} +#data "aws_s3_bucket" "terraform_logging" { +# bucket = "${var.project_name}-tf-state-log-${var.aws_region}" +#} +#data "aws_s3_bucket" "terraform_state" { +# bucket = "${var.project_name}-tf-state-${var.aws_region}" +#} -data "aws_cloudtrail_service_account" "main" {} +#data "aws_cloudtrail_service_account" "main" {} diff --git a/accounts/mdupont/logo.png b/accounts/mdupont/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..33d0f2d31fcbc0985cb86f5e358cb5d2223b3893 GIT binary patch literal 172 zcmeAS@N?(olHy`uVBq!ia0vp^RzR%7!3HEvOjfT0Qk(@Ik;M!Q+`=Ht$S`Y;1W+)} z)5S3)qw(!UMqUO5o+Af%>n$AYtR(alUPiIj|BNXPzaTC8_bP0l+XkKn8h=7 literal 0 HcmV?d00001 diff --git a/accounts/mdupont/main.tf b/accounts/mdupont/main.tf new file mode 100644 index 0000000..e89bd9c --- /dev/null +++ b/accounts/mdupont/main.tf @@ -0,0 +1,15 @@ +module cognito { + aws_account =var.aws_account_id + myemail ="jmdupont" + mydomain ="introspector" + mydomain_suffix = "meme" + #../../../17/ + aws_region = var.aws_region + env={ + region = var.aws_region + profile = var.profile + } + source = "../../environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool" + #source = "~/2024/12/17/cognito/terraform-aws-cognito-user-pool/examples/complete/" + #source = "git::https://github.com/meta-introspector/terraform-aws-cognito-user-pool.git?ref=feature/meta-meme" +} diff --git a/accounts/mdupont/output.tf b/accounts/mdupont/output.tf index a672016..ef3739e 100644 --- a/accounts/mdupont/output.tf +++ b/accounts/mdupont/output.tf @@ -6,9 +6,9 @@ output "iam_user" { value = var.iam_user } -output "table_name" { - value = var.table_name -} +#output "table_name" { +# value = var.table_name +#} output "project_name" { value = var.project_name @@ -19,15 +19,18 @@ output "aws_caller_identity_account_id" { } output "aws_s3_bucket_logging_name" { - value = data.aws_s3_bucket.terraform_logging.bucket + # value = data.aws_s3_bucket.terraform_logging.bucket + value = null } output "aws_s3_bucket_state_name" { - value = data.aws_s3_bucket.terraform_state.bucket + #value = data.aws_s3_bucket.terraform_state.bucket + value = null } output "aws_cloudtrail_service_account_id" { - value = data.aws_cloudtrail_service_account.main.id + #value = data.aws_cloudtrail_service_account.main.id + value = null } diff --git a/accounts/mdupont/variables.tf b/accounts/mdupont/variables.tf index 8560e4a..31b65a2 100644 --- a/accounts/mdupont/variables.tf +++ b/accounts/mdupont/variables.tf @@ -1,12 +1,17 @@ - variable "project_name" { - type = string - default = "swarms" - } +variable "project_name" { + type = string + default = "meta-meme" +} + +variable "profile" { + type = string + default = "default" +} variable "aws_region" { - type = string - default = "us-east-1" - } + type = string + default = "us-east-1" +} variable "aws_account_id" { type = string @@ -18,10 +23,10 @@ variable "aws_region" { default = "mdupont" } - variable "table_name" { - type = string - default = "swarms" - } +# variable "table_name" { +# type = string +# default = "meta-meme" +# } variable "lock_resource" { type = string diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/README.md b/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/README.md new file mode 100644 index 0000000..c025085 --- /dev/null +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/README.md @@ -0,0 +1,264 @@ +# This is a complete example + +``` +module "aws_cognito_user_pool_complete_example" { + + source = "lgallard/cognito-user-pool/aws" + + user_pool_name = "mypool_complete" + alias_attributes = ["email", "phone_number"] + auto_verified_attributes = ["email"] + sms_authentication_message = "Your username is {username} and temporary password is {####}." + sms_verification_message = "This is the verification message {####}." + + deletion_protection = "ACTIVE" + + mfa_configuration = "OPTIONAL" + software_token_mfa_configuration = { + enabled = true + } + + admin_create_user_config = { + email_message = "Dear {username}, your verification code is {####}." + email_subject = "Here, your verification code baby" + sms_message = "Your username is {username} and temporary password is {####}." + } + + device_configuration = { + challenge_required_on_new_device = true + device_only_remembered_on_user_prompt = true + } + + email_configuration = { + email_sending_account = "DEVELOPER" + reply_to_email_address = "email@mydomain.com" + source_arn = "arn:aws:ses:us-east-1:123456789012:identity/myemail@mydomain.com" + } + + lambda_config = { + create_auth_challenge = "arn:aws:lambda:us-east-1:123456789012:function:create_auth_challenge" + custom_message = "arn:aws:lambda:us-east-1:123456789012:function:custom_message" + define_auth_challenge = "arn:aws:lambda:us-east-1:123456789012:function:define_auth_challenge" + post_authentication = "arn:aws:lambda:us-east-1:123456789012:function:post_authentication" + post_confirmation = "arn:aws:lambda:us-east-1:123456789012:function:post_confirmation" + pre_authentication = "arn:aws:lambda:us-east-1:123456789012:function:pre_authentication" + pre_sign_up = "arn:aws:lambda:us-east-1:123456789012:function:pre_sign_up" + pre_token_generation = "arn:aws:lambda:us-east-1:123456789012:function:pre_token_generation" + user_migration = "arn:aws:lambda:us-east-1:123456789012:function:user_migration" + verify_auth_challenge_response = "arn:aws:lambda:us-east-1:123456789012:function:verify_auth_challenge_response" + } + + password_policy = { + minimum_length = 10 + require_lowercase = false + require_numbers = true + require_symbols = true + require_uppercase = true + temporary_password_validity_days = 120 + + } + + user_pool_add_ons = { + advanced_security_mode = "ENFORCED" + } + + verification_message_template = { + default_email_option = "CONFIRM_WITH_CODE" + } + + schemas = [ + { + attribute_data_type = "Boolean" + developer_only_attribute = false + mutable = true + name = "available" + required = false + }, + { + attribute_data_type = "Boolean" + developer_only_attribute = true + mutable = true + name = "registered" + required = false + } + ] + + string_schemas = [ + { + attribute_data_type = "String" + developer_only_attribute = false + mutable = false + name = "email" + required = true + + string_attribute_constraints = { + min_length = 7 + max_length = 15 + } + }, + { + attribute_data_type = "String" + developer_only_attribute = false + mutable = false + name = "gender" + required = true + + string_attribute_constraints = { + min_length = 7 + max_length = 15 + } + }, + ] + + number_schemas = [ + { + attribute_data_type = "Number" + developer_only_attribute = true + mutable = true + name = "mynumber1" + required = false + + number_attribute_constraints = { + min_value = 2 + max_value = 6 + } + }, + { + attribute_data_type = "Number" + developer_only_attribute = true + mutable = true + name = "mynumber2" + required = false + + number_attribute_constraints = { + min_value = 2 + max_value = 6 + } + }, + ] + + # user_pool_domain + domain = "mydomain-com" + + # clients + clients = [ + { + allowed_oauth_flows = [] + allowed_oauth_flows_user_pool_client = false + allowed_oauth_scopes = [] + callback_urls = ["https://mydomain.com/callback"] + default_redirect_uri = "https://mydomain.com/callback" + explicit_auth_flows = [] + generate_secret = true + logout_urls = [] + name = "test1" + read_attributes = ["email"] + supported_identity_providers = [] + write_attributes = [] + access_token_validity = 1 + id_token_validity = 1 + refresh_token_validity = 60 + token_validity_units = { + access_token = "hours" + id_token = "hours" + refresh_token = "days" + } + }, + { + allowed_oauth_flows = [] + allowed_oauth_flows_user_pool_client = false + allowed_oauth_scopes = [] + callback_urls = ["https://mydomain.com/callback"] + default_redirect_uri = "https://mydomain.com/callback" + explicit_auth_flows = [] + generate_secret = false + logout_urls = [] + name = "test2" + read_attributes = [] + supported_identity_providers = [] + write_attributes = [] + refresh_token_validity = 30 + }, + { + allowed_oauth_flows = ["code", "implicit"] + allowed_oauth_flows_user_pool_client = true + allowed_oauth_scopes = ["email", "openid"] + callback_urls = ["https://mydomain.com/callback"] + default_redirect_uri = "https://mydomain.com/callback" + explicit_auth_flows = ["CUSTOM_AUTH_FLOW_ONLY", "ADMIN_NO_SRP_AUTH"] + generate_secret = false + logout_urls = ["https://mydomain.com/logout"] + name = "test3" + read_attributes = ["email", "phone_number"] + supported_identity_providers = [] + write_attributes = ["email", "gender", "locale", ] + refresh_token_validity = 30 + } + ] + + # user_group + user_groups = [ + { name = "mygroup1" + description = "My group 1" + }, + { name = "mygroup2" + description = "My group 2" + }, + ] + + # resource_servers + resource_servers = [ + { + identifier = "https://mydomain.com" + name = "mydomain" + scope = [ + { + scope_name = "sample-scope-1" + scope_description = "A sample Scope Description for mydomain.com" + }, + { + scope_name = "sample-scope-2" + scope_description = "Another sample Scope Description for mydomain.com" + }, + ] + }, + { + identifier = "https://weather-read-app.com" + name = "weather-read" + scope = [ + { + scope_name = "weather.read" + scope_description = "Read weather forecasts" + } + ] + } + ] + + # identity_providers + identity_providers = [ + { + provider_name = "Google" + provider_type = "Google" + + provider_details = { + authorize_scopes = "email" + client_id = "your client_id" + client_secret = "your client_secret" + } + + attribute_mapping = { + email = "email" + username = "sub" + gender = "gender" + } + } + ] + + # tags + tags = { + Owner = "infra" + Environment = "production" + Terraform = true + } +} +``` diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/main.tf new file mode 100644 index 0000000..1a1161b --- /dev/null +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/main.tf @@ -0,0 +1,300 @@ +variable aws_region {} # us-east-1 +variable aws_account {} +variable myemail {} +variable mydomain {} +variable mydomain_suffix {} +locals { + mydomain=var.mydomain + mydomain_suffix = var.mydomain_suffix + mydomain_dot_com = "${local.mydomain}.${local.mydomain_suffix}" + myemail=var.myemail + myaccount=var.aws_account +} + +module "aws_cognito_user_pool_complete_example" { + + source = "lgallard/cognito-user-pool/aws" + + user_pool_name = "mypool_complete" + alias_attributes = ["email", "phone_number"] + auto_verified_attributes = ["email"] + sms_authentication_message = "Your username is {username} and temporary password is {####}." + sms_verification_message = "This is the verification message {####}." + + deletion_protection = "ACTIVE" + + mfa_configuration = "OPTIONAL" + software_token_mfa_configuration = { + enabled = true + } + + admin_create_user_config = { + email_message = "Dear {username}, your verification code is {####}." + email_subject = "Here, your verification code baby" + sms_message = "Your username is {username} and temporary password is {####}." + } + + device_configuration = { + challenge_required_on_new_device = true + device_only_remembered_on_user_prompt = true + } + + email_configuration = { + email_sending_account = "DEVELOPER" + reply_to_email_address = "email@${local.mydomain_dot_com}" + source_arn = "arn:aws:ses:${var.aws_region}:${var.aws_account}:identity/${local.myemail}@${local.mydomain_dot_com}" + } + + lambda_config = { + create_auth_challenge = "arn:aws:lambda:${var.aws_region}:${var.aws_account}:function:create_auth_challenge" + custom_message = "arn:aws:lambda:${var.aws_region}:${var.aws_account}:function:custom_message" + define_auth_challenge = "arn:aws:lambda:${var.aws_region}:${var.aws_account}:function:define_auth_challenge" + post_authentication = "arn:aws:lambda:${var.aws_region}:${var.aws_account}:function:post_authentication" + post_confirmation = "arn:aws:lambda:${var.aws_region}:${var.aws_account}:function:post_confirmation" + pre_authentication = "arn:aws:lambda:${var.aws_region}:${var.aws_account}:function:pre_authentication" + pre_sign_up = "arn:aws:lambda:${var.aws_region}:${var.aws_account}:function:pre_sign_up" + pre_token_generation = "arn:aws:lambda:${var.aws_region}:${var.aws_account}:function:pre_token_generation" + user_migration = "arn:aws:lambda:${var.aws_region}:${var.aws_account}:function:user_migration" + verify_auth_challenge_response = "arn:aws:lambda:${var.aws_region}:${var.aws_account}:function:verify_auth_challenge_response" + #kms_key_id = aws_kms_key.lambda-custom-sender.arn + pre_token_generation_config = { + lambda_arn = "arn:aws:lambda:${var.aws_region}:${var.aws_account}:function:pre_token_generation_config" + lambda_version = "V1_0" + } + #custom_email_sender = { + # lambda_arn = "arn:aws:lambda:${var.aws_region}:${var.aws_account}:function:custom_email_sender" + # lambda_version = "V1_0" + #} + #custom_sms_sender = { + # lambda_arn = "arn:aws:lambda:${var.aws_region}:${var.aws_account}:function:custom_sms_sender" + # lambda_version = "V1_0" + #} + } + + password_policy = { + minimum_length = 10 + require_lowercase = false + require_numbers = true + require_symbols = true + require_uppercase = true + temporary_password_validity_days = 120 + + } + + user_pool_add_ons = { + advanced_security_mode = "ENFORCED" + } + + verification_message_template = { + default_email_option = "CONFIRM_WITH_CODE" + } + + schemas = [ + { + attribute_data_type = "Boolean" + developer_only_attribute = false + mutable = true + name = "available" + required = false + }, + { + attribute_data_type = "Boolean" + developer_only_attribute = true + mutable = true + name = "registered" + required = false + } + ] + + string_schemas = [ + { + attribute_data_type = "String" + developer_only_attribute = false + mutable = false + name = "email" + required = true + + string_attribute_constraints = { + min_length = 7 + max_length = 15 + } + }, + { + attribute_data_type = "String" + developer_only_attribute = false + mutable = false + name = "gender" + required = true + + string_attribute_constraints = { + min_length = 7 + max_length = 15 + } + }, + ] + + number_schemas = [ + { + attribute_data_type = "Number" + developer_only_attribute = true + mutable = true + name = "mynumber1" + required = false + + number_attribute_constraints = { + min_value = 2 + max_value = 6 + } + }, + { + attribute_data_type = "Number" + developer_only_attribute = true + mutable = true + name = "mynumber2" + required = false + + number_attribute_constraints = { + min_value = 2 + max_value = 6 + } + }, + ] + + # user_pool_domain + domain = "${local.mydomain}-com" + + # clients + clients = [ + { + allowed_oauth_flows = [] + allowed_oauth_flows_user_pool_client = false + allowed_oauth_scopes = [] + callback_urls = ["https://${local.mydomain_dot_com}/callback"] + default_redirect_uri = "https://${local.mydomain_dot_com}/callback" + explicit_auth_flows = [] + generate_secret = true + logout_urls = [] + name = "test1" + read_attributes = ["email"] + supported_identity_providers = [] + write_attributes = [] + access_token_validity = 1 + id_token_validity = 1 + refresh_token_validity = 60 + token_validity_units = { + access_token = "hours" + id_token = "hours" + refresh_token = "days" + } + ui_customization_css = file("./custom_style.css") + ui_customization_image_file = filebase64("logo.png") + }, + { + allowed_oauth_flows = [] + allowed_oauth_flows_user_pool_client = false + allowed_oauth_scopes = [] + callback_urls = ["https://${local.mydomain_dot_com}/callback"] + default_redirect_uri = "https://${local.mydomain_dot_com}/callback" + explicit_auth_flows = [] + generate_secret = false + logout_urls = [] + name = "test2" + read_attributes = [] + supported_identity_providers = [] + write_attributes = [] + refresh_token_validity = 30 + }, + { + allowed_oauth_flows = ["code", "implicit"] + allowed_oauth_flows_user_pool_client = true + allowed_oauth_scopes = ["email", "openid"] + callback_urls = ["https://${local.mydomain_dot_com}/callback"] + default_redirect_uri = "https://${local.mydomain_dot_com}/callback" + explicit_auth_flows = ["CUSTOM_AUTH_FLOW_ONLY", "ADMIN_NO_SRP_AUTH"] + generate_secret = false + logout_urls = ["https://${local.mydomain_dot_com}/logout"] + name = "test3" + read_attributes = ["email", "phone_number"] + supported_identity_providers = [] + write_attributes = ["email", "gender", "locale", ] + refresh_token_validity = 30 + } + ] + + # user_group + user_groups = [ + { name = "mygroup1" + description = "My group 1" + }, + { name = "mygroup2" + description = "My group 2" + }, + ] + + # resource_servers + resource_servers = [ + { + identifier = "https://${local.mydomain_dot_com}" + name = "${local.mydomain}" + scope = [ + { + scope_name = "sample-scope-1" + scope_description = "A sample Scope Description for ${local.mydomain_dot_com}" + }, + { + scope_name = "sample-scope-2" + scope_description = "Another sample Scope Description for ${local.mydomain_dot_com}" + }, + ] + }, + { + identifier = "https://weather-read-app.com" + name = "weather-read" + scope = [ + { + scope_name = "weather.read" + scope_description = "Read weather forecasts" + } + ] + } + ] + + # identity_providers + identity_providers = [ + { + provider_name = "Google" + provider_type = "Google" + + provider_details = { + authorize_scopes = "email" + client_id = "your client_id" # This should be retrieved from AWS Secret Manager, otherwise Terraform will force an in-place replacement becuase is treated as a sensitive value + client_secret = "your client_secret" # # This should be retrieved from AWS Secret Manager, otherwise Terraform will force an in-place replacement becuase is treated as a sensitive value + attributes_url_add_attributes = "true" + authorize_url = "https://accounts.google.com/o/oauth2/v2/auth" + oidc_issuer = "https://accounts.google.com" + token_request_method = "POST" + token_url = "https://www.googleapis.com/oauth2/v4/token" + } + + attribute_mapping = { + email = "email" + username = "sub" + gender = "gender" + } + } + ] + + # tags + tags = { + Owner = "infra" + Environment = "production" + Terraform = true + } +} + + + # KMS key for lambda custom sender config" + resource "aws_kms_key" "lambda-custom-sender" { + count = 0 + description = "KMS key for lambda custom sender config" + } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/provider.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/provider.tf new file mode 100644 index 0000000..634c762 --- /dev/null +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/provider.tf @@ -0,0 +1,4 @@ +provider "aws" { + region = var.env["region"] + profile = var.env["profile"] +} diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/variables.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/variables.tf new file mode 100644 index 0000000..c7b7aed --- /dev/null +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/variables.tf @@ -0,0 +1,4 @@ +variable "env" { + type = map(any) + default = {} +} From e671f30c9146624b97a0e6be17558a21aacfdbd8 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Tue, 17 Dec 2024 10:09:40 -0500 Subject: [PATCH 032/158] apply --- .../dev/us-east-1/components/cognito_user_pool/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/main.tf index 1a1161b..c851cfc 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/main.tf @@ -53,7 +53,7 @@ module "aws_cognito_user_pool_complete_example" { post_confirmation = "arn:aws:lambda:${var.aws_region}:${var.aws_account}:function:post_confirmation" pre_authentication = "arn:aws:lambda:${var.aws_region}:${var.aws_account}:function:pre_authentication" pre_sign_up = "arn:aws:lambda:${var.aws_region}:${var.aws_account}:function:pre_sign_up" - pre_token_generation = "arn:aws:lambda:${var.aws_region}:${var.aws_account}:function:pre_token_generation" +# pre_token_generation = "arn:aws:lambda:${var.aws_region}:${var.aws_account}:function:pre_token_generation" user_migration = "arn:aws:lambda:${var.aws_region}:${var.aws_account}:function:user_migration" verify_auth_challenge_response = "arn:aws:lambda:${var.aws_region}:${var.aws_account}:function:verify_auth_challenge_response" #kms_key_id = aws_kms_key.lambda-custom-sender.arn From 3cd307aee6b2f766a844cd27372d4c17c1d699ea Mon Sep 17 00:00:00 2001 From: mike dupont Date: Tue, 17 Dec 2024 10:31:31 -0500 Subject: [PATCH 033/158] appplied --- accounts/mdupont/main.tf | 4 ++++ .../dev/us-east-1/components/Readme.md | 3 +++ .../dev/us-east-1/components/cognito_user_pool/main.tf | 8 ++++++-- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/accounts/mdupont/main.tf b/accounts/mdupont/main.tf index e89bd9c..42fe28b 100644 --- a/accounts/mdupont/main.tf +++ b/accounts/mdupont/main.tf @@ -13,3 +13,7 @@ module cognito { #source = "~/2024/12/17/cognito/terraform-aws-cognito-user-pool/examples/complete/" #source = "git::https://github.com/meta-introspector/terraform-aws-cognito-user-pool.git?ref=feature/meta-meme" } +output cognito{ + value = module.cognito + sensitive = true +} diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md b/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md index 92f42ec..2cb2a7a 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md @@ -23,3 +23,6 @@ terraform for ubuntu python uvicorn fastapi nginx systemd server with custom gi 8. create lambda_workers on queue 9. create resource_launchers to create new resources. 10. packer ami for ubuntu python uvicorn fastapi nginx systemd server with custom git modules +11. setup cicd +12. create jobt to generate diagrams and documentation from site +13. use swarms agent to improve code diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/main.tf index c851cfc..0a6ecbb 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/main.tf @@ -267,8 +267,8 @@ module "aws_cognito_user_pool_complete_example" { provider_details = { authorize_scopes = "email" - client_id = "your client_id" # This should be retrieved from AWS Secret Manager, otherwise Terraform will force an in-place replacement becuase is treated as a sensitive value - client_secret = "your client_secret" # # This should be retrieved from AWS Secret Manager, otherwise Terraform will force an in-place replacement becuase is treated as a sensitive value + client_id = var.google_oauth_client_id # This should be retrieved from AWS Secret Manager, otherwise Terraform will force an in-place replacement becuase is treated as a sensitive value + client_secret = var.google_oauth_client_secret #"your client_secret" # # This should be retrieved from AWS Secret Manager, otherwise Terraform will force an in-place replacement becuase is treated as a sensitive value attributes_url_add_attributes = "true" authorize_url = "https://accounts.google.com/o/oauth2/v2/auth" oidc_issuer = "https://accounts.google.com" @@ -298,3 +298,7 @@ module "aws_cognito_user_pool_complete_example" { count = 0 description = "KMS key for lambda custom sender config" } + +output cognito{ + value = module.aws_cognito_user_pool_complete_example +} From e64a39abc6cd24d4058700527fac5711dac46c94 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Tue, 17 Dec 2024 10:55:43 -0500 Subject: [PATCH 034/158] google --- accounts/mdupont/main.tf | 5 ++++ .../components/cognito_user_pool/README.md | 11 +++++++ .../components/cognito_user_pool/main.tf | 29 +++++++++++++++---- 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/accounts/mdupont/main.tf b/accounts/mdupont/main.tf index 42fe28b..8e621a9 100644 --- a/accounts/mdupont/main.tf +++ b/accounts/mdupont/main.tf @@ -1,3 +1,6 @@ +variable "google_oauth_client_secret" {} +variable "google_oauth_client_id" {} + module cognito { aws_account =var.aws_account_id myemail ="jmdupont" @@ -12,6 +15,8 @@ module cognito { source = "../../environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool" #source = "~/2024/12/17/cognito/terraform-aws-cognito-user-pool/examples/complete/" #source = "git::https://github.com/meta-introspector/terraform-aws-cognito-user-pool.git?ref=feature/meta-meme" + google_oauth_client_secret=var.google_oauth_client_secret + google_oauth_client_id=var.google_oauth_client_id } output cognito{ value = module.cognito diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/README.md b/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/README.md index c025085..505f95a 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/README.md +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/README.md @@ -1,3 +1,14 @@ +# manual setup + +1. Created auth client here +https://console.cloud.google.com/auth/clients/ + +2. stored password in +``` +export TF_VAR_google_oauth_client_id=DDDDD +export TF_VAR_google_oauth_client_secret=XXXX +``` + # This is a complete example ``` diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/main.tf index 0a6ecbb..23b583f 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/main.tf @@ -1,3 +1,5 @@ +variable "google_oauth_client_secret" {} +variable "google_oauth_client_id" {} variable aws_region {} # us-east-1 variable aws_account {} variable myemail {} @@ -166,7 +168,6 @@ module "aws_cognito_user_pool_complete_example" { # clients clients = [ { - allowed_oauth_flows = [] allowed_oauth_flows_user_pool_client = false allowed_oauth_scopes = [] callback_urls = ["https://${local.mydomain_dot_com}/callback"] @@ -190,9 +191,21 @@ module "aws_cognito_user_pool_complete_example" { ui_customization_image_file = filebase64("logo.png") }, { - allowed_oauth_flows = [] - allowed_oauth_flows_user_pool_client = false - allowed_oauth_scopes = [] + allowed_oauth_flows = [ + "code", + "implicit" + ] + allowed_oauth_flows_user_pool_client = true + allowed_oauth_scopes = [ + "aws.cognito.signin.user.admin", + "email", + "https://introspector.meme/sample-scope-1", + "https://introspector.meme/sample-scope-2", + "openid", + "phone", + "profile", + + ] callback_urls = ["https://${local.mydomain_dot_com}/callback"] default_redirect_uri = "https://${local.mydomain_dot_com}/callback" explicit_auth_flows = [] @@ -200,7 +213,11 @@ module "aws_cognito_user_pool_complete_example" { logout_urls = [] name = "test2" read_attributes = [] - supported_identity_providers = [] + supported_identity_providers = [ + "COGNITO", + "Google", + + ] write_attributes = [] refresh_token_validity = 30 }, @@ -267,7 +284,9 @@ module "aws_cognito_user_pool_complete_example" { provider_details = { authorize_scopes = "email" + #export TF_VAR_google_oauth_client_id=XXXX client_id = var.google_oauth_client_id # This should be retrieved from AWS Secret Manager, otherwise Terraform will force an in-place replacement becuase is treated as a sensitive value + # export TF_VAR_google_oauth_client_secret=YYY client_secret = var.google_oauth_client_secret #"your client_secret" # # This should be retrieved from AWS Secret Manager, otherwise Terraform will force an in-place replacement becuase is treated as a sensitive value attributes_url_add_attributes = "true" authorize_url = "https://accounts.google.com/o/oauth2/v2/auth" From e43831e0849b0ebe53ec4c9fd8509a43a746b9bd Mon Sep 17 00:00:00 2001 From: Mike DuPont Date: Tue, 17 Dec 2024 11:03:55 -0500 Subject: [PATCH 035/158] Delete accounts/swarms/.terraform.lock.hcl --- accounts/swarms/.terraform.lock.hcl | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 accounts/swarms/.terraform.lock.hcl diff --git a/accounts/swarms/.terraform.lock.hcl b/accounts/swarms/.terraform.lock.hcl deleted file mode 100644 index f639596..0000000 --- a/accounts/swarms/.terraform.lock.hcl +++ /dev/null @@ -1,20 +0,0 @@ -# This file is maintained automatically by "tofu init". -# Manual edits may be lost in future updates. - -provider "registry.opentofu.org/hashicorp/aws" { - version = "5.81.0" - constraints = ">= 3.29.0, >= 4.40.0, >= 5.55.0, >= 5.73.0" - hashes = [ - "h1:ird967uf44WBZ0u9rMlOdO4nCKJZMlUknLcD3lz4dWs=", - "zh:0d0c7cc1e16b16cd00ab36de35038162e1871b51a902e9016d08c55c9fc4de35", - "zh:0e4b8c6e46999015292ab4fb9a430ab00107a276f25243552cde66db53c58661", - "zh:30041314cdd4e877d75ee8c9d36aecfca094e276f7a3d8150f929cf5169b2fa5", - "zh:5ebd248ce3e5a7ef9cc2f41499668f182146325e10ea305c70469122f6161a13", - "zh:888a69d371373b418549e03f5922becb2b8074cb463552ecfa65f30455708db0", - "zh:8a21bb7fe60383ff5ca9db8630a287e41fd520d2514c08874a16dc74500fadd7", - "zh:9c4663dcbfe58544642d70ebfdc6c5fa91592ff04164f77c655e32e6024483e2", - "zh:b322873f1209443a8407d5f782d7d917de6a1391239dbd0e7f809ce6507bed76", - "zh:b7c9d5ca14b818b5932ac58a490646a425ebc41b33d149090aa5f48d1ca35c99", - "zh:e76cd202b03749f3082b0cbe849fd2e731cf3f9a6aa994d2d629602c3aede36c", - ] -} From ea2a6117b10c7a5f882fb79044c5543334bfe0a8 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Tue, 17 Dec 2024 11:24:27 -0500 Subject: [PATCH 036/158] more scopes --- .../components/cognito_user_pool/main.tf | 30 ++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/main.tf index 23b583f..d4de4ba 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/main.tf @@ -169,7 +169,16 @@ module "aws_cognito_user_pool_complete_example" { clients = [ { allowed_oauth_flows_user_pool_client = false - allowed_oauth_scopes = [] + allowed_oauth_scopes = [ + "aws.cognito.signin.user.admin", + "email", + "https://introspector.meme/sample-scope-1", + "https://introspector.meme/sample-scope-2", + "openid", + "phone", + "profile", + + ] callback_urls = ["https://${local.mydomain_dot_com}/callback"] default_redirect_uri = "https://${local.mydomain_dot_com}/callback" explicit_auth_flows = [] @@ -177,7 +186,11 @@ module "aws_cognito_user_pool_complete_example" { logout_urls = [] name = "test1" read_attributes = ["email"] - supported_identity_providers = [] + supported_identity_providers = [ + "COGNITO", + "Google", + + ] write_attributes = [] access_token_validity = 1 id_token_validity = 1 @@ -195,7 +208,7 @@ module "aws_cognito_user_pool_complete_example" { "code", "implicit" ] - allowed_oauth_flows_user_pool_client = true + allowed_oauth_flows_user_pool_client = false # FIXME why? allowed_oauth_scopes = [ "aws.cognito.signin.user.admin", "email", @@ -224,7 +237,16 @@ module "aws_cognito_user_pool_complete_example" { { allowed_oauth_flows = ["code", "implicit"] allowed_oauth_flows_user_pool_client = true - allowed_oauth_scopes = ["email", "openid"] + allowed_oauth_scopes = [ + "aws.cognito.signin.user.admin", + "email", + "https://introspector.meme/sample-scope-1", + "https://introspector.meme/sample-scope-2", + "openid", + "phone", + "profile", + + ] callback_urls = ["https://${local.mydomain_dot_com}/callback"] default_redirect_uri = "https://${local.mydomain_dot_com}/callback" explicit_auth_flows = ["CUSTOM_AUTH_FLOW_ONLY", "ADMIN_NO_SRP_AUTH"] From 0e86ce3aa4f6e1a3c7cfd35c33e884477b8c53c6 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Tue, 17 Dec 2024 12:35:53 -0500 Subject: [PATCH 037/158] adding ses --- accounts/mdupont/main.tf | 9 + .../dev/us-east-1/components/ses/context.tf | 279 ++++++++++++++++++ .../dev/us-east-1/components/ses/main.tf | 23 ++ 3 files changed, 311 insertions(+) create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/ses/context.tf create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/ses/main.tf diff --git a/accounts/mdupont/main.tf b/accounts/mdupont/main.tf index 8e621a9..c4ad754 100644 --- a/accounts/mdupont/main.tf +++ b/accounts/mdupont/main.tf @@ -1,6 +1,15 @@ variable "google_oauth_client_secret" {} variable "google_oauth_client_id" {} +module ses { + verify_dkim=true + # use another domain for the email so we dont have the root on ses yet + domain="mail.introspector.meme" + verify_domain =true + group_name="introspector" + source = "../../environments/swarms-aws-agent-api/dev/us-east-1/components/ses" +} + module cognito { aws_account =var.aws_account_id myemail ="jmdupont" diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/ses/context.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/ses/context.tf new file mode 100644 index 0000000..5e0ef88 --- /dev/null +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/ses/context.tf @@ -0,0 +1,279 @@ +# +# ONLY EDIT THIS FILE IN github.com/cloudposse/terraform-null-label +# All other instances of this file should be a copy of that one +# +# +# Copy this file from https://github.com/cloudposse/terraform-null-label/blob/master/exports/context.tf +# and then place it in your Terraform module to automatically get +# Cloud Posse's standard configuration inputs suitable for passing +# to Cloud Posse modules. +# +# curl -sL https://raw.githubusercontent.com/cloudposse/terraform-null-label/master/exports/context.tf -o context.tf +# +# Modules should access the whole context as `module.this.context` +# to get the input variables with nulls for defaults, +# for example `context = module.this.context`, +# and access individual variables as `module.this.`, +# with final values filled in. +# +# For example, when using defaults, `module.this.context.delimiter` +# will be null, and `module.this.delimiter` will be `-` (hyphen). +# + +module "this" { + source = "cloudposse/label/null" + version = "0.25.0" # requires Terraform >= 0.13.0 + + enabled = var.enabled + namespace = var.namespace + tenant = var.tenant + environment = var.environment + stage = var.stage + name = var.name + delimiter = var.delimiter + attributes = var.attributes + tags = var.tags + additional_tag_map = var.additional_tag_map + label_order = var.label_order + regex_replace_chars = var.regex_replace_chars + id_length_limit = var.id_length_limit + label_key_case = var.label_key_case + label_value_case = var.label_value_case + descriptor_formats = var.descriptor_formats + labels_as_tags = var.labels_as_tags + + context = var.context +} + +# Copy contents of cloudposse/terraform-null-label/variables.tf here + +variable "context" { + type = any + default = { + enabled = true + namespace = null + tenant = null + environment = null + stage = null + name = null + delimiter = null + attributes = [] + tags = {} + additional_tag_map = {} + regex_replace_chars = null + label_order = [] + id_length_limit = null + label_key_case = null + label_value_case = null + descriptor_formats = {} + # Note: we have to use [] instead of null for unset lists due to + # https://github.com/hashicorp/terraform/issues/28137 + # which was not fixed until Terraform 1.0.0, + # but we want the default to be all the labels in `label_order` + # and we want users to be able to prevent all tag generation + # by setting `labels_as_tags` to `[]`, so we need + # a different sentinel to indicate "default" + labels_as_tags = ["unset"] + } + description = <<-EOT + Single object for setting entire context at once. + See description of individual variables for details. + Leave string and numeric variables as `null` to use default value. + Individual variable settings (non-null) override settings in context object, + except for attributes, tags, and additional_tag_map, which are merged. + EOT + + validation { + condition = lookup(var.context, "label_key_case", null) == null ? true : contains(["lower", "title", "upper"], var.context["label_key_case"]) + error_message = "Allowed values: `lower`, `title`, `upper`." + } + + validation { + condition = lookup(var.context, "label_value_case", null) == null ? true : contains(["lower", "title", "upper", "none"], var.context["label_value_case"]) + error_message = "Allowed values: `lower`, `title`, `upper`, `none`." + } +} + +variable "enabled" { + type = bool + default = null + description = "Set to false to prevent the module from creating any resources" +} + +variable "namespace" { + type = string + default = null + description = "ID element. Usually an abbreviation of your organization name, e.g. 'eg' or 'cp', to help ensure generated IDs are globally unique" +} + +variable "tenant" { + type = string + default = null + description = "ID element _(Rarely used, not included by default)_. A customer identifier, indicating who this instance of a resource is for" +} + +variable "environment" { + type = string + default = null + description = "ID element. Usually used for region e.g. 'uw2', 'us-west-2', OR role 'prod', 'staging', 'dev', 'UAT'" +} + +variable "stage" { + type = string + default = null + description = "ID element. Usually used to indicate role, e.g. 'prod', 'staging', 'source', 'build', 'test', 'deploy', 'release'" +} + +variable "name" { + type = string + default = null + description = <<-EOT + ID element. Usually the component or solution name, e.g. 'app' or 'jenkins'. + This is the only ID element not also included as a `tag`. + The "name" tag is set to the full `id` string. There is no tag with the value of the `name` input. + EOT +} + +variable "delimiter" { + type = string + default = null + description = <<-EOT + Delimiter to be used between ID elements. + Defaults to `-` (hyphen). Set to `""` to use no delimiter at all. + EOT +} + +variable "attributes" { + type = list(string) + default = [] + description = <<-EOT + ID element. Additional attributes (e.g. `workers` or `cluster`) to add to `id`, + in the order they appear in the list. New attributes are appended to the + end of the list. The elements of the list are joined by the `delimiter` + and treated as a single ID element. + EOT +} + +variable "labels_as_tags" { + type = set(string) + default = ["default"] + description = <<-EOT + Set of labels (ID elements) to include as tags in the `tags` output. + Default is to include all labels. + Tags with empty values will not be included in the `tags` output. + Set to `[]` to suppress all generated tags. + **Notes:** + The value of the `name` tag, if included, will be the `id`, not the `name`. + Unlike other `null-label` inputs, the initial setting of `labels_as_tags` cannot be + changed in later chained modules. Attempts to change it will be silently ignored. + EOT +} + +variable "tags" { + type = map(string) + default = {} + description = <<-EOT + Additional tags (e.g. `{'BusinessUnit': 'XYZ'}`). + Neither the tag keys nor the tag values will be modified by this module. + EOT +} + +variable "additional_tag_map" { + type = map(string) + default = {} + description = <<-EOT + Additional key-value pairs to add to each map in `tags_as_list_of_maps`. Not added to `tags` or `id`. + This is for some rare cases where resources want additional configuration of tags + and therefore take a list of maps with tag key, value, and additional configuration. + EOT +} + +variable "label_order" { + type = list(string) + default = null + description = <<-EOT + The order in which the labels (ID elements) appear in the `id`. + Defaults to ["namespace", "environment", "stage", "name", "attributes"]. + You can omit any of the 6 labels ("tenant" is the 6th), but at least one must be present. + EOT +} + +variable "regex_replace_chars" { + type = string + default = null + description = <<-EOT + Terraform regular expression (regex) string. + Characters matching the regex will be removed from the ID elements. + If not set, `"/[^a-zA-Z0-9-]/"` is used to remove all characters other than hyphens, letters and digits. + EOT +} + +variable "id_length_limit" { + type = number + default = null + description = <<-EOT + Limit `id` to this many characters (minimum 6). + Set to `0` for unlimited length. + Set to `null` for keep the existing setting, which defaults to `0`. + Does not affect `id_full`. + EOT + validation { + condition = var.id_length_limit == null ? true : var.id_length_limit >= 6 || var.id_length_limit == 0 + error_message = "The id_length_limit must be >= 6 if supplied (not null), or 0 for unlimited length." + } +} + +variable "label_key_case" { + type = string + default = null + description = <<-EOT + Controls the letter case of the `tags` keys (label names) for tags generated by this module. + Does not affect keys of tags passed in via the `tags` input. + Possible values: `lower`, `title`, `upper`. + Default value: `title`. + EOT + + validation { + condition = var.label_key_case == null ? true : contains(["lower", "title", "upper"], var.label_key_case) + error_message = "Allowed values: `lower`, `title`, `upper`." + } +} + +variable "label_value_case" { + type = string + default = null + description = <<-EOT + Controls the letter case of ID elements (labels) as included in `id`, + set as tag values, and output by this module individually. + Does not affect values of tags passed in via the `tags` input. + Possible values: `lower`, `title`, `upper` and `none` (no transformation). + Set this to `title` and set `delimiter` to `""` to yield Pascal Case IDs. + Default value: `lower`. + EOT + + validation { + condition = var.label_value_case == null ? true : contains(["lower", "title", "upper", "none"], var.label_value_case) + error_message = "Allowed values: `lower`, `title`, `upper`, `none`." + } +} + +variable "descriptor_formats" { + type = any + default = {} + description = <<-EOT + Describe additional descriptors to be output in the `descriptors` output map. + Map of maps. Keys are names of descriptors. Values are maps of the form + `{ + format = string + labels = list(string) + }` + (Type is `any` so the map values can later be enhanced to provide additional options.) + `format` is a Terraform format string to be passed to the `format()` function. + `labels` is a list of labels, in order, to pass to `format()` function. + Label values will be normalized before being passed to `format()` so they will be + identical to how they appear in `id`. + Default is `{}` (`descriptors` output will be empty). + EOT +} + +#### End of copy of cloudposse/terraform-null-label/variables.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/ses/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/ses/main.tf new file mode 100644 index 0000000..1617f2f --- /dev/null +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/ses/main.tf @@ -0,0 +1,23 @@ +#variable "zone_id" {} # domain +variable "verify_domain" {} +variable "group_name" {} +variable "verify_dkim" {} +variable "domain" {} + +resource "aws_route53_zone" "private_dns_zone" { + name = var.domain + tags = module.this.tags +} + +module "ses" { + source = "git::https://github.com/cloudposse/terraform-aws-ses.git" + domain = var.domain + # zone_id = var.zone_id + zone_id = aws_route53_zone.private_dns_zone.zone_id + verify_dkim = var.verify_dkim + verify_domain = var.verify_domain + ses_group_name = var.group_name + context = module.this.context + #ses_group_enabled = false + ses_user_enabled = false +} From a2bff678ed58e980a16453af1c6a86a88e70435b Mon Sep 17 00:00:00 2001 From: mike dupont Date: Tue, 17 Dec 2024 13:26:43 -0500 Subject: [PATCH 038/158] starting on ses verification --- accounts/mdupont/main.tf | 15 ++++++++++++--- .../dev/us-east-1/components/ses/main.tf | 8 ++++---- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/accounts/mdupont/main.tf b/accounts/mdupont/main.tf index c4ad754..f5408b4 100644 --- a/accounts/mdupont/main.tf +++ b/accounts/mdupont/main.tf @@ -3,13 +3,22 @@ variable "google_oauth_client_id" {} module ses { verify_dkim=true - # use another domain for the email so we dont have the root on ses yet - domain="mail.introspector.meme" - verify_domain =true + domain="introspector.meme" # put the mail at the top level + #verify_domain =true + verify_domain =false # not on aws group_name="introspector" source = "../../environments/swarms-aws-agent-api/dev/us-east-1/components/ses" } +# module ses_verification { +# verify_dkim=true +# domain="introspector.meme" # put the mail at the top level +# #verify_domain =true +# verify_domain =false # not on aws +# group_name="introspector" +# source = "../../environments/swarms-aws-agent-api/dev/us-east-1/components/ses_verify" +# } + module cognito { aws_account =var.aws_account_id myemail ="jmdupont" diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/ses/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/ses/main.tf index 1617f2f..27d1c2c 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/ses/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/ses/main.tf @@ -4,10 +4,10 @@ variable "group_name" {} variable "verify_dkim" {} variable "domain" {} -resource "aws_route53_zone" "private_dns_zone" { - name = var.domain - tags = module.this.tags -} +#resource "aws_route53_zone" "private_dns_zone" { +# name = var.domain +# tags = module.this.tags +#} module "ses" { source = "git::https://github.com/cloudposse/terraform-aws-ses.git" From 464cc1c227a794949495b0cdba76b63a772e08c0 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Wed, 18 Dec 2024 09:32:43 -0500 Subject: [PATCH 039/158] insecure listener and updated instructions --- accounts/swarms/Readme.md | 117 ++++++++++++++++-- .../application_load_balancer/https/main.tf | 12 ++ .../us-east-1/components/dns_entry/Readme.md | 1 - 3 files changed, 121 insertions(+), 9 deletions(-) delete mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/dns_entry/Readme.md diff --git a/accounts/swarms/Readme.md b/accounts/swarms/Readme.md index db2cd84..fc96012 100644 --- a/accounts/swarms/Readme.md +++ b/accounts/swarms/Readme.md @@ -46,14 +46,115 @@ data "aws_route53_zone" "primary" { ``` so we need 4 records -1. NS api -> "ns-864.awsdns-44.net" -2. NS api -> "ns-1595.awsdns-07.co.uk" -3. NS api -> "ns-1331.awsdns-38.org" -4. NS api -> "ns-463.awsdns-57.com" - -see youtube or -https://youtu.be/3BI6_gq-lSU -https://dev.to/diegop0s/managing-your-godaddy-domain-with-route53-5f2p +1. NS api.swarms.ai -> "ns-864.awsdns-44.net" +2. NS api.swarms.ai -> "ns-1595.awsdns-07.co.uk" +3. NS api.swarms.ai -> "ns-1331.awsdns-38.org" +4. NS api.swarms.ai -> "ns-463.awsdns-57.com" + +see forum https://repost.aws/questions/QULXL3STgjQtefiJ_q0BixXA/configure-godaddy-subdomain-to-route53 + +it says ns records need fqdn! + +``` +dig NS api.swarms.ai @97.74.103.14 + +; <<>> DiG 9.18.28-0ubuntu0.22.04.1-Ubuntu <<>> NS api.swarms.ai @97.74.103.14 +;; global options: +cmd +;; Got answer: +;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 42722 +;; flags: qr rd; QUERY: 1, ANSWER: 0, AUTHORITY: 4, ADDITIONAL: 1 +;; WARNING: recursion requested but not available + +;; OPT PSEUDOSECTION: +; EDNS: version: 0, flags:; udp: 1232 +;; QUESTION SECTION: +;api.swarms.ai. IN NS + +;; AUTHORITY SECTION: +api.swarms.ai. 3600 IN NS ns-1912.awsdns-47.co.uk. +api.swarms.ai. 3600 IN NS ns-184.awsdns-23.com. +api.swarms.ai. 3600 IN NS ns-598.awsdns-10.net. +api.swarms.ai. 3600 IN NS ns-1175.awsdns-18.org. + +;; Query time: 5 msec +;; SERVER: 97.74.103.14#53(97.74.103.14) (UDP) +;; WHEN: Wed Dec 18 09:20:45 EST 2024 +;; MSG SIZE rcvd: 182 +``` + +``` +mdupont@mdupont-G470:~/swarms-terraform/accounts/swarms$ dig NS api.swarms.ai +trace + +; <<>> DiG 9.18.28-0ubuntu0.22.04.1-Ubuntu <<>> NS api.swarms.ai +trace +;; global options: +cmd +. 198100 IN NS b.root-servers.net. +. 198100 IN NS j.root-servers.net. +. 198100 IN NS a.root-servers.net. +. 198100 IN NS d.root-servers.net. +. 198100 IN NS c.root-servers.net. +. 198100 IN NS g.root-servers.net. +. 198100 IN NS l.root-servers.net. +. 198100 IN NS i.root-servers.net. +. 198100 IN NS h.root-servers.net. +. 198100 IN NS f.root-servers.net. +. 198100 IN NS e.root-servers.net. +. 198100 IN NS m.root-servers.net. +. 198100 IN NS k.root-servers.net. +;; Received 811 bytes from 127.0.0.53#53(127.0.0.53) in 10 ms + +ai. 172800 IN NS v0n0.nic.ai. +ai. 172800 IN NS v0n1.nic.ai. +ai. 172800 IN NS v0n2.nic.ai. +ai. 172800 IN NS v0n3.nic.ai. +ai. 172800 IN NS v2n0.nic.ai. +ai. 172800 IN NS v2n1.nic.ai. +ai. 86400 IN DS 44137 8 2 7886BD35ED745DCA983D951A643495B929B3A7676A88C682EF88EB6F EDBDB253 +ai. 86400 IN DS 3799 8 2 8A8030D4661AE6FCF417349682AC058648371002E70E717E4CF2F11F 83543385 +ai. 86400 IN RRSIG DS 8 1 86400 20241231050000 20241218040000 61050 . chqcZJHy4mAsB6DryQAHcvFBsUDVkhHQStDq65NbEXoeo+sfNsRWVpGV qyibbDL8nLY0QDOifh5EXu1Mnf6ZXqs8NPaPBEwCpA9oVmRA0t3vG2th jrDhKY77f4iL4ovMQLBSYbF5x61HnFZXcgyI22YDbbChsC6rCwmNJnwj sldGSNknyRy4ytEwbsWYquRmXIzSHJ2O9lMw1l/vUHpw9/xo6k26TyhZ 3bydt6Sg/e56zwevU0oW1sRpR9aKwn4x/0X0txKmUo+2wWtJr/GXLJ28 uWIuEF71Tvg2QKM0XqZ2CLeURCkU3v4sV92vKQ3rY0GkMiKKlWYFaC8Q Ev4+0A== +;; Received 807 bytes from 199.7.91.13#53(d.root-servers.net) in 11 ms + +;; communications error to 2001:500:a4::1#53: timed out +;; communications error to 2001:500:a4::1#53: timed out +;; communications error to 2001:500:a4::1#53: timed out +;; communications error to 2001:500:a1::1#53: timed out +;; communications error to 2001:500:a2::1#53: timed out +;; communications error to 2001:500:a5::1#53: timed out +swarms.ai. 3600 IN NS ns27.domaincontrol.com. +swarms.ai. 3600 IN NS ns28.domaincontrol.com. +58cj07tk4r4uuu6m10c83sia655jfil6.ai. 86400 IN NSEC3 1 1 0 73 58MDDLU23QVIIIQ5GPLB3A6K7OB4F5JH NS SOA TXT RRSIG DNSKEY NSEC3PARAM ZONEMD +jib3vggauf3u1alb3kfuqrcjo6a0v2hq.ai. 86400 IN NSEC3 1 1 0 73 JIGGLMUFEJ6D5CFLQAC5CFQICTP7IJTE NS DS RRSIG +58cj07tk4r4uuu6m10c83sia655jfil6.ai. 86400 IN RRSIG NSEC3 8 2 86400 20250108141827 20241218131827 6279 ai. r8VEiuIyhowQ2sXxszJEgCBMnMEkyboj418iO/jJfUKxWM408IJTSiuO aALz97JNhHMyzPxScRCO+Vcr3EOuoBknhiO5oO9w7UDnuzxNRyPuevV6 WdloLDUc3GRKSPxWom4/Dh+yaMTBXr2xiDDpIvmAElU5q1oGceB+5wWf 4i4= +jib3vggauf3u1alb3kfuqrcjo6a0v2hq.ai. 86400 IN RRSIG NSEC3 8 2 86400 20250107151955 20241217141955 6279 ai. vyYgTKyNXo+kYzRoc0zYeR544efw1GPI4br3GtS4lRaUwzc3sEFKtoyo /nNGBWKgnYxlWyhrAgTvCQTLO1Qt6uJWyHVcog+6hcVcbeFsL6whp/u8 LKHOtSFg2C/FzqP3JktiSPO5CcQh6WiBik2KXhkD00lMjXfStciqk9nP osk= +;; Received 583 bytes from 199.115.156.1#53(v2n0.nic.ai) in 22 ms + +api.swarms.ai. 3600 IN NS ns-1912.awsdns-47.co.uk. +api.swarms.ai. 3600 IN NS ns-184.awsdns-23.com. +api.swarms.ai. 3600 IN NS ns-598.awsdns-10.net. +api.swarms.ai. 3600 IN NS ns-1175.awsdns-18.org. +;; Received 182 bytes from 173.201.71.14#53(ns28.domaincontrol.com) in 5 ms + +;; Received 31 bytes from 205.251.199.120#53(ns-1912.awsdns-47.co.uk) in 4 ms +``` + +https://toolbox.googleapps.com/apps/dig/#ANY/ returns +for swarms.ai this: +``` +id 44700 +opcode QUERY +rcode NOERROR +flags QR RD RA +;QUESTION +swarms.ai. IN ANY +;ANSWER +swarms.ai. 3600 IN A 15.197.225.128 +swarms.ai. 3600 IN A 3.33.251.168 +swarms.ai. 3600 IN NS ns28.domaincontrol.com. +swarms.ai. 3600 IN NS ns27.domaincontrol.com. +swarms.ai. 3600 IN SOA ns27.domaincontrol.com. dns.jomax.net. 2024121702 28800 7200 604800 600 +swarms.ai. 3600 IN TXT "google-site-verification=VlUvNHJo0LQzJzm7SIwMzYLB7-Rexx4yxcSJKh0VtjE" +;AUTHORITY +;ADDITIONAL +``` # tofu apply diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf index 1d97095..3be5b7c 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf @@ -9,6 +9,7 @@ module "acm" { version = "~> 4.0" domain_name = var.domain_name zone_id = var.zone_id + wait_for_validation = false subject_alternative_names = [ "*.${var.domain_name}" ] @@ -16,6 +17,7 @@ module "acm" { ## now we just lift the listener code resource "aws_lb_listener" "this" { + count = 0 port = 443 protocol = "HTTPS" ssl_policy = "ELBSecurityPolicy-TLS13-1-2-Res-2021-06" @@ -34,3 +36,13 @@ resource "aws_lb_listener" "this" { type = "forward" } } + +resource "aws_lb_listener" "insecure" { + port = 80 + protocol = "HTTP" + load_balancer_arn = var.alb_arn + default_action { + target_group_arn =var.aws_lb_target_group_arn + type = "forward" + } +} diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/dns_entry/Readme.md b/environments/swarms-aws-agent-api/dev/us-east-1/components/dns_entry/Readme.md deleted file mode 100644 index f725191..0000000 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/dns_entry/Readme.md +++ /dev/null @@ -1 +0,0 @@ -dns_entry From 334207b9adda8ecaec001a79970d5be224bd62b4 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Wed, 18 Dec 2024 10:28:11 -0500 Subject: [PATCH 040/158] building ami --- .../components/machine_image/Readme.md | 9 +++++++ .../machine_image/ubuntu-fastapi.pkr.hcl | 26 +++++++++++-------- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/Readme.md b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/Readme.md index 69b85c3..61cd739 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/Readme.md +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/Readme.md @@ -7,3 +7,12 @@ aws ec2 describe-images --owners 099720109477 > images.json * tofu state rm "module.ec2.aws_spot_instance_request.this[0]" + +# packer build + +``` +packer init -upgrade ./ubuntu-fastapi.pkr.hcl +packer fmt ./ubuntu-fastapi.pkr.hcl +export AWS_DEFAULT_PROFILE=swarms +packer build ./ubuntu-fastapi.pkr.hcl +``` diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/ubuntu-fastapi.pkr.hcl b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/ubuntu-fastapi.pkr.hcl index c1d05aa..b0c95a5 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/ubuntu-fastapi.pkr.hcl +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/ubuntu-fastapi.pkr.hcl @@ -1,29 +1,33 @@ packer { required_plugins { - aws = { - source = "hashicorp/aws" - version = ">= 3.0.0" + amazon = { + version = ">= 1.2.8" + source = "github.com/hashicorp/amazon" } } } source "amazon-ebs" "ubuntu" { - ami_name = "ubuntu-fastapi-{{timestamp}}" - instance_type = "t2.micro" - region = "us-east-1" - source_ami = "ami-0c55b159cbfafe1f0" # Ubuntu 20.04 LTS + ami_name = "ubuntu-swarms-{{timestamp}}" + instance_type = "t2.medium" + region = "us-east-2" + source_ami = "ami-0325b9a2dfb474b2d" # Ubuntu 20.04 LTS ssh_username = "ubuntu" +} + +build { provisioner "shell" { inline = [ "sudo apt-get update", "sudo apt-get install -y python3 python3-pip git", - "pip3 install fastapi uvicorn", - "git clone https://github.com/yourusername/your-fastapi-module.git /app" + "export HOME=/root", + "sudo apt-get install -y ec2-instance-connect git virtualenv", + "sudo git clone https://github.com/jmikedupont2/swarms '/opt/swarms/'", + "cd /opt/swarms/; sudo git checkout --force feature/ec2", + "sudo bash -x /opt/swarms/api/install.sh" ] } -} -build { sources = ["source.amazon-ebs.ubuntu"] } From 79ec0b3a8f4f66a898f5dded9ba3048a3e44db7f Mon Sep 17 00:00:00 2001 From: mike dupont Date: Wed, 18 Dec 2024 10:36:21 -0500 Subject: [PATCH 041/158] now larger drive --- .../components/machine_image/ubuntu-fastapi.pkr.hcl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/ubuntu-fastapi.pkr.hcl b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/ubuntu-fastapi.pkr.hcl index b0c95a5..d7933e7 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/ubuntu-fastapi.pkr.hcl +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/ubuntu-fastapi.pkr.hcl @@ -13,7 +13,12 @@ source "amazon-ebs" "ubuntu" { region = "us-east-2" source_ami = "ami-0325b9a2dfb474b2d" # Ubuntu 20.04 LTS ssh_username = "ubuntu" - + launch_block_device_mappings { + device_name = "/dev/sda1" + volume_size = 30 + volume_type = "gp3" + delete_on_termination = true + } } build { From ada7442aa07b8ef7ee3c8fedd486b5e0470e5893 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Wed, 18 Dec 2024 10:57:53 -0500 Subject: [PATCH 042/158] new ami asg --- .../dev/us-east-1/main.tf | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index c811e52..3c18c88 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -16,9 +16,8 @@ variable name {} locals { -# ami_id = data.aws_ami.ami.id ami_id = var.ami_id - + new_ami_id = "ami-08093b6770af41b14" # environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/Readme.md } # SLOW @@ -145,3 +144,19 @@ module "asg_dynamic" { launch_template_id = module.lt_dynamic[each.key].launch_template_id target_group_arn = module.alb.alb_target_group_arn } + +module "asg_dynamic_new_ami" { + # built with packer + tags = local.tags + vpc_id = local.vpc_id + image_id = local.new_ami_id + ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 + for_each = toset(var.instance_types) + aws_iam_instance_profile_ssm_arn = aws_iam_instance_profile.ssm.arn + source = "./components/autoscaling_group" +# security_group_id = module.security.internal_security_group_id + instance_type = each.key + name = "swarms-ami-${each.key}" + launch_template_id = module.lt_dynamic[each.key].launch_template_id + target_group_arn = module.alb.alb_target_group_arn +} From 2e59cc291eaf5ac23deffbb1450954050d55ec72 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Wed, 18 Dec 2024 12:51:43 -0500 Subject: [PATCH 043/158] loading --- .../dev/us-east-1/components/Readme.md | 21 ++-- .../dev/us-east-1/components/roles/main.tf | 108 ++++++++++++++++++ .../dev/us-east-1/main.tf | 62 ++++------ 3 files changed, 143 insertions(+), 48 deletions(-) create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md b/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md index 92f42ec..2faa66c 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md @@ -2,24 +2,27 @@ ## Phase 1 0. create standard vpc with a private network to host ec2, -this uses a natgw that costs money https://aws.amazon.com/vpc/pricing/ so we will put the intances in public for now and use security groups to limit access. - 1. create minimal ec2 instance in machine_image - -terraform for ubuntu python uvicorn fastapi nginx systemd server with custom git modules - +terraform for ubuntu python uvicorn fastapi nginx systemd server with custom code pulled in via git 2. create minimal ec2 ami from instance in machine_image 3. create autoscaling_group of size 1 for image 4. create application load balancer 5. create dns_entry +6. packer ami for ubuntu python uvicorn fastapi nginx systemd server with custom git modules # todo +0. run_only to only run the server without installing everything 1. alb sticky sessions :send users back to server via sticky sessions or some id. 2. create cognito user pool for login -7. create work_queue -8. create lambda_workers on queue -9. create resource_launchers to create new resources. -10. packer ami for ubuntu python uvicorn fastapi nginx systemd server with custom git modules +4. create work_queue +5. create lambda_workers on queue +6. create resource_launchers to create new resources. +7. use fine grained roles +https://github.com/cloudposse/terraform-aws-ssm-iam-role.git +8. create user home directories for different agent +9. look at natgw alternatives +that costs money https://aws.amazon.com/vpc/pricing/ + diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf new file mode 100644 index 0000000..e57aca8 --- /dev/null +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf @@ -0,0 +1,108 @@ +variable tags {} +# data "aws_iam_policy_document" "assume_role" { +# statement { +# effect = "Allow" +# actions = ["sts:AssumeRole"] + +# principals { +# type = "Service" +# identifiers = ["ec2.amazonaws.com"] +# } +# } + +# statement { +# effect = "Allow" +# actions = ["sts:AssumeRole"] + +# principals { +# type = "AWS" +# identifiers = ["${var.assume_role_arns}"] +# } +# } +# } + +data "aws_iam_policy_document" "default" { + statement { + actions = ["ssm:DescribeParameters"] + resources = ["*"] + effect = "Allow" + } + +# statement { +# actions = ["${var.ssm_actions}"] +# resources = ["${formatlist("arn:aws:ssm:%s:%s:parameter/%s", var.region, var.account_id, var.ssm_parameters)}"] +# effect = "Allow" +# } + + # statement { + # actions = ["kms:Decrypt"] + # resources = ["${data.aws_kms_key.default.arn}"] + # effect = "Allow" + # } +} + +resource "aws_iam_policy" "default" { + name = "swarms-ssm" + description = "Allow SSM actions" + policy = data.aws_iam_policy_document.default.json +} + + +resource "aws_iam_role_policy_attachment" "default" { +# count = local.policy_only + role = join("", aws_iam_role.ssm.*.name) + policy_arn = join("", aws_iam_policy.default.*.arn) +} + +resource "aws_iam_role_policy_attachment" "SSM-role-policy-attach" { + role = join("", aws_iam_role.ssm.*.name) + policy_arn = data.aws_iam_policy.AmazonSSMManagedInstanceCore.arn +} + + +#### +# resource "aws_iam_role" "default" { +# # count = local.policy_only + +# name = "swarms-ssm" +# assume_role_policy = join("", data.aws_iam_policy_document.assume_role.*.json) +# description = "IAM Role with permissions to perform actions on SSM resources" +# max_session_duration = var.max_session_duration +# } +data "aws_iam_policy" "AmazonSSMManagedInstanceCore" { + arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" +} + +resource "aws_iam_role" "ssm" { + name = "ssm-swarms-role" + tags = var.tags + + assume_role_policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Action = "sts:AssumeRole", + Principal = { + Service = "ec2.amazonaws.com" + }, + Effect = "Allow", + Sid = "" + } + ] + }) +} + +resource "aws_iam_instance_profile" "ssm" { + name = "ssm-swarms-profile" + role = aws_iam_role.ssm.name + tags = var.tags +} + + +output ssm_profile_name { + value = aws_iam_instance_profile.ssm.name +} + +output ssm_profile_arn { + value = aws_iam_instance_profile.ssm.arn +} diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index 3c18c88..20fcd0c 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -69,29 +69,10 @@ variable "instance_types" { ] } -resource "aws_iam_role" "ssm" { - name = "ssm-${local.name}" - tags = var.tags - - assume_role_policy = jsonencode({ - Version = "2012-10-17", - Statement = [ - { - Action = "sts:AssumeRole", - Principal = { - Service = "ec2.amazonaws.com" - }, - Effect = "Allow", - Sid = "" - } - ] - }) -} -resource "aws_iam_instance_profile" "ssm" { - name = "ssm-${var.name}" - role = aws_iam_role.ssm.name - tags = var.tags +module "roles" { + source = "./components/roles" + tags = local.tags } module "lt_dynamic" { @@ -103,7 +84,8 @@ module "lt_dynamic" { ami_id = var.ami_id tags= local.tags source = "./components/launch_template" - iam_instance_profile_name = aws_iam_instance_profile.ssm.name + iam_instance_profile_name = module.roles.ssm_profile_name + #aws_iam_instance_profile.ssm.name } output security_group_id { @@ -136,7 +118,8 @@ module "asg_dynamic" { image_id = local.ami_id ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 for_each = toset(var.instance_types) - aws_iam_instance_profile_ssm_arn = aws_iam_instance_profile.ssm.arn + aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn + #iam_instance_profile_name = module.roles.ssm_profile_name source = "./components/autoscaling_group" # security_group_id = module.security.internal_security_group_id instance_type = each.key @@ -145,18 +128,19 @@ module "asg_dynamic" { target_group_arn = module.alb.alb_target_group_arn } -module "asg_dynamic_new_ami" { - # built with packer - tags = local.tags - vpc_id = local.vpc_id - image_id = local.new_ami_id - ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 - for_each = toset(var.instance_types) - aws_iam_instance_profile_ssm_arn = aws_iam_instance_profile.ssm.arn - source = "./components/autoscaling_group" -# security_group_id = module.security.internal_security_group_id - instance_type = each.key - name = "swarms-ami-${each.key}" - launch_template_id = module.lt_dynamic[each.key].launch_template_id - target_group_arn = module.alb.alb_target_group_arn -} +# module "asg_dynamic_new_ami" { +# # built with packer +# #count =0 +# tags = local.tags +# vpc_id = local.vpc_id +# image_id = local.new_ami_id +# ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 +# for_each = toset(var.instance_types) +# aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn +# source = "./components/autoscaling_group" +# # security_group_id = module.security.internal_security_group_id +# instance_type = each.key +# name = "swarms-ami-${each.key}" +# launch_template_id = module.lt_dynamic[each.key].launch_template_id +# target_group_arn = module.alb.alb_target_group_arn +# } From c38d39431778e6e66db09324cbfb94534f372dab Mon Sep 17 00:00:00 2001 From: mike dupont Date: Wed, 18 Dec 2024 13:17:37 -0500 Subject: [PATCH 044/158] new installer --- .../dev/us-east-1/components/Readme.md | 2 +- .../components/launch_template/main.tf | 3 +- .../dev/us-east-1/main.tf | 47 ++++++++++++------- 3 files changed, 34 insertions(+), 18 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md b/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md index 2faa66c..6d38230 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md @@ -25,4 +25,4 @@ https://github.com/cloudposse/terraform-aws-ssm-iam-role.git 8. create user home directories for different agent 9. look at natgw alternatives that costs money https://aws.amazon.com/vpc/pricing/ - +10. check in copy of swagger diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf index 09801f7..d73c20f 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf @@ -1,3 +1,4 @@ +variable install_script {} variable iam_instance_profile_name {} variable security_group_id {} variable name {} @@ -63,7 +64,7 @@ resource "aws_launch_template" "ec2_launch_template" { export BRANCH=feature/ec2 git stash git checkout --force $BRANCH - bash -x /opt/swarms/api/install.sh + bash -x ${var.install_script} EOF ) tags = var.tags diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index 20fcd0c..b2ce76f 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -86,6 +86,21 @@ module "lt_dynamic" { source = "./components/launch_template" iam_instance_profile_name = module.roles.ssm_profile_name #aws_iam_instance_profile.ssm.name + install_script = "/opt/swarms/api/install.sh" +} + +module "lt_dynamic_ami" { + vpc_id = local.vpc_id + for_each = toset(var.instance_types) + instance_type = each.key + name = "swarms-size-${each.key}" + security_group_id = module.security.internal_security_group_id + ami_id = var.ami_id + tags= local.tags + source = "./components/launch_template" + iam_instance_profile_name = module.roles.ssm_profile_name + #aws_iam_instance_profile.ssm.name + install_script = "/opt/swarms/api/just_run.sh" } output security_group_id { @@ -128,19 +143,19 @@ module "asg_dynamic" { target_group_arn = module.alb.alb_target_group_arn } -# module "asg_dynamic_new_ami" { -# # built with packer -# #count =0 -# tags = local.tags -# vpc_id = local.vpc_id -# image_id = local.new_ami_id -# ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 -# for_each = toset(var.instance_types) -# aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn -# source = "./components/autoscaling_group" -# # security_group_id = module.security.internal_security_group_id -# instance_type = each.key -# name = "swarms-ami-${each.key}" -# launch_template_id = module.lt_dynamic[each.key].launch_template_id -# target_group_arn = module.alb.alb_target_group_arn -# } +module "asg_dynamic_new_ami" { + # built with packer + #count =0 + tags = local.tags + vpc_id = local.vpc_id + image_id = local.new_ami_id + ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 + for_each = toset(var.instance_types) + aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn + source = "./components/autoscaling_group" +# security_group_id = module.security.internal_security_group_id + instance_type = each.key + name = "swarms-ami-${each.key}" + launch_template_id = module.lt_dynamic_ami[each.key].launch_template_id + target_group_arn = module.alb.alb_target_group_arn +} From 45f8b3886682e29875e784a49cc7a9535ace61c2 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Wed, 18 Dec 2024 14:22:25 -0500 Subject: [PATCH 045/158] update --- .../swarms-aws-agent-api/dev/us-east-1/components/Readme.md | 4 ++++ .../dev/us-east-1/components/launch_template/main.tf | 1 + environments/swarms-aws-agent-api/dev/us-east-1/main.tf | 4 ++-- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md b/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md index 6d38230..974840d 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md @@ -26,3 +26,7 @@ https://github.com/cloudposse/terraform-aws-ssm-iam-role.git 9. look at natgw alternatives that costs money https://aws.amazon.com/vpc/pricing/ 10. check in copy of swagger + +11. swarms router +12. fluid api +13. agent service discovery diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf index d73c20f..8a3a5b6 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf @@ -64,6 +64,7 @@ resource "aws_launch_template" "ec2_launch_template" { export BRANCH=feature/ec2 git stash git checkout --force $BRANCH + git pull # get the latest version bash -x ${var.install_script} EOF ) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index b2ce76f..7def66e 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -93,9 +93,9 @@ module "lt_dynamic_ami" { vpc_id = local.vpc_id for_each = toset(var.instance_types) instance_type = each.key - name = "swarms-size-${each.key}" + name = "swarms-ami-${each.key}" security_group_id = module.security.internal_security_group_id - ami_id = var.ami_id + ami_id = local.new_ami_id tags= local.tags source = "./components/launch_template" iam_instance_profile_name = module.roles.ssm_profile_name From 13621be1958536b23b438b2d6f4ab0e6b98b5c20 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Wed, 18 Dec 2024 14:42:30 -0500 Subject: [PATCH 046/158] remove the slow asg --- .../dev/us-east-1/main.tf | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index 7def66e..a9ebc0b 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -127,21 +127,22 @@ output alb { } -module "asg_dynamic" { - tags = local.tags - vpc_id = local.vpc_id - image_id = local.ami_id - ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 - for_each = toset(var.instance_types) - aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn - #iam_instance_profile_name = module.roles.ssm_profile_name - source = "./components/autoscaling_group" -# security_group_id = module.security.internal_security_group_id - instance_type = each.key - name = "swarms-size-${each.key}" - launch_template_id = module.lt_dynamic[each.key].launch_template_id - target_group_arn = module.alb.alb_target_group_arn -} +# this is the slow one, use the ami +# module "asg_dynamic" { +# tags = local.tags +# vpc_id = local.vpc_id +# image_id = local.ami_id +# ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 +# for_each = toset(var.instance_types) +# aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn +# #iam_instance_profile_name = module.roles.ssm_profile_name +# source = "./components/autoscaling_group" +# # security_group_id = module.security.internal_security_group_id +# instance_type = each.key +# name = "swarms-size-${each.key}" +# launch_template_id = module.lt_dynamic[each.key].launch_template_id +# target_group_arn = module.alb.alb_target_group_arn +# } module "asg_dynamic_new_ami" { # built with packer From 06a8b418ee708ba06a51866590bcee79105a995e Mon Sep 17 00:00:00 2001 From: mike dupont Date: Wed, 18 Dec 2024 19:50:30 -0500 Subject: [PATCH 047/158] simple autoscale policy --- .../dev/us-east-1/alb/main.tf | 0 .../dev/us-east-1/asg/main.tf | 0 .../components/autoscaling_group/main.tf | 62 +++++++++++++++++++ 3 files changed, 62 insertions(+) delete mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/alb/main.tf delete mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/asg/main.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/alb/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/alb/main.tf deleted file mode 100644 index e69de29..0000000 diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/asg/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/asg/main.tf deleted file mode 100644 index e69de29..0000000 diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf index e5e31ef..cfa945d 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf @@ -65,4 +65,66 @@ module "autoscaling" { } } + # Target scaling policy schedule based on average CPU load + scaling_policies = { + avg-cpu-policy-greater-than-50 = { + policy_type = "TargetTrackingScaling" + estimated_instance_warmup = 1200 + target_tracking_configuration = { + predefined_metric_specification = { + predefined_metric_type = "ASGAverageCPUUtilization" + } + target_value = 50.0 + } + }, + predictive-scaling = { + policy_type = "PredictiveScaling" + predictive_scaling_configuration = { + mode = "ForecastAndScale" + scheduling_buffer_time = 10 + max_capacity_breach_behavior = "IncreaseMaxCapacity" + max_capacity_buffer = 10 + metric_specification = { + target_value = 32 + predefined_scaling_metric_specification = { + predefined_metric_type = "ASGAverageCPUUtilization" + resource_label = "testLabel" + } + predefined_load_metric_specification = { + predefined_metric_type = "ASGTotalCPUUtilization" + resource_label = "testLabel" + } + } + } + } + # request-count-per-target = { + # policy_type = "TargetTrackingScaling" + # estimated_instance_warmup = 120 + # target_tracking_configuration = { + # predefined_metric_specification = { + # predefined_metric_type = "ALBRequestCountPerTarget" + # resource_label = "swarms1" + # #"${module.alb.arn_suffix}/${module.alb.target_groups["ex_asg"].arn_suffix}" + # } + # target_value = 800 + # } + # } + scale-out = { + name = "scale-out" + adjustment_type = "ExactCapacity" + policy_type = "StepScaling" + estimated_instance_warmup = 120 + step_adjustment = [ + { + scaling_adjustment = 1 + metric_interval_lower_bound = 0 + metric_interval_upper_bound = 10 + }, + { + scaling_adjustment = 2 + metric_interval_lower_bound = 10 + } + ] + } + } } From eedf4993b299342081c88f9f20affc5a229deb84 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Wed, 18 Dec 2024 20:56:54 -0500 Subject: [PATCH 048/158] swarmdeploy --- accounts/swarms/main.tf | 10 ++ .../components/autoscaling_group/main.tf | 130 ++++++++++++++ .../components/launch_template/main.tf | 80 +++++++++ .../swarms-deploy/dev/us-east-1/main.tf | 170 ++++++++++++++++++ environments/swarms-deploy/main.tf | 2 + 5 files changed, 392 insertions(+) create mode 100644 environments/swarms-deploy/dev/us-east-1/components/autoscaling_group/main.tf create mode 100644 environments/swarms-deploy/dev/us-east-1/components/launch_template/main.tf create mode 100644 environments/swarms-deploy/dev/us-east-1/main.tf create mode 100644 environments/swarms-deploy/main.tf diff --git a/accounts/swarms/main.tf b/accounts/swarms/main.tf index 2468b5b..66cff81 100644 --- a/accounts/swarms/main.tf +++ b/accounts/swarms/main.tf @@ -44,6 +44,16 @@ module "swarms_api" { } +module "swarmdeploy" { + source = "../../environments/swarms-deploy/dev/us-east-1" + domain = local.dns + ami_id = data.aws_ami.ami.id + name = "swarmdeploy" + tags = {project="swarmdeploy"} + vpc_id = "vpc-0b4cedd083227068d" + subnet_id = "subnet-04b3bdd4b0dc877f0" +} + output api { value = module.swarms_api } diff --git a/environments/swarms-deploy/dev/us-east-1/components/autoscaling_group/main.tf b/environments/swarms-deploy/dev/us-east-1/components/autoscaling_group/main.tf new file mode 100644 index 0000000..e82fa4a --- /dev/null +++ b/environments/swarms-deploy/dev/us-east-1/components/autoscaling_group/main.tf @@ -0,0 +1,130 @@ +variable aws_iam_instance_profile_ssm_arn {} +#variable target_group_arn{} +variable name {} +variable instance_type {} +variable launch_template_id {} +variable image_id {} +variable vpc_id {} +variable tags {} +variable ec2_subnet_id {} + +locals { + iam_instance_profile_arn = var.aws_iam_instance_profile_ssm_arn + instance_type = var.instance_type +} + +module "autoscaling" { + source = "terraform-aws-modules/autoscaling/aws" + version = "8.0.0" + name = var.name + + health_check_type = "EC2" + desired_capacity = 1 + max_size = 5 + min_size = 1 + + create_launch_template = false + update_default_version = true + + launch_template_id = var.launch_template_id + launch_template_version = "$Latest" + + vpc_zone_identifier = [var.ec2_subnet_id] + + instance_market_options = { + market_type = "spot" + } + network_interfaces = [{ + associate_public_ip_address=true + device_index = 0 + delete_on_termination = true + description = "interface1" +# security_groups = [var.security_group_id] + } + ] + instance_type = var.instance_type + image_id = var.image_id + + create_iam_instance_profile = true + iam_role_name = "ssm-${var.name}" + iam_role_path = "/ec2/" + iam_role_description = "SSM IAM role for swarms" + iam_role_tags = { + CustomIamRole = "Yes" + } + + iam_role_policies = { + AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" + } + + # # target_group_arn = + # traffic_source_attachments = { + # ex-alb = { + # traffic_source_identifier = var.target_group_arn + # traffic_source_type = "elbv2" # default + # } + # } + + # Target scaling policy schedule based on average CPU load + scaling_policies = { + avg-cpu-policy-greater-than-50 = { + policy_type = "TargetTrackingScaling" + estimated_instance_warmup = 1200 + target_tracking_configuration = { + predefined_metric_specification = { + predefined_metric_type = "ASGAverageCPUUtilization" + } + target_value = 50.0 + } + }, + predictive-scaling = { + policy_type = "PredictiveScaling" + predictive_scaling_configuration = { + mode = "ForecastAndScale" + scheduling_buffer_time = 10 + max_capacity_breach_behavior = "IncreaseMaxCapacity" + max_capacity_buffer = 10 + metric_specification = { + target_value = 32 + predefined_scaling_metric_specification = { + predefined_metric_type = "ASGAverageCPUUtilization" + resource_label = "testLabel" + } + predefined_load_metric_specification = { + predefined_metric_type = "ASGTotalCPUUtilization" + resource_label = "testLabel" + } + } + } + } + # request-count-per-target = { + # policy_type = "TargetTrackingScaling" + # estimated_instance_warmup = 120 + # target_tracking_configuration = { + # predefined_metric_specification = { + # predefined_metric_type = "ALBRequestCountPerTarget" + # resource_label = "swarms1" + # #"${module.alb.arn_suffix}/${module.alb.target_groups["ex_asg"].arn_suffix}" + # } + # target_value = 800 + # } + # } + scale-out = { + name = "scale-out" + adjustment_type = "ExactCapacity" + policy_type = "StepScaling" + estimated_instance_warmup = 120 + step_adjustment = [ + { + scaling_adjustment = 1 + metric_interval_lower_bound = 0 + metric_interval_upper_bound = 10 + }, + { + scaling_adjustment = 2 + metric_interval_lower_bound = 10 + } + ] + } + } +} diff --git a/environments/swarms-deploy/dev/us-east-1/components/launch_template/main.tf b/environments/swarms-deploy/dev/us-east-1/components/launch_template/main.tf new file mode 100644 index 0000000..ff953d8 --- /dev/null +++ b/environments/swarms-deploy/dev/us-east-1/components/launch_template/main.tf @@ -0,0 +1,80 @@ +variable install_script {} +variable iam_instance_profile_name {} +variable security_group_id {} +variable name {} +variable vpc_id {} +variable ami_id {} +variable tags {} +variable key_name { + default = "mdupont-deployer-key" +} + +# dont use this +variable instance_type {} + +locals { + tags = { + project="swarms" + instance_type = var.instance_type + name = var.name + } +} +resource "aws_launch_template" "ec2_launch_template" { + name_prefix = "${var.name}-launch-template-" + image_id = var.ami_id + key_name = var.key_name + instance_type = var.instance_type + network_interfaces { + associate_public_ip_address = true + delete_on_termination = true + security_groups = [var.security_group_id] + } + + iam_instance_profile { + # iam_instance_profile_arn = aws_iam_instance_profile.ssm.arn + name = var.iam_instance_profile_name #aws_iam_instance_profile.ec2_instance_profile.name + } + lifecycle { + create_before_destroy = true + } + block_device_mappings { + device_name = "/dev/sda1" + ebs { + volume_size = 30 + volume_type = "gp3" + encrypted = true + } + } + + user_data = base64encode(<<-EOF + #!/bin/bash + export HOME=/root + apt update + apt-get install -y ec2-instance-connect git virtualenv + snap install amazon-ssm-agent --classic || echo oops1 + snap start amazon-ssm-agent || echo oops2 + apt-get install -y --no-install-recommends ca-certificates=20230311 curl=7.88.1-10+deb12u7 | echo oops + curl -O "https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/$(dpkg --print-architecture)/latest/amazon-cloudwatch-agent.deb" + dpkg -i -E amazon-cloudwatch-agent.deb + + if [ ! -d "/opt/swarms/" ]; then + git clone https://github.com/jmikedupont2/SwarmDeploy "/opt/swarms/" + fi + cd "/opt/swarms/" || exit 1 + export BRANCH=main + git stash + git checkout --force $BRANCH + git pull # get the latest version + bash -x ${var.install_script} + EOF + ) + tags = var.tags +} + + +output "lt" { + value = resource.aws_launch_template.ec2_launch_template +} +output "launch_template_id" { + value = resource.aws_launch_template.ec2_launch_template.id +} diff --git a/environments/swarms-deploy/dev/us-east-1/main.tf b/environments/swarms-deploy/dev/us-east-1/main.tf new file mode 100644 index 0000000..e2d454f --- /dev/null +++ b/environments/swarms-deploy/dev/us-east-1/main.tf @@ -0,0 +1,170 @@ +variable vpc_id {} +variable subnet_id {} +locals { + # instance_type = "t3.large" + # instance_type = "t3.medium" + ami_name = "ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*" + name = "swarmdeploy" + region = "us-east-2" + domain = var.domain + tags = { + project="swarmdeploy" + } +} +variable domain {} +variable ami_id {} +variable tags {} +variable name {} + + +locals { + ami_id = var.ami_id + #new_ami_id = "ami-08093b6770af41b14" # environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/Readme.md +} + +# SLOW +locals { + root = "../../../swarms-aws-agent-api/dev/us-east-1/components/" +} +#module "vpc" { +# source = "${local.root}/vpc" +#} + +data "aws_vpc" "vpc" { + id = var.vpc_id +} + +locals { +# ec2_public_subnet_id_1 = module.vpc.ec2_public_subnet_id_1 +# ec2_public_subnet_id_2 = module.vpc.ec2_public_subnet_id_2 + #vpc_id = module.vpc.vpc_id + vpc_id = var.vpc_id +} + +module "security" { + source = "${local.root}/security" + vpc_id = local.vpc_id + tags = local.tags + name = local.name +} + +module "kp" { + source = "${local.root}/keypairs" +} + +# module "lt" { +# instance_type = local.instance_type +# security_group_id = module.security.security_group_id +# source = "./components/launch_template" +# } + +# module "asg" { +# source = "./components/autoscaling_group" +# name="swarms" +# security_group_id = module.security.security_group_id +# instance_type = local.instance_type +# launch_template_id = module.lt.launch_template_id +# } + +variable "instance_types" { + type = list(string) + default = [ + # "t4g.nano", "t3a.nano", "t3.nano", "t2.nano", + # "t4g.micro", "t3a.micro", "t3.micro", "t2.micro", "t1.micro", + #"t4g.small", "t3a.small", + #"t3.small", + #"t2.small", not working + # "t2.medium" # + "t3.medium" + ] +} + +module "roles" { + source = "${local.root}/roles" + + tags = local.tags +} + +module "lt_dynamic" { + vpc_id = local.vpc_id + for_each = toset(var.instance_types) + instance_type = each.key + name = "swarms-size-${each.key}" + security_group_id = module.security.internal_security_group_id + ami_id = var.ami_id + tags= local.tags + source = "./components/launch_template" + iam_instance_profile_name = module.roles.ssm_profile_name + #aws_iam_instance_profile.ssm.name + install_script = "/opt/swarms/api/install.sh" +} + +# module "lt_dynamic_ami" { +# vpc_id = local.vpc_id +# for_each = toset(var.instance_types) +# instance_type = each.key +# name = "swarms-ami-${each.key}" +# security_group_id = module.security.internal_security_group_id +# ami_id = local.new_ami_id +# tags= local.tags +# source = "./components/launch_template" +# iam_instance_profile_name = module.roles.ssm_profile_name +# #aws_iam_instance_profile.ssm.name +# install_script = "/opt/swarms/api/just_run.sh" +# } + +output security_group_id { + value = module.security.security_group_id +} + +#output vpc { +# value = module.vpc +#} + +# module "alb" { +# source = "${local.root}/application_load_balancer" +# domain_name = local.domain +# security_group_id = module.security.security_group_id # allowed to talk to internal +# public_subnets = [ +# local.ec2_public_subnet_id_1, +# local.ec2_public_subnet_id_2 ] +# vpc_id = local.vpc_id +# name = local.name +# } + +# output alb { +# value = module.alb +# } + +module "asg_dynamic" { + tags = local.tags + vpc_id = local.vpc_id + image_id = local.ami_id + ec2_subnet_id = var.subnet_id + for_each = toset(var.instance_types) + aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn + #iam_instance_profile_name = module.roles.ssm_profile_name + source = "./components/autoscaling_group" + # security_group_id = module.security.internal_security_group_id + instance_type = each.key + name = "swarmdeploy-${each.key}" + launch_template_id = module.lt_dynamic[each.key].launch_template_id +# target_group_arn = module.alb.alb_target_group_arn +} + +# module "asg_dynamic_new_ami" { +# # built with packer +# #count =0 +# tags = local.tags +# vpc_id = local.vpc_id +# image_id = local.new_ami_id +# ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 +# for_each = toset(var.instance_types) +# aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn +# source = "./components/autoscaling_group" +# # security_group_id = module.security.internal_security_group_id +# instance_type = each.key +# name = "swarms-ami-${each.key}" +# launch_template_id = module.lt_dynamic_ami[each.key].launch_template_id +# target_group_arn = module.alb.alb_target_group_arn +# } diff --git a/environments/swarms-deploy/main.tf b/environments/swarms-deploy/main.tf new file mode 100644 index 0000000..8fa2136 --- /dev/null +++ b/environments/swarms-deploy/main.tf @@ -0,0 +1,2 @@ +# + From cac4c79e2805f93f4dbdaeb94dab5a23d5e8cce0 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Wed, 18 Dec 2024 21:30:30 -0500 Subject: [PATCH 049/158] now just hacking this for now --- accounts/swarms/main.tf | 2 ++ .../swarms-deploy/dev/us-east-1/main.tf | 22 ++++++++++--------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/accounts/swarms/main.tf b/accounts/swarms/main.tf index 66cff81..60e68bb 100644 --- a/accounts/swarms/main.tf +++ b/accounts/swarms/main.tf @@ -52,6 +52,8 @@ module "swarmdeploy" { tags = {project="swarmdeploy"} vpc_id = "vpc-0b4cedd083227068d" subnet_id = "subnet-04b3bdd4b0dc877f0" + ssm_profile_arn= "arn:aws:iam::916723593639:instance-profile/ssm-swarms-profile" + ssm_profile_name = "ssm-swarms-profile" } output api { diff --git a/environments/swarms-deploy/dev/us-east-1/main.tf b/environments/swarms-deploy/dev/us-east-1/main.tf index e2d454f..60294c9 100644 --- a/environments/swarms-deploy/dev/us-east-1/main.tf +++ b/environments/swarms-deploy/dev/us-east-1/main.tf @@ -1,3 +1,5 @@ +variable "ssm_profile_arn" {} +variable "ssm_profile_name" {} variable vpc_id {} variable subnet_id {} locals { @@ -48,9 +50,9 @@ module "security" { name = local.name } -module "kp" { - source = "${local.root}/keypairs" -} +#module "kp" { +# source = "${local.root}/keypairs" +#} # module "lt" { # instance_type = local.instance_type @@ -79,11 +81,11 @@ variable "instance_types" { ] } -module "roles" { - source = "${local.root}/roles" - - tags = local.tags -} +#module "roles" { +# source = "${local.root}/roles" +# +# tags = local.tags +#} module "lt_dynamic" { vpc_id = local.vpc_id @@ -94,7 +96,7 @@ module "lt_dynamic" { ami_id = var.ami_id tags= local.tags source = "./components/launch_template" - iam_instance_profile_name = module.roles.ssm_profile_name + iam_instance_profile_name = var.ssm_profile_name #aws_iam_instance_profile.ssm.name install_script = "/opt/swarms/api/install.sh" } @@ -142,7 +144,7 @@ module "asg_dynamic" { image_id = local.ami_id ec2_subnet_id = var.subnet_id for_each = toset(var.instance_types) - aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn + aws_iam_instance_profile_ssm_arn = var.ssm_profile_arn #iam_instance_profile_name = module.roles.ssm_profile_name source = "./components/autoscaling_group" # security_group_id = module.security.internal_security_group_id From 554f9010340e3921a69574c71908210f1c951067 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Wed, 18 Dec 2024 21:46:26 -0500 Subject: [PATCH 050/158] bugfix: use the install in the root dir --- environments/swarms-deploy/dev/us-east-1/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environments/swarms-deploy/dev/us-east-1/main.tf b/environments/swarms-deploy/dev/us-east-1/main.tf index 60294c9..6e69834 100644 --- a/environments/swarms-deploy/dev/us-east-1/main.tf +++ b/environments/swarms-deploy/dev/us-east-1/main.tf @@ -98,7 +98,7 @@ module "lt_dynamic" { source = "./components/launch_template" iam_instance_profile_name = var.ssm_profile_name #aws_iam_instance_profile.ssm.name - install_script = "/opt/swarms/api/install.sh" + install_script = "/opt/swarms/install.sh" } # module "lt_dynamic_ami" { From f708fadef417093a0b2b3f6ba49910285980aaae Mon Sep 17 00:00:00 2001 From: mike dupont Date: Thu, 19 Dec 2024 08:08:49 -0500 Subject: [PATCH 051/158] proof of godaddy working --- accounts/swarms/Readme.md | 11 +++++++---- accounts/swarms/main.tf | 7 ++++++- .../application_load_balancer/https/main.tf | 4 ++-- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/accounts/swarms/Readme.md b/accounts/swarms/Readme.md index fc96012..bd08ebe 100644 --- a/accounts/swarms/Readme.md +++ b/accounts/swarms/Readme.md @@ -46,10 +46,13 @@ data "aws_route53_zone" "primary" { ``` so we need 4 records -1. NS api.swarms.ai -> "ns-864.awsdns-44.net" -2. NS api.swarms.ai -> "ns-1595.awsdns-07.co.uk" -3. NS api.swarms.ai -> "ns-1331.awsdns-38.org" -4. NS api.swarms.ai -> "ns-463.awsdns-57.com" +1. NS api -> ns-864.awsdns-44.net. +2. NS api -> ns-1595.awsdns-07.co.uk. +3. NS api -> ns-1331.awsdns-38.org. +4. NS api -> ns-463.awsdns-57.com. + + + see forum https://repost.aws/questions/QULXL3STgjQtefiJ_q0BixXA/configure-godaddy-subdomain-to-route53 diff --git a/accounts/swarms/main.tf b/accounts/swarms/main.tf index 60e68bb..4a8e0ea 100644 --- a/accounts/swarms/main.tf +++ b/accounts/swarms/main.tf @@ -1,11 +1,16 @@ locals { #ami_name = "ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*" ami_name = "ubuntu-minimal/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-minimal-*" - dns = "api.swarms.ai" + # dns = "api.swarms.ai" + dns = "api.arianakrasniqi.com" account = "916723593639" region = "us-east-2" } +#resource aws_route53_zone test{ +# name = local.dns +#} + provider aws { region = "us-east-2" profile = "swarms" diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf index 3be5b7c..d75f4a5 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf @@ -9,7 +9,7 @@ module "acm" { version = "~> 4.0" domain_name = var.domain_name zone_id = var.zone_id - wait_for_validation = false + #wait_for_validation = false subject_alternative_names = [ "*.${var.domain_name}" ] @@ -17,7 +17,7 @@ module "acm" { ## now we just lift the listener code resource "aws_lb_listener" "this" { - count = 0 +# count = 0 port = 443 protocol = "HTTPS" ssl_policy = "ELBSecurityPolicy-TLS13-1-2-Res-2021-06" From b6f89613729103ba14a2103400982aeb6df67bae Mon Sep 17 00:00:00 2001 From: mike dupont Date: Thu, 19 Dec 2024 10:40:00 -0500 Subject: [PATCH 052/158] secrets loading --- accounts/swarms/.terraform.lock.hcl | 45 +++++++++++++++++++++++++++ accounts/swarms/Readme.md | 7 ++++- accounts/swarms/github/main.tf | 29 +++++++++++++++++ accounts/swarms/main.tf | 48 ++++++++++++++++++++++++----- 4 files changed, 120 insertions(+), 9 deletions(-) create mode 100644 accounts/swarms/github/main.tf diff --git a/accounts/swarms/.terraform.lock.hcl b/accounts/swarms/.terraform.lock.hcl index f639596..b7e11c0 100644 --- a/accounts/swarms/.terraform.lock.hcl +++ b/accounts/swarms/.terraform.lock.hcl @@ -18,3 +18,48 @@ provider "registry.opentofu.org/hashicorp/aws" { "zh:e76cd202b03749f3082b0cbe849fd2e731cf3f9a6aa994d2d629602c3aede36c", ] } + +provider "registry.opentofu.org/hashicorp/github" { + version = "6.4.0" + hashes = [ + "h1:sJvuRMYWJ/ykZXTuoCuocHvx06hTwDVrXVVXq1814bw=", + "zh:00f431c2a2510efcb1115442dda5e90815bcb16e1a3301679ade0139fa963d3b", + "zh:12a862f4317b3cb65682c1b687650cd91eeee99e63774bdcfa8bcfc64bad097b", + "zh:226d5e09ff27f94cb9336089181d26f85cb30219b863a579597f2e107f37de49", + "zh:402ecaa5add568a52ee01d816810f3b90f693be35c680fcdc9b6284bf55326f1", + "zh:60e3bdd9fbefb3c1d790bc08889c1dc0e83636b82284faaa709411aa4f96bb9f", + "zh:625099eeff2f8aaecd22a24a451b326828435c8f9de86f2e5e99872e7b467fa7", + "zh:79e8b665421009df2260f50e10da1f7a7863b557ece96e2b07dfd2fad1e86fcd", + "zh:98e471fefc93dcfedeec750c694110db7d3331dc3a256191d30b9d2f70d12157", + "zh:a17702765e1fa92d1c288ddfd97075819ad61b344b341be7e09c554c841a6d9e", + "zh:ca72ccf40624ae26bf4660d8dd84a51638f0a1e78d5f19fdfaafaef97f838af6", + "zh:d009ab5527d45c44c424d26cd2eb51a5a6a6448f3fb1023b675789588cc08d64", + "zh:e5811be1e942a75b14dfcd3e03523d8df60cfbde0d7e24d75e78480a02a58949", + "zh:e6008ad28225ad6996b06bcd7f3070863329df406a56754e7fb9c31d6301ace4", + "zh:f1d93f56ea4f87183a5de4780704907605851d95a2d285a9ec755bf784c5569c", + "zh:fbd1fee2c9df3aa19cf8851ce134dea6e45ea01cb85695c1726670c285797e25", + ] +} + +provider "registry.opentofu.org/integrations/github" { + version = "6.4.0" + constraints = "6.4.0" + hashes = [ + "h1:sJvuRMYWJ/ykZXTuoCuocHvx06hTwDVrXVVXq1814bw=", + "zh:00f431c2a2510efcb1115442dda5e90815bcb16e1a3301679ade0139fa963d3b", + "zh:12a862f4317b3cb65682c1b687650cd91eeee99e63774bdcfa8bcfc64bad097b", + "zh:226d5e09ff27f94cb9336089181d26f85cb30219b863a579597f2e107f37de49", + "zh:402ecaa5add568a52ee01d816810f3b90f693be35c680fcdc9b6284bf55326f1", + "zh:60e3bdd9fbefb3c1d790bc08889c1dc0e83636b82284faaa709411aa4f96bb9f", + "zh:625099eeff2f8aaecd22a24a451b326828435c8f9de86f2e5e99872e7b467fa7", + "zh:79e8b665421009df2260f50e10da1f7a7863b557ece96e2b07dfd2fad1e86fcd", + "zh:98e471fefc93dcfedeec750c694110db7d3331dc3a256191d30b9d2f70d12157", + "zh:a17702765e1fa92d1c288ddfd97075819ad61b344b341be7e09c554c841a6d9e", + "zh:ca72ccf40624ae26bf4660d8dd84a51638f0a1e78d5f19fdfaafaef97f838af6", + "zh:d009ab5527d45c44c424d26cd2eb51a5a6a6448f3fb1023b675789588cc08d64", + "zh:e5811be1e942a75b14dfcd3e03523d8df60cfbde0d7e24d75e78480a02a58949", + "zh:e6008ad28225ad6996b06bcd7f3070863329df406a56754e7fb9c31d6301ace4", + "zh:f1d93f56ea4f87183a5de4780704907605851d95a2d285a9ec755bf784c5569c", + "zh:fbd1fee2c9df3aa19cf8851ce134dea6e45ea01cb85695c1726670c285797e25", + ] +} diff --git a/accounts/swarms/Readme.md b/accounts/swarms/Readme.md index bd08ebe..5146909 100644 --- a/accounts/swarms/Readme.md +++ b/accounts/swarms/Readme.md @@ -160,5 +160,10 @@ swarms.ai. 3600 IN TXT "google-site-verification=VlUvNHJo0LQzJzm7SIwMzYLB7-Rexx4 ``` # tofu apply +The new apply -`tofu apply` +``` +tofu plan -var "github_token=$(gh auth token)" +tofu apply -var "github_token=$(gh auth token)" + +``` diff --git a/accounts/swarms/github/main.tf b/accounts/swarms/github/main.tf new file mode 100644 index 0000000..e762631 --- /dev/null +++ b/accounts/swarms/github/main.tf @@ -0,0 +1,29 @@ + +variable aws_region {} +variable aws_account_id {} + +terraform { + required_providers { + github = { + source = "integrations/github" + version = "6.4.0" + } + } +} + +#data "github_actions_public_key" "public_key" { +# repository = "jmikedupont2/terraform-aws-oidc-github" +#} + + +resource "github_actions_secret" "region" { + repository = "terraform-aws-oidc-github" + secret_name = "AWS_REGION" + plaintext_value = var.aws_region +} + +resource "github_actions_secret" "account" { + repository = "terraform-aws-oidc-github" + secret_name = "AWS_ACCOUNT_ID" + plaintext_value = var.aws_account_id +} diff --git a/accounts/swarms/main.tf b/accounts/swarms/main.tf index 4a8e0ea..0df95d0 100644 --- a/accounts/swarms/main.tf +++ b/accounts/swarms/main.tf @@ -7,6 +7,22 @@ locals { region = "us-east-2" } + +variable "owner" { + description = "GitHub owner used to configure the provider" + default = "jmikedupont2" +} + +variable "github_token" { + description = "GitHub access token used to configure the provider" + type = string +} + +provider "github" { + owner = var.owner + token = var.github_token +} + #resource aws_route53_zone test{ # name = local.dns #} @@ -32,17 +48,20 @@ output region { } #SLOW - data "aws_ami" "ami" { - most_recent = true - name_regex = "^${local.ami_name}" - } +# data "aws_ami" "ami" { +# most_recent = true +# name_regex = "^${local.ami_name}" +# } +locals { +ami_id = "ami-0325b9a2dfb474b2d" +} module "swarms_api" { source = "../../environments/swarms-aws-agent-api/dev/us-east-1" domain = local.dns - ami_id = data.aws_ami.ami.id - #"ami-0ad5d6c7069ce56ac" - #ami_id = "ami-0ad5d6c7069ce56ac" + #ami_id = data.aws_ami.ami.id + ami_id = local.ami_id + name = "swarms" tags = {project="swarms"} @@ -52,7 +71,8 @@ module "swarms_api" { module "swarmdeploy" { source = "../../environments/swarms-deploy/dev/us-east-1" domain = local.dns - ami_id = data.aws_ami.ami.id + #ami_id = data.aws_ami.ami.id + ami_id = local.ami_id name = "swarmdeploy" tags = {project="swarmdeploy"} vpc_id = "vpc-0b4cedd083227068d" @@ -64,3 +84,15 @@ module "swarmdeploy" { output api { value = module.swarms_api } + + + +# setup the github tokens +module github { + source = "./github" + aws_account_id = local.account + aws_region = local.region +# github_token = var.github_token + +} + From e8aa0000148b2b4a5d09038349db4a1b0ab38a75 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Thu, 19 Dec 2024 12:33:27 -0500 Subject: [PATCH 053/158] version 1 --- environments/call-swarms/deploy.yaml | 38 +++++++++++++++++++ environments/call-swarms/main.tf | 11 ++++++ .../dev/us-east-1/components/Readme.md | 13 ++++++- 3 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 environments/call-swarms/deploy.yaml create mode 100644 environments/call-swarms/main.tf diff --git a/environments/call-swarms/deploy.yaml b/environments/call-swarms/deploy.yaml new file mode 100644 index 0000000..d72e610 --- /dev/null +++ b/environments/call-swarms/deploy.yaml @@ -0,0 +1,38 @@ +--- +description: | + ### Document name - Deploy Swarms + ## What does this document do? + It runs swarms commands + ## Input Parameters + * git_url: the source + * git_version: The version + * git_name: The remote name, only set once + ## Output Parameters + None. +schemaVersion: '2.2' +parameters: + GitUrl: + type: String + description: "(Required) The url." + allowedPattern: "^.+$" + GitName: + type: String + description: "(Required) The url." + allowedPattern: "^.+$" + GitVersion: + type: String + description: "(Required) The version." + allowedPattern: "^.+$" + +mainSteps: + - action: aws:runShellScript + name: Deploy + description:Run the script + inputs: + runCommand: + - | + #!/bin/bash + cd /opt/swarms/ + sudo git remote add {{GitName}} {{GitUrl}} || echo oops + sudo git checkout --force {{GitVersion}} + sudo bash -x /opt/swarms/api/install.sh diff --git a/environments/call-swarms/main.tf b/environments/call-swarms/main.tf new file mode 100644 index 0000000..f3ff1c3 --- /dev/null +++ b/environments/call-swarms/main.tf @@ -0,0 +1,11 @@ +# from +# https://github.com/shaikis/terraform-aws-ssm-document.git + +resource "aws_ssm_document" "deploy" { + name = "deploy" + document_format = "YAML" + document_type = "Command" + content = file("deploy.yaml") + tags = {env = "test"} +} + diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md b/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md index 974840d..60df5bc 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md @@ -10,10 +10,19 @@ terraform for ubuntu python uvicorn fastapi nginx systemd server with custom co 4. create application load balancer 5. create dns_entry 6. packer ami for ubuntu python uvicorn fastapi nginx systemd server with custom git modules - +7. oidc connect from github to aws # todo +0. create role for developers to run ssm job +to deploy new service. +1. call from github action. +2. have least privlege +3. allow for calling ssm with information about context +4. route to server +5. retrieve logs +6. provision other services to be used by the agents + 0. run_only to only run the server without installing everything 1. alb sticky sessions :send users back to server via sticky sessions or some id. 2. create cognito user pool for login @@ -30,3 +39,5 @@ that costs money https://aws.amazon.com/vpc/pricing/ 11. swarms router 12. fluid api 13. agent service discovery +14. setup ticketing interface +15. From aa37da0c66af88b8ad27d4ea7fb11d9a5a1df646 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Thu, 19 Dec 2024 12:40:15 -0500 Subject: [PATCH 054/158] fix the yaml --- accounts/swarms/main.tf | 5 +++++ environments/call-swarms/deploy.yaml | 23 +++++++++++------------ environments/call-swarms/main.tf | 2 +- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/accounts/swarms/main.tf b/accounts/swarms/main.tf index 0df95d0..1c5ed3f 100644 --- a/accounts/swarms/main.tf +++ b/accounts/swarms/main.tf @@ -96,3 +96,8 @@ module github { } + +# now create the ssm document +module call_swarms { + source = "../../environments/call-swarms" +} diff --git a/environments/call-swarms/deploy.yaml b/environments/call-swarms/deploy.yaml index d72e610..43b32f9 100644 --- a/environments/call-swarms/deploy.yaml +++ b/environments/call-swarms/deploy.yaml @@ -1,6 +1,6 @@ --- description: | - ### Document name - Deploy Swarms + ### Document name - Deploy Swarms ## What does this document do? It runs swarms commands ## Input Parameters @@ -25,14 +25,13 @@ parameters: allowedPattern: "^.+$" mainSteps: - - action: aws:runShellScript - name: Deploy - description:Run the script - inputs: - runCommand: - - | - #!/bin/bash - cd /opt/swarms/ - sudo git remote add {{GitName}} {{GitUrl}} || echo oops - sudo git checkout --force {{GitVersion}} - sudo bash -x /opt/swarms/api/install.sh +- action: aws:runShellScript + name: Deploy + inputs: + runCommand: + - | + #!/bin/bash + cd /opt/swarms/ + sudo git remote add {{GitName}} {{GitUrl}} || echo oops + sudo git checkout --force {{GitVersion}} + sudo bash -x /opt/swarms/api/install.sh diff --git a/environments/call-swarms/main.tf b/environments/call-swarms/main.tf index f3ff1c3..6e8b358 100644 --- a/environments/call-swarms/main.tf +++ b/environments/call-swarms/main.tf @@ -5,7 +5,7 @@ resource "aws_ssm_document" "deploy" { name = "deploy" document_format = "YAML" document_type = "Command" - content = file("deploy.yaml") + content = file("../../environments/call-swarms/deploy.yaml") tags = {env = "test"} } From cc6d1e854c81f37541e6af6c0a2071bff396ff61 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Thu, 19 Dec 2024 12:52:51 -0500 Subject: [PATCH 055/158] call swarms --- .github/workflows/call-swarms.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/workflows/call-swarms.yml diff --git a/.github/workflows/call-swarms.yml b/.github/workflows/call-swarms.yml new file mode 100644 index 0000000..fb2a898 --- /dev/null +++ b/.github/workflows/call-swarms.yml @@ -0,0 +1,19 @@ +name: CallSwarms +on: + workflow_dispatch: +jobs: + caller-identity: + name: Return the IAM user + permissions: + contents: read + id-token: write + runs-on: ubuntu-latest + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ secrets.AWS_REGION }} + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/github + - run: | + aws sts get-caller-identity + aws ssm send-command --document-name "deploy" --document-version "1" --targets '[{"Key":"InstanceIds","Values":["i-060238b47044e8dd3"]}]' --parameters '{"GitUrl":["https://github.com/kyegomez/swarms"],"GitName":["kye"],"GitVersion":["master"]}' --timeout-seconds 600 --max-concurrency "50" --max-errors "0" --region us-east-2 From 57d111ebca26caeffe9733d0e9f96eeb839ee5c2 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Thu, 19 Dec 2024 15:13:40 -0500 Subject: [PATCH 056/158] notes from todays meeting --- README.md | 414 +++++++++++++++++++++++++++++++++ accounts/swarms/github/main.tf | 13 +- accounts/swarms/main.tf | 11 +- 3 files changed, 427 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index afa820f..6206132 100644 --- a/README.md +++ b/README.md @@ -15,3 +15,417 @@ terraform-template/ # Root directory of the Terraform template │ ├── swarms ``` + +``` +help me create a c4/semantic web/mermaid plantuml deployment diagram +for a self service cognito enabled, smart agent server hosting, +with many different deployment options ranging from t2-small to any larger size. +using terraform, aws, ec2, alb,diagram +accounts/swarms + -> environments/swarms-aws-agent-api/dev/us-east-1 + godaddy -> ns api.swarms.ai -> + route53 -> cname ->application load balancer + -> autoscaling group -> launch template -> ami + AMI : components/machine_image/ubuntu-fastapi.pkr.hcl + + swarms -> ec2, user data + github clone branch run api/install.sh, swarms user, app/boot.sh as swarms user (virtual env) + instance profile (our server, customer server) -> + todo :read access to specific ssm secrets -> openai key + servers tagged as user X can access secrets tagged Y. + outgoing ip address : (can talk to other servers) (allow certain ip ranges) + , nginx, systemd (isolate user process, write access to home), uvicorn -> swarms 4 processes fast api. + + githubactions -> oidc connection -> aws -> ssm document -> update your part of the server. + githubactions -> oidc connection -> aws -> ssm document -> terraform -> create new clusters + +shared hosted aws account : + profiles for each customer, + tags for customer resources + which api calls they can use + tags for cost tracking + custom vpc, subsets network resources. + + on servers, the agents will have own aws role/profiles (like lamda) + either each customer/agent gets own unix user, homedir, storage, email , backup. + containerization, k8s, docker. + + +Here is a concise PlantUML use case diagram based on your specifications: + +```plantuml +@startuml + +!define RECTANGLE class + +RECTANGLE "Self-Service Cognito-Enabled Smart Agent Server" { + (User) +} + +rectangle "Accounts/Swarm" { + usecase "Create Environments" as U1 + usecase "Deploy EC2 Instances" as U2 + usecase "Manage Load Balancer" as U3 + usecase "Configure Auto-Scaling" as U4 + usecase "Provision AMI" as U5 + usecase "Access SSM Secrets" as U6 + usecase "Update Server Configuration" as U7 + usecase "Track Customer API Calls" as U8 + usecase "Provision Shared Resources" as U9 +} + +User --> U1 : "Creates / Manages" +User --> U2 : "Deploys Instances" +User --> U3 : "Configures Load Balancer" +User --> U4 : "Sets Auto-Scaling Parameters" +User --> U5 : "Uses AMI for Deployment" +User --> U6 : "Requests Secrets for Access" +User --> U7 : "Updates Configuration" +User --> U8 : "Monitors API Usage" +User --> U9 : "Shares Resources among Customers" + +U1 --> "Route53" : "Configures CNAME" +U2 --> "EC2" : "Launches Instances" +U3 --> "ALB" : "Routes Traffic" +U4 --> "AutoScaling Group" : "Manages Scalability" +U5 --> "AMI" : "Provides Machine Image" +U7 --> "GitHub Actions" : "Automates Deploy" +U6 --> "Instance Profile" : "Manages Permissions" +U8 --> "Tags" : "Tracks Costs" + +note right of U5 : Component: \ncomponents/machine_image/ubuntu-fastapi.pkr.hcl +note right of U8 : Custom VPC \n\n Customer Profiles + +@enduml +``` + +### Explanation: +- This diagram represents the primary use cases associated with your smart agent server project. +- Each use case such as creating environments, deploying EC2 instances, and accessing SSM secrets represents functionalities your system supports. +- Annotations explain the components (like AMI) and setups (like custom VPC) tied to specific activities. + + +```plantuml +@startuml + +!define RECTANGLE(x) rectangle x as x +!define DIAMOND(x) diamond x as x + +package "Accounts/Swarm" { + RECTANGLE(swarms-aws-agent-api) { + package "Environments" { + RECTANGLE(dev) { + RECTANGLE(us_east_1) { + RECTANGLE(Godaddy) + RECTANGLE(Route53) + RECTANGLE(ALB) as alb + RECTANGLE(AutoScalingGroup) as asg + RECTANGLE(LaunchTemplate) as lt + RECTANGLE(AMI) as ami + } + } + } + } + Godaddy -> Route53: ns api.swarms.ai + Route53 -> alb: CNAME + alb -> asg + asg -> lt + lt -> ami: "components/machine_image/ubuntu-fastapi.pkr.hcl" + + package "EC2 Instance" { + RECTANGLE(EC2) { + RECTANGLE(UserData) + RECTANGLE(InstanceProfile) + RECTANGLE(Nginx) + RECTANGLE(Systemd) + RECTANGLE(Uvicorn) + RECTANGLE(FastAPI) + + EC2 -> UserData: github clone branch run + EC2 -> InstanceProfile: read access to SSM secrets + EC2 -> Nginx: isolates user process + EC2 -> Systemd: manage services + EC2 -> Uvicorn: "4 instances of FastAPI" + } + } + + UserData -> EC2: "run api/install.sh" + InstanceProfile -> EC2: "access tagged secrets" + Nginx -> Systemd: "manage processes" +} + +package "GitHub Actions" { + RECTANGLE(OIDC) { + RECTANGLE(SSM_Document) + RECTANGLE(Terraform) + } + OIDC -> AWS + AWS -> SSM_Document: "update server" + OIDC -> AWS: "create new clusters" + AWS -> Terraform +} + +package "Shared Hosted AWS Account" { + RECTANGLE(Profiles) { + RECTANGLE(Tags) + RECTANGLE(VPC) + } + Profiles -> Tags: "customer resources" + Profiles -> VPC: "subnet resources" + Profiles -> EC2: "instance roles" +} + +@enduml + + +Here's an updated PlantUML deployment diagram reflecting your architecture with key components: + +```plantuml +@startuml + +!define RECTANGLE(x) rectangle x as x +!define DIAMOND(x) diamond x as x + +package "Accounts/Swarm" { + RECTANGLE(swarms-aws-agent-api) { + package "Environments" { + RECTANGLE(dev) { + RECTANGLE(us_east_1) { + RECTANGLE(Godaddy) + RECTANGLE(Route53) + RECTANGLE(ALB) as alb + RECTANGLE(AutoScalingGroup) as asg + RECTANGLE(LaunchTemplate) as lt + RECTANGLE(AMI) as ami + } + } + } + } + Godaddy -> Route53: ns api.swarms.ai + Route53 -> alb: CNAME + alb -> asg + asg -> lt + lt -> ami: "components/machine_image/ubuntu-fastapi.pkr.hcl" + + package "EC2 Instance" { + RECTANGLE(EC2) { + RECTANGLE(UserData) + RECTANGLE(InstanceProfile) + RECTANGLE(Nginx) + RECTANGLE(Systemd) + RECTANGLE(Uvicorn) + RECTANGLE(FastAPI) + + EC2 -> UserData: github clone branch run + EC2 -> InstanceProfile: read access to SSM secrets + EC2 -> Nginx: isolates user process + EC2 -> Systemd: manage services + EC2 -> Uvicorn: "4 instances of FastAPI" + } + } + + UserData -> EC2: "run api/install.sh" + InstanceProfile -> EC2: "access tagged secrets" + Nginx -> Systemd: "manage processes" +} + +package "GitHub Actions" { + RECTANGLE(OIDC) { + RECTANGLE(SSM_Document) + RECTANGLE(Terraform) + } + OIDC -> AWS + AWS -> SSM_Document: "update server" + OIDC -> AWS: "create new clusters" + AWS -> Terraform +} + +package "Shared Hosted AWS Account" { + RECTANGLE(Profiles) { + RECTANGLE(Tags) + RECTANGLE(VPC) + } + Profiles -> Tags: "customer resources" + Profiles -> VPC: "subnet resources" + Profiles -> EC2: "instance roles" +} + +@enduml +``` + + +It seems there was some confusion. Here’s a refined version of the complete PlantUML deployment diagram for your smart agent server setup: + +```plantuml +@startuml + +!define RECTANGLE(x) rectangle x as x +!define DIAMOND(x) diamond x as x + +package "Accounts/Swarms" { + RECTANGLE("swarms-aws-agent-api") { + package "Environments" { + RECTANGLE("dev") { + RECTANGLE("us-east-1") { + RECTANGLE(Godaddy) + RECTANGLE(Route53) + RECTANGLE(ALB) as alb + RECTANGLE("AutoScalingGroup") as asg + RECTANGLE("LaunchTemplate") as lt + RECTANGLE("AMI") as ami + } + } + } + } + Godaddy -> Route53: ns api.swarms.ai + Route53 -> alb: CNAME + alb -> asg + asg -> lt + lt -> ami: "components/machine_image/ubuntu-fastapi.pkr.hcl" + + package "EC2 Instance" { + RECTANGLE("EC2") { + RECTANGLE("UserData") as userdata + RECTANGLE("InstanceProfile") as ip + RECTANGLE("Nginx") as nginx + RECTANGLE("Systemd") as systemd + RECTANGLE("Uvicorn") as uvicorn + RECTANGLE("FastAPI") as fastapi + + EC2 -> userdata: "github clone branch run api/install.sh" + EC2 -> ip: "read access to SSM secrets" + EC2 -> nginx: "isolates user process" + EC2 -> systemd: "manages services" + EC2 -> uvicorn: "4 instances of FastAPI" + } + } + + userdata -> EC2: "execute boot.sh" + ip -> EC2: "access tagged secrets" + nginx -> systemd: "manage processes" +} + +package "GitHub Actions" { + RECTANGLE("OIDC") { + RECTANGLE("AWS") + RECTANGLE("SSM Document") + RECTANGLE("Terraform") + } + OIDC -> AWS + AWS -> SSM Document: "update server" + OIDC -> AWS: "create new clusters" +} + +package "Shared Hosted AWS Account" { + RECTANGLE("Profiles") { + RECTANGLE("Tags") + RECTANGLE("VPC") + } + Profiles -> Tags: "customer resources" + Profiles -> VPC: "subnet resources" + Profiles -> EC2: "instance roles" +} + +note right of ami: "Designed for\nFastAPI deployments" +@enduml +``` + +### Key Components: +1. **Accounts/Swarms**: Represents the main framework for your smart agent server. +2. **Environments**: Contains different environments such as `dev` in `us-east-1`. +3. **EC2 Instance**: Details the EC2 setup, including user data and necessary components (Nginx, Systemd, Uvicorn). +4. **GitHub Actions**: Integrates CI/CD using OIDC, SSM, and Terraform. +5. **Shared Hosted AWS Account**: Manages profiles, tags, and VPC configurations for customer resources. + +You can copy this code into a PlantUML tool to visualize the deployment diagram. +### Explanation: +- **Packages and Rectangles**: The diagram encapsulates components like Route 53, ALB, and Auto Scaling Group within a structured format. +- **Interactions**: Arrows show the relationships and interactions among components, such as the CNAME setup with Route 53 and the EC2 instances launching user data scripts. +- **GitHub Actions**: Represents automation activities and connections to AWS services. +- **Shared AWS Account**: Displays customer management and tagging for resources. + +This structure captures your specifications for the deployment of a self-service Cognito-enabled smart agent server using Terraform and AWS components effectively. Adjust as necessary for your specific documentation needs!``` + +### Explanation: +- The main structure includes the accounts, environments, EC2 instances, GitHub Actions, and shared AWS resources. +- The individual components and their interactions are represented using rectangles for entities and arrows for relationships. +- Customize further by adding details and refining the relationships based on your specific needs.``` + +To manage a shared hosted AWS account for multiple customers with the requirements you've laid out, you can consider the following architecture: + +### AWS Account Structure: +1. **Profiles**: Utilize IAM roles and users for each customer. Create separate profiles in the AWS CLI for easy access. + +2. **Resource Tagging**: Implement a consistent tagging strategy for all resources (e.g., `CustomerName`, `Project`, `Environment`, `CostCenter`) to facilitate cost management and resource identification. + +3. **API Call Restrictions**: Use IAM policies to specify which API calls each customer's role can perform based on their needs. + +4. **Cost Tracking**: Leverage AWS Cost Explorer and AWS Budgets, filtering by tags to track costs per customer. + +5. **Networking**: Set up custom VPCs for customers if isolation is required. Use subnets and security groups to segment resources. + +### Server and Agent Management: +1. **AWS Roles for Agents**: Each agent (server) should have an IAM role assigned that grants necessary permissions. If using Lambda, this can be done via execution roles. + +2. **Unix User Management**: Depending on your requirements: + - One option is to create a Unix user for each customer/agent. + - Assign unique home directories, storage solutions (e.g., EBS, S3), and ensure proper permissions. + +3. **Containerization**: Use Docker and Kubernetes (EKS) to manage applications. This allows for easier resource allocation and isolation between different customers. + +4. **Email & Backup**: Consider using Amazon SES for email needs and AWS Backup/CloudFormation for automated backups of customer data. + +### Best Practices: +- **Security**: Regularly audit IAM roles and policies for least privilege access. +- **Monitoring**: Use CloudWatch for logging and monitoring resource utilization. +- **Cost Management**: Use AWS Cost Allocation Reports to analyze customer costs effectively. + +This architecture ensures streamlined operations, cost tracking, and enhanced security across customer resources in a shared AWS account. + + + +### Repo Structure Overview + +- **Root Directory**: The main container of your Terraform project (`terraform-template/`). + +- **README.md**: Provides an overview and documentation for the project, helping users understand its purpose and how to use it. + +- **Environments**: Contains specific configurations for different environments (e.g., development, staging). + - **swarms-aws-agent-api**: An example service with various setups. + - **dev/us-east-1**: The development environment tailored for the US East (N. Virginia) region. + +- **Modules**: Encapsulates reusable Terraform code to ensure DRY (Don't Repeat Yourself) principles. + - **swarms**: A collection of Terraform modules related to the swarm infrastructure. + +### Components Explained + +- **DNS Management**: + - Using GoDaddy for domain management and AWS Route 53 for DNS routing (e.g., `api.swarms.ai`). + +- **Application Layer**: + - Includes an Application Load Balancer directing traffic to an Auto Scaling Group, which manages instances defined by a launch template. + +- **Instance Management**: + - AMIs (Amazon Machine Images) created using Packer (e.g., `ubuntu-fastapi.pkr.hcl`). + +- **Settings for EC2 Instances**: + - User data scripts to set up necessary configurations, such as cloning from GitHub and running scripts as the `swarms` user to set up the FastAPI application. + +- **Access Management**: + - IAM roles and instance profiles are created to manage permissions. SSM (AWS Systems Manager) secrets are tagged, allowing specific servers to access configurations, like OpenAI keys. + +- **Networking**: + - Outbound traffic settings defined to control which IP ranges can communicate with the infrastructure. + +- **Application Deployment**: + - Utilizes Nginx and Systemd to manage FastAPI processes, ensuring multiple instances run smoothly. + +### CI/CD with GitHub Actions +- Implements OIDC connections to AWS for secure deployments and updates using GitHub Actions. +- SSM documents are utilized for executing specific management tasks, like creating new clusters and updating configurations seamlessly. + +This structure facilitates a modular, maintainable, and automated approach to infrastructure as code using Terraform, alongside solid CI/CD practices. + + + +It seems you might want to continue with a task or scenario related to the previous content but haven't specified what you'd like next. If you need more details or a specific aspect of the Terraform setup, architecture diagram, or other inquiries, please let me know, and I'll be glad to assist! diff --git a/accounts/swarms/github/main.tf b/accounts/swarms/github/main.tf index e762631..f3136b4 100644 --- a/accounts/swarms/github/main.tf +++ b/accounts/swarms/github/main.tf @@ -1,6 +1,6 @@ - variable aws_region {} variable aws_account_id {} +variable repos {} terraform { required_providers { @@ -11,19 +11,16 @@ terraform { } } -#data "github_actions_public_key" "public_key" { -# repository = "jmikedupont2/terraform-aws-oidc-github" -#} - - resource "github_actions_secret" "region" { - repository = "terraform-aws-oidc-github" + for_each = var.repos + repository = each.key # secret_name = "AWS_REGION" plaintext_value = var.aws_region } resource "github_actions_secret" "account" { - repository = "terraform-aws-oidc-github" + for_each = var.repos + repository = each.key secret_name = "AWS_ACCOUNT_ID" plaintext_value = var.aws_account_id } diff --git a/accounts/swarms/main.tf b/accounts/swarms/main.tf index 1c5ed3f..23382a4 100644 --- a/accounts/swarms/main.tf +++ b/accounts/swarms/main.tf @@ -1,8 +1,8 @@ locals { #ami_name = "ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*" ami_name = "ubuntu-minimal/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-minimal-*" - # dns = "api.swarms.ai" - dns = "api.arianakrasniqi.com" + dns = "api.swarms.ai" + #dns = "api.arianakrasniqi.com" account = "916723593639" region = "us-east-2" } @@ -93,11 +93,16 @@ module github { aws_account_id = local.account aws_region = local.region # github_token = var.github_token - + repos = toset([ + "terraform-aws-oidc-github", + "swarms", + "swarms-terraform" + ]) } # now create the ssm document module call_swarms { source = "../../environments/call-swarms" + } From 301184822ea5bedab1ca7488e2f19e9f35bc087c Mon Sep 17 00:00:00 2001 From: Mike DuPont Date: Thu, 19 Dec 2024 15:16:12 -0500 Subject: [PATCH 057/158] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6206132..54caf9a 100644 --- a/README.md +++ b/README.md @@ -49,9 +49,9 @@ shared hosted aws account : on servers, the agents will have own aws role/profiles (like lamda) either each customer/agent gets own unix user, homedir, storage, email , backup. containerization, k8s, docker. +``` - -Here is a concise PlantUML use case diagram based on your specifications: +# diagram1 ```plantuml @startuml From b9cd6066e6af7f6f62f091915294ca4c5ca91895 Mon Sep 17 00:00:00 2001 From: Mike DuPont Date: Thu, 19 Dec 2024 15:18:12 -0500 Subject: [PATCH 058/158] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 54caf9a..ddf23fb 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,8 @@ note right of U8 : Custom VPC \n\n Customer Profiles @enduml ``` +![1000019322](https://github.com/user-attachments/assets/7f2c8f90-8222-4344-a007-59ea518b0f00) +https://www.planttext.com/?text=RPJVRzem4CVVyrVSUDeUA1gK_dgOccQeGu8AaRAdIfCnbs0riPalRbTD-zztjfCre3q6V_pr-NE7RwaHjd5rbOYgCg-r-gLqcIGVTbWeZR2UPO_fm_rY1h8IH4do_iCWnweOvcXVb4J8JAbLOuOpBROLxY2lEHRI4dK3da4xWDy9mCSDeVsK_4aIYx8HkgmG1gcKnkc6Hlchi7K47Q4Kn8kPHT4WpFIBiaRN74W340IRpttgAnugymQpR0nppNVI4bjitCUMGeiIOM743kv4vJ4RiSdHcKONgdGM8NLygLoAYhL4yBmFhgnvKQICXdGvZ-lJ_ZhV38aWpvTiGLhiQh_eGvl3pjymQeBkRD5mRAGl-_IZ5V8PCaUDgHbFLtF8k7TjyDN_gyltmd9NraZ6sNzghdqVV4DWEFp2Gk6cqqqmWkYA1ZrYx8cecdgUUyGaP7JwwCHq3pptOCv2ZXJ3IbXHOuFsA7NgOVuahnaAOo4MuQUXprruReq_7H87jyO37nlHPoz3Pb1F9z9xr9MdbiRFjgKWUqDifOzSXvjumVGk64Hj-3fTc_ZaXFrwVh0shi7fP71YAdj8bpFE7KUA9paG2-6qtEEnxabaqiLTO48Y0Kz6KIZ9w-VjEfljnDvRumw1RliAVaLiLQMQDzuoRRVuCRvfHsV7r5B8lNy9vdMyzcNaxbNppMrPgnT6OOi_hGusOtCOleXmLpfmQg4gF2fZXRPMH24cOTzXnwCeOsYOTjyVRYWxr8R_A5QL-mPC4USYC_N1Q1w46tZIilkWKHqB_w5yuIEtvGV63Gl1bHpKsU7PdkrbTPhdknrR1jzN6USzwVT_MbiyFuAtg7VyPFu5 ### Explanation: - This diagram represents the primary use cases associated with your smart agent server project. From 112773ee85c3e2f8a732477f21a64d850e4a1df0 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Thu, 19 Dec 2024 15:37:20 -0500 Subject: [PATCH 059/158] test --- .github/workflows/call-swarms.yml | 108 ++++++++++++++++++++++++++++++ environments/call-swarms/main.tf | 94 ++++++++++++++++++++++++++ 2 files changed, 202 insertions(+) diff --git a/.github/workflows/call-swarms.yml b/.github/workflows/call-swarms.yml index fb2a898..ce43743 100644 --- a/.github/workflows/call-swarms.yml +++ b/.github/workflows/call-swarms.yml @@ -1,3 +1,16 @@ +# create iam policy to allow this to execute + +## ERROR: +# Run aws sts get-caller-identity +# { +# "UserId": "AROA5K4H36GT5YYCJUOGV:GitHubActions", +# "Account": "***", +# "Arn": "arn:aws:sts::***:assumed-role/github/GitHubActions" +# } + +# An error occurred (AccessDeniedException) when calling the SendCommand operation: User: arn:aws:sts::***:assumed-role/github/GitHubActions is not authorized to perform: ssm:SendCommand on resource: arn:aws:ec2:***:***:instance/i-060238b47044e8dd3 because no identity-based policy allows the ssm:SendCommand action +# Error: Process completed with exit code 254. + name: CallSwarms on: workflow_dispatch: @@ -17,3 +30,98 @@ jobs: - run: | aws sts get-caller-identity aws ssm send-command --document-name "deploy" --document-version "1" --targets '[{"Key":"InstanceIds","Values":["i-060238b47044e8dd3"]}]' --parameters '{"GitUrl":["https://github.com/kyegomez/swarms"],"GitName":["kye"],"GitVersion":["master"]}' --timeout-seconds 600 --max-concurrency "50" --max-errors "0" --region us-east-2 + + +create a terraform code to deploy this and attach +To allow the specified `ssm:SendCommand` operation, you need to create an IAM policy that grants the necessary permissions for the assumed role. The policy should be attached to the role `github`. Here’s a sample IAM policy: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ssm:SendCommand", + "ssm:ListCommands", + "ssm:GetCommandInvocation" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeInstances" + ], + "Resource": "*" + } + ] +} +``` + +### Steps to implement: + +1. **Go to the IAM Console** in your AWS Management Console. +2. **Locate the Role**: Search for the `github` role. +3. **Attach Policy**: + - Go to the "Permissions" tab and click "Add inline policy." + - Choose "JSON" and paste the policy above into the policy editor. + - Review and give the policy a name, then save it. + +### Additional Notes: +- Adjust the `Resource` element if you want to restrict access to specific resources rather than all (`*`). For example, you can specify the ARNs of specific EC2 instances or SSM documents. +- Always follow the principle of least privilege to ensure you only grant the permissions that are necessary. + +To deploy an IAM policy for the `github` role that allows the `ssm:SendCommand` operation, you can use Terraform. Below is a sample Terraform code snippet to create the IAM policy and attach it to the `github` role: + +```hcl +provider "aws" { + region = "us-east-1" # Change to your desired region +} + +resource "aws_iam_policy" "github_ssm_policy" { + name = "GitHubSSMPolicy" + description = "Policy to allow SSM commands for GitHub role" + + policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Effect = "Allow", + Action = [ + "ssm:SendCommand", + "ssm:ListCommands", + "ssm:GetCommandInvocation" + ], + Resource = "*" + }, + { + Effect = "Allow", + Action = [ + "ec2:DescribeInstances" + ], + Resource = "*" + } + ] + }) +} + +resource "aws_iam_role_policy_attachment" "attach_github_ssm_policy" { + policy_arn = aws_iam_policy.github_ssm_policy.arn + role = "github" # Ensure this matches your IAM role +} + +output "policy_arn" { + value = aws_iam_policy.github_ssm_policy.arn +} +``` + +### Instructions: + +1. **Install Terraform** if you haven't already. +2. **Configure AWS Credentials**: Ensure your AWS credentials are set up properly (e.g., using `aws configure`). +3. **Create a new Terraform file** (e.g., `main.tf`) and paste the above code into it. +4. **Initialize Terraform**: Run `terraform init` in your terminal to initialize the working directory. +5. **Apply the Terraform configuration**: Run `terraform apply`, and confirm the changes when prompted. + +This code will create a new IAM policy that allows the specified actions and automatically attach it to the existing `github` role. Adjust the policy as needed for your security requirements. diff --git a/environments/call-swarms/main.tf b/environments/call-swarms/main.tf index 6e8b358..ff87217 100644 --- a/environments/call-swarms/main.tf +++ b/environments/call-swarms/main.tf @@ -9,3 +9,97 @@ resource "aws_ssm_document" "deploy" { tags = {env = "test"} } + +# create a terraform code to deploy this and attach +# To allow the specified `ssm:SendCommand` operation, you need to create an IAM policy that grants the necessary permissions for the assumed role. The policy should be attached to the role `github`. Here’s a sample IAM policy: + +# ```json +# { +# "Version": "2012-10-17", +# "Statement": [ +# { +# "Effect": "Allow", +# "Action": [ +# "ssm:SendCommand", +# "ssm:ListCommands", +# "ssm:GetCommandInvocation" +# ], +# "Resource": "*" +# }, +# { +# "Effect": "Allow", +# "Action": [ +# "ec2:DescribeInstances" +# ], +# "Resource": "*" +# } +# ] +# } +# ``` + +# ### Steps to implement: + +# 1. **Go to the IAM Console** in your AWS Management Console. +# 2. **Locate the Role**: Search for the `github` role. +# 3. **Attach Policy**: +# - Go to the "Permissions" tab and click "Add inline policy." +# - Choose "JSON" and paste the policy above into the policy editor. +# - Review and give the policy a name, then save it. + +# ### Additional Notes: +# - Adjust the `Resource` element if you want to restrict access to specific resources rather than all (`*`). For example, you can specify the ARNs of specific EC2 instances or SSM documents. +# - Always follow the principle of least privilege to ensure you only grant the permissions that are necessary. + +# To deploy an IAM policy for the `github` role that allows the `ssm:SendCommand` operation, you can use Terraform. Below is a sample Terraform code snippet to create the IAM policy and attach it to the `github` role: + +# ```hcl +provider "aws" { + region = "us-east-1" # Change to your desired region +} + +resource "aws_iam_policy" "github_ssm_policy" { + name = "GitHubSSMPolicy" + description = "Policy to allow SSM commands for GitHub role" + + policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Effect = "Allow", + Action = [ + "ssm:SendCommand", + "ssm:ListCommands", + "ssm:GetCommandInvocation" + ], + Resource = "*" + }, + { + Effect = "Allow", + Action = [ + "ec2:DescribeInstances" + ], + Resource = "*" + } + ] + }) +} + +resource "aws_iam_role_policy_attachment" "attach_github_ssm_policy" { + policy_arn = aws_iam_policy.github_ssm_policy.arn + role = "github" # Ensure this matches your IAM role +} + +output "policy_arn" { + value = aws_iam_policy.github_ssm_policy.arn +} +# ``` + +# ### Instructions: + +# 1. **Install Terraform** if you haven't already. +# 2. **Configure AWS Credentials**: Ensure your AWS credentials are set up properly (e.g., using `aws configure`). +# 3. **Create a new Terraform file** (e.g., `main.tf`) and paste the above code into it. +# 4. **Initialize Terraform**: Run `terraform init` in your terminal to initialize the working directory. +# 5. **Apply the Terraform configuration**: Run `terraform apply`, and confirm the changes when prompted. + +# This code will create a new IAM policy that allows the specified actions and automatically attach it to the existing `github` role. Adjust the policy as needed for your security requirements. From c512375469724e1f6f3891efa0a87bd351c70fc7 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Thu, 19 Dec 2024 15:56:15 -0500 Subject: [PATCH 060/158] first version of call swarms calling https://github.com/jmikedupont2/terraform-aws-oidc-github/pull/1 --- .github/workflows/call-swarms.yml | 94 ------------------------------- accounts/swarms/Readme.md | 5 ++ environments/call-swarms/main.tf | 2 +- 3 files changed, 6 insertions(+), 95 deletions(-) diff --git a/.github/workflows/call-swarms.yml b/.github/workflows/call-swarms.yml index ce43743..0faf362 100644 --- a/.github/workflows/call-swarms.yml +++ b/.github/workflows/call-swarms.yml @@ -31,97 +31,3 @@ jobs: aws sts get-caller-identity aws ssm send-command --document-name "deploy" --document-version "1" --targets '[{"Key":"InstanceIds","Values":["i-060238b47044e8dd3"]}]' --parameters '{"GitUrl":["https://github.com/kyegomez/swarms"],"GitName":["kye"],"GitVersion":["master"]}' --timeout-seconds 600 --max-concurrency "50" --max-errors "0" --region us-east-2 - -create a terraform code to deploy this and attach -To allow the specified `ssm:SendCommand` operation, you need to create an IAM policy that grants the necessary permissions for the assumed role. The policy should be attached to the role `github`. Here’s a sample IAM policy: - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "ssm:SendCommand", - "ssm:ListCommands", - "ssm:GetCommandInvocation" - ], - "Resource": "*" - }, - { - "Effect": "Allow", - "Action": [ - "ec2:DescribeInstances" - ], - "Resource": "*" - } - ] -} -``` - -### Steps to implement: - -1. **Go to the IAM Console** in your AWS Management Console. -2. **Locate the Role**: Search for the `github` role. -3. **Attach Policy**: - - Go to the "Permissions" tab and click "Add inline policy." - - Choose "JSON" and paste the policy above into the policy editor. - - Review and give the policy a name, then save it. - -### Additional Notes: -- Adjust the `Resource` element if you want to restrict access to specific resources rather than all (`*`). For example, you can specify the ARNs of specific EC2 instances or SSM documents. -- Always follow the principle of least privilege to ensure you only grant the permissions that are necessary. - -To deploy an IAM policy for the `github` role that allows the `ssm:SendCommand` operation, you can use Terraform. Below is a sample Terraform code snippet to create the IAM policy and attach it to the `github` role: - -```hcl -provider "aws" { - region = "us-east-1" # Change to your desired region -} - -resource "aws_iam_policy" "github_ssm_policy" { - name = "GitHubSSMPolicy" - description = "Policy to allow SSM commands for GitHub role" - - policy = jsonencode({ - Version = "2012-10-17", - Statement = [ - { - Effect = "Allow", - Action = [ - "ssm:SendCommand", - "ssm:ListCommands", - "ssm:GetCommandInvocation" - ], - Resource = "*" - }, - { - Effect = "Allow", - Action = [ - "ec2:DescribeInstances" - ], - Resource = "*" - } - ] - }) -} - -resource "aws_iam_role_policy_attachment" "attach_github_ssm_policy" { - policy_arn = aws_iam_policy.github_ssm_policy.arn - role = "github" # Ensure this matches your IAM role -} - -output "policy_arn" { - value = aws_iam_policy.github_ssm_policy.arn -} -``` - -### Instructions: - -1. **Install Terraform** if you haven't already. -2. **Configure AWS Credentials**: Ensure your AWS credentials are set up properly (e.g., using `aws configure`). -3. **Create a new Terraform file** (e.g., `main.tf`) and paste the above code into it. -4. **Initialize Terraform**: Run `terraform init` in your terminal to initialize the working directory. -5. **Apply the Terraform configuration**: Run `terraform apply`, and confirm the changes when prompted. - -This code will create a new IAM policy that allows the specified actions and automatically attach it to the existing `github` role. Adjust the policy as needed for your security requirements. diff --git a/accounts/swarms/Readme.md b/accounts/swarms/Readme.md index 5146909..176495a 100644 --- a/accounts/swarms/Readme.md +++ b/accounts/swarms/Readme.md @@ -167,3 +167,8 @@ tofu plan -var "github_token=$(gh auth token)" tofu apply -var "github_token=$(gh auth token)" ``` + +# oidc + +see this branch for the pike generated policies +https://github.com/jmikedupont2/terraform-aws-oidc-github/pull/1 diff --git a/environments/call-swarms/main.tf b/environments/call-swarms/main.tf index ff87217..0c04f61 100644 --- a/environments/call-swarms/main.tf +++ b/environments/call-swarms/main.tf @@ -56,7 +56,6 @@ resource "aws_ssm_document" "deploy" { provider "aws" { region = "us-east-1" # Change to your desired region } - resource "aws_iam_policy" "github_ssm_policy" { name = "GitHubSSMPolicy" description = "Policy to allow SSM commands for GitHub role" @@ -96,6 +95,7 @@ output "policy_arn" { # ### Instructions: + # 1. **Install Terraform** if you haven't already. # 2. **Configure AWS Credentials**: Ensure your AWS credentials are set up properly (e.g., using `aws configure`). # 3. **Create a new Terraform file** (e.g., `main.tf`) and paste the above code into it. From 69de2934d992265ba0e54f26ca995d428635aa1c Mon Sep 17 00:00:00 2001 From: mike dupont Date: Thu, 19 Dec 2024 16:12:31 -0500 Subject: [PATCH 061/158] update job version --- .github/workflows/call-swarms.yml | 2 +- environments/call-swarms/deploy.yaml | 5 +- environments/call-swarms/main.tf | 70 ++++++++++++++-------------- 3 files changed, 40 insertions(+), 37 deletions(-) diff --git a/.github/workflows/call-swarms.yml b/.github/workflows/call-swarms.yml index 0faf362..31699d9 100644 --- a/.github/workflows/call-swarms.yml +++ b/.github/workflows/call-swarms.yml @@ -29,5 +29,5 @@ jobs: role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/github - run: | aws sts get-caller-identity - aws ssm send-command --document-name "deploy" --document-version "1" --targets '[{"Key":"InstanceIds","Values":["i-060238b47044e8dd3"]}]' --parameters '{"GitUrl":["https://github.com/kyegomez/swarms"],"GitName":["kye"],"GitVersion":["master"]}' --timeout-seconds 600 --max-concurrency "50" --max-errors "0" --region us-east-2 + aws ssm send-command --document-name "deploy" --document-version "2" --targets '[{"Key":"InstanceIds","Values":["i-060238b47044e8dd3"]}]' --parameters '{"GitUrl":["https://github.com/kyegomez/swarms"],"GitName":["kye"],"GitVersion":["master"]}' --timeout-seconds 600 --max-concurrency "50" --max-errors "0" --region us-east-2 diff --git a/environments/call-swarms/deploy.yaml b/environments/call-swarms/deploy.yaml index 43b32f9..e3e6d30 100644 --- a/environments/call-swarms/deploy.yaml +++ b/environments/call-swarms/deploy.yaml @@ -34,4 +34,7 @@ mainSteps: cd /opt/swarms/ sudo git remote add {{GitName}} {{GitUrl}} || echo oops sudo git checkout --force {{GitVersion}} - sudo bash -x /opt/swarms/api/install.sh + #sudo bash -x /opt/swarms/api/install.sh + sudo su -c "bash -e -x /var/swarms/agent_workspace/boot.sh" swarms + sudo systemctl daemon-reload + systemctl restart swarms-uvicorn || journalctl -xeu swarms-uvicorn.service diff --git a/environments/call-swarms/main.tf b/environments/call-swarms/main.tf index 0c04f61..7c929a0 100644 --- a/environments/call-swarms/main.tf +++ b/environments/call-swarms/main.tf @@ -53,44 +53,44 @@ resource "aws_ssm_document" "deploy" { # To deploy an IAM policy for the `github` role that allows the `ssm:SendCommand` operation, you can use Terraform. Below is a sample Terraform code snippet to create the IAM policy and attach it to the `github` role: # ```hcl -provider "aws" { - region = "us-east-1" # Change to your desired region -} -resource "aws_iam_policy" "github_ssm_policy" { - name = "GitHubSSMPolicy" - description = "Policy to allow SSM commands for GitHub role" +# provider "aws" { +# region = "us-east-1" # Change to your desired region +# } +# resource "aws_iam_policy" "github_ssm_policy" { +# name = "GitHubSSMPolicy" +# description = "Policy to allow SSM commands for GitHub role" - policy = jsonencode({ - Version = "2012-10-17", - Statement = [ - { - Effect = "Allow", - Action = [ - "ssm:SendCommand", - "ssm:ListCommands", - "ssm:GetCommandInvocation" - ], - Resource = "*" - }, - { - Effect = "Allow", - Action = [ - "ec2:DescribeInstances" - ], - Resource = "*" - } - ] - }) -} +# policy = jsonencode({ +# Version = "2012-10-17", +# Statement = [ +# { +# Effect = "Allow", +# Action = [ +# "ssm:SendCommand", +# "ssm:ListCommands", +# "ssm:GetCommandInvocation" +# ], +# Resource = "*" +# }, +# { +# Effect = "Allow", +# Action = [ +# "ec2:DescribeInstances" +# ], +# Resource = "*" +# } +# ] +# }) +# } -resource "aws_iam_role_policy_attachment" "attach_github_ssm_policy" { - policy_arn = aws_iam_policy.github_ssm_policy.arn - role = "github" # Ensure this matches your IAM role -} +# resource "aws_iam_role_policy_attachment" "attach_github_ssm_policy" { +# policy_arn = aws_iam_policy.github_ssm_policy.arn +# role = "github" # Ensure this matches your IAM role +# } -output "policy_arn" { - value = aws_iam_policy.github_ssm_policy.arn -} +# output "policy_arn" { +# value = aws_iam_policy.github_ssm_policy.arn +# } # ``` # ### Instructions: From 3c4a68ba8f73d3718f240a8d5576e42f04d61515 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Thu, 19 Dec 2024 17:04:56 -0500 Subject: [PATCH 062/158] starting on a new autoscaling group --- .../application_load_balancer/https/main.tf | 29 ++++++++++++------- .../application_load_balancer/main.tf | 13 ++++++++- .../target_group/main.tf | 5 ++-- .../dev/us-east-1/main.tf | 17 +++++++++++ 4 files changed, 51 insertions(+), 13 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf index d75f4a5..4166e25 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf @@ -2,6 +2,7 @@ variable alb_arn{} variable domain_name{} variable zone_id{} variable aws_lb_target_group_arn{} +variable new_target_group_arn{} module "acm" { # count = 0 @@ -15,28 +16,36 @@ module "acm" { ] } -## now we just lift the listener code +## add a rule for matching urls for /v1/// and routing that to a new target group +#like var.aws_lb_target_group{ { user, agent, api, target group name}} resource "aws_lb_listener" "this" { -# count = 0 port = 443 protocol = "HTTPS" ssl_policy = "ELBSecurityPolicy-TLS13-1-2-Res-2021-06" certificate_arn = module.acm.acm_certificate_arn load_balancer_arn = var.alb_arn - #additional_certificate_arns = [module.wildcard_cert.acm_certificate_arn] - # #forward = { - # #target_group_key = "ex-swarms-instance" - # target_group_arn = "ex-swarms-instance" - # #target_group = [] - default_action { target_group_arn =var.aws_lb_target_group_arn - #module.alb.target_groups["ex-lambda-with-trigger"].arn - #length(try(default_action.value.target_groups, [])) > 0 ? null : try(default_action.value.arn, aws_lb_target_group.this[default_action.value.target_group_key].arn, null) type = "forward" } } +resource "aws_lb_listener_rule" "route_v1_api" { + listener_arn = aws_lb_listener.this.arn + priority = 100 # Set priority as needed, must be unique + + action { + type = "forward" + target_group_arn = var.new_target_group_arn # New target group's ARN + } + + condition { + path_pattern { + values = ["/v1/*/*/*"] + } + } +} + resource "aws_lb_listener" "insecure" { port = 80 protocol = "HTTP" diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf index 5762415..941b8d4 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf @@ -49,6 +49,13 @@ module "route53" { module "tg" { source = "./target_group/" + name_prefix = "swarms" + vpc_id = var.vpc_id # module.vpc.vpc_id +} + +module "tg_test" { + source = "./target_group/" + name_prefix = "test" vpc_id = var.vpc_id # module.vpc.vpc_id } @@ -59,10 +66,14 @@ module "https" { domain_name = var.domain_name alb_arn = module.alb.arn aws_lb_target_group_arn = module.tg.alb_target_group_arn - #aws_lb_target_group.this.arn + new_target_group_arn = module.tg_test.alb_target_group_arn } output alb_target_group_arn { value = module.tg.alb_target_group_arn } + +output test_alb_target_group_arn { + value = module.tg_test.alb_target_group_arn +} diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/target_group/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/target_group/main.tf index 017efed..a971dea 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/target_group/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/target_group/main.tf @@ -1,6 +1,7 @@ -variable vpc_id {} +variable vpc_id {} +variable name_prefix {} resource "aws_lb_target_group" "this" { - name_prefix = "swarms" + name_prefix = var.name_prefix protocol = "HTTP" port = 80 target_type = "instance" diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index a9ebc0b..d394938 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -160,3 +160,20 @@ module "asg_dynamic_new_ami" { launch_template_id = module.lt_dynamic_ami[each.key].launch_template_id target_group_arn = module.alb.alb_target_group_arn } + +module "asg_dynamic_new_ami_test" { + # built with packer + #count =0 + tags = local.tags + vpc_id = local.vpc_id + image_id = local.new_ami_id + ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 + for_each = toset(var.instance_types) + aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn + source = "./components/autoscaling_group" +# security_group_id = module.security.internal_security_group_id + instance_type = each.key + name = "swarms-ami-${each.key}" + launch_template_id = module.lt_dynamic_ami[each.key].launch_template_id + target_group_arn = module.alb.test_alb_target_group_arn +} From ea44b5f87269356f1c472478de8a924189062c91 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Thu, 19 Dec 2024 17:17:42 -0500 Subject: [PATCH 063/158] customer test --- .../application_load_balancer/https/main.tf | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf index 4166e25..ef1f5db 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf @@ -30,6 +30,7 @@ resource "aws_lb_listener" "this" { } } +# make a new one for checking for ?customer=name in the rules instead of the path resource "aws_lb_listener_rule" "route_v1_api" { listener_arn = aws_lb_listener.this.arn priority = 100 # Set priority as needed, must be unique @@ -46,6 +47,24 @@ resource "aws_lb_listener_rule" "route_v1_api" { } } + +To check for `?customer=name` in the rules instead of the path, modify the condition as follows: + +resource "aws_lb_listener_rule" "route_v1_api" { + listener_arn = aws_lb_listener.this.arn + priority = 100 # Set priority as needed, must be unique + + action { + type = "forward" + target_group_arn = var.new_target_group_arn # New target group's ARN + } + + condition { + field = "query-string" + values = ["customer=test"] + } +} + resource "aws_lb_listener" "insecure" { port = 80 protocol = "HTTP" From eca56903d0a28a8e11479bfa2175f64a1a338a2a Mon Sep 17 00:00:00 2001 From: mike dupont Date: Thu, 19 Dec 2024 17:26:53 -0500 Subject: [PATCH 064/158] working it seems --- .../application_load_balancer/https/main.tf | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf index ef1f5db..3717ee7 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf @@ -48,11 +48,9 @@ resource "aws_lb_listener_rule" "route_v1_api" { } -To check for `?customer=name` in the rules instead of the path, modify the condition as follows: - -resource "aws_lb_listener_rule" "route_v1_api" { +resource "aws_lb_listener_rule" "route_v1_api_customer" { listener_arn = aws_lb_listener.this.arn - priority = 100 # Set priority as needed, must be unique + priority = 200 # Set priority as needed, must be unique action { type = "forward" @@ -60,8 +58,10 @@ resource "aws_lb_listener_rule" "route_v1_api" { } condition { - field = "query-string" - values = ["customer=test"] + query_string { + key = "customer" + value ="test" + } } } From db93c71718b41d8d1b2a5e3a05fab7a2dad00596 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Fri, 20 Dec 2024 14:45:34 -0500 Subject: [PATCH 065/158] now adding a call swarms.sh --- .github/workflows/call-swarms.yml | 27 ++++++-------- actions/call_swarms.sh | 62 +++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 15 deletions(-) create mode 100644 actions/call_swarms.sh diff --git a/.github/workflows/call-swarms.yml b/.github/workflows/call-swarms.yml index 31699d9..6c1daf6 100644 --- a/.github/workflows/call-swarms.yml +++ b/.github/workflows/call-swarms.yml @@ -1,15 +1,12 @@ -# create iam policy to allow this to execute - -## ERROR: -# Run aws sts get-caller-identity -# { -# "UserId": "AROA5K4H36GT5YYCJUOGV:GitHubActions", -# "Account": "***", -# "Arn": "arn:aws:sts::***:assumed-role/github/GitHubActions" -# } - -# An error occurred (AccessDeniedException) when calling the SendCommand operation: User: arn:aws:sts::***:assumed-role/github/GitHubActions is not authorized to perform: ssm:SendCommand on resource: arn:aws:ec2:***:***:instance/i-060238b47044e8dd3 because no identity-based policy allows the ssm:SendCommand action -# Error: Process completed with exit code 254. +on: + push: + branches: + - "feature/call_swarms" + pull_request: + branches: + - "main" + paths: + - ".github/workflows/call-swarms.yaml" name: CallSwarms on: @@ -27,7 +24,7 @@ jobs: with: aws-region: ${{ secrets.AWS_REGION }} role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/github + - uses: actions/checkout@v2 - run: | - aws sts get-caller-identity - aws ssm send-command --document-name "deploy" --document-version "2" --targets '[{"Key":"InstanceIds","Values":["i-060238b47044e8dd3"]}]' --parameters '{"GitUrl":["https://github.com/kyegomez/swarms"],"GitName":["kye"],"GitVersion":["master"]}' --timeout-seconds 600 --max-concurrency "50" --max-errors "0" --region us-east-2 - + chmod +x ./actions/call_swarms.sh + ./actions/call_swarms.sh \ No newline at end of file diff --git a/actions/call_swarms.sh b/actions/call_swarms.sh new file mode 100644 index 0000000..7f4d6da --- /dev/null +++ b/actions/call_swarms.sh @@ -0,0 +1,62 @@ +#!/bin/bash +set -e +set -x +# Constants +export REGION="${REGION:-us-east-2}" +export AWS_PROFILE="${AWS_PROFILE:-swarms}" +TAG_KEY="${TAG_KEY:-sandbox}" +TAG_VALUE="${TAG_VALUE:-kye}" +GIT_URL="${GIT_URL:-https://github.com/kyegomez/swarms}" +export GIT_NAME="${GIT_NAME:-kye}" +export GIT_VERSION="${GIT_VERSION:-master}" + +DOCUMENT_NAME="${DOCUMENT_NAME:-deploy}" +DOCUMENT_VERSION="${DOCUMENT_VERSION:-2}" +TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-600}" +MAX_CONCURRENCY="${MAX_CONCURRENCY:-50}" +MAX_ERRORS="${MAX_ERRORS:-0}" + +# Function to get instance IDs +get_instance_ids() { + aws ec2 describe-instances \ + --filters "Name=tag:$TAG_KEY,Values=$TAG_VALUE" \ + --query "Reservations[*].Instances[*].InstanceId" \ + --output text \ + --region $REGION +} + +# Function to send command to instance +send_command() { + local instance_id="$1" + aws ssm send-command \ + --document-name "$DOCUMENT_NAME" \ + --document-version "$DOCUMENT_VERSION" \ + --targets "[{\"Key\":\"InstanceIds\",\"Values\":[\"$instance_id\"]}]" \ + --parameters "{\"GitUrl\":[\"$GIT_URL\"],\"GitName\":[\"$GIT_NAME\"],\"GitVersion\":[\"$GIT_VERSION\"]}" \ + --timeout-seconds $TIMEOUT_SECONDS \ + --max-concurrency "$MAX_CONCURRENCY" \ + --max-errors "$MAX_ERRORS" \ + --region $REGION +} + +# Function to fetch command output +fetch_command_output() { + local command_id="$1" + aws ssm list-command-invocations \ + --command-id "$command_id" \ + --details \ + --region $REGION | jq -r '.CommandInvocations[] | {InstanceId, Status, Output}' +} + +# Main script execution +for instance in $(get_instance_ids); do + echo "Instance ID: $instance" + result=$(send_command "$instance") + command_id=$(echo $result | jq -r '.Command.CommandId') + + # Wait for the command to complete + aws ssm wait command-executed --command-id "$command_id" --region $REGION --instance $instance + + # Fetch and print the command output + fetch_command_output "$command_id" +done From 749092dea7f06869b09bde56e950789275655ecb Mon Sep 17 00:00:00 2001 From: mike dupont Date: Fri, 20 Dec 2024 14:50:11 -0500 Subject: [PATCH 066/158] update --- .github/workflows/call-swarms.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/call-swarms.yml b/.github/workflows/call-swarms.yml index 6c1daf6..29ec6bc 100644 --- a/.github/workflows/call-swarms.yml +++ b/.github/workflows/call-swarms.yml @@ -25,6 +25,4 @@ jobs: aws-region: ${{ secrets.AWS_REGION }} role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/github - uses: actions/checkout@v2 - - run: | - chmod +x ./actions/call_swarms.sh - ./actions/call_swarms.sh \ No newline at end of file + - run: bash -x -e ./actions/call_swarms.sh \ No newline at end of file From dcf219ec80dc5f5b24dd572f5574fe0f66131c3f Mon Sep 17 00:00:00 2001 From: mike dupont Date: Fri, 20 Dec 2024 14:54:14 -0500 Subject: [PATCH 067/158] fixing bug --- .github/workflows/call-swarms.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/call-swarms.yml b/.github/workflows/call-swarms.yml index 29ec6bc..e7b5f05 100644 --- a/.github/workflows/call-swarms.yml +++ b/.github/workflows/call-swarms.yml @@ -1,4 +1,6 @@ on: + workflow_dispatch: + push: branches: - "feature/call_swarms" @@ -9,8 +11,6 @@ on: - ".github/workflows/call-swarms.yaml" name: CallSwarms -on: - workflow_dispatch: jobs: caller-identity: name: Return the IAM user @@ -25,4 +25,4 @@ jobs: aws-region: ${{ secrets.AWS_REGION }} role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/github - uses: actions/checkout@v2 - - run: bash -x -e ./actions/call_swarms.sh \ No newline at end of file + - run: bash -x -e ./actions/call_swarms.sh From b6b8460e7874ede4f07c2a41424df97379900a86 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Fri, 20 Dec 2024 14:55:37 -0500 Subject: [PATCH 068/158] removing test code --- actions/call_swarms.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/actions/call_swarms.sh b/actions/call_swarms.sh index 7f4d6da..70778ff 100644 --- a/actions/call_swarms.sh +++ b/actions/call_swarms.sh @@ -3,7 +3,8 @@ set -e set -x # Constants export REGION="${REGION:-us-east-2}" -export AWS_PROFILE="${AWS_PROFILE:-swarms}" +#export AWS_PROFILE="${AWS_PROFILE:-swarms}" +#export AWS_PROFILE="${AWS_PROFILE}" only needed for testing locally TAG_KEY="${TAG_KEY:-sandbox}" TAG_VALUE="${TAG_VALUE:-kye}" GIT_URL="${GIT_URL:-https://github.com/kyegomez/swarms}" From d482d1c037432cae60c8efea0d5761f8a7c1fe24 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Fri, 20 Dec 2024 16:04:42 -0500 Subject: [PATCH 069/158] deploy --- accounts/swarms/main.tf | 1 + .../application_load_balancer/https/main.tf | 25 +++---------------- .../application_load_balancer/route53/main.tf | 17 +++++++++++++ 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/accounts/swarms/main.tf b/accounts/swarms/main.tf index 23382a4..7d846ea 100644 --- a/accounts/swarms/main.tf +++ b/accounts/swarms/main.tf @@ -69,6 +69,7 @@ module "swarms_api" { } module "swarmdeploy" { +count =0 source = "../../environments/swarms-deploy/dev/us-east-1" domain = local.dns #ami_id = data.aws_ami.ami.id diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf index 3717ee7..4843e16 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf @@ -30,7 +30,8 @@ resource "aws_lb_listener" "this" { } } -# make a new one for checking for ?customer=name in the rules instead of the path + +# make a new one for checking for name is test.api.swarms.ai resource "aws_lb_listener_rule" "route_v1_api" { listener_arn = aws_lb_listener.this.arn priority = 100 # Set priority as needed, must be unique @@ -41,26 +42,8 @@ resource "aws_lb_listener_rule" "route_v1_api" { } condition { - path_pattern { - values = ["/v1/*/*/*"] - } - } -} - - -resource "aws_lb_listener_rule" "route_v1_api_customer" { - listener_arn = aws_lb_listener.this.arn - priority = 200 # Set priority as needed, must be unique - - action { - type = "forward" - target_group_arn = var.new_target_group_arn # New target group's ARN - } - - condition { - query_string { - key = "customer" - value ="test" + host_header { + values = ["test.api.swarms.ai"] } } } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf index 697bffa..c140e2f 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf @@ -25,6 +25,23 @@ resource "aws_route53_record" "api-cname" { } } +resource "aws_route53_record" "api-cname-test" { + zone_id = data.aws_route53_zone.primary.zone_id + name = "test.${var.domain_name}" + type = "CNAME" + records = [aws_route53_record.api-cname.fqdn] + ttl= 300 +} + +resource "aws_route53_record" "api-cname-dev" { + zone_id = data.aws_route53_zone.primary.zone_id + name = "dev.${var.domain_name}" + type = "CNAME" + records = [aws_route53_record.api-cname.fqdn] + ttl= 300 + +} + output cname { value = aws_route53_record.api-cname.fqdn } From 832898b6c1f978d7bd6a7d166253da276228235c Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sat, 21 Dec 2024 08:48:17 -0500 Subject: [PATCH 070/158] start of the observability --- accounts/swarms/main.tf | 19 +- .../components/launch_template/main.tf | 21 +- .../dev/us-east-1/components/roles/main.tf | 5 + .../dev/us-east-1/main.tf | 7 +- modules/aws/ssm/observability/install.sh | 30 ++ modules/aws/ssm/observability/jobs/main.tf | 251 +++++++++ modules/aws/ssm/observability/main.tf | 496 ++++++++++++++++++ modules/github/actions/aws/policy/main.json | 85 +++ 8 files changed, 905 insertions(+), 9 deletions(-) create mode 100644 modules/aws/ssm/observability/install.sh create mode 100644 modules/aws/ssm/observability/jobs/main.tf create mode 100755 modules/aws/ssm/observability/main.tf create mode 100644 modules/github/actions/aws/policy/main.json diff --git a/accounts/swarms/main.tf b/accounts/swarms/main.tf index 7d846ea..1425cc2 100644 --- a/accounts/swarms/main.tf +++ b/accounts/swarms/main.tf @@ -1,8 +1,7 @@ locals { #ami_name = "ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*" - ami_name = "ubuntu-minimal/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-minimal-*" + ami_name = "ubuntu-minimal/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-minimal-*" # useast2id= dns = "api.swarms.ai" - #dns = "api.arianakrasniqi.com" account = "916723593639" region = "us-east-2" } @@ -16,6 +15,7 @@ variable "owner" { variable "github_token" { description = "GitHub access token used to configure the provider" type = string + default = "" } provider "github" { @@ -48,19 +48,20 @@ output region { } #SLOW -# data "aws_ami" "ami" { +#data "aws_ami" "ami" { # slow # most_recent = true # name_regex = "^${local.ami_name}" # } locals { -ami_id = "ami-0325b9a2dfb474b2d" + us_east_2_swarms_ami_id = "ami-0325b9a2dfb474b2d" + us_east_2_ami_id = "ami-0325b9a2dfb474b2d" } module "swarms_api" { source = "../../environments/swarms-aws-agent-api/dev/us-east-1" domain = local.dns #ami_id = data.aws_ami.ami.id - ami_id = local.ami_id + ami_id = local.us_east_2_swarms_ami_id name = "swarms" @@ -73,7 +74,7 @@ count =0 source = "../../environments/swarms-deploy/dev/us-east-1" domain = local.dns #ami_id = data.aws_ami.ami.id - ami_id = local.ami_id + ami_id = local.us_east_2_swarms_ami_id name = "swarmdeploy" tags = {project="swarmdeploy"} vpc_id = "vpc-0b4cedd083227068d" @@ -107,3 +108,9 @@ module call_swarms { source = "../../environments/call-swarms" } + +module ssm_observer { + source = "../../modules/aws/ssm/observability" + ami_id = local.us_east_2_ami_id +} + diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf index 8a3a5b6..cfa07e6 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf @@ -19,6 +19,10 @@ locals { name = var.name } } +data "aws_ssm_parameter" "cw_agent_config" { + #arn:aws:ssm:us-east-2:916723593639:parameter/cloudwatch-agent/config + name = "/cloudwatch-agent/config" +} resource "aws_launch_template" "ec2_launch_template" { name_prefix = "${var.name}-launch-template-" image_id = var.ami_id @@ -56,7 +60,22 @@ resource "aws_launch_template" "ec2_launch_template" { apt-get install -y --no-install-recommends ca-certificates=20230311 curl=7.88.1-10+deb12u7 | echo oops curl -O "https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/$(dpkg --print-architecture)/latest/amazon-cloudwatch-agent.deb" dpkg -i -E amazon-cloudwatch-agent.deb - + # Install prerequisite packages + apt-get install -y wget unzip systemd + # In case of missing dependencies + # apt-get install -f -y + # Configure and start the CloudWatch agent + /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c "ssm:${data.aws_ssm_parameter.cw_agent_config.name}" + # Enable and start the service using systemctl + systemctl enable amazon-cloudwatch-agent + systemctl start amazon-cloudwatch-agent + + # Clean up downloaded files + rm -f amazon-cloudwatch-agent.deb + # Verify installation + systemctl status amazon-cloudwatch-agent + + if [ ! -d "/opt/swarms/" ]; then git clone https://github.com/jmikedupont2/swarms "/opt/swarms/" fi diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf index e57aca8..2ee73c6 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf @@ -47,6 +47,11 @@ resource "aws_iam_policy" "default" { policy = data.aws_iam_policy_document.default.json } +resource "aws_iam_role_policy_attachment" "AmazonSSMManagedEC2InstanceDefaultPolicy" { + role = join("", aws_iam_role.ssm.*.name) + policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedEC2InstanceDefaultPolicy" +} + resource "aws_iam_role_policy_attachment" "default" { # count = local.policy_only diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index d394938..b40e59a 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -8,6 +8,9 @@ locals { tags = { project="swarms" } + dev_tags = { + sandbox="kye" + } } variable domain {} variable ami_id {} @@ -164,7 +167,7 @@ module "asg_dynamic_new_ami" { module "asg_dynamic_new_ami_test" { # built with packer #count =0 - tags = local.tags + tags = merge(local.tags, local.dev_tags) vpc_id = local.vpc_id image_id = local.new_ami_id ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 @@ -173,7 +176,7 @@ module "asg_dynamic_new_ami_test" { source = "./components/autoscaling_group" # security_group_id = module.security.internal_security_group_id instance_type = each.key - name = "swarms-ami-${each.key}" + name = "test-swarms-ami-${each.key}" launch_template_id = module.lt_dynamic_ami[each.key].launch_template_id target_group_arn = module.alb.test_alb_target_group_arn } diff --git a/modules/aws/ssm/observability/install.sh b/modules/aws/ssm/observability/install.sh new file mode 100644 index 0000000..3c5bc20 --- /dev/null +++ b/modules/aws/ssm/observability/install.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# lets make this bash script into a terraform template +#aws_ssm_parameter_cw_agent_config_name=${aws_ssm_parameter.cw_agent_config.name} +# Update package lists +apt-get update +apt-get upgrade -y + +# Install prerequisite packages +apt-get install -y wget unzip systemd + +# Download and install the CloudWatch agent +wget https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb +dpkg -i -E ./amazon-cloudwatch-agent.deb + +# In case of missing dependencies +apt-get install -f -y + +# Configure and start the CloudWatch agent +/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c "ssm:${aws_ssm_parameter_cw_agent_config_name}" + + +# Enable and start the service using systemctl +systemctl enable amazon-cloudwatch-agent +systemctl start amazon-cloudwatch-agent + +# Clean up downloaded files +rm -f amazon-cloudwatch-agent.deb + +# Verify installation +systemctl status amazon-cloudwatch-agent diff --git a/modules/aws/ssm/observability/jobs/main.tf b/modules/aws/ssm/observability/jobs/main.tf new file mode 100644 index 0000000..cb7a273 --- /dev/null +++ b/modules/aws/ssm/observability/jobs/main.tf @@ -0,0 +1,251 @@ +# S3 Bucket for SSM logs and outputs +resource "aws_s3_bucket" "ssm_logs" { + bucket = "ssm-operation-logs-${data.aws_caller_identity.current.account_id}" +} + +resource "aws_s3_bucket_versioning" "ssm_logs" { + bucket = aws_s3_bucket.ssm_logs.id + versioning_configuration { + status = "Enabled" + } +} + +resource "aws_s3_bucket_lifecycle_configuration" "ssm_logs" { + bucket = aws_s3_bucket.ssm_logs.id + + rule { + id = "cleanup_old_logs" + status = "Enabled" + + transition { + days = 30 + storage_class = "STANDARD_IA" + } + + expiration { + days = 90 + } + } +} + +# SSM Session logging to S3 +resource "aws_s3_bucket_policy" "ssm_logs" { + bucket = aws_s3_bucket.ssm_logs.id + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "SSMBucketPermission" + Effect = "Allow" + Principal = { + Service = "ssm.amazonaws.com" + } + Action = [ + "s3:PutObject", + "s3:PutObjectAcl" + ] + Resource = "${aws_s3_bucket.ssm_logs.arn}/*" + Condition = { + StringEquals = { + "s3:x-amz-acl" = "bucket-owner-full-control" + } + } + } + ] + }) +} + +# CloudWatch Log Group for SSM +resource "aws_cloudwatch_log_group" "ssm_logs" { + name = "/aws/ssm/operations" + retention_in_days = 30 +} + +# X-Ray tracing configuration +resource "aws_xray_sampling_rule" "ssm_tracing" { + rule_name = "SSMOperations" + priority = 1000 + reservoir_size = 1 + fixed_rate = 0.05 + host = "*" + http_method = "*" + url_path = "*" + service_name = "*" + service_type = "*" + version = 1 +} + +# IAM role updates for X-Ray and enhanced logging +resource "aws_iam_role_policy_attachment" "xray_policy" { + role = aws_iam_role.maintenance_window_role.name + policy_arn = "arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess" +} + +# Custom policy for S3 and CloudWatch access +resource "aws_iam_role_policy" "ssm_logging" { + name = "ssm-logging-policy" + role = aws_iam_role.maintenance_window_role.name + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "s3:PutObject", + "s3:GetObject", + "s3:PutObjectAcl" + ] + Resource = "${aws_s3_bucket.ssm_logs.arn}/*" + }, + { + Effect = "Allow" + Action = [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + "logs:DescribeLogStreams" + ] + Resource = "${aws_cloudwatch_log_group.ssm_logs.arn}:*" + } + ] + }) +} + +# Updated SSM Document for Python script execution with X-Ray tracing +resource "aws_ssm_document" "python_with_xray" { + name = "RunPythonWithXRay" + document_type = "Command" + document_format = "YAML" + + content = < /tmp/xray-daemon.json << 'EOF' + { + "Version": 1, + "RecordingName": "SSMOperation", + "RecordingType": "ServiceType", + "SamplingRule": { + "FixedRate": 1, + "ReservoirSize": 5 + } + } + EOF + - curl https://s3.us-east-2.amazonaws.com/aws-xray-assets.us-east-2/xray-daemon/aws-xray-daemon-linux-3.x.zip -o /tmp/xray-daemon.zip + - unzip /tmp/xray-daemon.zip -d /opt/xray + - /opt/xray/xray-daemon -c /tmp/xray-daemon.json & + - action: aws:runShellScript + name: ExecuteScript + inputs: + runCommand: + - | + cat > /tmp/wrapper.py << 'EOF' + import boto3 + from aws_xray_sdk.core import xray_recorder + from aws_xray_sdk.core import patch_all + import os + import sys + import json + + # Initialize X-Ray + xray_recorder.configure( + context_missing='LOG_ERROR', + service='SSMPythonOperation' + ) + patch_all() + + # Start X-Ray segment + segment = xray_recorder.begin_segment('SSMPythonScript') + + try: + # Execute the actual script + with open('/tmp/script.py', 'r') as f: + exec(f.read()) + except Exception as e: + segment.put_annotation('error', str(e)) + raise + finally: + xray_recorder.end_segment() + EOF + - echo "{{ ScriptContent }}" > /tmp/script.py + - python3 /tmp/wrapper.py +DOC +} + +# CloudWatch Dashboard for SSM Operations +resource "aws_cloudwatch_dashboard" "ssm_operations" { + dashboard_name = "SSMOperations" + + dashboard_body = jsonencode({ + widgets = [ + { + type = "metric" + x = 0 + y = 0 + width = 12 + height = 6 + properties = { + metrics = [ + ["AWS/SSM", "CommandSuccess", "DocumentName", "RunPythonWithXRay"], + ["AWS/SSM", "CommandFailed", "DocumentName", "RunPythonWithXRay"] + ] + period = 300 + stat = "Sum" + region = var.aws_region + title = "SSM Command Execution Status" + } + }, + { + type = "log" + x = 0 + y = 6 + width = 12 + height = 6 + properties = { + query = "fields @timestamp, @message | sort @timestamp desc | limit 20" + region = var.aws_region + title = "Recent SSM Operation Logs" + view = "table" + logGroupName = aws_cloudwatch_log_group.ssm_logs.name + } + } + ] + }) +} + +# Get current account ID +data "aws_caller_identity" "current" {} + +# CloudWatch Metric Alarm for Failed Commands +resource "aws_cloudwatch_metric_alarm" "ssm_failures" { + alarm_name = "ssm-command-failures" + comparison_operator = "GreaterThanThreshold" + evaluation_periods = "1" + metric_name = "CommandFailed" + namespace = "AWS/SSM" + period = "300" + statistic = "Sum" + threshold = "0" + alarm_description = "This metric monitors failed SSM commands" + + dimensions = { + DocumentName = aws_ssm_document.python_with_xray.name + } +} \ No newline at end of file diff --git a/modules/aws/ssm/observability/main.tf b/modules/aws/ssm/observability/main.tf new file mode 100755 index 0000000..7020338 --- /dev/null +++ b/modules/aws/ssm/observability/main.tf @@ -0,0 +1,496 @@ +variable ami_id {} +# Previous provider and variables configuration remains the same +#provider "aws" { +# region = var.aws_region +#} + +variable "aws_region" { + description = "AWS region" + type = string + default = "us-east-2" +} + +variable "instance_type" { + description = "EC2 instance type" + type = string + default = "t3.micro" +} + +variable "patch_schedule" { + description = "Cron expression for patch schedule" + type = string + default = "cron(0 0 ? * SUN *)" # Run at midnight every Sunday +} + +# Update EC2 role to include SSM permissions +resource "aws_iam_role" "ec2_monitoring_role" { + name = "ec2-monitoring-role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Principal = { + Service = "ec2.amazonaws.com" + } + } + ] + }) +} + +# Add SSM policy attachments +resource "aws_iam_role_policy_attachment" "ssm_policy" { + role = aws_iam_role.ec2_monitoring_role.name + policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" +} + +resource "aws_iam_role_policy_attachment" "cloudwatch_agent_policy" { + role = aws_iam_role.ec2_monitoring_role.name + policy_arn = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy" +} + +# Previous instance profile configuration remains the same +resource "aws_iam_instance_profile" "monitoring_profile" { + name = "ec2-monitoring-profile" + role = aws_iam_role.ec2_monitoring_role.name +} + +# SSM Patch Baseline +resource "aws_ssm_patch_baseline" "os_patches" { + name = "ec2-patch-baseline" + operating_system = "AMAZON_LINUX_2" + + approval_rule { + approve_after_days = 7 + compliance_level = "HIGH" + + patch_filter { + key = "CLASSIFICATION" + values = ["Security", "Bugfix", "Recommended"] + #valid values are: Security, Bugfix, Enhancement, Recommended, Newpackage + } + + patch_filter { + key = "SEVERITY" + values = ["Critical", "Important"] + } + } + + tags = { + Environment = "Production" + } +} + +# SSM Patch Group +resource "aws_ssm_patch_group" "patch_group" { + baseline_id = aws_ssm_patch_baseline.os_patches.id + patch_group = "production-servers" +} + +# SSM Maintenance Window +resource "aws_ssm_maintenance_window" "patch_window"{ + cutoff = 1 + name = "production-patch-window" + schedule = var.patch_schedule + duration = 4 #"PT4H" # 4 hours + allow_unassociated_targets = false +} + +# Maintenance Window Target +resource "aws_ssm_maintenance_window_target" "patch_target" { + resource_type = "INSTANCE" + window_id = aws_ssm_maintenance_window.patch_window.id + name = "patch-production-servers" + + targets { + key = "tag:PatchGroup" + values = ["production-servers"] + } +} + +# Maintenance Window Task +resource "aws_ssm_maintenance_window_task" "patch_task" { + window_id = aws_ssm_maintenance_window.patch_window.id + task_type = "RUN_COMMAND" + task_arn = "AWS-RunPatchBaseline" + service_role_arn = aws_iam_role.maintenance_window_role.arn + priority = 1 + max_concurrency = "50%" + max_errors = "25%" + + targets { + key = "WindowTargetIds" + values = [aws_ssm_maintenance_window_target.patch_target.id] + } + + task_invocation_parameters { + run_command_parameters { + parameter { + name = "Operation" + values = ["Install"] + } + parameter { + name = "RebootOption" + values = ["RebootIfNeeded"] + } + } + } +} + +# Maintenance Window Role +resource "aws_iam_role" "maintenance_window_role" { + name = "ssm-maintenance-window-role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Principal = { + Service = "ssm.amazonaws.com" + } + } + ] + }) +} + +# Attach required policies for Maintenance Window Role +resource "aws_iam_role_policy_attachment" "maintenance_window_policy" { + role = aws_iam_role.maintenance_window_role.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonSSMMaintenanceWindowRole" +} + +# # Update EC2 instance configuration with patch group tag +# resource "aws_instance" "monitored_instance" { +# ami = var.ami_id +# instance_type = var.instance_type + +# iam_instance_profile = aws_iam_instance_profile.monitoring_profile.name +# monitoring = true + +# user_data = <<-EOF +# #!/bin/bash +# yum update -y +# yum install -y amazon-cloudwatch-agent +# amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c ssm:${aws_ssm_parameter.cw_agent_config.name} +# systemctl start amazon-cloudwatch-agent +# systemctl enable amazon-cloudwatch-agent +# EOF + +# tags = { +# Name = "monitored-instance" +# PatchGroup = "production-servers" +# } +# } + +# Add CloudWatch Event Rule for Patch Compliance Monitoring +resource "aws_cloudwatch_event_rule" "patch_compliance" { + name = "patch-compliance-monitoring" + description = "Monitor patch compliance state changes" + + event_pattern = jsonencode({ + source = ["aws.ssm"] + detail-type = ["Patch Compliance State Change"] + }) +} + +resource "aws_cloudwatch_event_target" "patch_compliance_sns" { + rule = aws_cloudwatch_event_rule.patch_compliance.name + target_id = "PatchComplianceNotification" + arn = aws_sns_topic.patch_notifications.arn +} + +# SNS Topic for Patch Notifications +resource "aws_sns_topic" "patch_notifications" { + name = "patch-compliance-notifications" +} + + +# SSM State Manager association for CloudWatch agent updates +resource "aws_ssm_association" "cloudwatch_agent_update" { + name = "AWS-ConfigureAWSPackage" + + targets { + key = "tag:Monitoring" + values = ["enabled"] + } + + parameters = { + action = "Install" + name = "AmazonCloudWatchAgent" + } +} + +# Updated CloudWatch Agent configuration in SSM Parameter Store +resource "aws_ssm_parameter" "cw_agent_config" { + name = "/cloudwatch-agent/config" + description = "CloudWatch agent configuration" + type = "SecureString" + value = jsonencode({ + agent = { + metrics_collection_interval = 60 + run_as_user = "root" + } + metrics = { + namespace = "CustomEC2Metrics" + metrics_collected = { + cpu = { + resources = ["*"] + measurement = [ + "cpu_usage_idle", + "cpu_usage_user", + "cpu_usage_system", + "cpu_usage_iowait" + ] + totalcpu = true + metrics_collection_interval = 60 + } + mem = { + measurement = [ + "mem_used_percent", + "mem_total", + "mem_used", + "mem_cached", + "mem_buffered" + ] + metrics_collection_interval = 60 + } + disk = { + resources = ["/"] + measurement = [ + "disk_used_percent", + "disk_free", + "disk_total", + "disk_inodes_free", + "disk_inodes_used" + ] + metrics_collection_interval = 60 + } + netstat = { + metrics_collection_interval = 60 + measurement = [ + "tcp_established", + "tcp_time_wait" + ] + } + } + } + logs = { + logs_collected = { + files = { + collect_list = [ + { + file_path = "/var/log/messages" + log_group_name = "/ec2/system" + log_stream_name = "{instance_id}" + timezone = "UTC" + }, + { + file_path = "/var/log/secure" + log_group_name = "/ec2/secure" + log_stream_name = "{instance_id}" + timezone = "UTC" + } + ] + } + } + } + }) +} + +# SSM command to update CloudWatch configuration on running instances +resource "aws_ssm_association" "update_cloudwatch_config" { + name = "AWS-RunShellScript" + + targets { + key = "tag:Monitoring" + values = ["enabled"] + } + + parameters = { + commands = "amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c ssm:${aws_ssm_parameter.cw_agent_config.name}\n systemctl restart amazon-cloudwatch-agent" + + } +} + +# CloudWatch Log Groups for collected logs +resource "aws_cloudwatch_log_group" "system_logs" { + name = "/ec2/system" + retention_in_days = 30 +} + +resource "aws_cloudwatch_log_group" "secure_logs" { + name = "/ec2/secure" + retention_in_days = 30 +} + +# SSM Document for CloudWatch agent troubleshooting +resource "aws_ssm_document" "cloudwatch_agent_troubleshoot" { + name = "TroubleshootCloudWatchAgent" + document_type = "Command" + document_format = "YAML" + + content = < Date: Sat, 21 Dec 2024 10:23:25 -0500 Subject: [PATCH 071/158] applied --- accounts/swarms/.terraform.lock.hcl | 26 +++++++++++++------------- accounts/swarms/main.tf | 4 ++++ 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/accounts/swarms/.terraform.lock.hcl b/accounts/swarms/.terraform.lock.hcl index b7e11c0..c8c6a16 100644 --- a/accounts/swarms/.terraform.lock.hcl +++ b/accounts/swarms/.terraform.lock.hcl @@ -2,20 +2,20 @@ # Manual edits may be lost in future updates. provider "registry.opentofu.org/hashicorp/aws" { - version = "5.81.0" - constraints = ">= 3.29.0, >= 4.40.0, >= 5.55.0, >= 5.73.0" + version = "5.82.2" + constraints = ">= 3.29.0, >= 4.6.0, >= 4.40.0, >= 5.46.0, >= 5.55.0, >= 5.73.0" hashes = [ - "h1:ird967uf44WBZ0u9rMlOdO4nCKJZMlUknLcD3lz4dWs=", - "zh:0d0c7cc1e16b16cd00ab36de35038162e1871b51a902e9016d08c55c9fc4de35", - "zh:0e4b8c6e46999015292ab4fb9a430ab00107a276f25243552cde66db53c58661", - "zh:30041314cdd4e877d75ee8c9d36aecfca094e276f7a3d8150f929cf5169b2fa5", - "zh:5ebd248ce3e5a7ef9cc2f41499668f182146325e10ea305c70469122f6161a13", - "zh:888a69d371373b418549e03f5922becb2b8074cb463552ecfa65f30455708db0", - "zh:8a21bb7fe60383ff5ca9db8630a287e41fd520d2514c08874a16dc74500fadd7", - "zh:9c4663dcbfe58544642d70ebfdc6c5fa91592ff04164f77c655e32e6024483e2", - "zh:b322873f1209443a8407d5f782d7d917de6a1391239dbd0e7f809ce6507bed76", - "zh:b7c9d5ca14b818b5932ac58a490646a425ebc41b33d149090aa5f48d1ca35c99", - "zh:e76cd202b03749f3082b0cbe849fd2e731cf3f9a6aa994d2d629602c3aede36c", + "h1:65B6GjFmKobkuaO58AcDPUu/otnTR7qRTynWTgU5A7Q=", + "zh:0c662b1f27119941e2329155070c8a81f6979e3d5e98296949f373c46487d84e", + "zh:56cbaeaf050174bb9ba9a00f31fe441365ffece6ad39254885c9d4e6517e27e3", + "zh:6e7f66c9f9955deb9ce0a2ae4b6df366b3b6bc3e23bca1fcdd4351ecab3ec907", + "zh:7971a22df020391ca20936584282d01f3380bcfc79260afbdf04d6b4d50cf75a", + "zh:7b8f2175447e64c9a9c9624818a597cd6c1609f379f7f8e3e179ee7394a467b7", + "zh:a805e26eb7f58e46b130decab79c0871dfd73206d7bd6c1793e27c2dc35fd8b6", + "zh:b6163e746c77d47269c0e3014bdc1f805152b3d94471dd6443373d91b848cf45", + "zh:c46872fd0c304852bd506c036f31dbe047a97036992319a3c594949538651680", + "zh:d586f1ef412f336ede54d720a78f11a64952a39bfe401e141d24826530cb2449", + "zh:ddaf2e1de0f1660ac31c5f1bebd40cc2b29e19f64bf41591c3b5df00dc268367", ] } diff --git a/accounts/swarms/main.tf b/accounts/swarms/main.tf index 1425cc2..dffee1e 100644 --- a/accounts/swarms/main.tf +++ b/accounts/swarms/main.tf @@ -114,3 +114,7 @@ module ssm_observer { ami_id = local.us_east_2_ami_id } +module ssm_setup { + source = "../../modules/aws/ssm/setup" +} + From 08e85bdeef940d713241848e04773cf08247aae7 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sat, 21 Dec 2024 10:31:11 -0500 Subject: [PATCH 072/158] allow decrypt --- .../dev/us-east-1/components/roles/main.tf | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf index 2ee73c6..21a9c67 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf @@ -28,6 +28,12 @@ data "aws_iam_policy_document" "default" { effect = "Allow" } + statement { + actions = ["kms:Decrypt"] + resources = [ "arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590" ] + effect = "Allow" + } + # statement { # actions = ["${var.ssm_actions}"] # resources = ["${formatlist("arn:aws:ssm:%s:%s:parameter/%s", var.region, var.account_id, var.ssm_parameters)}"] @@ -53,6 +59,8 @@ resource "aws_iam_role_policy_attachment" "AmazonSSMManagedEC2InstanceDefaultPol } + + resource "aws_iam_role_policy_attachment" "default" { # count = local.policy_only role = join("", aws_iam_role.ssm.*.name) From 06b9a22461910b2280167998f199bccb2b231003 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sat, 21 Dec 2024 10:36:46 -0500 Subject: [PATCH 073/158] wip --- .../dev/us-east-1/components/roles/main.tf | 40 +------------------ 1 file changed, 1 insertion(+), 39 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf index 21a9c67..eca0441 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf @@ -1,25 +1,4 @@ variable tags {} -# data "aws_iam_policy_document" "assume_role" { -# statement { -# effect = "Allow" -# actions = ["sts:AssumeRole"] - -# principals { -# type = "Service" -# identifiers = ["ec2.amazonaws.com"] -# } -# } - -# statement { -# effect = "Allow" -# actions = ["sts:AssumeRole"] - -# principals { -# type = "AWS" -# identifiers = ["${var.assume_role_arns}"] -# } -# } -# } data "aws_iam_policy_document" "default" { statement { @@ -40,11 +19,6 @@ data "aws_iam_policy_document" "default" { # effect = "Allow" # } - # statement { - # actions = ["kms:Decrypt"] - # resources = ["${data.aws_kms_key.default.arn}"] - # effect = "Allow" - # } } resource "aws_iam_policy" "default" { @@ -58,9 +32,6 @@ resource "aws_iam_role_policy_attachment" "AmazonSSMManagedEC2InstanceDefaultPol policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedEC2InstanceDefaultPolicy" } - - - resource "aws_iam_role_policy_attachment" "default" { # count = local.policy_only role = join("", aws_iam_role.ssm.*.name) @@ -72,20 +43,11 @@ resource "aws_iam_role_policy_attachment" "SSM-role-policy-attach" { policy_arn = data.aws_iam_policy.AmazonSSMManagedInstanceCore.arn } - -#### -# resource "aws_iam_role" "default" { -# # count = local.policy_only - -# name = "swarms-ssm" -# assume_role_policy = join("", data.aws_iam_policy_document.assume_role.*.json) -# description = "IAM Role with permissions to perform actions on SSM resources" -# max_session_duration = var.max_session_duration -# } data "aws_iam_policy" "AmazonSSMManagedInstanceCore" { arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" } + resource "aws_iam_role" "ssm" { name = "ssm-swarms-role" tags = var.tags From 3c23717295f1c05a8e757781f0b0c5767565f6ac Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sat, 21 Dec 2024 15:34:37 -0500 Subject: [PATCH 074/158] work in progress --- actions/Makefile | 2 + actions/call_swarms.sh | 9 +- .../dev/us-east-1/components/roles/main.tf | 48 +++++++++++ modules/aws/ssm/setup/main.tf | 85 +++++++++++++++++++ 4 files changed, 141 insertions(+), 3 deletions(-) create mode 100644 actions/Makefile create mode 100644 modules/aws/ssm/setup/main.tf diff --git a/actions/Makefile b/actions/Makefile new file mode 100644 index 0000000..838d5de --- /dev/null +++ b/actions/Makefile @@ -0,0 +1,2 @@ +test: + AWS_PROFILE=swarms bash ./call_swarms.sh diff --git a/actions/call_swarms.sh b/actions/call_swarms.sh index 70778ff..f91c765 100644 --- a/actions/call_swarms.sh +++ b/actions/call_swarms.sh @@ -5,8 +5,8 @@ set -x export REGION="${REGION:-us-east-2}" #export AWS_PROFILE="${AWS_PROFILE:-swarms}" #export AWS_PROFILE="${AWS_PROFILE}" only needed for testing locally -TAG_KEY="${TAG_KEY:-sandbox}" -TAG_VALUE="${TAG_VALUE:-kye}" +TAG_KEY="${TAG_KEY:-Name}" +TAG_VALUE="${TAG_VALUE:-test-swarms-ami-t3.medium}" GIT_URL="${GIT_URL:-https://github.com/kyegomez/swarms}" export GIT_NAME="${GIT_NAME:-kye}" export GIT_VERSION="${GIT_VERSION:-master}" @@ -37,7 +37,10 @@ send_command() { --timeout-seconds $TIMEOUT_SECONDS \ --max-concurrency "$MAX_CONCURRENCY" \ --max-errors "$MAX_ERRORS" \ - --region $REGION + --region $REGION \ + --output-s3-bucket-name "swarms-session-logs-20241221151754799300000003" \ + --cloud-watch-output-config '{"CloudWatchOutputEnabled":true,"CloudWatchLogGroupName":"/ssm/session-logs-20241221151803393300000006"}' + } # Function to fetch command output diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf index eca0441..6ed2e7e 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf @@ -12,6 +12,54 @@ data "aws_iam_policy_document" "default" { resources = [ "arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590" ] effect = "Allow" } + + { + "Effect": "Allow", + "Principal": { + "Service": "logs.us-east-1.amazonaws.com" + }, + "Action": [ + "kms:Encrypt*", + "kms:Decrypt*", + "kms:ReEncrypt*", + "kms:GenerateDataKey*", + "kms:Describe*" + ], + "Resource": "*", + "Condition": { + "ArnLike": { + "kms:EncryptionContext:aws:logs:arn": "arn:aws:logs:us-east-1:xxxxx:log-group:SSM" + } + } +}, +{ + "Effect": "Allow", + "Principal": { + "Service": "ssm.amazonaws.com" + }, + "Action": [ + "kms:Encrypt*", + "kms:Decrypt*", + "kms:ReEncrypt*", + "kms:GenerateDataKey*", + "kms:Describe*" + ], + "Resource": "*" +}, +{ + "Effect": "Allow", + "Action": [ + "kms:Encrypt*", + "kms:Decrypt*", + "kms:ReEncrypt*", + "kms:GenerateDataKey*", + "kms:Describe*" + ], + "Resource": "*", + "Principal": { + "AWS": "arn:aws:iam::xxxxx:role/SSMRole" + } +} # statement { # actions = ["${var.ssm_actions}"] diff --git a/modules/aws/ssm/setup/main.tf b/modules/aws/ssm/setup/main.tf new file mode 100644 index 0000000..3ba4fef --- /dev/null +++ b/modules/aws/ssm/setup/main.tf @@ -0,0 +1,85 @@ +module "ssm" { +# source = "bridgecrewio/session-manager/aws" + # version = "0.4.2" + source = "git::https://github.com/jmikedupont2/terraform-aws-session-manager.git?ref=master" + bucket_name = "swarms-session-logs" + access_log_bucket_name = "swarms-session-access-logs" + enable_log_to_s3 = true + enable_log_to_cloudwatch = true + tags = {project="swarms"} + #linux_shell_profile = "date" +} + +#https://github.com/gazoakley/terraform-aws-session-manager-settings + + +resource "aws_cloudwatch_log_group" "app_signals" { + for_each = toset(["ec2","eks","generic","k8s","data"]) + name = "/aws/appsignals/${each.key}" + retention_in_days = 30 + kms_key_id = "arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590" +} + +# module "session-manager-settings" { +# source = "gazoakley/session-manager-settings/aws" +# s3_bucket_name = "swarms-session-logs-bucket" +# cloudwatch_log_group_name = "/ssm/swarms-session-logs" +# #kms_key_id = arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590 +# kms_key_id = "cc8e1ee7-a05b-4642-bd81-ba5548635590" +# } +# Configure the AWS provider +# provider "aws" { +# region = "us-east-2" # Matching the region from your CloudTrail event +# } + +# # Create or update the S3 bucket with tags +# resource "aws_s3_bucket" "session_logs" { +# bucket = "swarms-session-logs-20241221151754799300000003" + +# # Force destroy can be set to true if you want to allow Terraform to delete the bucket even if it contains objects +# force_destroy = false + +# tags = { +# Environment = "Production" # Example tag +# Project = "Swarms" # Example tag +# Created = "2024-12-21" # Example tag +# } +# } + +# # Add bucket versioning (recommended for logging buckets) +# resource "aws_s3_bucket_versioning" "session_logs_versioning" { +# bucket = aws_s3_bucket.session_logs.id +# versioning_configuration { +# status = "Enabled" +# } +# } + +# # Add bucket encryption (recommended) +# resource "aws_s3_bucket_server_side_encryption_configuration" "session_logs_encryption" { +# bucket = aws_s3_bucket.session_logs.id + +# rule { +# apply_server_side_encryption_by_default { +# sse_algorithm = "AES256" +# } +# } +# } + +# # Add lifecycle rules (optional, but recommended for log buckets) +# resource "aws_s3_bucket_lifecycle_configuration" "session_logs_lifecycle" { +# bucket = aws_s3_bucket.session_logs.id + +# rule { +# id = "log_retention" +# status = "Enabled" + +# transition { +# days = 30 +# storage_class = "STANDARD_IA" +# } + +# expiration { +# days = 90 # Adjust retention period as needed +# } +# } +# } From dd2b6fdb6a69082b3164b21458c5d7acf57e9d6b Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sat, 21 Dec 2024 16:48:56 -0500 Subject: [PATCH 075/158] call swarms --- .../dev/us-east-1/components/roles/main.tf | 100 ++++++++++-------- logs/Makefile | 2 + logs/analyse.sh | 6 ++ logs/getlogs.sh | 43 ++++++++ modules/aws/ssm/setup/main.tf | 9 +- 5 files changed, 112 insertions(+), 48 deletions(-) create mode 100644 logs/Makefile create mode 100644 logs/analyse.sh create mode 100644 logs/getlogs.sh diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf index 6ed2e7e..ef6ec8f 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf @@ -13,53 +13,59 @@ data "aws_iam_policy_document" "default" { effect = "Allow" } - { - "Effect": "Allow", - "Principal": { - "Service": "logs.us-east-1.amazonaws.com" - }, - "Action": [ - "kms:Encrypt*", - "kms:Decrypt*", - "kms:ReEncrypt*", - "kms:GenerateDataKey*", - "kms:Describe*" - ], - "Resource": "*", - "Condition": { - "ArnLike": { - "kms:EncryptionContext:aws:logs:arn": "arn:aws:logs:us-east-1:xxxxx:log-group:SSM" - } - } -}, -{ - "Effect": "Allow", - "Principal": { - "Service": "ssm.amazonaws.com" - }, - "Action": [ - "kms:Encrypt*", - "kms:Decrypt*", - "kms:ReEncrypt*", - "kms:GenerateDataKey*", - "kms:Describe*" - ], - "Resource": "*" -}, -{ - "Effect": "Allow", - "Action": [ - "kms:Encrypt*", - "kms:Decrypt*", - "kms:ReEncrypt*", - "kms:GenerateDataKey*", - "kms:Describe*" - ], - "Resource": "*", - "Principal": { - "AWS": "arn:aws:iam::xxxxx:role/SSMRole" - } -} + statement { + actions = [ + "logs:DescribeLogGroups", + "logs:DescribeLogStreams", + "logs:CreateLogGroup", + "logs:CreateLogStream", + ] + resources = [ "*" ] + effect = "Allow" + } + + statement { + effect = "Allow" + resources = [ "arn:aws:s3:::swarms-session-logs*" ] + actions = [ + "s3:GetEncryptionConfiguration" + ] + } + + statement { + effect = "Allow" + resources = [ "*" ] + actions = [ + "kms:Encrypt*", + "kms:Decrypt*", + "kms:ReEncrypt*", + "kms:GenerateDataKey*", + "kms:Describe*" + ] + } + + # statement { + # sid = "Allow CloudWatch access" + # effect = "Allow" + # principals { + # type = "Service" + # identifiers = ["logs.us-east-2.amazonaws.com"] + # } + # actions = [ + # "kms:Encrypt*", + # "kms:Decrypt*", + # "kms:ReEncrypt*", + # "kms:GenerateDataKey*", + # "kms:Describe*" + # ] + # condition { + # test = "ArnLike" + # values = ["arn:aws:logs:region:${data.aws_caller_identity.current.account_id}:*"] + # variable = "kms:EncryptionContext:aws:logs:arn" + # } + # } + + #arn:aws:logs:us-east-2:916723593639:log-group::log-stream # statement { # actions = ["${var.ssm_actions}"] diff --git a/logs/Makefile b/logs/Makefile new file mode 100644 index 0000000..d72a5ef --- /dev/null +++ b/logs/Makefile @@ -0,0 +1,2 @@ +logs1: + bash ./getlogs.sh diff --git a/logs/analyse.sh b/logs/analyse.sh new file mode 100644 index 0000000..1132f2e --- /dev/null +++ b/logs/analyse.sh @@ -0,0 +1,6 @@ +#!/bin/bash +echo "Bash version ${BASH_VERSION}..." +for i in {1..100} +do + grep error errors.txt | grep "does not exist for account"| cut -d: -f${i} | sort | uniq -c | sort -n +done diff --git a/logs/getlogs.sh b/logs/getlogs.sh new file mode 100644 index 0000000..62ff606 --- /dev/null +++ b/logs/getlogs.sh @@ -0,0 +1,43 @@ + + +# [--lookup-attributes ] + +# [--end-time ] +# [--event-category ] +# [--cli-input-json ] +# [--starting-token ] +# [--page-size ] +# [--max-items ] +# [--generate-cli-skeleton ] +# [--debug] +# [--endpoint-url ] +# [--no-verify-ssl] +# [--no-paginate] +# [--output ] +# [--query ] +# [--profile ] +# [--region ] +# [--version ] +# [--color ] +# [--no-sign-request] +# [--ca-bundle ] +# [--cli-read-timeout ] +# [--cli-connect-timeout ] + + +# Get today's date in YYYYMMDDHHMM format +#LOG_DATE=$(date -d "today" +"%Y%m%d%H%M") + +# Find the latest log file +LATEST_LOG=$(ls logs/*.log 2>/dev/null | tail -n 1) + +echo LATEST_LOG $LATEST_LOG + + +FILENAME=$(ls -t logs/*.log | head -1) +# Extract the start time of the latest log file +NEW_DATE=$(date -u -r ${FILENAME} +%FT%TZ) +echo NEW_DATE $NEW_DATE + +# now look for the latest logs/$(date -d "today" +"%Y%m%d%H%M").log files and get the start time of them and use that -10 seconds to start the new one, we want all events newer than our latest +aws cloudtrail lookup-events --profile swarms --region us-east-2 --max-items 1000 --start-time $NEW_DATE --output json >> logs/$(date -d "today" +"%Y%m%d%H%M").log diff --git a/modules/aws/ssm/setup/main.tf b/modules/aws/ssm/setup/main.tf index 3ba4fef..18a9f85 100644 --- a/modules/aws/ssm/setup/main.tf +++ b/modules/aws/ssm/setup/main.tf @@ -14,12 +14,19 @@ module "ssm" { resource "aws_cloudwatch_log_group" "app_signals" { - for_each = toset(["ec2","eks","generic","k8s","data"]) + for_each = toset(["ec2","eks","generic","k8s"]) name = "/aws/appsignals/${each.key}" retention_in_days = 30 kms_key_id = "arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590" } +resource "aws_cloudwatch_log_group" "app_signals2" { + for_each = toset(["data"]) + name = "/aws/application-signals/${each.key}" + retention_in_days = 30 + kms_key_id = "arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590" +} + # module "session-manager-settings" { # source = "gazoakley/session-manager-settings/aws" # s3_bucket_name = "swarms-session-logs-bucket" From c849407fd9b292f25c086c03c887127581834e3e Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sat, 21 Dec 2024 16:50:36 -0500 Subject: [PATCH 076/158] gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 0f90f0f..54867ed 100644 --- a/.gitignore +++ b/.gitignore @@ -88,3 +88,4 @@ flycheck_*.el # network security /network-security.data +/logs/logs/ From 45427cdf60ede83a363c059b8632edf866fbf9dd Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sat, 21 Dec 2024 18:33:35 -0500 Subject: [PATCH 077/158] fixing more perms --- accounts/swarms/main.tf | 31 +++++++------- .../components/launch_template/main.tf | 7 +++- .../dev/us-east-1/components/roles/main.tf | 3 ++ .../dev/us-east-1/main.tf | 2 + modules/aws/ssm/observability/main.tf | 41 +++++++++++++++++++ 5 files changed, 67 insertions(+), 17 deletions(-) diff --git a/accounts/swarms/main.tf b/accounts/swarms/main.tf index dffee1e..d279a65 100644 --- a/accounts/swarms/main.tf +++ b/accounts/swarms/main.tf @@ -31,21 +31,21 @@ provider aws { region = "us-east-2" profile = "swarms" } -output dns { - value = local.dns -} +#output dns { +# value = local.dns +#} -output profile { - value = "swarms" -} +#output profile { +# value = "swarms" +#} -output account { - value = "916723593639" -} +#output account { +# value = "916723593639" +#} -output region { - value = "us-east-2" -} +#output region { +# value = "us-east-2" +#} #SLOW #data "aws_ami" "ami" { # slow @@ -83,10 +83,9 @@ count =0 ssm_profile_name = "ssm-swarms-profile" } -output api { - value = module.swarms_api -} - +#output api { +# value = module.swarms_api +#} # setup the github tokens diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf index cfa07e6..6ee21ed 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf @@ -1,3 +1,7 @@ +variable branch { +# default = "feature/cloudwatch" +} + variable install_script {} variable iam_instance_profile_name {} variable security_group_id {} @@ -23,6 +27,7 @@ data "aws_ssm_parameter" "cw_agent_config" { #arn:aws:ssm:us-east-2:916723593639:parameter/cloudwatch-agent/config name = "/cloudwatch-agent/config" } +# defined resource "aws_launch_template" "ec2_launch_template" { name_prefix = "${var.name}-launch-template-" image_id = var.ami_id @@ -80,7 +85,7 @@ resource "aws_launch_template" "ec2_launch_template" { git clone https://github.com/jmikedupont2/swarms "/opt/swarms/" fi cd "/opt/swarms/" || exit 1 - export BRANCH=feature/ec2 + export BRANCH=${var.branch} git stash git checkout --force $BRANCH git pull # get the latest version diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf index ef6ec8f..83f140b 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf @@ -19,6 +19,9 @@ data "aws_iam_policy_document" "default" { "logs:DescribeLogStreams", "logs:CreateLogGroup", "logs:CreateLogStream", + "logs:PutLogEvents", + "logs:PutLogEventsBatch", + "cloudwatch:PutMetricData", ] resources = [ "*" ] effect = "Allow" diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index b40e59a..8b1fa0b 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -80,6 +80,7 @@ module "roles" { module "lt_dynamic" { vpc_id = local.vpc_id + branch = "feature/ec2" for_each = toset(var.instance_types) instance_type = each.key name = "swarms-size-${each.key}" @@ -93,6 +94,7 @@ module "lt_dynamic" { } module "lt_dynamic_ami" { + branch = "feature/cloudwatch" vpc_id = local.vpc_id for_each = toset(var.instance_types) instance_type = each.key diff --git a/modules/aws/ssm/observability/main.tf b/modules/aws/ssm/observability/main.tf index 7020338..f672e09 100755 --- a/modules/aws/ssm/observability/main.tf +++ b/modules/aws/ssm/observability/main.tf @@ -279,6 +279,8 @@ resource "aws_ssm_parameter" "cw_agent_config" { } } logs = { + log_stream_name="logs" + force_flush_interval=60 logs_collected = { files = { collect_list = [ @@ -288,6 +290,33 @@ resource "aws_ssm_parameter" "cw_agent_config" { log_stream_name = "{instance_id}" timezone = "UTC" }, + + # nginx + { + file_path = "/var/log/nginx/swarms/access.log" + log_group_name = "/swarms/ngnix_access" + log_stream_name = "{instance_id}" + timezone = "UTC" + }, + { + file_path = "/var/log/nginx/swarms/error.log" + log_group_name = "/swarms/nginx_error" + log_stream_name = "{instance_id}" + timezone = "UTC" + }, + + { + file_path = "/var/log/cloud-init-output.log" + log_group_name = "/ec2/init" + log_stream_name = "{instance_id}" + timezone = "UTC" + }, + { + file_path = "/var/log/swarms_systemd.log" + log_group_name = "/swarms/systemd" + log_stream_name = "{instance_id}" + timezone = "UTC" + }, { file_path = "/var/log/secure" log_group_name = "/ec2/secure" @@ -316,15 +345,27 @@ resource "aws_ssm_association" "update_cloudwatch_config" { } } +resource "aws_cloudwatch_log_group" "log_groups" { + for_each = toset([ + "/swarms/ngnix_access", + "/swarms/nginx_error", + "/swarms/systemd", "/ec2/init" ]) + name = each.key + retention_in_days = 30 + kms_key_id = "arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590" +} + # CloudWatch Log Groups for collected logs resource "aws_cloudwatch_log_group" "system_logs" { name = "/ec2/system" retention_in_days = 30 + kms_key_id = "arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590" } resource "aws_cloudwatch_log_group" "secure_logs" { name = "/ec2/secure" retention_in_days = 30 + kms_key_id = "arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590" } # SSM Document for CloudWatch agent troubleshooting From fcbecca955fe0e49e2268bb40d6093680fc90d94 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sat, 21 Dec 2024 20:19:37 -0500 Subject: [PATCH 078/158] wip --- .../swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf index 83f140b..0885274 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf @@ -22,6 +22,7 @@ data "aws_iam_policy_document" "default" { "logs:PutLogEvents", "logs:PutLogEventsBatch", "cloudwatch:PutMetricData", + "ec2:DescribeTags", ] resources = [ "*" ] effect = "Allow" From d0b08dafec4f2a68c0fb6a87358d9866d763aa3f Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sat, 21 Dec 2024 20:19:56 -0500 Subject: [PATCH 079/158] report --- logs/report.sh | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 logs/report.sh diff --git a/logs/report.sh b/logs/report.sh new file mode 100644 index 0000000..2bb39d9 --- /dev/null +++ b/logs/report.sh @@ -0,0 +1,8 @@ +# get the lines +gron ./logs/*.log | grep -i error > report.txt + +# now we decode +gron -u < report.txt | jq -r ".Events[]|.CloudTrailEvent" | jq | sort | uniq -c | sort -n + + +#gron -u < report.txt | jq -r ".Events[]|.CloudTrailEvent" | jq From 1cb80d1f358c34ad1090026052f156f6f9ca2104 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sun, 22 Dec 2024 04:10:00 -0500 Subject: [PATCH 080/158] adding branch name --- accounts/swarms/main.tf | 6 +- .../components/launch_template/main.tf | 73 ++++++++++--------- .../dev/us-east-1/main.tf | 52 +++++++++---- 3 files changed, 80 insertions(+), 51 deletions(-) diff --git a/accounts/swarms/main.tf b/accounts/swarms/main.tf index d279a65..b526c59 100644 --- a/accounts/swarms/main.tf +++ b/accounts/swarms/main.tf @@ -62,7 +62,7 @@ module "swarms_api" { domain = local.dns #ami_id = data.aws_ami.ami.id ami_id = local.us_east_2_swarms_ami_id - + name = "swarms" tags = {project="swarms"} @@ -117,3 +117,7 @@ module ssm_setup { source = "../../modules/aws/ssm/setup" } +output user_data_new { + value = module.swarms_api.user_data_new +} + diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf index 6ee21ed..059d78b 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf @@ -22,40 +22,7 @@ locals { instance_type = var.instance_type name = var.name } -} -data "aws_ssm_parameter" "cw_agent_config" { - #arn:aws:ssm:us-east-2:916723593639:parameter/cloudwatch-agent/config - name = "/cloudwatch-agent/config" -} -# defined -resource "aws_launch_template" "ec2_launch_template" { - name_prefix = "${var.name}-launch-template-" - image_id = var.ami_id - key_name = var.key_name - instance_type = var.instance_type - network_interfaces { - associate_public_ip_address = true - delete_on_termination = true - security_groups = [var.security_group_id] - } - - iam_instance_profile { - # iam_instance_profile_arn = aws_iam_instance_profile.ssm.arn - name = var.iam_instance_profile_name #aws_iam_instance_profile.ec2_instance_profile.name - } - lifecycle { - create_before_destroy = true - } - block_device_mappings { - device_name = "/dev/sda1" - ebs { - volume_size = 30 - volume_type = "gp3" - encrypted = true - } - } - - user_data = base64encode(<<-EOF + user_data = <<-EOF #!/bin/bash export HOME=/root apt update @@ -91,7 +58,40 @@ resource "aws_launch_template" "ec2_launch_template" { git pull # get the latest version bash -x ${var.install_script} EOF - ) + +} +data "aws_ssm_parameter" "cw_agent_config" { + #arn:aws:ssm:us-east-2:916723593639:parameter/cloudwatch-agent/config + name = "/cloudwatch-agent/config" +} +# defined +resource "aws_launch_template" "ec2_launch_template" { + name_prefix = "${var.name}-launch-template-" + image_id = var.ami_id + key_name = var.key_name + instance_type = var.instance_type + network_interfaces { + associate_public_ip_address = true + delete_on_termination = true + security_groups = [var.security_group_id] + } + + iam_instance_profile { + # iam_instance_profile_arn = aws_iam_instance_profile.ssm.arn + name = var.iam_instance_profile_name #aws_iam_instance_profile.ec2_instance_profile.name + } + lifecycle { + create_before_destroy = true + } + block_device_mappings { + device_name = "/dev/sda1" + ebs { + volume_size = 30 + volume_type = "gp3" + encrypted = true + } + } + user_data = base64encode(local.user_data) tags = var.tags } @@ -102,3 +102,6 @@ output "lt" { output "launch_template_id" { value = resource.aws_launch_template.ec2_launch_template.id } +output "user_data" { + value = local.user_data +} diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index 8b1fa0b..1282cb8 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -78,22 +78,38 @@ module "roles" { tags = local.tags } -module "lt_dynamic" { - vpc_id = local.vpc_id +# module "lt_dynamic" { +# vpc_id = local.vpc_id +# branch = "feature/ec2" +# for_each = toset(var.instance_types) +# instance_type = each.key +# name = "swarms-size-${each.key}" +# security_group_id = module.security.internal_security_group_id +# ami_id = var.ami_id +# tags= local.tags +# source = "./components/launch_template" +# iam_instance_profile_name = module.roles.ssm_profile_name +# #aws_iam_instance_profile.ssm.name +# install_script = "/opt/swarms/api/install.sh" +# } + +module "lt_dynamic_ami_prod" { + #branch = "feature/cloudwatch" branch = "feature/ec2" + vpc_id = local.vpc_id for_each = toset(var.instance_types) instance_type = each.key - name = "swarms-size-${each.key}" + name = "swarms-ami-${each.key}" security_group_id = module.security.internal_security_group_id - ami_id = var.ami_id + ami_id = local.new_ami_id tags= local.tags source = "./components/launch_template" iam_instance_profile_name = module.roles.ssm_profile_name #aws_iam_instance_profile.ssm.name - install_script = "/opt/swarms/api/install.sh" + install_script = "/opt/swarms/api/just_run.sh" } -module "lt_dynamic_ami" { +module "lt_dynamic_ami_test" { branch = "feature/cloudwatch" vpc_id = local.vpc_id for_each = toset(var.instance_types) @@ -108,13 +124,6 @@ module "lt_dynamic_ami" { install_script = "/opt/swarms/api/just_run.sh" } -output security_group_id { - value = module.security.security_group_id -} - -output vpc { - value = module.vpc -} module "alb" { @@ -162,7 +171,7 @@ module "asg_dynamic_new_ami" { # security_group_id = module.security.internal_security_group_id instance_type = each.key name = "swarms-ami-${each.key}" - launch_template_id = module.lt_dynamic_ami[each.key].launch_template_id + launch_template_id = module.lt_dynamic_ami_prod[each.key].launch_template_id target_group_arn = module.alb.alb_target_group_arn } @@ -179,6 +188,19 @@ module "asg_dynamic_new_ami_test" { # security_group_id = module.security.internal_security_group_id instance_type = each.key name = "test-swarms-ami-${each.key}" - launch_template_id = module.lt_dynamic_ami[each.key].launch_template_id + launch_template_id = module.lt_dynamic_ami_test[each.key].launch_template_id target_group_arn = module.alb.test_alb_target_group_arn } + +output security_group_id { + value = module.security.security_group_id +} + +output vpc { + value = module.vpc +} + + +output user_data_new { + value = module.lt_dynamic_ami_test["t3.medium"].user_data +} From 151c3e8905fadfc38c47bcdee17b08352ce43410 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sun, 22 Dec 2024 05:47:10 -0500 Subject: [PATCH 081/158] now fixing the logging config --- accounts/swarms/main.tf | 2 ++ .../components/launch_template/main.tf | 21 +++++------ .../dev/us-east-1/main.tf | 35 ++++++++++++------- 3 files changed, 32 insertions(+), 26 deletions(-) diff --git a/accounts/swarms/main.tf b/accounts/swarms/main.tf index b526c59..68eb48a 100644 --- a/accounts/swarms/main.tf +++ b/accounts/swarms/main.tf @@ -58,6 +58,8 @@ locals { } module "swarms_api" { + aws_account_id = local.account + region = local.region source = "../../environments/swarms-aws-agent-api/dev/us-east-1" domain = local.dns #ami_id = data.aws_ami.ami.id diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf index 059d78b..4c14176 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf @@ -1,19 +1,13 @@ -variable branch { -# default = "feature/cloudwatch" -} - +variable ssm_parameter_name_cw_agent_config{} +variable branch {} variable install_script {} variable iam_instance_profile_name {} variable security_group_id {} variable name {} -variable vpc_id {} -variable ami_id {} -variable tags {} -variable key_name { - default = "mdupont-deployer-key" -} - -# dont use this +variable vpc_id {} +variable ami_id {} +variable tags {} +variable key_name {} variable instance_type {} locals { @@ -62,7 +56,8 @@ locals { } data "aws_ssm_parameter" "cw_agent_config" { #arn:aws:ssm:us-east-2:916723593639:parameter/cloudwatch-agent/config - name = "/cloudwatch-agent/config" + name = var.ssm_parameter_name_cw_agent_config + #"/cloudwatch-agent/config" } # defined resource "aws_launch_template" "ec2_launch_template" { diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index 1282cb8..f1feea0 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -1,9 +1,12 @@ +variable region {} +variable key_name { + default = "mdupont-deployer-key" # FIXME: move to settings +} locals { # instance_type = "t3.large" # instance_type = "t3.medium" ami_name = "ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*" name = "swarms" - region = "us-east-2" domain = var.domain tags = { project="swarms" @@ -13,6 +16,7 @@ locals { } } variable domain {} +variable aws_account_id {} variable ami_id {} variable tags {} variable name {} @@ -75,7 +79,7 @@ variable "instance_types" { module "roles" { source = "./components/roles" - tags = local.tags + tags = local.tags } # module "lt_dynamic" { @@ -94,19 +98,21 @@ module "roles" { # } module "lt_dynamic_ami_prod" { - #branch = "feature/cloudwatch" - branch = "feature/ec2" vpc_id = local.vpc_id for_each = toset(var.instance_types) instance_type = each.key name = "swarms-ami-${each.key}" security_group_id = module.security.internal_security_group_id ami_id = local.new_ami_id - tags= local.tags + key_name = var.key_name + tags= merge(local.tags, { + environment = "production" + }) source = "./components/launch_template" iam_instance_profile_name = module.roles.ssm_profile_name - #aws_iam_instance_profile.ssm.name install_script = "/opt/swarms/api/just_run.sh" + ssm_parameter_name_cw_agent_config= "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/cloudwatch-agent/config" + branch = "feature/ec2" } module "lt_dynamic_ami_test" { @@ -117,22 +123,25 @@ module "lt_dynamic_ami_test" { name = "swarms-ami-${each.key}" security_group_id = module.security.internal_security_group_id ami_id = local.new_ami_id - tags= local.tags + tags= merge(local.tags, { + environment = "test" + }) source = "./components/launch_template" + key_name = var.key_name #"mdupont-deployer-key" + ssm_parameter_name_cw_agent_config= "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/cloudwatch-agent/config/details" iam_instance_profile_name = module.roles.ssm_profile_name - #aws_iam_instance_profile.ssm.name install_script = "/opt/swarms/api/just_run.sh" } -module "alb" { +module "alb" { source = "./components/application_load_balancer" domain_name = local.domain security_group_id = module.security.security_group_id # allowed to talk to internal public_subnets = [ local.ec2_public_subnet_id_1, - local.ec2_public_subnet_id_2 ] + local.ec2_public_subnet_id_2 ] vpc_id = local.vpc_id name = local.name } @@ -141,7 +150,7 @@ output alb { } -# this is the slow one, use the ami +# this is the slow one, use the ami # module "asg_dynamic" { # tags = local.tags # vpc_id = local.vpc_id @@ -166,7 +175,7 @@ module "asg_dynamic_new_ami" { image_id = local.new_ami_id ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 for_each = toset(var.instance_types) - aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn + aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn source = "./components/autoscaling_group" # security_group_id = module.security.internal_security_group_id instance_type = each.key @@ -183,7 +192,7 @@ module "asg_dynamic_new_ami_test" { image_id = local.new_ami_id ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 for_each = toset(var.instance_types) - aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn + aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn source = "./components/autoscaling_group" # security_group_id = module.security.internal_security_group_id instance_type = each.key From aee885641d2b12fdcfa22f6309fc03961cb08e96 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sun, 22 Dec 2024 06:38:50 -0500 Subject: [PATCH 082/158] swap --- modules/aws/ssm/observability/main.tf | 285 +++++++++++++------------- 1 file changed, 144 insertions(+), 141 deletions(-) diff --git a/modules/aws/ssm/observability/main.tf b/modules/aws/ssm/observability/main.tf index f672e09..a6a4fee 100755 --- a/modules/aws/ssm/observability/main.tf +++ b/modules/aws/ssm/observability/main.tf @@ -224,12 +224,141 @@ resource "aws_ssm_association" "cloudwatch_agent_update" { } } -# Updated CloudWatch Agent configuration in SSM Parameter Store -resource "aws_ssm_parameter" "cw_agent_config" { - name = "/cloudwatch-agent/config" - description = "CloudWatch agent configuration" - type = "SecureString" - value = jsonencode({ +locals { + normal_config = { + "agent": { + "metrics_collection_interval": 60 + }, + "metrics": { + "namespace": "CWAgent", + "append_dimensions": { + "InstanceId": "$${aws:InstanceId}" + }, + "metrics_collected": { + "mem": { + "measurement": [ + "used_percent", + "used", + "total", + "inactive", + "free", + "cached", + "buffered", + "available_percent", + "available", + "active" + ] + }, + "cpu": { + "measurement": [ + "usage_active", + "time_guest_nice", + "time_idle", + "time_irq", + "time_iowait", + "time_guest", + "time_active", + "time_softirq", + "time_nice", + "time_system", + "time_user", + "time_steal", + "usage_guest", + "usage_guest_nice", + "usage_idle", + "usage_iowait", + "usage_irq", + "usage_nice", + "usage_softirq", + "usage_steal" + ] + }, + "disk": { + "measurement": [ + "used_percent", + "free", + "inodes_free", + "inodes_total", + "inodes_used", + "total", + "used" + ] + }, + "diskio": { + "measurement": [ + "read_bytes", + "write_bytes", + "iops_in_progress", + "io_time", + "reads", + "writes", + "read_time", + "write_time" + ] + }, + "net": { + "measurement": [ + "bytes_sent", + "bytes_recv", + "drop_in", + "drop_out", + "err_in", + "err_out", + "packets_sent", + "packets_recv" + ] + }, + "netstat": { + "measurement": [ + "tcp_established", + "tcp_close", + "tcp_close_wait", + "tcp_closing", + "tcp_fin_wait1", + "tcp_last_ack", + "tcp_listen", + "tcp_fin_wait2", + "tcp_none", + "tcp_syn_recv", + "tcp_time_wait", + "tcp_syn_sent", + "udp_socket" + ] + }, + "processes": { + "measurement": [ + "running", + "wait", + "zombies", + "total_threads", + "total", + "paging", + "sleeping", + "stopped", + "dead", + "blocked", + "idle" + ] + }, + "swap": { + "measurement": [ + "used_percent", + "free", + "used" + ] + } + } + }, + "traces": { + "traces_collected": { + "xray": {}, + "otlp": {}, + "application_signals": {} + } + } + } + + detailed_config = { agent = { metrics_collection_interval = 60 run_as_user = "root" @@ -327,7 +456,14 @@ resource "aws_ssm_parameter" "cw_agent_config" { } } } - }) + } +} +# Updated CloudWatch Agent configuration in SSM Parameter Store +resource "aws_ssm_parameter" "cw_agent_config" { + name = "/cloudwatch-agent/config" + description = "CloudWatch agent configuration" + type = "SecureString" + value = jsonencode(local.normal_config) } # SSM command to update CloudWatch configuration on running instances @@ -395,143 +531,10 @@ output "troubleshoot_command" { value = "aws ssm start-automation-execution --document-name ${aws_ssm_document.cloudwatch_agent_troubleshoot.name} --target Key=tag:Monitoring,Values=enabled" } - - # Updated CloudWatch Agent configuration in SSM Parameter Store resource "aws_ssm_parameter" "cw_agent_config_details" { name = "/cloudwatch-agent/config/details" description = "CloudWatch agent configuration with details" type = "SecureString" - value = jsonencode({ - "agent": { - "metrics_collection_interval": 60 - }, - "metrics": { - "namespace": "CWAgent", - "append_dimensions": { - "InstanceId": "$${aws:InstanceId}" - }, - "metrics_collected": { - "mem": { - "measurement": [ - "used_percent", - "used", - "total", - "inactive", - "free", - "cached", - "buffered", - "available_percent", - "available", - "active" - ] - }, - "cpu": { - "measurement": [ - "usage_active", - "time_guest_nice", - "time_idle", - "time_irq", - "time_iowait", - "time_guest", - "time_active", - "time_softirq", - "time_nice", - "time_system", - "time_user", - "time_steal", - "usage_guest", - "usage_guest_nice", - "usage_idle", - "usage_iowait", - "usage_irq", - "usage_nice", - "usage_softirq", - "usage_steal" - ] - }, - "disk": { - "measurement": [ - "used_percent", - "free", - "inodes_free", - "inodes_total", - "inodes_used", - "total", - "used" - ] - }, - "diskio": { - "measurement": [ - "read_bytes", - "write_bytes", - "iops_in_progress", - "io_time", - "reads", - "writes", - "read_time", - "write_time" - ] - }, - "net": { - "measurement": [ - "bytes_sent", - "bytes_recv", - "drop_in", - "drop_out", - "err_in", - "err_out", - "packets_sent", - "packets_recv" - ] - }, - "netstat": { - "measurement": [ - "tcp_established", - "tcp_close", - "tcp_close_wait", - "tcp_closing", - "tcp_fin_wait1", - "tcp_last_ack", - "tcp_listen", - "tcp_fin_wait2", - "tcp_none", - "tcp_syn_recv", - "tcp_time_wait", - "tcp_syn_sent", - "udp_socket" - ] - }, - "processes": { - "measurement": [ - "running", - "wait", - "zombies", - "total_threads", - "total", - "paging", - "sleeping", - "stopped", - "dead", - "blocked", - "idle" - ] - }, - "swap": { - "measurement": [ - "used_percent", - "free", - "used" - ] - } - } - }, - "traces": { - "traces_collected": { - "xray": {}, - "otlp": {}, - "application_signals": {} - } - } - }) + value = jsonencode(local.detailed_config) } From 41a652df519ea60bf344214d4a5524fb20dcd410 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sun, 22 Dec 2024 07:30:20 -0500 Subject: [PATCH 083/158] update the code to fetch then checkout --- .../dev/us-east-1/components/launch_template/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf index 4c14176..648dc97 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf @@ -41,15 +41,15 @@ locals { # Verify installation systemctl status amazon-cloudwatch-agent - if [ ! -d "/opt/swarms/" ]; then git clone https://github.com/jmikedupont2/swarms "/opt/swarms/" fi cd "/opt/swarms/" || exit 1 export BRANCH=${var.branch} git stash + git fetch --all # get the latest version git checkout --force $BRANCH - git pull # get the latest version + bash -x ${var.install_script} EOF From f1da299ab0a2542086779247b3e6b8440eba122c Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sun, 22 Dec 2024 15:27:00 -0500 Subject: [PATCH 084/158] pdev --- accounts/swarms/main.tf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/accounts/swarms/main.tf b/accounts/swarms/main.tf index 68eb48a..389bd60 100644 --- a/accounts/swarms/main.tf +++ b/accounts/swarms/main.tf @@ -123,3 +123,7 @@ output user_data_new { value = module.swarms_api.user_data_new } + +module pdev { + source = "./pdev" +} From c930829768c0f941bc9fb0f214c17e62378ecc76 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sun, 22 Dec 2024 15:28:52 -0500 Subject: [PATCH 085/158] pdev --- accounts/swarms/pdev/main.tf | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 accounts/swarms/pdev/main.tf diff --git a/accounts/swarms/pdev/main.tf b/accounts/swarms/pdev/main.tf new file mode 100644 index 0000000..37f22ce --- /dev/null +++ b/accounts/swarms/pdev/main.tf @@ -0,0 +1,20 @@ + +resource "aws_iam_role" "pdev" { + name = "pdev" + description = "pdev role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Principal = { + AWS = "arn:aws:iam::354918380242:root" + } + Action = "sts:AssumeRole" + "Condition": { "Bool": { "aws:MultiFactorAuthPresent": "true" } } + } + + ] + }) +} From 9dd1a4155ea78379a12c203e8b0675d5b47271b9 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sun, 22 Dec 2024 16:51:53 -0500 Subject: [PATCH 086/158] pdev --- accounts/swarms/main.tf | 4 ++++ accounts/swarms/pdev/main.tf | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/accounts/swarms/main.tf b/accounts/swarms/main.tf index 389bd60..9ab41f5 100644 --- a/accounts/swarms/main.tf +++ b/accounts/swarms/main.tf @@ -127,3 +127,7 @@ output user_data_new { module pdev { source = "./pdev" } + +module mdupont { + source = "./mdupont" +} diff --git a/accounts/swarms/pdev/main.tf b/accounts/swarms/pdev/main.tf index 37f22ce..fb359b0 100644 --- a/accounts/swarms/pdev/main.tf +++ b/accounts/swarms/pdev/main.tf @@ -12,7 +12,7 @@ resource "aws_iam_role" "pdev" { AWS = "arn:aws:iam::354918380242:root" } Action = "sts:AssumeRole" - "Condition": { "Bool": { "aws:MultiFactorAuthPresent": "true" } } +# "Condition": { "Bool": { "aws:MultiFactorAuthPresent": "true" } } } ] From cb9a924f71df0550c253627b598a5ca4428c8b85 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sun, 22 Dec 2024 16:52:15 -0500 Subject: [PATCH 087/158] adding main --- accounts/swarms/mdupont/main.tf | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 accounts/swarms/mdupont/main.tf diff --git a/accounts/swarms/mdupont/main.tf b/accounts/swarms/mdupont/main.tf new file mode 100644 index 0000000..3a3f62d --- /dev/null +++ b/accounts/swarms/mdupont/main.tf @@ -0,0 +1,21 @@ +# aws sts assume-role --role-arn arn:aws:iam::916723593639:role/github --profile mdupont --role-session-name mdupont +resource "aws_iam_role" "pdev" { + name = "mdupont_external" + description = "mdupont external role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Principal = { + #AWS = "arn:aws:iam::354918380242:root" + AWS = "arn:aws:iam::767503528736:user/mdupont" + } + Action = "sts:AssumeRole" + #"Condition": { "Bool": { "aws:MultiFactorAuthPresent": "true" } } + } + + ] + }) +} From f4326f3739c334cd5306542c81f03f4e0db6321f Mon Sep 17 00:00:00 2001 From: mike dupont Date: Mon, 23 Dec 2024 12:05:59 -0500 Subject: [PATCH 088/158] starting with docker --- .../components/launch_template_docker/main.tf | 120 ++++++++++++++++++ .../dev/us-east-1/main.tf | 19 +++ logs/parse.py | 107 ++++++++++++++++ 3 files changed, 246 insertions(+) create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template_docker/main.tf create mode 100644 logs/parse.py diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template_docker/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template_docker/main.tf new file mode 100644 index 0000000..359a741 --- /dev/null +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template_docker/main.tf @@ -0,0 +1,120 @@ +# launch template for running agent api in docker +variable ssm_parameter_name_cw_agent_config{} +variable branch {} +variable install_script {} +variable iam_instance_profile_name {} +variable security_group_id {} +variable name {} +variable vpc_id {} +variable ami_id {} +variable tags {} +variable key_name {} +variable instance_type {} + +locals { + tags = { + project="swarms" + instance_type = var.instance_type + name = var.name + } + # FIXME refactor launch template to pass in user data as template parameter, + # split up user data into reusable chunks that we can use in different forms like docker files + user_data = <<-EOF + #!/bin/bash + export HOME=/root + apt update + apt-get install -y ec2-instance-connect git + + # Install docker + apt-get install -y cloud-utils apt-transport-https ca-certificates curl software-properties-common + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) \ + stable" + apt-get update + apt-get install -y docker-ce + usermod -aG docker ubuntu + + # Install docker-compose + curl -L https://github.com/docker/compose/releases/download/1.21.0/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose + chmod +x /usr/local/bin/docker-compose + + snap install amazon-ssm-agent --classic || echo oops1 + snap start amazon-ssm-agent || echo oops2 + apt-get install -y --no-install-recommends ca-certificates=20230311 curl=7.88.1-10+deb12u7 | echo oops + curl -O "https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/$(dpkg --print-architecture)/latest/amazon-cloudwatch-agent.deb" + dpkg -i -E amazon-cloudwatch-agent.deb + # Install prerequisite packages + apt-get install -y wget unzip systemd + # In case of missing dependencies + # apt-get install -f -y + # Configure and start the CloudWatch agent + /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c "ssm:${data.aws_ssm_parameter.cw_agent_config.name}" + # Enable and start the service using systemctl + systemctl enable amazon-cloudwatch-agent + systemctl start amazon-cloudwatch-agent + + # Clean up downloaded files + rm -f amazon-cloudwatch-agent.deb + # Verify installation + systemctl status amazon-cloudwatch-agent + + if [ ! -d "/opt/swarms/" ]; then + git clone https://github.com/jmikedupont2/swarms "/opt/swarms/" + fi + cd "/opt/swarms/" || exit 1 + export BRANCH=${var.branch} + git stash + git fetch --all # get the latest version + git checkout --force $BRANCH + + bash -x ${var.install_script} + EOF + +} +data "aws_ssm_parameter" "cw_agent_config" { + name = var.ssm_parameter_name_cw_agent_config +} + +# defined +resource "aws_launch_template" "ec2_launch_template" { + name_prefix = "${var.name}-launch-template-" + image_id = var.ami_id + key_name = var.key_name + instance_type = var.instance_type + network_interfaces { + associate_public_ip_address = true + delete_on_termination = true + security_groups = [var.security_group_id] + } + + iam_instance_profile { + # iam_instance_profile_arn = aws_iam_instance_profile.ssm.arn + name = var.iam_instance_profile_name #aws_iam_instance_profile.ec2_instance_profile.name + } + lifecycle { + create_before_destroy = true + } + block_device_mappings { + device_name = "/dev/sda1" + ebs { + volume_size = 30 + volume_type = "gp3" + encrypted = true + } + } + user_data = base64encode(local.user_data) + tags = var.tags +} + + +output "lt" { + value = resource.aws_launch_template.ec2_launch_template +} +output "launch_template_id" { + value = resource.aws_launch_template.ec2_launch_template.id +} +output "user_data" { + value = local.user_data +} diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index f1feea0..e3f090b 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -134,6 +134,25 @@ module "lt_dynamic_ami_test" { } +module "lt_dynamic_ami_docker" { + branch = "feature/squash2-docker" + vpc_id = local.vpc_id + for_each = toset(var.instance_types) + instance_type = each.key + name = "swarms-docker-${each.key}" + security_group_id = module.security.internal_security_group_id + ami_id = local.new_ami_id + tags= merge(local.tags, { + environment = "test" + }) + source = "./components/launch_template_docker" + key_name = var.key_name #"mdupont-deployer-key" + ssm_parameter_name_cw_agent_config= "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/cloudwatch-agent/config/details" + iam_instance_profile_name = module.roles.ssm_profile_name + install_script = "/opt/swarms/api/rundocker.sh" +} + + module "alb" { source = "./components/application_load_balancer" diff --git a/logs/parse.py b/logs/parse.py new file mode 100644 index 0000000..e508356 --- /dev/null +++ b/logs/parse.py @@ -0,0 +1,107 @@ +# # in python +# open up logs/*.log and read out json + +# { +# "Events": [ +# { +# "EventId": "3ea90966-2bc7-4d82-a208-91c4b8446f23", +# "EventName": "GetLogEvents", +# "ReadOnly": "true", +# "AccessKeyId": "ASIA5K4H36GT4C65HRZX", +# "EventTime": 1734818869.0, +# "EventSource": "logs.amazonaws.com", +# "Username": "dupont", +# "Resources": [], +# "CloudTrailEvent": "{\"eventVersion\":\"1.11\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"AIDA5K4H36GT5MJLQHPRT\",\"arn\":\"arn:aws:iam::916723593639:user/dupont\",\"accountId\":\"916723593639\",\"accessKeyId\" + +# CloudTrailEvent is a nested json + +# To read JSON data from log files in a directory using Python, you can use the following code snippet: +import json +import glob +log_files = glob.glob('logs/*.log') +all_events = [] +for log_file in log_files: + with open(log_file, 'r') as f: + try: + event_data = json.load(f) + except Exception as e: + print(log_file,e) + e1 = event_data.get("Events", []) + for e in e1: + print(e1) + #all_events.extend(e1) + +#print(all_events) # or process the events as needed + +# ### Explanation: +# 1. **glob** is used to match all log files in the `logs` directory. +# 2. The code reads each log file line by line and attempts to decode each line as JSON. +# 3. Any parsed events are collected in the `all_events` list. +# 4. Handle JSON decoding errors gracefully. + + + +# import boto3, argparse, json +# from datetime import datetime, timedelta + +# # Parse command line arguments +# parser = argparse.ArgumentParser(description='Generate an IAM policy based on CloudTrail events.') +# parser.add_argument('--service', dest='service_name', required=True, +# help='The name of the AWS service to generate a policy for (e.g., "ec2", "s3", "lambda", etc.)') +# parser.add_argument('--region', dest='region_name', required=True, +# help='The AWS region to search for CloudTrail events in') +# parser.add_argument('--hours', dest='hours_ago', type=int, default=2, +# help='The number of hours to look back in the CloudTrail events (default is 2)') +# args = parser.parse_args() + +# # Initialize CloudTrail client +# client = boto3.client('cloudtrail', region_name=args.region_name) + +# # Calculate start time for CloudTrail lookup +# start_time = datetime.utcnow() - timedelta(hours=args.hours_ago) + +# # Dictionary to store permissions by service +# permissions_by_service = {} + +# # Paginate through CloudTrail events +# for response in client.get_paginator('lookup_events').paginate( +# StartTime=start_time, +# EndTime=datetime.utcnow(), +# LookupAttributes=[ +# { +# 'AttributeKey': 'EventSource', +# 'AttributeValue': f'{args.service_name}.amazonaws.com' +# } +# ] +# ): +# # Iterate through events and extract permissions +# for event in response['Events']: +# permission = event['EventName'] +# if ":" in permission: +# service, action = permission.split(':') +# else: +# service = args.service_name +# action = permission +# permissions_by_service.setdefault(service, set()).add(action) + +# # Create policy statement +# policy = { +# "Version": "2012-10-17", +# "Statement": [] +# } + +# # Iterate through permissions by service and add to policy statement +# for service, actions in permissions_by_service.items(): +# statement = { +# "Sid": "VisualEditor0", +# "Effect": "Allow", +# "Action": [f"{service}:{action}" for action in actions], +# "Resource": "*" +# } +# policy["Statement"].append(statement) + +# # Print policy in JSON format +# print(f"last: {args.hours_ago}h") +# print(f"service name filter: {args.service_name}") +# print(json.dumps(policy, indent=4)) From 3899d697fc2b872fa6e87c66690f7fba5425902f Mon Sep 17 00:00:00 2001 From: mike dupont Date: Mon, 23 Dec 2024 12:17:08 -0500 Subject: [PATCH 089/158] update readme --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ddf23fb..d6de0c8 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,9 @@ Terraform project template for deploying infrastructure across multiple environm ``` terraform-template/ # Root directory of the Terraform template repository ├── README.md # Project documentation and overview -├── environments +├── accounts +│ ├── swarms # the main swarms account +├── environments │ ├── swarms-aws-agent-api │ │ ├── dev # Development environment configuration for the demo-azure-vm setup │ │ │ └── us-east-1 From 43420045b2fd7ef178c00c6d008bb67cd1a51c4f Mon Sep 17 00:00:00 2001 From: mike dupont Date: Mon, 23 Dec 2024 12:46:09 -0500 Subject: [PATCH 090/158] normalizing the names we have three target groups prod,test and dev --- .../application_load_balancer/https/main.tf | 25 ++++++++++++++++--- .../application_load_balancer/main.tf | 15 +++++++---- .../dev/us-east-1/main.tf | 17 +++++++++++++ 3 files changed, 48 insertions(+), 9 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf index 4843e16..3e5030a 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf @@ -1,8 +1,9 @@ variable alb_arn{} variable domain_name{} variable zone_id{} -variable aws_lb_target_group_arn{} -variable new_target_group_arn{} +variable prod_target_group_arn{} +variable test_target_group_arn{} +variable dev_target_group_arn{} module "acm" { # count = 0 @@ -25,7 +26,7 @@ resource "aws_lb_listener" "this" { certificate_arn = module.acm.acm_certificate_arn load_balancer_arn = var.alb_arn default_action { - target_group_arn =var.aws_lb_target_group_arn + target_group_arn =var.prod_target_group_arn type = "forward" } } @@ -38,7 +39,7 @@ resource "aws_lb_listener_rule" "route_v1_api" { action { type = "forward" - target_group_arn = var.new_target_group_arn # New target group's ARN + target_group_arn = var.target_target_group_arn # New target group's ARN } condition { @@ -48,6 +49,22 @@ resource "aws_lb_listener_rule" "route_v1_api" { } } +resource "aws_lb_listener_rule" "route_v1_api_dev" { + listener_arn = aws_lb_listener.this.arn + priority = 100 # Set priority as needed, must be unique + + action { + type = "forward" + target_group_arn = var.dev_target_group_arn # New target group's ARN + } + + condition { + host_header { + values = ["dev.api.swarms.ai"] + } + } +} + resource "aws_lb_listener" "insecure" { port = 80 protocol = "HTTP" diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf index 941b8d4..7505080 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf @@ -34,8 +34,6 @@ module "alb" { - - output dns { value = module.alb.dns_name } @@ -47,7 +45,7 @@ module "route53" { domain_name = var.domain_name } -module "tg" { +module "tg_prod" { source = "./target_group/" name_prefix = "swarms" vpc_id = var.vpc_id # module.vpc.vpc_id @@ -59,14 +57,21 @@ module "tg_test" { vpc_id = var.vpc_id # module.vpc.vpc_id } +module "tg_dev" { + source = "./target_group/" + name_prefix = "dev" + vpc_id = var.vpc_id # module.vpc.vpc_id +} + module "https" { source = "./https/" # vpc_id = var.vpc_id # module.vpc.vpc_id zone_id = module.route53.primary_zone_id domain_name = var.domain_name alb_arn = module.alb.arn - aws_lb_target_group_arn = module.tg.alb_target_group_arn - new_target_group_arn = module.tg_test.alb_target_group_arn + prod_target_group_arn = module.tg.alb_target_group_arn + test_target_group_arn = module.tg_test.alb_target_group_arn + dev_target_group_arn = module.tg_dev.alb_target_group_arn } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index e3f090b..5adcc4b 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -220,6 +220,23 @@ module "asg_dynamic_new_ami_test" { target_group_arn = module.alb.test_alb_target_group_arn } +module "asg_dynamic_new_ami_test" { + # built with packer + #count =0 + tags = merge(local.tags, local.dev_tags) + vpc_id = local.vpc_id + image_id = local.new_ami_id + ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 + for_each = toset(var.instance_types) + aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn + source = "./components/autoscaling_group" +# security_group_id = module.security.internal_security_group_id + instance_type = each.key + name = "docker-swarms-ami-${each.key}" + launch_template_id = module.lt_dynamic_ami_docker[each.key].launch_template_id + target_group_arn = module.alb.dev_alb_target_group_arn +} + output security_group_id { value = module.security.security_group_id } From 42a947fb5c84479d62058fb8451f870c89d8d57a Mon Sep 17 00:00:00 2001 From: mike dupont Date: Mon, 23 Dec 2024 12:54:20 -0500 Subject: [PATCH 091/158] adding status --- README.md | 6 ++++++ .../components/application_load_balancer/https/main.tf | 4 ++-- .../components/application_load_balancer/main.tf | 10 +++++++--- .../swarms-aws-agent-api/dev/us-east-1/main.tf | 4 ++-- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index d6de0c8..bdbcb19 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,12 @@ Terraform project template for deploying infrastructure across multiple environments and regions, following best practices with modular structure and automated syntax checks (GitHub Actions) +## Status report + +Currently we are building out a dev infrastructure using our existing code +and adding in a docker target, this has the advantage that we can build and test +the docker outside of aws and then deploy it easily. + ## Repo structure ``` diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf index 3e5030a..eb432f6 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf @@ -39,7 +39,7 @@ resource "aws_lb_listener_rule" "route_v1_api" { action { type = "forward" - target_group_arn = var.target_target_group_arn # New target group's ARN + target_group_arn = var.test_target_group_arn # New target group's ARN } condition { @@ -70,7 +70,7 @@ resource "aws_lb_listener" "insecure" { protocol = "HTTP" load_balancer_arn = var.alb_arn default_action { - target_group_arn =var.aws_lb_target_group_arn + target_group_arn = var.prod_target_group_arn type = "forward" } } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf index 7505080..a03a2a6 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf @@ -69,16 +69,20 @@ module "https" { zone_id = module.route53.primary_zone_id domain_name = var.domain_name alb_arn = module.alb.arn - prod_target_group_arn = module.tg.alb_target_group_arn + prod_target_group_arn = module.tg_prod.alb_target_group_arn test_target_group_arn = module.tg_test.alb_target_group_arn dev_target_group_arn = module.tg_dev.alb_target_group_arn } -output alb_target_group_arn { - value = module.tg.alb_target_group_arn +output prod_alb_target_group_arn { + value = module.tg_prod.alb_target_group_arn } output test_alb_target_group_arn { value = module.tg_test.alb_target_group_arn } + +output dev_alb_target_group_arn { + value = module.tg_dev.alb_target_group_arn +} diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index 5adcc4b..ad36c00 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -200,7 +200,7 @@ module "asg_dynamic_new_ami" { instance_type = each.key name = "swarms-ami-${each.key}" launch_template_id = module.lt_dynamic_ami_prod[each.key].launch_template_id - target_group_arn = module.alb.alb_target_group_arn + target_group_arn = module.alb.prod_alb_target_group_arn } module "asg_dynamic_new_ami_test" { @@ -220,7 +220,7 @@ module "asg_dynamic_new_ami_test" { target_group_arn = module.alb.test_alb_target_group_arn } -module "asg_dynamic_new_ami_test" { +module "asg_dynamic_new_ami_dev" { # built with packer #count =0 tags = merge(local.tags, local.dev_tags) From b2c2099d63fa924078e65bc84a799356fc45b574 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Mon, 23 Dec 2024 20:43:17 -0500 Subject: [PATCH 092/158] top level module --- .github/workflows/call-swarms.yml | 13 +++--- .gitignore | 1 + actions/call_swarms.sh | 44 ++++++++++++++++--- environments/call-swarms/deploy-docker.yaml | 39 ++++++++++++++++ environments/call-swarms/main.tf | 16 ++++++- .../application_load_balancer/https/main.tf | 2 +- .../dev/us-east-1/main.tf | 1 + main.tf | 4 ++ 8 files changed, 104 insertions(+), 16 deletions(-) create mode 100644 environments/call-swarms/deploy-docker.yaml create mode 100644 main.tf diff --git a/.github/workflows/call-swarms.yml b/.github/workflows/call-swarms.yml index e7b5f05..1797f9f 100644 --- a/.github/workflows/call-swarms.yml +++ b/.github/workflows/call-swarms.yml @@ -1,14 +1,13 @@ on: workflow_dispatch: - push: branches: - - "feature/call_swarms" - pull_request: - branches: - - "main" - paths: - - ".github/workflows/call-swarms.yaml" + - "feature/docker" +# pull_request: +# branches: +# - "main" +# paths: +# - ".github/workflows/call-swarms.yaml" name: CallSwarms jobs: diff --git a/.gitignore b/.gitignore index 54867ed..cbff7d2 100644 --- a/.gitignore +++ b/.gitignore @@ -89,3 +89,4 @@ flycheck_*.el /network-security.data /logs/logs/ +.aider* diff --git a/actions/call_swarms.sh b/actions/call_swarms.sh index f91c765..69f5091 100644 --- a/actions/call_swarms.sh +++ b/actions/call_swarms.sh @@ -1,18 +1,48 @@ #!/bin/bash +echo < Date: Mon, 23 Dec 2024 20:51:19 -0500 Subject: [PATCH 093/158] tofu apply -target module.call_swarms.aws_ssm_document.deploy-docker --- environments/call-swarms/deploy-docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environments/call-swarms/deploy-docker.yaml b/environments/call-swarms/deploy-docker.yaml index cece0cd..e141089 100644 --- a/environments/call-swarms/deploy-docker.yaml +++ b/environments/call-swarms/deploy-docker.yaml @@ -26,7 +26,7 @@ parameters: mainSteps: - action: aws:runShellScript - name: Deploy Docker + name: DeployDocker inputs: runCommand: - | From 4c8eba2840fe876695998d340dc101f7c3c7648c Mon Sep 17 00:00:00 2001 From: mike dupont Date: Mon, 23 Dec 2024 21:00:28 -0500 Subject: [PATCH 094/158] example logs --- actions/call_swarms.org | 272 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 272 insertions(+) create mode 100644 actions/call_swarms.org diff --git a/actions/call_swarms.org b/actions/call_swarms.org new file mode 100644 index 0000000..15a5aa3 --- /dev/null +++ b/actions/call_swarms.org @@ -0,0 +1,272 @@ +mdupont@mdupont-G470:~/swarms-terraform/actions$ AWS_PROFILE=swarms bash ./call_swarms.sh + ++ export REGION=us-east-2 ++ REGION=us-east-2 ++ TAG_KEY=Name ++ TAG_VALUE=docker-swarms-ami-t3.medium ++ GIT_URL=https://github.com/jmikedupont/swarms ++ export GIT_NAME=mdupont ++ GIT_NAME=mdupont ++ export GIT_VERSION=feature/squash2-docker ++ GIT_VERSION=feature/squash2-docker ++ DOCUMENT_NAME=deploy-docker ++ DOCUMENT_VERSION=1 ++ TIMEOUT_SECONDS=600 ++ MAX_CONCURRENCY=50 ++ MAX_ERRORS=0 +++ get_instance_ids +++ aws ec2 describe-instances --filters Name=tag:Name,Values=docker-swarms-ami-t3.medium --query 'Reservations[*].Instances[*].InstanceId' --output text --region us-east-2 ++ for instance in $(get_instance_ids) ++ echo 'Instance ID: i-0a3dae164f8f3c09a' +Instance ID: i-0a3dae164f8f3c09a +++ send_command i-0a3dae164f8f3c09a +++ local instance_id=i-0a3dae164f8f3c09a +++ aws ssm send-command --document-name deploy-docker --document-version 1 --targets '[{"Key":"InstanceIds","Values":["i-0a3dae164f8f3c09a"]}]' --parameters '{"GitUrl":["https://github.com/jmikedupont/swarms"],"GitName":["mdupont"],"GitVersion":["feature/squash2-docker"]}' --timeout-seconds 600 --max-concurrency 50 --max-errors 0 --region us-east-2 --output-s3-bucket-name swarms-session-logs-20241221151754799300000003 --cloud-watch-output-config '{"CloudWatchOutputEnabled":true,"CloudWatchLogGroupName":"/ssm/session-logs-20241221151803393300000006"}' ++ result='{ + "Command": { + "CommandId": "82d43144-a4f4-4b6d-a507-23ad5179e0b4", + "DocumentName": "deploy-docker", + "DocumentVersion": "1", + "Comment": "", + "ExpiresAfter": 1735009432.928, + "Parameters": { + "GitName": [ + "mdupont" + ], + "GitUrl": [ + "https://github.com/jmikedupont/swarms" + ], + "GitVersion": [ + "feature/squash2-docker" + ] + }, + "InstanceIds": [], + "Targets": [ + { + "Key": "InstanceIds", + "Values": [ + "i-0a3dae164f8f3c09a" + ] + } + ], + "RequestedDateTime": 1735005232.928, + "Status": "Pending", + "StatusDetails": "Pending", + "OutputS3Region": "us-east-2", + "OutputS3BucketName": "swarms-session-logs-20241221151754799300000003", + "OutputS3KeyPrefix": "", + "MaxConcurrency": "50", + "MaxErrors": "0", + "TargetCount": 0, + "CompletedCount": 0, + "ErrorCount": 0, + "DeliveryTimedOutCount": 0, + "ServiceRole": "", + "NotificationConfig": { + "NotificationArn": "", + "NotificationEvents": [], + "NotificationType": "" + }, + "CloudWatchOutputConfig": { + "CloudWatchLogGroupName": "/ssm/session-logs-20241221151803393300000006", + "CloudWatchOutputEnabled": true + }, + "TimeoutSeconds": 600, + "AlarmConfiguration": { + "IgnorePollAlarmFailure": false, + "Alarms": [] + }, + "TriggeredAlarms": [] + } +}' +++ jq -r .Command.CommandId +++ echo '{' '"Command":' '{' '"CommandId":' '"82d43144-a4f4-4b6d-a507-23ad5179e0b4",' '"DocumentName":' '"deploy-docker",' '"DocumentVersion":' '"1",' '"Comment":' '"",' '"ExpiresAfter":' 1735009432.928, '"Parameters":' '{' '"GitName":' '[' '"mdupont"' '],' '"GitUrl":' '[' '"https://github.com/jmikedupont/swarms"' '],' '"GitVersion":' '[' '"feature/squash2-docker"' ']' '},' '"InstanceIds":' '[],' '"Targets":' '[' '{' '"Key":' '"InstanceIds",' '"Values":' '[' '"i-0a3dae164f8f3c09a"' ']' '}' '],' '"RequestedDateTime":' 1735005232.928, '"Status":' '"Pending",' '"StatusDetails":' '"Pending",' '"OutputS3Region":' '"us-east-2",' '"OutputS3BucketName":' '"swarms-session-logs-20241221151754799300000003",' '"OutputS3KeyPrefix":' '"",' '"MaxConcurrency":' '"50",' '"MaxErrors":' '"0",' '"TargetCount":' 0, '"CompletedCount":' 0, '"ErrorCount":' 0, '"DeliveryTimedOutCount":' 0, '"ServiceRole":' '"",' '"NotificationConfig":' '{' '"NotificationArn":' '"",' '"NotificationEvents":' '[],' '"NotificationType":' '""' '},' '"CloudWatchOutputConfig":' '{' '"CloudWatchLogGroupName":' '"/ssm/session-logs-20241221151803393300000006",' '"CloudWatchOutputEnabled":' true '},' '"TimeoutSeconds":' 600, '"AlarmConfiguration":' '{' '"IgnorePollAlarmFailure":' false, '"Alarms":' '[]' '},' '"TriggeredAlarms":' '[]' '}' '}' ++ command_id=82d43144-a4f4-4b6d-a507-23ad5179e0b4 ++ aws ssm wait command-executed --command-id 82d43144-a4f4-4b6d-a507-23ad5179e0b4 --region us-east-2 --instance i-0a3dae164f8f3c09a ++ fetch_command_output 82d43144-a4f4-4b6d-a507-23ad5179e0b4 ++ local command_id=82d43144-a4f4-4b6d-a507-23ad5179e0b4 ++ jq -r '.CommandInvocations[] | {InstanceId, Status, Output}' + +Lets get the log outputs and the cloudtrails and the cloudwatch logs for this command ++ aws ssm list-command-invocations --command-id 82d43144-a4f4-4b6d-a507-23ad5179e0b4 --details --region us-east-2 +{ + "InstanceId": "i-0a3dae164f8f3c09a", + "Status": "Success", + "Output": null +} + + +To obtain the log outputs, cloudtrail logs, and cloudwatch logs for the specified AWS Systems +Manager (SSM) command, follow these steps: + +*** Step 1: Retrieve the Output from the Command Execution + +Since the output is ~null~ in your provided response, we'll first try to fetch the output directly. +#+BEGIN_SRC sh +aws ssm list-command-invocations --command-id 82d43144-a4f4-4b6d-a507-23ad5179e0b4 --details --region us-east-2 --profile swarms +#+END_SRC + +Lets fetch the results +#+RESULTS: +| { | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| "CommandInvocations": | [ | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| { | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| "CommandId": | "82d43144-a4f4-4b6d-a507-23ad5179e0b4", | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| "InstanceId": | "i-0a3dae164f8f3c09a", | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| "InstanceName": | "ip-10-0-4-93.us-east-2.compute.internal", | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| "Comment": | "", | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| "DocumentName": | "deploy-docker", | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| "DocumentVersion": | "1", | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| "RequestedDateTime": | 1735005232.999, | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| "Status": | "Success", | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| "StatusDetails": | "Success", | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| "StandardOutputUrl": | "https://s3.us-east-2.amazonaws.com/swarms-session-logs-20241221151754799300000003/82d43144-a4f4-4b6d-a507-23ad5179e0b4/i-0a3dae164f8f3c09a/awsrunShellScript/DeployDocker/stdout", | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| "StandardErrorUrl": | "https://s3.us-east-2.amazonaws.com/swarms-session-logs-20241221151754799300000003/82d43144-a4f4-4b6d-a507-23ad5179e0b4/i-0a3dae164f8f3c09a/awsrunShellScript/DeployDocker/stderr", | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| "CommandPlugins": | [ | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| { | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| "Name": | "DeployDocker", | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| "Status": | "Success", | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| "StatusDetails": | "Success", | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| "ResponseCode": | 0, | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| "ResponseStartDateTime": | 1735005233.167, | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| "ResponseFinishDateTime": | 1735005237.002, | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| "Output": | "oops\nYour | branch | is | up | to | date | with | 'origin/feature/squash2-docker'.\n-- | No | entries | --\n\n----------ERROR-------\nerror: | remote | mdupont | already | exists.\nAlready | on | 'feature/squash2-docker'\nbash: | /opt/swarms/api/docker-boot.sh: | No | such | file | or | directory\nFailed | to | restart | swarms-docker.service: | Unit | swarms-docker.service | not | found.\n", | +| "StandardOutputUrl": | "https://s3.us-east-2.amazonaws.com/swarms-session-logs-20241221151754799300000003/82d43144-a4f4-4b6d-a507-23ad5179e0b4/i-0a3dae164f8f3c09a/awsrunShellScript/DeployDocker/stdout", | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| "StandardErrorUrl": | "https://s3.us-east-2.amazonaws.com/swarms-session-logs-20241221151754799300000003/82d43144-a4f4-4b6d-a507-23ad5179e0b4/i-0a3dae164f8f3c09a/awsrunShellScript/DeployDocker/stderr", | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| "OutputS3Region": | "us-east-2", | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| "OutputS3BucketName": | "swarms-session-logs-20241221151754799300000003", | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| "OutputS3KeyPrefix": | 82d43144-a4f4-4b6d-a507-23ad5179e0b4/i-0a3dae164f8f3c09a/awsrunShellScript | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| } | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| ], | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| "ServiceRole": | "", | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| "NotificationConfig": | { | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| "NotificationArn": | "", | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| "NotificationEvents": | [], | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| "NotificationType": | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| }, | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| "CloudWatchOutputConfig": | { | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| "CloudWatchLogGroupName": | "/ssm/session-logs-20241221151803393300000006", | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| "CloudWatchOutputEnabled": | true | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| } | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| } | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| ] | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +| } | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | + +If the ~--details~ flag doesn't provide the output, you may need to retrieve it using another +method. One common approach is to use CloudWatch Logs. + +*** Step 2: Retrieve Output from CloudWatch Logs + +Assuming the SSM command execution generated logs in CloudWatch Logs, you can find the specific log +group and stream associated with your command ID. + +1. *Get the Log Group Name*: + The log group name typically follows a pattern like ~/aws/ssm//~. You need to + determine which exact account ID is used for this output. + +2. *Search for Logs in CloudWatch Logs*: + Use the AWS Management Console or ~aws logs~ command line tool to search for logs in the + appropriate log group. + +**** Using AWS CLI: +#+BEGIN_SRC sh +aws logs get-log-events --log-group-name /aws/ssm/us-east-2/123456789012 --start-time 1672531200000 +--end-time 1675123200000 +#+END_SRC + +Replace ~123456789012~ with your actual account ID and set the time range to match when the command +was executed. + +**** Using AWS Management Console: + +1. Open the CloudWatch Logs console. +2. Navigate to the log group ~/aws/ssm/us-east-2/123456789012~. +3. Use the search bar to filter logs for the command ID ~82d43144-a4f4-4b6d-a507-23ad5179e0b4~. + +*** Step 3: Retrieve CloudTrail Logs + +CloudTrail records all API calls made by AWS users. You can find the logs that correspond to your +SSM command execution. + +1. *Open the CloudTrail Console*: + Navigate to the CloudTrail console and select your trail or go directly to the event management + page. + +2. *Search for Events*: + Use the search bar to filter events based on ~EventName~ being ~StartCommand~, ~CancelCommand~, + etc., and use the command ID ~82d43144-a4f4-4b6d-a507-23ad5179e0b4~. + +*** Step 4: Analyze Logs + +Once you have the logs, analyze them to understand the outcome of your SSM command execution. Look +for any errors or exceptions that might have occurred during the process. + +By following these steps, you should be able to retrieve the necessary log outputs and related data +for your AWS Systems Manager command execution. + + +The output you provided contains details from a command invocation executed in an AWS +environment. The command was part of the ~deploy-docker~ document and targeted the instance +~i-0a3dae164f8f3c09a~. Here is a summary of the results: + +*** Command Summary + +- *Command ID*: ~82d43144-a4f4-4b6d-a507-23ad5179e0b4~ +- *Instance ID*: ~i-0a3dae164f8f3c09a~ +- *Instance Name*: ~ip-10-0-4-93.us-east-2.compute.internal~ +- *Document Name*: ~deploy-docker~ +- *Document Version*: ~1~ +- *Requested DateTime*: ~1735005232.999~ (Epoch time, approximately December 18, 2024) +- *Status*: ~Success~ +- *Output S3 Bucket and Key*: + - ~Region~: ~us-east-2~ + - ~Bucket Name~: ~swarms-session-logs-20241221151754799300000003~ + - ~Key Prefix~: ~82d43144-a4f4-4b6d-a507-23ad5179e0b4/i-0a3dae164f8f3c09a/awsrunShellScript~ + +*** Output and Error Details + +The output of the command contained several issues: + +1. *Git Status*: +#+END_SRC + Your branch is up to date with 'origin/feature/squash2-docker'. +#+END_SRC + +2. *Git Remote Issue*: +#+END_SRC + -- + No entries -- + ----------ERROR------- + error: remote mdupont already exists. + Already on 'feature/squash2-docker' + bash: /opt/swarms/api/docker-boot.sh: No such file or directory + Failed to restart swarms-docker.service: Unit swarms-docker.service not found. +#+END_SRC + +*** Summary of Errors: + +- *Git Remote Conflict*: The ~mdupont~ remote already exists in the repository. You need to either + remove or update this remote. +- *Missing Script File*: ~/opt/swarms/api/docker-boot.sh~ does not exist on the server. +- *Service Not Found*: ~swarms-docker.service~ is not found on the system. + +*** Recommendations: + +1. *Remove Duplicate Remote*: +#+BEGIN_SRC sh + git remote remove mdupont +#+END_SRC + +2. *Verify Script Location*: + - Ensure that ~/opt/swarms/api/docker-boot.sh~ exists and has the correct permissions. + - If it doesn't exist, you may need to upload or create this script on the server. + +3. *Check Service Availability*: + - Verify if ~swarms-docker.service~ is available on your system. + - If not, you might need to install or create this service unit file. + +4. *Review Logs*: + - Access the logs from S3 for more detailed error messages and context. + +This should help in troubleshooting and resolving the issues encountered during the command +execution. From 0761940ebfbda562723d019741814a66aa5ccdb6 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Tue, 24 Dec 2024 09:53:51 -0500 Subject: [PATCH 095/158] creating readme --- actions/Readme.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 actions/Readme.md diff --git a/actions/Readme.md b/actions/Readme.md new file mode 100644 index 0000000..0df4431 --- /dev/null +++ b/actions/Readme.md @@ -0,0 +1,35 @@ +Welcome to the the call swarms ssm framework, +it allows invocation of the swarms agent on a remote ssm server via the aws api. +This can be almost any cloud server that is reachable via ssm, that can be on prem +and in theory on another cloud. + +The file actions/call_swarms.sh is a Bash script designed to interact with AWS +services. It sets up environment variables for AWS region, tag keys and values, +Git repository details, and other parameters. The script defines functions to +retrieve instance IDs, send commands to instances via AWS SSM, and fetch command +outputs. It iterates over the instances, sends commands, waits for execution, +and retrieves outputs, logging them to CloudWatch. The script is structured to +facilitate remote invocation of a swarms agent on cloud servers. + +The script called is defined in `environments/call-swarms/main.tf` and applied in accounts/swarms/ like +```bash +pushd ../../accounts/swarms/ +tofu apply +# terraform apply +popd +``` + +It includes the file +resource "aws_ssm_document" "deploy" { + content = file("${local.codebase_root_path}/environments/call-swarms/deploy.yaml") +it will call + sudo su -c "bash -e -x /var/swarms/agent_workspace/boot.sh" swarms + + +and resource "aws_ssm_document" "deploy-docker" { + ~/swarms-terraform/environments/call-swarms/deploy-docker.yaml + It will call + sudo bash -e -x /opt/swarms/api/docker-boot.sh # needs root access + +The permissions to call aws from github is applied here in this terraform code +https://github.com/jmikedupont2/terraform-aws-oidc-github/pull/1 From b96bfdd6b684ad096245e9f515d0e2d0195c5f86 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Tue, 24 Dec 2024 09:53:59 -0500 Subject: [PATCH 096/158] creating readme --- actions/call_swarms.sh | 17 ++--------------- environments/call-swarms/deploy-docker.yaml | 7 ++++--- environments/call-swarms/deploy.yaml | 1 + 3 files changed, 7 insertions(+), 18 deletions(-) diff --git a/actions/call_swarms.sh b/actions/call_swarms.sh index 69f5091..3625eb2 100644 --- a/actions/call_swarms.sh +++ b/actions/call_swarms.sh @@ -1,17 +1,5 @@ #!/bin/bash echo < Date: Tue, 24 Dec 2024 10:33:56 -0500 Subject: [PATCH 097/158] fix fmt --- accounts/mdupont/output.tf | 18 +- accounts/mdupont/variables.tf | 104 +++--- accounts/swarms/github/main.tf | 24 +- accounts/swarms/main.tf | 66 ++-- accounts/swarms/mdupont/main.tf | 6 +- accounts/swarms/pdev/main.tf | 4 +- environments/call-swarms/main.tf | 10 +- .../application_load_balancer/https/main.tf | 40 +-- .../application_load_balancer/main.tf | 62 ++-- .../application_load_balancer/route53/main.tf | 36 +- .../target_group/main.tf | 34 +- .../components/autoscaling_group/main.tf | 50 +-- .../components/launch_template/main.tf | 46 +-- .../components/launch_template_docker/main.tf | 46 +-- .../components/machine_image/main.tf | 42 +-- .../components/machine_image/variables.tf | 2 +- .../components/resource_launchers/main.tf | 31 +- .../dev/us-east-1/components/roles/main.tf | 70 ++-- .../dev/us-east-1/components/security/main.tf | 18 +- .../dev/us-east-1/components/vpc/main.tf | 28 +- .../dev/us-east-1/components/vpc/outputs.tf | 4 +- .../dev/us-east-1/main.tf | 192 +++++----- .../components/autoscaling_group/main.tf | 48 +-- .../components/launch_template/main.tf | 40 +-- .../swarms-deploy/dev/us-east-1/main.tf | 74 ++-- main.tf | 2 +- modules/aws/ssm/observability/jobs/main.tf | 18 +- modules/aws/ssm/observability/main.tf | 330 +++++++++--------- modules/aws/ssm/setup/main.tf | 18 +- 29 files changed, 720 insertions(+), 743 deletions(-) diff --git a/accounts/mdupont/output.tf b/accounts/mdupont/output.tf index a672016..8990f26 100644 --- a/accounts/mdupont/output.tf +++ b/accounts/mdupont/output.tf @@ -1,33 +1,33 @@ output "aws_region" { - value = var.aws_region + value = var.aws_region } output "iam_user" { - value = var.iam_user + value = var.iam_user } output "table_name" { - value = var.table_name + value = var.table_name } output "project_name" { - value = var.project_name + value = var.project_name } output "aws_caller_identity_account_id" { - value = data.aws_caller_identity.current.account_id + value = data.aws_caller_identity.current.account_id } output "aws_s3_bucket_logging_name" { - value = data.aws_s3_bucket.terraform_logging.bucket + value = data.aws_s3_bucket.terraform_logging.bucket } output "aws_s3_bucket_state_name" { - value = data.aws_s3_bucket.terraform_state.bucket + value = data.aws_s3_bucket.terraform_state.bucket } output "aws_cloudtrail_service_account_id" { - value = data.aws_cloudtrail_service_account.main.id + value = data.aws_cloudtrail_service_account.main.id } - + diff --git a/accounts/mdupont/variables.tf b/accounts/mdupont/variables.tf index 8560e4a..1d16725 100644 --- a/accounts/mdupont/variables.tf +++ b/accounts/mdupont/variables.tf @@ -1,56 +1,56 @@ - variable "project_name" { - type = string - default = "swarms" - } +variable "project_name" { + type = string + default = "swarms" +} variable "aws_region" { - type = string - default = "us-east-1" - } - - variable "aws_account_id" { - type = string - default = "767503528736" - } - - variable "iam_user" { - type = string - default = "mdupont" - } - - variable "table_name" { - type = string - default = "swarms" - } - - variable "lock_resource" { - type = string - default = "terraform/state/lock" - } - - variable "partition" { - type = string - default = "aws" - } - - variable "logs_resource" { - type = string - default = "aws_logs" - } - - variable "permissions_check" { - type = string - default = "config-permissions-check" - } - - variable "delivery_service" { - type = string - default = "delivery.logs.amazonaws.com" - } - - variable "logging_service" { - type = string - default = "logging.s3.amazonaws.com" - } + type = string + default = "us-east-1" +} + +variable "aws_account_id" { + type = string + default = "767503528736" +} + +variable "iam_user" { + type = string + default = "mdupont" +} + +variable "table_name" { + type = string + default = "swarms" +} + +variable "lock_resource" { + type = string + default = "terraform/state/lock" +} + +variable "partition" { + type = string + default = "aws" +} + +variable "logs_resource" { + type = string + default = "aws_logs" +} + +variable "permissions_check" { + type = string + default = "config-permissions-check" +} + +variable "delivery_service" { + type = string + default = "delivery.logs.amazonaws.com" +} + +variable "logging_service" { + type = string + default = "logging.s3.amazonaws.com" +} # swarms-tf-state-log-us-east-1 # swarms-tf-state-us-east-1 diff --git a/accounts/swarms/github/main.tf b/accounts/swarms/github/main.tf index f3136b4..af0c4af 100644 --- a/accounts/swarms/github/main.tf +++ b/accounts/swarms/github/main.tf @@ -1,26 +1,26 @@ -variable aws_region {} -variable aws_account_id {} -variable repos {} +variable "aws_region" {} +variable "aws_account_id" {} +variable "repos" {} terraform { required_providers { github = { - source = "integrations/github" + source = "integrations/github" version = "6.4.0" } } } resource "github_actions_secret" "region" { - for_each = var.repos - repository = each.key # - secret_name = "AWS_REGION" - plaintext_value = var.aws_region + for_each = var.repos + repository = each.key # + secret_name = "AWS_REGION" + plaintext_value = var.aws_region } resource "github_actions_secret" "account" { - for_each = var.repos - repository = each.key - secret_name = "AWS_ACCOUNT_ID" - plaintext_value = var.aws_account_id + for_each = var.repos + repository = each.key + secret_name = "AWS_ACCOUNT_ID" + plaintext_value = var.aws_account_id } diff --git a/accounts/swarms/main.tf b/accounts/swarms/main.tf index 9ab41f5..9609b8d 100644 --- a/accounts/swarms/main.tf +++ b/accounts/swarms/main.tf @@ -1,21 +1,21 @@ locals { #ami_name = "ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*" - ami_name = "ubuntu-minimal/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-minimal-*" # useast2id= - dns = "api.swarms.ai" - account = "916723593639" - region = "us-east-2" + ami_name = "ubuntu-minimal/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-minimal-*" # useast2id= + dns = "api.swarms.ai" + account = "916723593639" + region = "us-east-2" } variable "owner" { description = "GitHub owner used to configure the provider" - default = "jmikedupont2" + default = "jmikedupont2" } variable "github_token" { description = "GitHub access token used to configure the provider" type = string - default = "" + default = "" } provider "github" { @@ -27,8 +27,8 @@ provider "github" { # name = local.dns #} -provider aws { - region = "us-east-2" +provider "aws" { + region = "us-east-2" profile = "swarms" } #output dns { @@ -53,35 +53,35 @@ provider aws { # name_regex = "^${local.ami_name}" # } locals { - us_east_2_swarms_ami_id = "ami-0325b9a2dfb474b2d" - us_east_2_ami_id = "ami-0325b9a2dfb474b2d" + us_east_2_swarms_ami_id = "ami-0325b9a2dfb474b2d" + us_east_2_ami_id = "ami-0325b9a2dfb474b2d" } module "swarms_api" { aws_account_id = local.account - region = local.region - source = "../../environments/swarms-aws-agent-api/dev/us-east-1" - domain = local.dns + region = local.region + source = "../../environments/swarms-aws-agent-api/dev/us-east-1" + domain = local.dns #ami_id = data.aws_ami.ami.id ami_id = local.us_east_2_swarms_ami_id name = "swarms" - tags = {project="swarms"} - + tags = { project = "swarms" } + } module "swarmdeploy" { -count =0 + count = 0 source = "../../environments/swarms-deploy/dev/us-east-1" domain = local.dns #ami_id = data.aws_ami.ami.id - ami_id = local.us_east_2_swarms_ami_id - name = "swarmdeploy" - tags = {project="swarmdeploy"} - vpc_id = "vpc-0b4cedd083227068d" - subnet_id = "subnet-04b3bdd4b0dc877f0" - ssm_profile_arn= "arn:aws:iam::916723593639:instance-profile/ssm-swarms-profile" + ami_id = local.us_east_2_swarms_ami_id + name = "swarmdeploy" + tags = { project = "swarmdeploy" } + vpc_id = "vpc-0b4cedd083227068d" + subnet_id = "subnet-04b3bdd4b0dc877f0" + ssm_profile_arn = "arn:aws:iam::916723593639:instance-profile/ssm-swarms-profile" ssm_profile_name = "ssm-swarms-profile" } @@ -91,11 +91,11 @@ count =0 # setup the github tokens -module github { - source = "./github" +module "github" { + source = "./github" aws_account_id = local.account - aws_region = local.region -# github_token = var.github_token + aws_region = local.region + # github_token = var.github_token repos = toset([ "terraform-aws-oidc-github", "swarms", @@ -105,29 +105,29 @@ module github { # now create the ssm document -module call_swarms { +module "call_swarms" { source = "../../environments/call-swarms" } -module ssm_observer { +module "ssm_observer" { source = "../../modules/aws/ssm/observability" - ami_id = local.us_east_2_ami_id + ami_id = local.us_east_2_ami_id } -module ssm_setup { +module "ssm_setup" { source = "../../modules/aws/ssm/setup" } -output user_data_new { +output "user_data_new" { value = module.swarms_api.user_data_new } -module pdev { +module "pdev" { source = "./pdev" } -module mdupont { +module "mdupont" { source = "./mdupont" } diff --git a/accounts/swarms/mdupont/main.tf b/accounts/swarms/mdupont/main.tf index 3a3f62d..e8eef16 100644 --- a/accounts/swarms/mdupont/main.tf +++ b/accounts/swarms/mdupont/main.tf @@ -10,12 +10,12 @@ resource "aws_iam_role" "pdev" { Effect = "Allow" Principal = { #AWS = "arn:aws:iam::354918380242:root" - AWS = "arn:aws:iam::767503528736:user/mdupont" + AWS = "arn:aws:iam::767503528736:user/mdupont" } Action = "sts:AssumeRole" - #"Condition": { "Bool": { "aws:MultiFactorAuthPresent": "true" } } + #"Condition": { "Bool": { "aws:MultiFactorAuthPresent": "true" } } } - + ] }) } diff --git a/accounts/swarms/pdev/main.tf b/accounts/swarms/pdev/main.tf index fb359b0..848b7ef 100644 --- a/accounts/swarms/pdev/main.tf +++ b/accounts/swarms/pdev/main.tf @@ -12,9 +12,9 @@ resource "aws_iam_role" "pdev" { AWS = "arn:aws:iam::354918380242:root" } Action = "sts:AssumeRole" -# "Condition": { "Bool": { "aws:MultiFactorAuthPresent": "true" } } + # "Condition": { "Bool": { "aws:MultiFactorAuthPresent": "true" } } } - + ] }) } diff --git a/environments/call-swarms/main.tf b/environments/call-swarms/main.tf index 042b2fd..4829396 100644 --- a/environments/call-swarms/main.tf +++ b/environments/call-swarms/main.tf @@ -10,9 +10,9 @@ resource "aws_ssm_document" "deploy" { name = "deploy" document_format = "YAML" document_type = "Command" - - content = file("${local.codebase_root_path}/environments/call-swarms/deploy.yaml") - tags = {env = "test"} + + content = file("${local.codebase_root_path}/environments/call-swarms/deploy.yaml") + tags = { env = "test" } } resource "aws_ssm_document" "deploy-docker" { @@ -20,7 +20,7 @@ resource "aws_ssm_document" "deploy-docker" { document_format = "YAML" document_type = "Command" content = file("${local.codebase_root_path}/environments/call-swarms/deploy-docker.yaml") - tags = {env = "test"} + tags = { env = "test" } } @@ -73,7 +73,7 @@ resource "aws_ssm_document" "deploy-docker" { # resource "aws_iam_policy" "github_ssm_policy" { # name = "GitHubSSMPolicy" # description = "Policy to allow SSM commands for GitHub role" - + # policy = jsonencode({ # Version = "2012-10-17", # Statement = [ diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf index 8362a6b..7767d3f 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf @@ -1,14 +1,14 @@ -variable alb_arn{} -variable domain_name{} -variable zone_id{} -variable prod_target_group_arn{} -variable test_target_group_arn{} -variable dev_target_group_arn{} +variable "alb_arn" {} +variable "domain_name" {} +variable "zone_id" {} +variable "prod_target_group_arn" {} +variable "test_target_group_arn" {} +variable "dev_target_group_arn" {} module "acm" { -# count = 0 - source = "terraform-aws-modules/acm/aws" - version = "~> 4.0" + # count = 0 + source = "terraform-aws-modules/acm/aws" + version = "~> 4.0" domain_name = var.domain_name zone_id = var.zone_id #wait_for_validation = false @@ -20,13 +20,13 @@ module "acm" { ## add a rule for matching urls for /v1/// and routing that to a new target group #like var.aws_lb_target_group{ { user, agent, api, target group name}} resource "aws_lb_listener" "this" { - port = 443 - protocol = "HTTPS" - ssl_policy = "ELBSecurityPolicy-TLS13-1-2-Res-2021-06" - certificate_arn = module.acm.acm_certificate_arn + port = 443 + protocol = "HTTPS" + ssl_policy = "ELBSecurityPolicy-TLS13-1-2-Res-2021-06" + certificate_arn = module.acm.acm_certificate_arn load_balancer_arn = var.alb_arn default_action { - target_group_arn =var.prod_target_group_arn + target_group_arn = var.prod_target_group_arn type = "forward" } } @@ -35,11 +35,11 @@ resource "aws_lb_listener" "this" { # make a new one for checking for name is test.api.swarms.ai resource "aws_lb_listener_rule" "route_v1_api" { listener_arn = aws_lb_listener.this.arn - priority = 200 # Set priority as needed, must be unique + priority = 200 # Set priority as needed, must be unique action { type = "forward" - target_group_arn = var.test_target_group_arn # New target group's ARN + target_group_arn = var.test_target_group_arn # New target group's ARN } condition { @@ -51,11 +51,11 @@ resource "aws_lb_listener_rule" "route_v1_api" { resource "aws_lb_listener_rule" "route_v1_api_dev" { listener_arn = aws_lb_listener.this.arn - priority = 100 # Set priority as needed, must be unique + priority = 100 # Set priority as needed, must be unique action { type = "forward" - target_group_arn = var.dev_target_group_arn # New target group's ARN + target_group_arn = var.dev_target_group_arn # New target group's ARN } condition { @@ -66,8 +66,8 @@ resource "aws_lb_listener_rule" "route_v1_api_dev" { } resource "aws_lb_listener" "insecure" { - port = 80 - protocol = "HTTP" + port = 80 + protocol = "HTTP" load_balancer_arn = var.alb_arn default_action { target_group_arn = var.prod_target_group_arn diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf index a03a2a6..9d5e4bc 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf @@ -1,13 +1,13 @@ -variable vpc_id {} -variable security_group_id {} # = local.name -variable name {} # = local.name -variable domain_name {} -variable public_subnets {} #= module.vpc.public_subnets +variable "vpc_id" {} +variable "security_group_id" {} # = local.name +variable "name" {} # = local.name +variable "domain_name" {} +variable "public_subnets" {} #= module.vpc.public_subnets data "aws_availability_zones" "available" {} locals { - name = "ex-${basename(path.cwd)}" + name = "ex-${basename(path.cwd)}" vpc_cidr = "10.0.0.0/16" azs = slice(data.aws_availability_zones.available.names, 0, 3) @@ -20,69 +20,69 @@ locals { } module "alb" { - source = "terraform-aws-modules/alb/aws" - version = "9.12.0" - name = "${var.name}-api" # local.name - vpc_id = var.vpc_id # module.vpc.vpc_id - subnets = var.public_subnets # module.vpc.public_subnets + source = "terraform-aws-modules/alb/aws" + version = "9.12.0" + name = "${var.name}-api" # local.name + vpc_id = var.vpc_id # module.vpc.vpc_id + subnets = var.public_subnets # module.vpc.public_subnets enable_deletion_protection = false - create_security_group=false - security_groups = [var.security_group_id] - client_keep_alive = 7200 - tags = local.tags + create_security_group = false + security_groups = [var.security_group_id] + client_keep_alive = 7200 + tags = local.tags } -output dns { +output "dns" { value = module.alb.dns_name } module "route53" { - source = "./route53/" + source = "./route53/" alb_dns_name = module.alb.dns_name alb_dns_zone = module.alb.zone_id - domain_name = var.domain_name + domain_name = var.domain_name } module "tg_prod" { - source = "./target_group/" + source = "./target_group/" name_prefix = "swarms" - vpc_id = var.vpc_id # module.vpc.vpc_id + vpc_id = var.vpc_id # module.vpc.vpc_id } module "tg_test" { - source = "./target_group/" + source = "./target_group/" name_prefix = "test" - vpc_id = var.vpc_id # module.vpc.vpc_id + vpc_id = var.vpc_id # module.vpc.vpc_id } module "tg_dev" { - source = "./target_group/" + source = "./target_group/" name_prefix = "dev" - vpc_id = var.vpc_id # module.vpc.vpc_id + vpc_id = var.vpc_id # module.vpc.vpc_id } module "https" { source = "./https/" # vpc_id = var.vpc_id # module.vpc.vpc_id - zone_id = module.route53.primary_zone_id - domain_name = var.domain_name - alb_arn = module.alb.arn + zone_id = module.route53.primary_zone_id + domain_name = var.domain_name + alb_arn = module.alb.arn prod_target_group_arn = module.tg_prod.alb_target_group_arn test_target_group_arn = module.tg_test.alb_target_group_arn - dev_target_group_arn = module.tg_dev.alb_target_group_arn + dev_target_group_arn = module.tg_dev.alb_target_group_arn } -output prod_alb_target_group_arn { +output "prod_alb_target_group_arn" { value = module.tg_prod.alb_target_group_arn } -output test_alb_target_group_arn { +output "test_alb_target_group_arn" { value = module.tg_test.alb_target_group_arn } -output dev_alb_target_group_arn { +output "dev_alb_target_group_arn" { value = module.tg_dev.alb_target_group_arn } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf index c140e2f..98c3b96 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf @@ -1,26 +1,26 @@ -variable domain_name {} # = local.name -variable alb_dns_name {} -variable alb_dns_zone {} +variable "domain_name" {} # = local.name +variable "alb_dns_name" {} +variable "alb_dns_zone" {} data "aws_route53_zone" "primary" { - name = var.domain_name + name = var.domain_name } resource "aws_route53_record" "api-cname" { zone_id = data.aws_route53_zone.primary.zone_id name = var.domain_name # type = "CNAME" - type = "A" -# ttl = 5 + type = "A" + # ttl = 5 -# weighted_routing_policy { -# weight = 10 -# } + # weighted_routing_policy { + # weight = 10 + # } #set_identifier = "dev" - alias { - name = var.alb_dns_name - zone_id = var.alb_dns_zone + alias { + name = var.alb_dns_name + zone_id = var.alb_dns_zone evaluate_target_health = true - + # } } @@ -30,7 +30,7 @@ resource "aws_route53_record" "api-cname-test" { name = "test.${var.domain_name}" type = "CNAME" records = [aws_route53_record.api-cname.fqdn] - ttl= 300 + ttl = 300 } resource "aws_route53_record" "api-cname-dev" { @@ -38,16 +38,16 @@ resource "aws_route53_record" "api-cname-dev" { name = "dev.${var.domain_name}" type = "CNAME" records = [aws_route53_record.api-cname.fqdn] - ttl= 300 + ttl = 300 } -output cname { +output "cname" { value = aws_route53_record.api-cname.fqdn } -output zone { +output "zone" { value = data.aws_route53_zone.primary } -output primary_zone_id { +output "primary_zone_id" { value = data.aws_route53_zone.primary.zone_id } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/target_group/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/target_group/main.tf index a971dea..bba27f5 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/target_group/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/target_group/main.tf @@ -1,19 +1,19 @@ -variable vpc_id {} -variable name_prefix {} +variable "vpc_id" {} +variable "name_prefix" {} resource "aws_lb_target_group" "this" { - name_prefix = var.name_prefix - protocol = "HTTP" - port = 80 - target_type = "instance" - vpc_id = var.vpc_id - deregistration_delay = 10 + name_prefix = var.name_prefix + protocol = "HTTP" + port = 80 + target_type = "instance" + vpc_id = var.vpc_id + deregistration_delay = 10 #load_balancing_algorithm_type = "weighted_random" #load_balancing_anomaly_mitigation = "on" #load_balancing_cross_zone_enabled = false protocol_version = "HTTP1" # health_check { - path = "/v1/docs" # the docs api + path = "/v1/docs" # the docs api enabled = true healthy_threshold = 10 interval = 130 @@ -22,14 +22,14 @@ resource "aws_lb_target_group" "this" { timeout = 120 unhealthy_threshold = 10 } - -# stickiness { -# cookie_duration = 86400 -# enabled = true -# type = "lb_cookie" -# } - + + # stickiness { + # cookie_duration = 86400 + # enabled = true + # type = "lb_cookie" + # } + } -output alb_target_group_arn { +output "alb_target_group_arn" { value = aws_lb_target_group.this.arn } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf index cfa945d..27f5e58 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf @@ -1,33 +1,33 @@ -variable aws_iam_instance_profile_ssm_arn {} -variable target_group_arn{} -variable name {} -variable instance_type {} -variable launch_template_id {} -variable image_id {} -variable vpc_id {} -variable tags {} -variable ec2_subnet_id {} +variable "aws_iam_instance_profile_ssm_arn" {} +variable "target_group_arn" {} +variable "name" {} +variable "instance_type" {} +variable "launch_template_id" {} +variable "image_id" {} +variable "vpc_id" {} +variable "tags" {} +variable "ec2_subnet_id" {} locals { iam_instance_profile_arn = var.aws_iam_instance_profile_ssm_arn - instance_type = var.instance_type + instance_type = var.instance_type } module "autoscaling" { source = "terraform-aws-modules/autoscaling/aws" version = "8.0.0" - name = var.name + name = var.name - health_check_type = "EC2" - desired_capacity = 1 - max_size = 5 - min_size = 1 + health_check_type = "EC2" + desired_capacity = 1 + max_size = 5 + min_size = 1 create_launch_template = false - update_default_version = true - - launch_template_id = var.launch_template_id - launch_template_version = "$Latest" + update_default_version = true + + launch_template_id = var.launch_template_id + launch_template_version = "$Latest" vpc_zone_identifier = [var.ec2_subnet_id] @@ -35,16 +35,16 @@ module "autoscaling" { market_type = "spot" } network_interfaces = [{ - associate_public_ip_address=true + associate_public_ip_address = true device_index = 0 delete_on_termination = true description = "interface1" -# security_groups = [var.security_group_id] - } + # security_groups = [var.security_group_id] + } ] instance_type = var.instance_type - image_id = var.image_id - + image_id = var.image_id + create_iam_instance_profile = true iam_role_name = "ssm-${var.name}" iam_role_path = "/ec2/" @@ -65,7 +65,7 @@ module "autoscaling" { } } - # Target scaling policy schedule based on average CPU load + # Target scaling policy schedule based on average CPU load scaling_policies = { avg-cpu-policy-greater-than-50 = { policy_type = "TargetTrackingScaling" diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf index 648dc97..95c36c7 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf @@ -1,20 +1,20 @@ -variable ssm_parameter_name_cw_agent_config{} -variable branch {} -variable install_script {} -variable iam_instance_profile_name {} -variable security_group_id {} -variable name {} -variable vpc_id {} -variable ami_id {} -variable tags {} -variable key_name {} -variable instance_type {} +variable "ssm_parameter_name_cw_agent_config" {} +variable "branch" {} +variable "install_script" {} +variable "iam_instance_profile_name" {} +variable "security_group_id" {} +variable "name" {} +variable "vpc_id" {} +variable "ami_id" {} +variable "tags" {} +variable "key_name" {} +variable "instance_type" {} locals { tags = { - project="swarms" + project = "swarms" instance_type = var.instance_type - name = var.name + name = var.name } user_data = <<-EOF #!/bin/bash @@ -52,25 +52,25 @@ locals { bash -x ${var.install_script} EOF - + } data "aws_ssm_parameter" "cw_agent_config" { #arn:aws:ssm:us-east-2:916723593639:parameter/cloudwatch-agent/config - name = var.ssm_parameter_name_cw_agent_config + name = var.ssm_parameter_name_cw_agent_config #"/cloudwatch-agent/config" } # defined resource "aws_launch_template" "ec2_launch_template" { - name_prefix = "${var.name}-launch-template-" - image_id = var.ami_id - key_name = var.key_name - instance_type = var.instance_type + name_prefix = "${var.name}-launch-template-" + image_id = var.ami_id + key_name = var.key_name + instance_type = var.instance_type network_interfaces { associate_public_ip_address = true - delete_on_termination = true - security_groups = [var.security_group_id] + delete_on_termination = true + security_groups = [var.security_group_id] } - + iam_instance_profile { # iam_instance_profile_arn = aws_iam_instance_profile.ssm.arn name = var.iam_instance_profile_name #aws_iam_instance_profile.ec2_instance_profile.name @@ -87,7 +87,7 @@ resource "aws_launch_template" "ec2_launch_template" { } } user_data = base64encode(local.user_data) - tags = var.tags + tags = var.tags } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template_docker/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template_docker/main.tf index 359a741..58fff78 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template_docker/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template_docker/main.tf @@ -1,21 +1,21 @@ # launch template for running agent api in docker -variable ssm_parameter_name_cw_agent_config{} -variable branch {} -variable install_script {} -variable iam_instance_profile_name {} -variable security_group_id {} -variable name {} -variable vpc_id {} -variable ami_id {} -variable tags {} -variable key_name {} -variable instance_type {} +variable "ssm_parameter_name_cw_agent_config" {} +variable "branch" {} +variable "install_script" {} +variable "iam_instance_profile_name" {} +variable "security_group_id" {} +variable "name" {} +variable "vpc_id" {} +variable "ami_id" {} +variable "tags" {} +variable "key_name" {} +variable "instance_type" {} locals { tags = { - project="swarms" + project = "swarms" instance_type = var.instance_type - name = var.name + name = var.name } # FIXME refactor launch template to pass in user data as template parameter, # split up user data into reusable chunks that we can use in different forms like docker files @@ -71,24 +71,24 @@ locals { bash -x ${var.install_script} EOF - + } data "aws_ssm_parameter" "cw_agent_config" { - name = var.ssm_parameter_name_cw_agent_config + name = var.ssm_parameter_name_cw_agent_config } # defined resource "aws_launch_template" "ec2_launch_template" { - name_prefix = "${var.name}-launch-template-" - image_id = var.ami_id - key_name = var.key_name - instance_type = var.instance_type + name_prefix = "${var.name}-launch-template-" + image_id = var.ami_id + key_name = var.key_name + instance_type = var.instance_type network_interfaces { associate_public_ip_address = true - delete_on_termination = true - security_groups = [var.security_group_id] + delete_on_termination = true + security_groups = [var.security_group_id] } - + iam_instance_profile { # iam_instance_profile_arn = aws_iam_instance_profile.ssm.arn name = var.iam_instance_profile_name #aws_iam_instance_profile.ec2_instance_profile.name @@ -105,7 +105,7 @@ resource "aws_launch_template" "ec2_launch_template" { } } user_data = base64encode(local.user_data) - tags = var.tags + tags = var.tags } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf index 2b7fbaa..c4eb0c7 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf @@ -1,24 +1,24 @@ locals { - name = "swarms" + name = "swarms" tags = { - project="swarms" + project = "swarms" } } module "security_group_instance" { - source = "terraform-aws-modules/security-group/aws" - version = "~> 5.0" + source = "terraform-aws-modules/security-group/aws" + version = "~> 5.0" name = "${local.name}-ec2" description = "Security Group for EC2 Instance" - vpc_id = local.vpc_id + vpc_id = local.vpc_id ingress_with_cidr_blocks = [ - { - from_port = 443 - to_port = 443 - protocol = "tcp" - cidr_blocks = "0.0.0.0/0" - }, + { + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = "0.0.0.0/0" + }, { from_port = 80 to_port = 80 @@ -28,15 +28,15 @@ module "security_group_instance" { ] egress_rules = ["all-all"] - tags = local.tags + tags = local.tags } module "ec2" { - source = "terraform-aws-modules/ec2-instance/aws" + source = "terraform-aws-modules/ec2-instance/aws" associate_public_ip_address = true # for now - name = local.name - ami = local.ami # data.aws_ami.ubuntu.id - instance_type = "t3.large" + name = local.name + ami = local.ami # data.aws_ami.ubuntu.id + instance_type = "t3.large" create_iam_instance_profile = true iam_role_description = "IAM role for EC2 instance" iam_role_policies = { @@ -48,11 +48,11 @@ module "ec2" { { encrypted = true volume_size = 30 - volume_type = "gp3" + volume_type = "gp3" } ] - user_data = <<-EOF + user_data = <<-EOF #!/bin/bash export HOME=/root apt update @@ -67,9 +67,9 @@ export BRANCH=feature/ec2 git checkout --force $BRANCH bash -x /opt/swarms/api/install.sh EOF - tags = local.tags + tags = local.tags create_spot_instance = true - subnet_id = local.ec2_subnet_id + subnet_id = local.ec2_subnet_id } @@ -78,6 +78,6 @@ output "ec2_data" { } output "iam_instance_profile_name" { - value = module.ec2.iam_instance_profile_id + value = module.ec2.iam_instance_profile_id description = "IAM Instance Profile Name created for EC2 instance" } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/variables.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/variables.tf index e9e30b2..428bf9e 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/variables.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/variables.tf @@ -1,3 +1,3 @@ -variable test_server_count { +variable "test_server_count" { default = 1 # how many test servers to run } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/resource_launchers/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/resource_launchers/main.tf index 70a86cd..1b0de80 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/resource_launchers/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/resource_launchers/main.tf @@ -25,7 +25,7 @@ resource "aws_iam_policy" "lambda_scaling_policy" { "autoscaling:SetDesiredCapacity" ] Resource = "*" - Effect = "Allow" + Effect = "Allow" }] }) } @@ -39,36 +39,13 @@ resource "aws_lambda_function" "scale_lambda" { function_name = "${local.name}-scale-lambda" role = aws_iam_role.lambda_role.arn - runtime = "python3.8" - handler = "scale_lambda.handler" + runtime = "python3.8" + handler = "scale_lambda.handler" source_code_hash = filebase64sha256("lambda/scale_lambda.zip") - + # Environment variables for the Lambda function environment = { AUTO_SCALING_GROUP_NAME = aws_autoscaling_group.ec2_autoscaling_group.name } } -``` - -### 4. Lambda Function Code (Python) - -You'll need to create a file named `scale_lambda.py` in the `lambda/` directory with content like the following: - -```python -import boto3 -import os - -def handler(event, context): - asg_name = os.environ['AUTO_SCALING_GROUP_NAME'] - client = boto3.client('autoscaling') - - response = client.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_name]) - desired_capacity = response['AutoScalingGroups'][0]['DesiredCapacity'] - - # Example scaling logic - client.set_desired_capacity( - AutoScalingGroupName=asg_name, - DesiredCapacity=desired_capacity + 1, # Increment desired instances - HonorCooldown=True - ) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf index 0885274..c320d8b 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf @@ -1,4 +1,4 @@ -variable tags {} +variable "tags" {} data "aws_iam_policy_document" "default" { statement { @@ -7,38 +7,38 @@ data "aws_iam_policy_document" "default" { effect = "Allow" } - statement { - actions = ["kms:Decrypt"] - resources = [ "arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590" ] - effect = "Allow" - } - - statement { - actions = [ - "logs:DescribeLogGroups", - "logs:DescribeLogStreams", - "logs:CreateLogGroup", - "logs:CreateLogStream", - "logs:PutLogEvents", - "logs:PutLogEventsBatch", - "cloudwatch:PutMetricData", - "ec2:DescribeTags", - ] - resources = [ "*" ] - effect = "Allow" - } + statement { + actions = ["kms:Decrypt"] + resources = ["arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590"] + effect = "Allow" + } statement { - effect = "Allow" - resources = [ "arn:aws:s3:::swarms-session-logs*" ] + actions = [ + "logs:DescribeLogGroups", + "logs:DescribeLogStreams", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + "logs:PutLogEventsBatch", + "cloudwatch:PutMetricData", + "ec2:DescribeTags", + ] + resources = ["*"] + effect = "Allow" + } + + statement { + effect = "Allow" + resources = ["arn:aws:s3:::swarms-session-logs*"] actions = [ "s3:GetEncryptionConfiguration" ] } - + statement { - effect = "Allow" - resources = [ "*" ] + effect = "Allow" + resources = ["*"] actions = [ "kms:Encrypt*", "kms:Decrypt*", @@ -70,12 +70,12 @@ data "aws_iam_policy_document" "default" { # } #arn:aws:logs:us-east-2:916723593639:log-group::log-stream - -# statement { -# actions = ["${var.ssm_actions}"] -# resources = ["${formatlist("arn:aws:ssm:%s:%s:parameter/%s", var.region, var.account_id, var.ssm_parameters)}"] -# effect = "Allow" -# } + + # statement { + # actions = ["${var.ssm_actions}"] + # resources = ["${formatlist("arn:aws:ssm:%s:%s:parameter/%s", var.region, var.account_id, var.ssm_parameters)}"] + # effect = "Allow" + # } } @@ -91,7 +91,7 @@ resource "aws_iam_role_policy_attachment" "AmazonSSMManagedEC2InstanceDefaultPol } resource "aws_iam_role_policy_attachment" "default" { -# count = local.policy_only + # count = local.policy_only role = join("", aws_iam_role.ssm.*.name) policy_arn = join("", aws_iam_policy.default.*.arn) } @@ -132,10 +132,10 @@ resource "aws_iam_instance_profile" "ssm" { } -output ssm_profile_name { +output "ssm_profile_name" { value = aws_iam_instance_profile.ssm.name } -output ssm_profile_arn { +output "ssm_profile_arn" { value = aws_iam_instance_profile.ssm.arn } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/security/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/security/main.tf index 5572aba..f099b61 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/security/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/security/main.tf @@ -1,6 +1,6 @@ -variable vpc_id { } -variable tags { } -variable name { } +variable "vpc_id" {} +variable "tags" {} +variable "name" {} module "asg_sg" { source = "terraform-aws-modules/security-group/aws" @@ -14,7 +14,7 @@ module "asg_sg" { ingress_rules = [ "https-443-tcp", "http-80-tcp", -# "ssh-tcp" dont need this now + # "ssh-tcp" dont need this now ] egress_rules = ["all-all"] @@ -30,11 +30,11 @@ module "asg_sg_internal" { description = "An internal security group" vpc_id = var.vpc_id # see ~/2024/12/13/terraform-aws-security-group/examples/complete/main.tf - ingress_with_source_security_group_id = [ + ingress_with_source_security_group_id = [ { - rule = "http-80-tcp", + rule = "http-80-tcp", # only allow from load balancer for security - source_security_group_id = module.asg_sg.security_group_id + source_security_group_id = module.asg_sg.security_group_id } ] egress_rules = ["all-all"] @@ -43,11 +43,11 @@ module "asg_sg_internal" { } output "security_group_id" { - value = module.asg_sg.security_group_id + value = module.asg_sg.security_group_id } output "internal_security_group_id" { - value = module.asg_sg_internal.security_group_id + value = module.asg_sg_internal.security_group_id } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/main.tf index 303c8fc..daba127 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/main.tf @@ -3,15 +3,15 @@ data "aws_availability_zones" "available" {} locals { - name = "swarms" + name = "swarms" vpc_cidr = "10.0.0.0/16" azs = slice(data.aws_availability_zones.available.names, 0, 3) tags = { - project = local.name -# GithubRepo = "terraform-aws-vpc" -# GithubOrg = "terraform-aws-modules" + project = local.name + # GithubRepo = "terraform-aws-vpc" + # GithubOrg = "terraform-aws-modules" } } @@ -41,12 +41,12 @@ locals { ################################################################################ module "vpc" { - source = "terraform-aws-modules/vpc/aws" - name = local.name - cidr = local.vpc_cidr - azs = local.azs - private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k)] - public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 4)] + source = "terraform-aws-modules/vpc/aws" + name = local.name + cidr = local.vpc_cidr + azs = local.azs + private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k)] + public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 4)] # database_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 8)] # elasticache_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 12)] # redshift_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 16)] @@ -80,7 +80,7 @@ module "vpc" { # enable_dhcp_options = true # dhcp_options_domain_name = "service.consul" # dhcp_options_domain_name_servers = ["127.0.0.1", "10.10.0.2"] - + # # VPC Flow Logs (Cloudwatch log group and IAM role will be created) # vpc_flow_log_iam_role_name = "vpc-complete-example-role" # vpc_flow_log_iam_role_use_name_prefix = false @@ -88,8 +88,8 @@ module "vpc" { # create_flow_log_cloudwatch_log_group = true # create_flow_log_cloudwatch_iam_role = true # flow_log_max_aggregation_interval = 60 - - tags = local.tags + + tags = local.tags } # ################################################################################ @@ -230,5 +230,5 @@ module "vpc" { # tags = local.tags # } output "vpc" { -value = module.vpc + value = module.vpc } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/outputs.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/outputs.tf index 32381e3..470ceae 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/outputs.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/outputs.tf @@ -549,10 +549,10 @@ output "vpc_flow_log_cloudwatch_iam_role_arn" { # description = "ID of the security group" # value = module.vpc_endpoints.security_group_id # } -output ec2_public_subnet_id_2 { +output "ec2_public_subnet_id_2" { value = module.vpc.public_subnets[1] } -output ec2_public_subnet_id_1 { +output "ec2_public_subnet_id_1" { value = module.vpc.public_subnets[0] } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index 5aa27ea..f26cd70 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -1,29 +1,29 @@ -variable region {} -variable key_name { +variable "region" {} +variable "key_name" { default = "mdupont-deployer-key" # FIXME: move to settings } locals { # instance_type = "t3.large" # instance_type = "t3.medium" ami_name = "ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*" - name = "swarms" - domain = var.domain + name = "swarms" + domain = var.domain tags = { - project="swarms" + project = "swarms" } dev_tags = { - sandbox="kye" + sandbox = "kye" } } -variable domain {} -variable aws_account_id {} -variable ami_id {} -variable tags {} -variable name {} +variable "domain" {} +variable "aws_account_id" {} +variable "ami_id" {} +variable "tags" {} +variable "name" {} locals { - ami_id = var.ami_id + ami_id = var.ami_id new_ami_id = "ami-08093b6770af41b14" # environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/Readme.md } @@ -35,14 +35,14 @@ module "vpc" { locals { ec2_public_subnet_id_1 = module.vpc.ec2_public_subnet_id_1 ec2_public_subnet_id_2 = module.vpc.ec2_public_subnet_id_2 - vpc_id = module.vpc.vpc_id + vpc_id = module.vpc.vpc_id } module "security" { source = "./components/security" vpc_id = local.vpc_id - tags = local.tags - name = local.name + tags = local.tags + name = local.name } module "kp" { @@ -64,10 +64,10 @@ module "kp" { # } variable "instance_types" { - type = list(string) + type = list(string) default = [ - # "t4g.nano", "t3a.nano", "t3.nano", "t2.nano", - # "t4g.micro", "t3a.micro", "t3.micro", "t2.micro", "t1.micro", + # "t4g.nano", "t3a.nano", "t3.nano", "t2.nano", + # "t4g.micro", "t3a.micro", "t3.micro", "t2.micro", "t1.micro", #"t4g.small", "t3a.small", #"t3.small", #"t2.small", not working @@ -79,7 +79,7 @@ variable "instance_types" { module "roles" { source = "./components/roles" - tags = local.tags + tags = local.tags } # module "lt_dynamic" { @@ -98,57 +98,57 @@ module "roles" { # } module "lt_dynamic_ami_prod" { - vpc_id = local.vpc_id - for_each = toset(var.instance_types) - instance_type = each.key - name = "swarms-ami-${each.key}" + vpc_id = local.vpc_id + for_each = toset(var.instance_types) + instance_type = each.key + name = "swarms-ami-${each.key}" security_group_id = module.security.internal_security_group_id - ami_id = local.new_ami_id - key_name = var.key_name - tags= merge(local.tags, { + ami_id = local.new_ami_id + key_name = var.key_name + tags = merge(local.tags, { environment = "production" }) - source = "./components/launch_template" - iam_instance_profile_name = module.roles.ssm_profile_name - install_script = "/opt/swarms/api/just_run.sh" - ssm_parameter_name_cw_agent_config= "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/cloudwatch-agent/config" - branch = "feature/ec2" + source = "./components/launch_template" + iam_instance_profile_name = module.roles.ssm_profile_name + install_script = "/opt/swarms/api/just_run.sh" + ssm_parameter_name_cw_agent_config = "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/cloudwatch-agent/config" + branch = "feature/ec2" } module "lt_dynamic_ami_test" { - branch = "feature/cloudwatch" - vpc_id = local.vpc_id - for_each = toset(var.instance_types) - instance_type = each.key - name = "swarms-ami-${each.key}" + branch = "feature/cloudwatch" + vpc_id = local.vpc_id + for_each = toset(var.instance_types) + instance_type = each.key + name = "swarms-ami-${each.key}" security_group_id = module.security.internal_security_group_id - ami_id = local.new_ami_id - tags= merge(local.tags, { + ami_id = local.new_ami_id + tags = merge(local.tags, { environment = "test" }) - source = "./components/launch_template" - key_name = var.key_name #"mdupont-deployer-key" - ssm_parameter_name_cw_agent_config= "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/cloudwatch-agent/config/details" - iam_instance_profile_name = module.roles.ssm_profile_name - install_script = "/opt/swarms/api/just_run.sh" + source = "./components/launch_template" + key_name = var.key_name #"mdupont-deployer-key" + ssm_parameter_name_cw_agent_config = "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/cloudwatch-agent/config/details" + iam_instance_profile_name = module.roles.ssm_profile_name + install_script = "/opt/swarms/api/just_run.sh" } module "lt_dynamic_ami_docker" { - branch = "feature/squash2-docker" - vpc_id = local.vpc_id - for_each = toset(var.instance_types) - instance_type = each.key - name = "swarms-docker-${each.key}" + branch = "feature/squash2-docker" + vpc_id = local.vpc_id + for_each = toset(var.instance_types) + instance_type = each.key + name = "swarms-docker-${each.key}" security_group_id = module.security.internal_security_group_id - ami_id = local.new_ami_id - tags= merge(local.tags, { + ami_id = local.new_ami_id + tags = merge(local.tags, { environment = "test" }) - source = "./components/launch_template_docker" - key_name = var.key_name #"mdupont-deployer-key" - ssm_parameter_name_cw_agent_config= "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/cloudwatch-agent/config/details" - iam_instance_profile_name = module.roles.ssm_profile_name + source = "./components/launch_template_docker" + key_name = var.key_name #"mdupont-deployer-key" + ssm_parameter_name_cw_agent_config = "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/cloudwatch-agent/config/details" + iam_instance_profile_name = module.roles.ssm_profile_name #install_script = "/opt/swarms/api/docker-boot.sh" this is called from ssm for a refresh install_script = "/opt/swarms/api/rundocker.sh" } @@ -156,16 +156,16 @@ module "lt_dynamic_ami_docker" { module "alb" { - source = "./components/application_load_balancer" - domain_name = local.domain - security_group_id = module.security.security_group_id # allowed to talk to internal + source = "./components/application_load_balancer" + domain_name = local.domain + security_group_id = module.security.security_group_id # allowed to talk to internal public_subnets = [ local.ec2_public_subnet_id_1, - local.ec2_public_subnet_id_2 ] + local.ec2_public_subnet_id_2] vpc_id = local.vpc_id - name = local.name + name = local.name } -output alb { +output "alb" { value = module.alb } @@ -190,63 +190,63 @@ output alb { module "asg_dynamic_new_ami" { # built with packer #count =0 - tags = local.tags - vpc_id = local.vpc_id - image_id = local.new_ami_id - ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 - for_each = toset(var.instance_types) + tags = local.tags + vpc_id = local.vpc_id + image_id = local.new_ami_id + ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 + for_each = toset(var.instance_types) aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn - source = "./components/autoscaling_group" -# security_group_id = module.security.internal_security_group_id - instance_type = each.key - name = "swarms-ami-${each.key}" - launch_template_id = module.lt_dynamic_ami_prod[each.key].launch_template_id - target_group_arn = module.alb.prod_alb_target_group_arn + source = "./components/autoscaling_group" + # security_group_id = module.security.internal_security_group_id + instance_type = each.key + name = "swarms-ami-${each.key}" + launch_template_id = module.lt_dynamic_ami_prod[each.key].launch_template_id + target_group_arn = module.alb.prod_alb_target_group_arn } module "asg_dynamic_new_ami_test" { # built with packer #count =0 - tags = merge(local.tags, local.dev_tags) - vpc_id = local.vpc_id - image_id = local.new_ami_id - ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 - for_each = toset(var.instance_types) + tags = merge(local.tags, local.dev_tags) + vpc_id = local.vpc_id + image_id = local.new_ami_id + ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 + for_each = toset(var.instance_types) aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn - source = "./components/autoscaling_group" -# security_group_id = module.security.internal_security_group_id - instance_type = each.key - name = "test-swarms-ami-${each.key}" - launch_template_id = module.lt_dynamic_ami_test[each.key].launch_template_id - target_group_arn = module.alb.test_alb_target_group_arn + source = "./components/autoscaling_group" + # security_group_id = module.security.internal_security_group_id + instance_type = each.key + name = "test-swarms-ami-${each.key}" + launch_template_id = module.lt_dynamic_ami_test[each.key].launch_template_id + target_group_arn = module.alb.test_alb_target_group_arn } module "asg_dynamic_new_ami_dev" { # built with packer #count =0 - tags = merge(local.tags, local.dev_tags) - vpc_id = local.vpc_id - image_id = local.new_ami_id - ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 - for_each = toset(var.instance_types) + tags = merge(local.tags, local.dev_tags) + vpc_id = local.vpc_id + image_id = local.new_ami_id + ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 + for_each = toset(var.instance_types) aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn - source = "./components/autoscaling_group" -# security_group_id = module.security.internal_security_group_id - instance_type = each.key - name = "docker-swarms-ami-${each.key}" - launch_template_id = module.lt_dynamic_ami_docker[each.key].launch_template_id - target_group_arn = module.alb.dev_alb_target_group_arn + source = "./components/autoscaling_group" + # security_group_id = module.security.internal_security_group_id + instance_type = each.key + name = "docker-swarms-ami-${each.key}" + launch_template_id = module.lt_dynamic_ami_docker[each.key].launch_template_id + target_group_arn = module.alb.dev_alb_target_group_arn } -output security_group_id { +output "security_group_id" { value = module.security.security_group_id } -output vpc { +output "vpc" { value = module.vpc } -output user_data_new { +output "user_data_new" { value = module.lt_dynamic_ami_test["t3.medium"].user_data } diff --git a/environments/swarms-deploy/dev/us-east-1/components/autoscaling_group/main.tf b/environments/swarms-deploy/dev/us-east-1/components/autoscaling_group/main.tf index e82fa4a..3d2d550 100644 --- a/environments/swarms-deploy/dev/us-east-1/components/autoscaling_group/main.tf +++ b/environments/swarms-deploy/dev/us-east-1/components/autoscaling_group/main.tf @@ -1,33 +1,33 @@ -variable aws_iam_instance_profile_ssm_arn {} +variable "aws_iam_instance_profile_ssm_arn" {} #variable target_group_arn{} -variable name {} -variable instance_type {} -variable launch_template_id {} -variable image_id {} -variable vpc_id {} -variable tags {} -variable ec2_subnet_id {} +variable "name" {} +variable "instance_type" {} +variable "launch_template_id" {} +variable "image_id" {} +variable "vpc_id" {} +variable "tags" {} +variable "ec2_subnet_id" {} locals { iam_instance_profile_arn = var.aws_iam_instance_profile_ssm_arn - instance_type = var.instance_type + instance_type = var.instance_type } module "autoscaling" { source = "terraform-aws-modules/autoscaling/aws" version = "8.0.0" - name = var.name + name = var.name - health_check_type = "EC2" - desired_capacity = 1 - max_size = 5 - min_size = 1 + health_check_type = "EC2" + desired_capacity = 1 + max_size = 5 + min_size = 1 create_launch_template = false - update_default_version = true - - launch_template_id = var.launch_template_id - launch_template_version = "$Latest" + update_default_version = true + + launch_template_id = var.launch_template_id + launch_template_version = "$Latest" vpc_zone_identifier = [var.ec2_subnet_id] @@ -35,16 +35,16 @@ module "autoscaling" { market_type = "spot" } network_interfaces = [{ - associate_public_ip_address=true + associate_public_ip_address = true device_index = 0 delete_on_termination = true description = "interface1" -# security_groups = [var.security_group_id] - } + # security_groups = [var.security_group_id] + } ] instance_type = var.instance_type - image_id = var.image_id - + image_id = var.image_id + create_iam_instance_profile = true iam_role_name = "ssm-${var.name}" iam_role_path = "/ec2/" @@ -65,7 +65,7 @@ module "autoscaling" { # } # } - # Target scaling policy schedule based on average CPU load + # Target scaling policy schedule based on average CPU load scaling_policies = { avg-cpu-policy-greater-than-50 = { policy_type = "TargetTrackingScaling" diff --git a/environments/swarms-deploy/dev/us-east-1/components/launch_template/main.tf b/environments/swarms-deploy/dev/us-east-1/components/launch_template/main.tf index ff953d8..511bb41 100644 --- a/environments/swarms-deploy/dev/us-east-1/components/launch_template/main.tf +++ b/environments/swarms-deploy/dev/us-east-1/components/launch_template/main.tf @@ -1,35 +1,35 @@ -variable install_script {} -variable iam_instance_profile_name {} -variable security_group_id {} -variable name {} -variable vpc_id {} -variable ami_id {} -variable tags {} -variable key_name { +variable "install_script" {} +variable "iam_instance_profile_name" {} +variable "security_group_id" {} +variable "name" {} +variable "vpc_id" {} +variable "ami_id" {} +variable "tags" {} +variable "key_name" { default = "mdupont-deployer-key" } # dont use this -variable instance_type {} +variable "instance_type" {} locals { tags = { - project="swarms" + project = "swarms" instance_type = var.instance_type - name = var.name + name = var.name } } resource "aws_launch_template" "ec2_launch_template" { - name_prefix = "${var.name}-launch-template-" - image_id = var.ami_id - key_name = var.key_name - instance_type = var.instance_type + name_prefix = "${var.name}-launch-template-" + image_id = var.ami_id + key_name = var.key_name + instance_type = var.instance_type network_interfaces { associate_public_ip_address = true - delete_on_termination = true - security_groups = [var.security_group_id] + delete_on_termination = true + security_groups = [var.security_group_id] } - + iam_instance_profile { # iam_instance_profile_arn = aws_iam_instance_profile.ssm.arn name = var.iam_instance_profile_name #aws_iam_instance_profile.ec2_instance_profile.name @@ -67,8 +67,8 @@ resource "aws_launch_template" "ec2_launch_template" { git pull # get the latest version bash -x ${var.install_script} EOF - ) - tags = var.tags + ) + tags = var.tags } diff --git a/environments/swarms-deploy/dev/us-east-1/main.tf b/environments/swarms-deploy/dev/us-east-1/main.tf index 6e69834..4d87cec 100644 --- a/environments/swarms-deploy/dev/us-east-1/main.tf +++ b/environments/swarms-deploy/dev/us-east-1/main.tf @@ -1,26 +1,26 @@ variable "ssm_profile_arn" {} variable "ssm_profile_name" {} -variable vpc_id {} -variable subnet_id {} +variable "vpc_id" {} +variable "subnet_id" {} locals { # instance_type = "t3.large" # instance_type = "t3.medium" ami_name = "ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*" - name = "swarmdeploy" - region = "us-east-2" - domain = var.domain + name = "swarmdeploy" + region = "us-east-2" + domain = var.domain tags = { - project="swarmdeploy" + project = "swarmdeploy" } } -variable domain {} -variable ami_id {} -variable tags {} -variable name {} +variable "domain" {} +variable "ami_id" {} +variable "tags" {} +variable "name" {} locals { - ami_id = var.ami_id + ami_id = var.ami_id #new_ami_id = "ami-08093b6770af41b14" # environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/Readme.md } @@ -37,8 +37,8 @@ data "aws_vpc" "vpc" { } locals { -# ec2_public_subnet_id_1 = module.vpc.ec2_public_subnet_id_1 -# ec2_public_subnet_id_2 = module.vpc.ec2_public_subnet_id_2 + # ec2_public_subnet_id_1 = module.vpc.ec2_public_subnet_id_1 + # ec2_public_subnet_id_2 = module.vpc.ec2_public_subnet_id_2 #vpc_id = module.vpc.vpc_id vpc_id = var.vpc_id } @@ -46,8 +46,8 @@ locals { module "security" { source = "${local.root}/security" vpc_id = local.vpc_id - tags = local.tags - name = local.name + tags = local.tags + name = local.name } #module "kp" { @@ -69,10 +69,10 @@ module "security" { # } variable "instance_types" { - type = list(string) + type = list(string) default = [ - # "t4g.nano", "t3a.nano", "t3.nano", "t2.nano", - # "t4g.micro", "t3a.micro", "t3.micro", "t2.micro", "t1.micro", + # "t4g.nano", "t3a.nano", "t3.nano", "t2.nano", + # "t4g.micro", "t3a.micro", "t3.micro", "t2.micro", "t1.micro", #"t4g.small", "t3a.small", #"t3.small", #"t2.small", not working @@ -88,14 +88,14 @@ variable "instance_types" { #} module "lt_dynamic" { - vpc_id = local.vpc_id - for_each = toset(var.instance_types) - instance_type = each.key - name = "swarms-size-${each.key}" - security_group_id = module.security.internal_security_group_id - ami_id = var.ami_id - tags= local.tags - source = "./components/launch_template" + vpc_id = local.vpc_id + for_each = toset(var.instance_types) + instance_type = each.key + name = "swarms-size-${each.key}" + security_group_id = module.security.internal_security_group_id + ami_id = var.ami_id + tags = local.tags + source = "./components/launch_template" iam_instance_profile_name = var.ssm_profile_name #aws_iam_instance_profile.ssm.name install_script = "/opt/swarms/install.sh" @@ -115,7 +115,7 @@ module "lt_dynamic" { # install_script = "/opt/swarms/api/just_run.sh" # } -output security_group_id { +output "security_group_id" { value = module.security.security_group_id } @@ -139,19 +139,19 @@ output security_group_id { # } module "asg_dynamic" { - tags = local.tags - vpc_id = local.vpc_id - image_id = local.ami_id - ec2_subnet_id = var.subnet_id - for_each = toset(var.instance_types) + tags = local.tags + vpc_id = local.vpc_id + image_id = local.ami_id + ec2_subnet_id = var.subnet_id + for_each = toset(var.instance_types) aws_iam_instance_profile_ssm_arn = var.ssm_profile_arn #iam_instance_profile_name = module.roles.ssm_profile_name - source = "./components/autoscaling_group" + source = "./components/autoscaling_group" # security_group_id = module.security.internal_security_group_id - instance_type = each.key - name = "swarmdeploy-${each.key}" - launch_template_id = module.lt_dynamic[each.key].launch_template_id -# target_group_arn = module.alb.alb_target_group_arn + instance_type = each.key + name = "swarmdeploy-${each.key}" + launch_template_id = module.lt_dynamic[each.key].launch_template_id + # target_group_arn = module.alb.alb_target_group_arn } # module "asg_dynamic_new_ami" { diff --git a/main.tf b/main.tf index cc6fa7b..d31853b 100644 --- a/main.tf +++ b/main.tf @@ -1,4 +1,4 @@ # this -module swarms { +module "swarms" { source = "./accounts/swarms" } diff --git a/modules/aws/ssm/observability/jobs/main.tf b/modules/aws/ssm/observability/jobs/main.tf index cb7a273..34b23c8 100644 --- a/modules/aws/ssm/observability/jobs/main.tf +++ b/modules/aws/ssm/observability/jobs/main.tf @@ -219,10 +219,10 @@ resource "aws_cloudwatch_dashboard" "ssm_operations" { width = 12 height = 6 properties = { - query = "fields @timestamp, @message | sort @timestamp desc | limit 20" - region = var.aws_region - title = "Recent SSM Operation Logs" - view = "table" + query = "fields @timestamp, @message | sort @timestamp desc | limit 20" + region = var.aws_region + title = "Recent SSM Operation Logs" + view = "table" logGroupName = aws_cloudwatch_log_group.ssm_logs.name } } @@ -240,11 +240,11 @@ resource "aws_cloudwatch_metric_alarm" "ssm_failures" { evaluation_periods = "1" metric_name = "CommandFailed" namespace = "AWS/SSM" - period = "300" - statistic = "Sum" - threshold = "0" - alarm_description = "This metric monitors failed SSM commands" - + period = "300" + statistic = "Sum" + threshold = "0" + alarm_description = "This metric monitors failed SSM commands" + dimensions = { DocumentName = aws_ssm_document.python_with_xray.name } diff --git a/modules/aws/ssm/observability/main.tf b/modules/aws/ssm/observability/main.tf index a6a4fee..a9fa252 100755 --- a/modules/aws/ssm/observability/main.tf +++ b/modules/aws/ssm/observability/main.tf @@ -1,4 +1,4 @@ -variable ami_id {} +variable "ami_id" {} # Previous provider and variables configuration remains the same #provider "aws" { # region = var.aws_region @@ -19,7 +19,7 @@ variable "instance_type" { variable "patch_schedule" { description = "Cron expression for patch schedule" type = string - default = "cron(0 0 ? * SUN *)" # Run at midnight every Sunday + default = "cron(0 0 ? * SUN *)" # Run at midnight every Sunday } # Update EC2 role to include SSM permissions @@ -61,7 +61,7 @@ resource "aws_iam_instance_profile" "monitoring_profile" { resource "aws_ssm_patch_baseline" "os_patches" { name = "ec2-patch-baseline" operating_system = "AMAZON_LINUX_2" - + approval_rule { approve_after_days = 7 compliance_level = "HIGH" @@ -90,8 +90,8 @@ resource "aws_ssm_patch_group" "patch_group" { } # SSM Maintenance Window -resource "aws_ssm_maintenance_window" "patch_window"{ - cutoff = 1 +resource "aws_ssm_maintenance_window" "patch_window" { + cutoff = 1 name = "production-patch-window" schedule = var.patch_schedule duration = 4 #"PT4H" # 4 hours @@ -101,9 +101,9 @@ resource "aws_ssm_maintenance_window" "patch_window"{ # Maintenance Window Target resource "aws_ssm_maintenance_window_target" "patch_target" { resource_type = "INSTANCE" - window_id = aws_ssm_maintenance_window.patch_window.id - name = "patch-production-servers" - + window_id = aws_ssm_maintenance_window.patch_window.id + name = "patch-production-servers" + targets { key = "tag:PatchGroup" values = ["production-servers"] @@ -167,7 +167,7 @@ resource "aws_iam_role_policy_attachment" "maintenance_window_policy" { # resource "aws_instance" "monitored_instance" { # ami = var.ami_id # instance_type = var.instance_type - + # iam_instance_profile = aws_iam_instance_profile.monitoring_profile.name # monitoring = true @@ -226,142 +226,142 @@ resource "aws_ssm_association" "cloudwatch_agent_update" { locals { normal_config = { - "agent": { - "metrics_collection_interval": 60 + "agent" : { + "metrics_collection_interval" : 60 }, - "metrics": { - "namespace": "CWAgent", - "append_dimensions": { - "InstanceId": "$${aws:InstanceId}" + "metrics" : { + "namespace" : "CWAgent", + "append_dimensions" : { + "InstanceId" : "$${aws:InstanceId}" + }, + "metrics_collected" : { + "mem" : { + "measurement" : [ + "used_percent", + "used", + "total", + "inactive", + "free", + "cached", + "buffered", + "available_percent", + "available", + "active" + ] }, - "metrics_collected": { - "mem": { - "measurement": [ - "used_percent", - "used", - "total", - "inactive", - "free", - "cached", - "buffered", - "available_percent", - "available", - "active" - ] - }, - "cpu": { - "measurement": [ - "usage_active", - "time_guest_nice", - "time_idle", - "time_irq", - "time_iowait", - "time_guest", - "time_active", - "time_softirq", - "time_nice", - "time_system", - "time_user", - "time_steal", - "usage_guest", - "usage_guest_nice", - "usage_idle", - "usage_iowait", - "usage_irq", - "usage_nice", - "usage_softirq", - "usage_steal" - ] - }, - "disk": { - "measurement": [ - "used_percent", - "free", - "inodes_free", - "inodes_total", - "inodes_used", - "total", - "used" - ] - }, - "diskio": { - "measurement": [ - "read_bytes", - "write_bytes", - "iops_in_progress", - "io_time", - "reads", - "writes", - "read_time", - "write_time" - ] - }, - "net": { - "measurement": [ - "bytes_sent", - "bytes_recv", - "drop_in", - "drop_out", - "err_in", - "err_out", - "packets_sent", - "packets_recv" - ] - }, - "netstat": { - "measurement": [ - "tcp_established", - "tcp_close", - "tcp_close_wait", - "tcp_closing", - "tcp_fin_wait1", - "tcp_last_ack", - "tcp_listen", - "tcp_fin_wait2", - "tcp_none", - "tcp_syn_recv", - "tcp_time_wait", - "tcp_syn_sent", - "udp_socket" - ] - }, - "processes": { - "measurement": [ - "running", - "wait", - "zombies", - "total_threads", - "total", - "paging", - "sleeping", - "stopped", - "dead", - "blocked", - "idle" - ] - }, - "swap": { - "measurement": [ - "used_percent", - "free", - "used" - ] - } + "cpu" : { + "measurement" : [ + "usage_active", + "time_guest_nice", + "time_idle", + "time_irq", + "time_iowait", + "time_guest", + "time_active", + "time_softirq", + "time_nice", + "time_system", + "time_user", + "time_steal", + "usage_guest", + "usage_guest_nice", + "usage_idle", + "usage_iowait", + "usage_irq", + "usage_nice", + "usage_softirq", + "usage_steal" + ] + }, + "disk" : { + "measurement" : [ + "used_percent", + "free", + "inodes_free", + "inodes_total", + "inodes_used", + "total", + "used" + ] + }, + "diskio" : { + "measurement" : [ + "read_bytes", + "write_bytes", + "iops_in_progress", + "io_time", + "reads", + "writes", + "read_time", + "write_time" + ] + }, + "net" : { + "measurement" : [ + "bytes_sent", + "bytes_recv", + "drop_in", + "drop_out", + "err_in", + "err_out", + "packets_sent", + "packets_recv" + ] + }, + "netstat" : { + "measurement" : [ + "tcp_established", + "tcp_close", + "tcp_close_wait", + "tcp_closing", + "tcp_fin_wait1", + "tcp_last_ack", + "tcp_listen", + "tcp_fin_wait2", + "tcp_none", + "tcp_syn_recv", + "tcp_time_wait", + "tcp_syn_sent", + "udp_socket" + ] + }, + "processes" : { + "measurement" : [ + "running", + "wait", + "zombies", + "total_threads", + "total", + "paging", + "sleeping", + "stopped", + "dead", + "blocked", + "idle" + ] + }, + "swap" : { + "measurement" : [ + "used_percent", + "free", + "used" + ] } + } }, - "traces": { - "traces_collected": { - "xray": {}, - "otlp": {}, - "application_signals": {} - } + "traces" : { + "traces_collected" : { + "xray" : {}, + "otlp" : {}, + "application_signals" : {} + } } } - + detailed_config = { agent = { metrics_collection_interval = 60 - run_as_user = "root" + run_as_user = "root" } metrics = { namespace = "CustomEC2Metrics" @@ -374,7 +374,7 @@ locals { "cpu_usage_system", "cpu_usage_iowait" ] - totalcpu = true + totalcpu = true metrics_collection_interval = 60 } mem = { @@ -408,49 +408,49 @@ locals { } } logs = { - log_stream_name="logs" - force_flush_interval=60 + log_stream_name = "logs" + force_flush_interval = 60 logs_collected = { files = { collect_list = [ { - file_path = "/var/log/messages" - log_group_name = "/ec2/system" + file_path = "/var/log/messages" + log_group_name = "/ec2/system" log_stream_name = "{instance_id}" - timezone = "UTC" + timezone = "UTC" }, - - # nginx + + # nginx { - file_path = "/var/log/nginx/swarms/access.log" - log_group_name = "/swarms/ngnix_access" + file_path = "/var/log/nginx/swarms/access.log" + log_group_name = "/swarms/ngnix_access" log_stream_name = "{instance_id}" - timezone = "UTC" + timezone = "UTC" }, { - file_path = "/var/log/nginx/swarms/error.log" - log_group_name = "/swarms/nginx_error" + file_path = "/var/log/nginx/swarms/error.log" + log_group_name = "/swarms/nginx_error" log_stream_name = "{instance_id}" - timezone = "UTC" + timezone = "UTC" }, { - file_path = "/var/log/cloud-init-output.log" - log_group_name = "/ec2/init" + file_path = "/var/log/cloud-init-output.log" + log_group_name = "/ec2/init" log_stream_name = "{instance_id}" - timezone = "UTC" + timezone = "UTC" }, { - file_path = "/var/log/swarms_systemd.log" - log_group_name = "/swarms/systemd" + file_path = "/var/log/swarms_systemd.log" + log_group_name = "/swarms/systemd" log_stream_name = "{instance_id}" - timezone = "UTC" + timezone = "UTC" }, { - file_path = "/var/log/secure" - log_group_name = "/ec2/secure" + file_path = "/var/log/secure" + log_group_name = "/ec2/secure" log_stream_name = "{instance_id}" - timezone = "UTC" + timezone = "UTC" } ] } @@ -477,7 +477,7 @@ resource "aws_ssm_association" "update_cloudwatch_config" { parameters = { commands = "amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c ssm:${aws_ssm_parameter.cw_agent_config.name}\n systemctl restart amazon-cloudwatch-agent" - + } } @@ -485,23 +485,23 @@ resource "aws_cloudwatch_log_group" "log_groups" { for_each = toset([ "/swarms/ngnix_access", "/swarms/nginx_error", - "/swarms/systemd", "/ec2/init" ]) - name = each.key + "/swarms/systemd", "/ec2/init"]) + name = each.key retention_in_days = 30 - kms_key_id = "arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590" + kms_key_id = "arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590" } # CloudWatch Log Groups for collected logs resource "aws_cloudwatch_log_group" "system_logs" { name = "/ec2/system" retention_in_days = 30 - kms_key_id = "arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590" + kms_key_id = "arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590" } resource "aws_cloudwatch_log_group" "secure_logs" { name = "/ec2/secure" retention_in_days = 30 - kms_key_id = "arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590" + kms_key_id = "arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590" } # SSM Document for CloudWatch agent troubleshooting diff --git a/modules/aws/ssm/setup/main.tf b/modules/aws/ssm/setup/main.tf index 18a9f85..b5eb5f2 100644 --- a/modules/aws/ssm/setup/main.tf +++ b/modules/aws/ssm/setup/main.tf @@ -1,12 +1,12 @@ module "ssm" { -# source = "bridgecrewio/session-manager/aws" + # source = "bridgecrewio/session-manager/aws" # version = "0.4.2" - source = "git::https://github.com/jmikedupont2/terraform-aws-session-manager.git?ref=master" + source = "git::https://github.com/jmikedupont2/terraform-aws-session-manager.git?ref=master" bucket_name = "swarms-session-logs" access_log_bucket_name = "swarms-session-access-logs" enable_log_to_s3 = true enable_log_to_cloudwatch = true - tags = {project="swarms"} + tags = { project = "swarms" } #linux_shell_profile = "date" } @@ -14,17 +14,17 @@ module "ssm" { resource "aws_cloudwatch_log_group" "app_signals" { - for_each = toset(["ec2","eks","generic","k8s"]) + for_each = toset(["ec2", "eks", "generic", "k8s"]) name = "/aws/appsignals/${each.key}" retention_in_days = 30 - kms_key_id = "arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590" + kms_key_id = "arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590" } resource "aws_cloudwatch_log_group" "app_signals2" { - for_each = toset(["data"]) + for_each = toset(["data"]) name = "/aws/application-signals/${each.key}" retention_in_days = 30 - kms_key_id = "arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590" + kms_key_id = "arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590" } # module "session-manager-settings" { @@ -42,10 +42,10 @@ resource "aws_cloudwatch_log_group" "app_signals2" { # # Create or update the S3 bucket with tags # resource "aws_s3_bucket" "session_logs" { # bucket = "swarms-session-logs-20241221151754799300000003" - + # # Force destroy can be set to true if you want to allow Terraform to delete the bucket even if it contains objects # force_destroy = false - + # tags = { # Environment = "Production" # Example tag # Project = "Swarms" # Example tag From 8138b4a379569e95d417aaf737713d1e7757c2e7 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Tue, 24 Dec 2024 11:38:27 -0500 Subject: [PATCH 098/158] now its pulling --- actions/Readme.md | 4 +++- actions/call_swarms.sh | 11 ++++----- environments/call-swarms/deploy-docker.yaml | 25 +++++++++++++++++---- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/actions/Readme.md b/actions/Readme.md index 0df4431..99efffb 100644 --- a/actions/Readme.md +++ b/actions/Readme.md @@ -14,7 +14,9 @@ facilitate remote invocation of a swarms agent on cloud servers. The script called is defined in `environments/call-swarms/main.tf` and applied in accounts/swarms/ like ```bash pushd ../../accounts/swarms/ -tofu apply +#tofu apply +# apply only the one change +tofu apply -auto-approve --target module.call_swarms.aws_ssm_document.deploy-docker # terraform apply popd ``` diff --git a/actions/call_swarms.sh b/actions/call_swarms.sh index 3625eb2..2e37d0e 100644 --- a/actions/call_swarms.sh +++ b/actions/call_swarms.sh @@ -1,7 +1,5 @@ #!/bin/bash -echo < Date: Wed, 25 Dec 2024 10:09:07 -0500 Subject: [PATCH 099/158] resize --- actions/call_swarms.org | 130 +++++++++--------- .../dev/us-east-1/main.tf | 67 ++++++--- 2 files changed, 113 insertions(+), 84 deletions(-) diff --git a/actions/call_swarms.org b/actions/call_swarms.org index 15a5aa3..d7bf80c 100644 --- a/actions/call_swarms.org +++ b/actions/call_swarms.org @@ -196,77 +196,77 @@ SSM command execution. Use the search bar to filter events based on ~EventName~ being ~StartCommand~, ~CancelCommand~, etc., and use the command ID ~82d43144-a4f4-4b6d-a507-23ad5179e0b4~. -*** Step 4: Analyze Logs -Once you have the logs, analyze them to understand the outcome of your SSM command execution. Look -for any errors or exceptions that might have occurred during the process. -By following these steps, you should be able to retrieve the necessary log outputs and related data -for your AWS Systems Manager command execution. ++ aws ssm list-command-invocations --command-id 6c96c523-2970-4071-9d5d-f95a041703ef --details --region us-east-2 +{ + "CommandInvocations": [ + { + "CommandId": "6c96c523-2970-4071-9d5d-f95a041703ef", + "InstanceId": "i-0a3dae164f8f3c09a", + "InstanceName": "ip-10-0-4-93.us-east-2.compute.internal", + "Comment": "", + "DocumentName": "deploy-docker", + "DocumentVersion": "$LATEST", + "RequestedDateTime": 1735058288.722, + "Status": "Success", + "StatusDetails": "Success", + "StandardOutputUrl": "https://s3.us-east-2.amazonaws.com/swarms-session-logs-20241221151754799300000003/6c96c523-2970-4071-9d5d-f95a041703ef/i-0a3dae164f8f3c09a/awsrunShellScript/DeployDocker/stdout", + "StandardErrorUrl": "https://s3.us-east-2.amazonaws.com/swarms-session-logs-20241221151754799300000003/6c96c523-2970-4071-9d5d-f95a041703ef/i-0a3dae164f8f3c09a/awsrunShellScript/DeployDocker/stderr", + "CommandPlugins": [ + { + "Name": "DeployDocker", + "Status": "Success", + "StatusDetails": "Success", + "ResponseCode": 0, + "ResponseStartDateTime": 1735058288.981, + "ResponseFinishDateTime": 1735058293.354, + "Output": "cd /opt/swarms/\nexport HOME=/home/swarms/\ngit config --global user.email \"git-agent@swarms.ai\"\ngit config --global user.name \"git agent\"\ngit remote add mdupont https://github.com/jmikedupont2/swarms || echo oops\ngit remote set-url mdupont https://github.com/jmikedupont2/swarms || echo oops2\necho git url https://github.com/jmikedupont2/swarms\ngit fetch --all|| echo oops \ngit pull mdupont feature/squash2-docker|| echo oops \ngit checkout --force feature/squash2-docker || echo checkout\ngit log -3 || echo git fail\nls /opt/swarms/api/*.sh || echo no files\noops\ngit url https://github.com/jmikedupont2/swarms\nFetching origin\nFetching mdupont\nUpdating 62a7b307..68755906\nFast-forward\n api/docker-boot.sh | 16 +++++++++\n api/rundocker.sh | 76 +++++++++++++++++++++++++++++++++++++++\n api/systemd/swarms-docker.service | 25 +++++++++++++\n 3 files changed, 117 insertions(+)\n create mode 100644 api/docker-boot.sh\n create mode 100644 api/rundocker.sh\n create mode 100644 api/systemd/swarms-docker.service\nYour branch is up to date with 'origin/feature/squash2-docker'.\ncommit 68755906be19d73deb0c8c5144323ecd3c203068\nAuthor: mike dupont \nD\n---Output truncated---\n----------ERROR-------\n+ chown -R swarms:swarms /opt/swarms\n+ tee /opt/swarms/update_docker_swarms.sh\n+ sudo chown swarms /opt/swarms/update_docker_swarms.sh\n+ su -c 'bash -e -x /opt/swarms/update_docker_swarms.sh' swarms\n+ cd /opt/swarms/\n+ export HOME=/home/swarms/\n+ HOME=/home/swarms/\n+ git config --global user.email git-agent@swarms.ai\n+ git config --global user.name 'git agent'\n+ git remote add mdupont https://github.com/jmikedupont2/swarms\nerror: remote mdupont already exists.\n+ echo oops\n+ git remote set-url mdupont https://github.com/jmikedupont2/swarms\n+ echo git url https://github.com/jmikedupont2/swarms\n+ git fetch --all\nFrom https://github.com/jmikedupont2/swarms\n 62a7b307..68755906 feature/squash2-docker -> origin/feature/squash2-docker\n+ git pull mdupont feature/squash2-docker\nFrom https://github.com/jmikedupont2/swarms\n * branch feature/squash2-docker -> FETCH_HEAD\n+ git checkout --force feature/squash2-docker\nAlready on 'feature/squash2-docker'\n+ git log -3\n+ ls /opt/swarms/api/boot.sh /opt/swarms/api/boot_fast.sh /opt/swarms/api/docker-boot.sh /opt/swarms/api/install.sh /opt/swarms/api/just_run.sh /opt/swarms/api/rerun-git.sh /opt/swarms/api/rerun.sh /opt/swarms/api/run_logs.sh /opt/swa\n---Error truncated----", + "StandardOutputUrl": "https://s3.us-east-2.amazonaws.com/swarms-session-logs-20241221151754799300000003/6c96c523-2970-4071-9d5d-f95a041703ef/i-0a3dae164f8f3c09a/awsrunShellScript/DeployDocker/stdout", + "StandardErrorUrl": " + "OutputS3Region": "us-east-2", + "OutputS3BucketName": "swarms-session-logs-20241221151754799300000003", + "OutputS3KeyPrefix": "6c96c523-2970-4071-9d5d-f95a041703ef/i-0a3dae164f8f3c09a/awsrunShellScript" + } + ], + "ServiceRole": "", + "NotificationConfig": { + "NotificationArn": "", + "NotificationEvents": [], + "NotificationType": "" + }, + "CloudWatchOutputConfig": { + "CloudWatchLogGroupName": "/ssm/session-logs-20241221151803393300000006", + "CloudWatchOutputEnabled": true + } + } + ] +} -The output you provided contains details from a command invocation executed in an AWS -environment. The command was part of the ~deploy-docker~ document and targeted the instance -~i-0a3dae164f8f3c09a~. Here is a summary of the results: - -*** Command Summary - -- *Command ID*: ~82d43144-a4f4-4b6d-a507-23ad5179e0b4~ -- *Instance ID*: ~i-0a3dae164f8f3c09a~ -- *Instance Name*: ~ip-10-0-4-93.us-east-2.compute.internal~ -- *Document Name*: ~deploy-docker~ -- *Document Version*: ~1~ -- *Requested DateTime*: ~1735005232.999~ (Epoch time, approximately December 18, 2024) -- *Status*: ~Success~ -- *Output S3 Bucket and Key*: - - ~Region~: ~us-east-2~ - - ~Bucket Name~: ~swarms-session-logs-20241221151754799300000003~ - - ~Key Prefix~: ~82d43144-a4f4-4b6d-a507-23ad5179e0b4/i-0a3dae164f8f3c09a/awsrunShellScript~ - -*** Output and Error Details - -The output of the command contained several issues: - -1. *Git Status*: -#+END_SRC - Your branch is up to date with 'origin/feature/squash2-docker'. -#+END_SRC - -2. *Git Remote Issue*: -#+END_SRC - -- - No entries -- - ----------ERROR------- - error: remote mdupont already exists. - Already on 'feature/squash2-docker' - bash: /opt/swarms/api/docker-boot.sh: No such file or directory - Failed to restart swarms-docker.service: Unit swarms-docker.service not found. -#+END_SRC - -*** Summary of Errors: - -- *Git Remote Conflict*: The ~mdupont~ remote already exists in the repository. You need to either - remove or update this remote. -- *Missing Script File*: ~/opt/swarms/api/docker-boot.sh~ does not exist on the server. -- *Service Not Found*: ~swarms-docker.service~ is not found on the system. - -*** Recommendations: +aws logs get-log-events --log-group-name /ssm/session-logs-20241221151803393300000006 +#aws logs describe-log-streams --log-group-name "CloudWatchLogGroupName": "/ssm/session-logs-20241221151803393300000006 --profile swarms --region us-east-2 +#aws logs describe-log-streams --log-group-name /ssm/session-logs-20241221151803393300000006 --profile swarms --region us-east-2 +aws logs describe-log-streams --log-group-name /ssm/session-logs-20241221151803393300000006 --profile swarms --region us-east-2 --output json | jq . > logstreams.json +aws logs get-log-events --log-group-name /ssm/session-logs-20241221151803393300000006 --log-stream-name $(jq ".logStreams[0].logStreamName" logstreams.json) --region us-east-2 --profile swarms -1. *Remove Duplicate Remote*: -#+BEGIN_SRC sh - git remote remove mdupont -#+END_SRC +#jq ".logStreams[0].logStreamName" logstreams.json + awslogs groups -2. *Verify Script Location*: - - Ensure that ~/opt/swarms/api/docker-boot.sh~ exists and has the correct permissions. - - If it doesn't exist, you may need to upload or create this script on the server. + /aws/application-signals/data +/aws/appsignals/ec2 +/aws/appsignals/eks +/aws/appsignals/generic +/aws/appsignals/k8s -3. *Check Service Availability*: - - Verify if ~swarms-docker.service~ is available on your system. - - If not, you might need to install or create this service unit file. + /ec2/init +/ec2/secure +/ec2/system +/ssm/session-logs-20241221151803393300000006 +/swarms/nginx_error +/swarms/ngnix_access +/swarms/systemd -4. *Review Logs*: - - Access the logs from S3 for more detailed error messages and context. + awslogs get /swarms/systemd -This should help in troubleshooting and resolving the issues encountered during the command -execution. + awslogs get /ssm/session-logs-20241221151803393300000006 diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index f26cd70..218769c 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -76,6 +76,22 @@ variable "instance_types" { ] } +variable "test_instance_types" { + type = list(string) + default = [ + # "t4g.nano", "t3a.nano", "t3.nano", "t2.nano", + # "t4g.micro", "t3a.micro", "t3.micro", "t2.micro", "t1.micro", + #"t4g.small", "t3a.small", + #"t3.small", + #"t2.small", not working + # "t2.medium" # + + + + #"t3.medium" # no instances for now, this is commented out + ] +} + module "roles" { source = "./components/roles" @@ -134,10 +150,23 @@ module "lt_dynamic_ami_test" { } +variable "dev_instance_types" { + type = list(string) + default = [ + # "t4g.nano", "t3a.nano", "t3.nano", "t2.nano", + # "t4g.micro", "t3a.micro", "t3.micro", "t2.micro", "t1.micro", + #"t4g.small", "t3a.small", + #"t3.small", + #"t2.small", not working + # "t2.medium" # + "t3.small" + ] +} + module "lt_dynamic_ami_docker" { branch = "feature/squash2-docker" vpc_id = local.vpc_id - for_each = toset(var.instance_types) + for_each = toset(var.dev_instance_types) instance_type = each.key name = "swarms-docker-${each.key}" security_group_id = module.security.internal_security_group_id @@ -153,6 +182,22 @@ module "lt_dynamic_ami_docker" { install_script = "/opt/swarms/api/rundocker.sh" } +module "asg_dynamic_new_ami_dev" { + # built with packer +# count =0 + tags = merge(local.tags, local.dev_tags) + vpc_id = local.vpc_id + image_id = local.new_ami_id + ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 + for_each = toset(var.dev_instance_types) + aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn + source = "./components/autoscaling_group" + # security_group_id = module.security.internal_security_group_id + instance_type = each.key + name = "docker-swarms-ami-${each.key}" + launch_template_id = module.lt_dynamic_ami_docker[each.key].launch_template_id + target_group_arn = module.alb.dev_alb_target_group_arn +} module "alb" { @@ -206,12 +251,12 @@ module "asg_dynamic_new_ami" { module "asg_dynamic_new_ami_test" { # built with packer - #count =0 + for_each = toset(var.test_instance_types) tags = merge(local.tags, local.dev_tags) vpc_id = local.vpc_id image_id = local.new_ami_id ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 - for_each = toset(var.instance_types) + aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn source = "./components/autoscaling_group" # security_group_id = module.security.internal_security_group_id @@ -221,22 +266,6 @@ module "asg_dynamic_new_ami_test" { target_group_arn = module.alb.test_alb_target_group_arn } -module "asg_dynamic_new_ami_dev" { - # built with packer - #count =0 - tags = merge(local.tags, local.dev_tags) - vpc_id = local.vpc_id - image_id = local.new_ami_id - ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 - for_each = toset(var.instance_types) - aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn - source = "./components/autoscaling_group" - # security_group_id = module.security.internal_security_group_id - instance_type = each.key - name = "docker-swarms-ami-${each.key}" - launch_template_id = module.lt_dynamic_ami_docker[each.key].launch_template_id - target_group_arn = module.alb.dev_alb_target_group_arn -} output "security_group_id" { value = module.security.security_group_id From f71554b022ec2a69c755d82f17600cad1bc7131e Mon Sep 17 00:00:00 2001 From: mike dupont Date: Wed, 25 Dec 2024 11:42:20 -0500 Subject: [PATCH 100/158] max price 0.0275 --- .../components/autoscaling_group/main.tf | 19 +++++ .../dev/us-east-1/main.tf | 77 ++++++++++++++----- 2 files changed, 76 insertions(+), 20 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf index 27f5e58..cfe5893 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf @@ -1,3 +1,16 @@ +variable use_mixed_instances_policy { + default = false +} +variable mixed_instances_policy { + default = null + type = any +} +variable instance_requirements { + #default = null + type = any + default = {} +} + variable "aws_iam_instance_profile_ssm_arn" {} variable "target_group_arn" {} variable "name" {} @@ -43,6 +56,7 @@ module "autoscaling" { } ] instance_type = var.instance_type + #instance_requirements = var.instance_requirements description = "The attribute requirements for the type of instance. If present then `instance_type` cannot be present" image_id = var.image_id create_iam_instance_profile = true @@ -109,6 +123,11 @@ module "autoscaling" { # target_value = 800 # } # } + #use_mixed_instances_policy = var.use_mixed_instances_policy + + #mixed_instances_policy = var.mixed_instances_policy + + scale-out = { name = "scale-out" adjustment_type = "ExactCapacity" diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index 218769c..53c3722 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -1,3 +1,6 @@ +variable spot_max_price { + default = 0.0275 +} variable "region" {} variable "key_name" { default = "mdupont-deployer-key" # FIXME: move to settings @@ -182,24 +185,6 @@ module "lt_dynamic_ami_docker" { install_script = "/opt/swarms/api/rundocker.sh" } -module "asg_dynamic_new_ami_dev" { - # built with packer -# count =0 - tags = merge(local.tags, local.dev_tags) - vpc_id = local.vpc_id - image_id = local.new_ami_id - ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 - for_each = toset(var.dev_instance_types) - aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn - source = "./components/autoscaling_group" - # security_group_id = module.security.internal_security_group_id - instance_type = each.key - name = "docker-swarms-ami-${each.key}" - launch_template_id = module.lt_dynamic_ami_docker[each.key].launch_template_id - target_group_arn = module.alb.dev_alb_target_group_arn -} - - module "alb" { source = "./components/application_load_balancer" domain_name = local.domain @@ -258,7 +243,7 @@ module "asg_dynamic_new_ami_test" { ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn - source = "./components/autoscaling_group" + source = "./components/autoscaling_group/spot" # security_group_id = module.security.internal_security_group_id instance_type = each.key name = "test-swarms-ami-${each.key}" @@ -266,6 +251,59 @@ module "asg_dynamic_new_ami_test" { target_group_arn = module.alb.test_alb_target_group_arn } +module "asg_dynamic_new_ami_dev" { + # built with packer +# count =0 + tags = merge(local.tags, local.dev_tags) + vpc_id = local.vpc_id + image_id = local.new_ami_id + ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 + for_each = toset(var.dev_instance_types) + aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn + + source = "./components/autoscaling_group/spot" + # security_group_id = module.security.internal_security_group_id + instance_type = each.key + name = "docker-swarms-ami-${each.key}" + launch_template_id = module.lt_dynamic_ami_docker[each.key].launch_template_id + target_group_arn = module.alb.dev_alb_target_group_arn + + use_mixed_instances_policy = true + mixed_instances_policy = { + instances_distribution = { + on_demand_base_capacity = 0 + on_demand_percentage_above_base_capacity = 0 + spot_instance_pools = 2 + spot_max_price = var.spot_max_price +# spot_allocation_strategy = "capacity-optimized" + } + + override = [ + { + instance_requirements = { + cpu_manufacturers = ["amazon-web-services", "amd", "intel"] + #cpu_manufacturers = ["amd"] + #local_storage_types = ["ssd"] + max_spot_price_as_percentage_of_optimal_on_demand_price = 60 + memory_gib_per_vcpu = { + min = 2 + max = 4 + } + memory_mib = { + min = 2048 + }, + vcpu_count = { + min = 2 + max = 4 + } + } + } + ] + } + instance_requirements = { + } + +} output "security_group_id" { value = module.security.security_group_id @@ -275,7 +313,6 @@ output "vpc" { value = module.vpc } - output "user_data_new" { value = module.lt_dynamic_ami_test["t3.medium"].user_data } From 81564cbca8b3ad552c070789936370f0814cfaa9 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Wed, 25 Dec 2024 11:43:51 -0500 Subject: [PATCH 101/158] fmt --- .../components/autoscaling_group/main.tf | 20 ++++++++--------- .../dev/us-east-1/main.tf | 22 +++++++++---------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf index cfe5893..ad5997c 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf @@ -1,16 +1,16 @@ -variable use_mixed_instances_policy { +variable "use_mixed_instances_policy" { default = false } -variable mixed_instances_policy { +variable "mixed_instances_policy" { default = null - type = any + type = any } -variable instance_requirements { +variable "instance_requirements" { #default = null - type = any - default = {} + type = any + default = {} } - + variable "aws_iam_instance_profile_ssm_arn" {} variable "target_group_arn" {} variable "name" {} @@ -57,7 +57,7 @@ module "autoscaling" { ] instance_type = var.instance_type #instance_requirements = var.instance_requirements description = "The attribute requirements for the type of instance. If present then `instance_type` cannot be present" - image_id = var.image_id + image_id = var.image_id create_iam_instance_profile = true iam_role_name = "ssm-${var.name}" @@ -124,10 +124,10 @@ module "autoscaling" { # } # } #use_mixed_instances_policy = var.use_mixed_instances_policy - + #mixed_instances_policy = var.mixed_instances_policy - + scale-out = { name = "scale-out" adjustment_type = "ExactCapacity" diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index 53c3722..625e6fd 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -1,4 +1,4 @@ -variable spot_max_price { +variable "spot_max_price" { default = 0.0275 } variable "region" {} @@ -90,7 +90,7 @@ variable "test_instance_types" { # "t2.medium" # - + #"t3.medium" # no instances for now, this is commented out ] } @@ -236,11 +236,11 @@ module "asg_dynamic_new_ami" { module "asg_dynamic_new_ami_test" { # built with packer - for_each = toset(var.test_instance_types) - tags = merge(local.tags, local.dev_tags) - vpc_id = local.vpc_id - image_id = local.new_ami_id - ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 + for_each = toset(var.test_instance_types) + tags = merge(local.tags, local.dev_tags) + vpc_id = local.vpc_id + image_id = local.new_ami_id + ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn source = "./components/autoscaling_group/spot" @@ -253,7 +253,7 @@ module "asg_dynamic_new_ami_test" { module "asg_dynamic_new_ami_dev" { # built with packer -# count =0 + # count =0 tags = merge(local.tags, local.dev_tags) vpc_id = local.vpc_id image_id = local.new_ami_id @@ -261,7 +261,7 @@ module "asg_dynamic_new_ami_dev" { for_each = toset(var.dev_instance_types) aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn - source = "./components/autoscaling_group/spot" + source = "./components/autoscaling_group/spot" # security_group_id = module.security.internal_security_group_id instance_type = each.key name = "docker-swarms-ami-${each.key}" @@ -275,13 +275,13 @@ module "asg_dynamic_new_ami_dev" { on_demand_percentage_above_base_capacity = 0 spot_instance_pools = 2 spot_max_price = var.spot_max_price -# spot_allocation_strategy = "capacity-optimized" + # spot_allocation_strategy = "capacity-optimized" } override = [ { instance_requirements = { - cpu_manufacturers = ["amazon-web-services", "amd", "intel"] + cpu_manufacturers = ["amazon-web-services", "amd", "intel"] #cpu_manufacturers = ["amd"] #local_storage_types = ["ssd"] max_spot_price_as_percentage_of_optimal_on_demand_price = 60 From 66dda6e13109d1d23f5d37a3d52eb53594d1ed10 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Wed, 25 Dec 2024 11:47:58 -0500 Subject: [PATCH 102/158] adding spot --- actions/call_swarms.org | 137 ++++++++++++++++ .../components/autoscaling_group/spot/main.tf | 148 ++++++++++++++++++ .../resource_launchers/lambda/scale_lambda.py | 19 +++ .../dev/us-east-1/main.tf | 22 +-- 4 files changed, 315 insertions(+), 11 deletions(-) create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/spot/main.tf create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/resource_launchers/lambda/scale_lambda.py diff --git a/actions/call_swarms.org b/actions/call_swarms.org index d7bf80c..bf11307 100644 --- a/actions/call_swarms.org +++ b/actions/call_swarms.org @@ -270,3 +270,140 @@ aws logs get-log-events --log-group-name /ssm/session-logs-202412211518033933000 awslogs get /swarms/systemd awslogs get /ssm/session-logs-20241221151803393300000006 + + + + + + mdupont@mdupont-G470:~/swarms-terraform/actions$ bash ./call_swarms.sh ++ export REGION=us-east-2 ++ REGION=us-east-2 ++ TAG_KEY=Name ++ TAG_VALUE=docker-swarms-ami-t3.medium ++ GIT_URL=https://github.com/jmikedupont2/swarms ++ export GIT_NAME=mdupont ++ GIT_NAME=mdupont ++ export GIT_VERSION=feature/squash2-docker ++ GIT_VERSION=feature/squash2-docker ++ DOCUMENT_NAME=deploy-docker ++ DOCUMENT_VERSION='$LATEST' ++ TIMEOUT_SECONDS=600 ++ MAX_CONCURRENCY=50 ++ MAX_ERRORS=0 +++ get_instance_ids +++ aws ec2 describe-instances --filters Name=tag:Name,Values=docker-swarms-ami-t3.medium --query 'Reservations[*].Instances[*].InstanceId' --output text --region us-east-2 ++ for instance in $(get_instance_ids) ++ echo 'Instance ID: i-0a3dae164f8f3c09a' +Instance ID: i-0a3dae164f8f3c09a +++ send_command i-0a3dae164f8f3c09a +++ local instance_id=i-0a3dae164f8f3c09a +++ aws ssm send-command --document-name deploy-docker --document-version '$LATEST' --targets '[{"Key":"InstanceIds","Values":["i-0a3dae164f8f3c09a"]}]' --parameters '{"GitUrl":["https://github.com/jmikedupont2/swarms"],"GitName":["mdupont"],"GitVersion":["feature/squash2-docker"]}' --timeout-seconds 600 --max-concurrency 50 --max-errors 0 --region us-east-2 --output-s3-bucket-name swarms-session-logs-20241221151754799300000003 --cloud-watch-output-config '{"CloudWatchOutputEnabled":true,"CloudWatchLogGroupName":"/ssm/session-logs-20241221151803393300000006"}' ++ result='{ + "Command": { + "CommandId": "11548a23-624f-4fac-a085-d813e91f5fff", + "DocumentName": "deploy-docker", + "DocumentVersion": "$LATEST", + "Comment": "", + "ExpiresAfter": 1735139787.028, + "Parameters": { + "GitName": [ + "mdupont" + ], + "GitUrl": [ + "https://github.com/jmikedupont2/swarms" + ], + "GitVersion": [ + "feature/squash2-docker" + ] + }, + "InstanceIds": [], + "Targets": [ + { + "Key": "InstanceIds", + "Values": [ + "i-0a3dae164f8f3c09a" + ] + } + ], + "RequestedDateTime": 1735135587.028, + "Status": "Pending", + "StatusDetails": "Pending", + "OutputS3Region": "us-east-2", + "OutputS3BucketName": "swarms-session-logs-20241221151754799300000003", + "OutputS3KeyPrefix": "", + "MaxConcurrency": "50", + "MaxErrors": "0", + "TargetCount": 0, + "CompletedCount": 0, + "ErrorCount": 0, + "DeliveryTimedOutCount": 0, + "ServiceRole": "", + "NotificationConfig": { + "NotificationArn": "", + "NotificationEvents": [], + "NotificationType": "" + }, + "CloudWatchOutputConfig": { + "CloudWatchLogGroupName": "/ssm/session-logs-20241221151803393300000006", + "CloudWatchOutputEnabled": true + }, + "TimeoutSeconds": 600, + "AlarmConfiguration": { + "IgnorePollAlarmFailure": false, + "Alarms": [] + }, + "TriggeredAlarms": [] + } +}' +++ jq -r .Command.CommandId +lets fetch the full log +++ echo '{' '"Command":' '{' '"CommandId":' '"11548a23-624f-4fac-a085-d813e91f5fff",' '"DocumentName":' '"deploy-docker",' '"DocumentVersion":' '"$LATEST",' '"Comment":' '"",' '"ExpiresAfter":' 1735139787.028, '"Parameters":' '{' '"GitName":' '[' '"mdupont"' '],' '"GitUrl":' '[' '"https://github.com/jmikedupont2/swarms"' '],' '"GitVersion":' '[' '"feature/squash2-docker"' ']' '},' '"InstanceIds":' '[],' '"Targets":' '[' '{' '"Key":' '"InstanceIds",' '"Values":' '[' '"i-0a3dae164f8f3c09a"' ']' '}' '],' '"RequestedDateTime":' 1735135587.028, '"Status":' '"Pending",' '"StatusDetails":' '"Pending",' '"OutputS3Region":' '"us-east-2",' '"OutputS3BucketName":' '"swarms-session-logs-20241221151754799300000003",' '"OutputS3KeyPrefix":' '"",' '"MaxConcurrency":' '"50",' '"MaxErrors":' '"0",' '"TargetCount":' 0, '"CompletedCount":' 0, '"ErrorCount":' 0, '"DeliveryTimedOutCount":' 0, '"ServiceRole":' '"",' '"NotificationConfig":' '{' '"NotificationArn":' '"",' '"NotificationEvents":' '[],' '"NotificationType":' '""' '},' '"CloudWatchOutputConfig":' '{' '"CloudWatchLogGroupName":' '"/ssm/session-logs-20241221151803393300000006",' '"CloudWatchOutputEnabled":' true '},' '"TimeoutSeconds":' 600, '"AlarmConfiguration":' '{' '"IgnorePollAlarmFailure":' false, '"Alarms":' '[]' '},' '"TriggeredAlarms":' '[]' '}' '}' ++ command_id=11548a23-624f-4fac-a085-d813e91f5fff ++ aws ssm wait command-executed --command-id 11548a23-624f-4fac-a085-d813e91f5fff --region us-east-2 --instance i-0a3dae164f8f3c09a ++ fetch_command_output 11548a23-624f-4fac-a085-d813e91f5fff ++ local command_id=11548a23-624f-4fac-a085-d813e91f5fff ++ jq . ++ aws ssm list-command-invocations --command-id 11548a23-624f-4fac-a085-d813e91f5fff --details --region us-east-2 +{ + "CommandInvocations": [ + { + "CommandId": "11548a23-624f-4fac-a085-d813e91f5fff", + "InstanceId": "i-0a3dae164f8f3c09a", + "InstanceName": "ip-10-0-4-93.us-east-2.compute.internal", + "Comment": "", + "DocumentName": "deploy-docker", + "DocumentVersion": "$LATEST", + "RequestedDateTime": 1735135587.107, + "Status": "Success", + "StatusDetails": "Success", + "StandardOutputUrl": "https://s3.us-east-2.amazonaws.com/swarms-session-logs-20241221151754799300000003/11548a23-624f-4fac-a085-d813e91f5fff/i-0a3dae164f8f3c09a/awsrunShellScript/DeployDocker/stdout", + "StandardErrorUrl": "https://s3.us-east-2.amazonaws.com/swarms-session-logs-20241221151754799300000003/11548a23-624f-4fac-a085-d813e91f5fff/i-0a3dae164f8f3c09a/awsrunShellScript/DeployDocker/stderr", + "CommandPlugins": [ + { + "Name": "DeployDocker", + "Status": "Success", + "StatusDetails": "Success", + "ResponseCode": 0, + "ResponseStartDateTime": 1735135587.335, + "ResponseFinishDateTime": 1735135668.743, + "Output": "cd /opt/swarms/\nexport HOME=/home/swarms/\ngit config --global user.email \"git-agent@swarms.ai\"\ngit config --global user.name \"git agent\"\ngit remote add mdupont https://github.com/jmikedupont2/swarms || echo oops\ngit remote set-url mdupont https://github.com/jmikedupont2/swarms || echo oops2\necho git url https://github.com/jmikedupont2/swarms\ngit fetch --all|| echo oops \ngit pull mdupont feature/squash2-docker|| echo oops \ngit checkout --force feature/squash2-docker || echo checkout\ngit log -3 || echo git fail\nls /opt/swarms/api/*.sh || echo no files\noops\ngit url https://github.com/jmikedupont2/swarms\nFetching origin\nFetching mdupont\nUpdating 68755906..3e0a7ead\nFast-forward\n api/docker-boot.sh | 19 +++++++++++++++----\n api/rundocker.sh | 12 ++++++++++--\n 2 files changed, 25 insertions(+), 6 deletions(-)\nYour branch is up to date with 'origin/feature/squash2-docker'.\ncommit 3e0a7ead57e7f53cfd5ac0034b5ec40ac7cbfafc\nAuthor: mike dupont \nDate: Tue Dec 24 12:19:11 2024 -0500\n\n shell check\n\ncommit 24af710cb49da489ae8e1aa86a641e02a5e01f2c\nAuthor: mike dupont \nDate: Tue Dec 24 12:17:53 2024 -0500\n\n going to test\n\n\n---Output truncated---\n----------ERROR-------\n+ chown -R swarms:swarms /opt/swarms\n+ tee /opt/swarms/update_docker_swarms.sh\n+ sudo chown swarms /opt/swarms/update_docker_swarms.sh\n+ su -c 'bash -e -x /opt/swarms/update_docker_swarms.sh' swarms\n+ cd /opt/swarms/\n+ export HOME=/home/swarms/\n+ HOME=/home/swarms/\n+ git config --global user.email git-agent@swarms.ai\n+ git config --global user.name 'git agent'\n+ git remote add mdupont https://github.com/jmikedupont2/swarms\nerror: remote mdupont already exists.\n+ echo oops\n+ git remote set-url mdupont https://github.com/jmikedupont2/swarms\n+ echo git url https://github.com/jmikedupont2/swarms\n+ git fetch --all\nFrom https://github.com/jmikedupont2/swarms\n 68755906..3e0a7ead feature/squash2-docker -> origin/feature/squash2-docker\nFrom https://github.com/jmikedupont2/swarms\n 68755906..3e0a7ead feature/squash2-docker -> mdupont/feature/squash2-docker\n+ git pull mdupont feature/squash2-docker\nFrom https://github.com/jmikedupont2/swarms\n * branch feature/squash2-docker -> FETCH_HEAD\n+ git checkout --force feature/squash2-docker\nAlready on 'feature/squash2-docker'\n+ git log -3\n+ ls /opt/swarms/api/boot.sh /opt/swarms/api/boot_fast.sh /opt/swarms/api/docker-boot.sh /opt/swarms/api/insta\n---Error truncated----", + "StandardOutputUrl": "https://s3.us-east-2.amazonaws.com/swarms-session-logs-20241221151754799300000003/11548a23-624f-4fac-a085-d813e91f5fff/i-0a3dae164f8f3c09a/awsrunShellScript/DeployDocker/stdout", + "StandardErrorUrl": "https://s3.us-east-2.amazonaws.com/swarms-session-logs-20241221151754799300000003/11548a23-624f-4fac-a085-d813e91f5fff/i-0a3dae164f8f3c09a/awsrunShellScript/DeployDocker/stderr", + "OutputS3Region": "us-east-2", + "OutputS3BucketName": "swarms-session-logs-20241221151754799300000003", + "OutputS3KeyPrefix": "11548a23-624f-4fac-a085-d813e91f5fff/i-0a3dae164f8f3c09a/awsrunShellScript" + } + ], + "ServiceRole": "", + "NotificationConfig": { + "NotificationArn": "", + "NotificationEvents": [], + "NotificationType": "" + }, + "CloudWatchOutputConfig": { + "CloudWatchLogGroupName": "/ssm/session-logs-20241221151803393300000006", + "CloudWatchOutputEnabled": true + } + } + ] +} diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/spot/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/spot/main.tf new file mode 100644 index 0000000..7adb4e9 --- /dev/null +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/spot/main.tf @@ -0,0 +1,148 @@ +variable "use_mixed_instances_policy" { + default = true +} +#variable spot_max_price {} +variable "mixed_instances_policy" { + default = null + type = any +} +variable "instance_requirements" { + #default = null + type = any + default = {} +} + +variable "aws_iam_instance_profile_ssm_arn" {} +variable "target_group_arn" {} +variable "name" {} +variable "instance_type" {} +variable "launch_template_id" {} +variable "image_id" {} +variable "vpc_id" {} +variable "tags" {} +variable "ec2_subnet_id" {} + +locals { + iam_instance_profile_arn = var.aws_iam_instance_profile_ssm_arn + instance_type = var.instance_type +} + +module "autoscaling" { + source = "terraform-aws-modules/autoscaling/aws" + version = "8.0.0" + name = var.name + + health_check_type = "EC2" + desired_capacity = 1 + max_size = 5 + min_size = 1 + capacity_rebalance = true + create_launch_template = false + update_default_version = true + + launch_template_id = var.launch_template_id + launch_template_version = "$Latest" + + vpc_zone_identifier = [var.ec2_subnet_id] + + instance_market_options = { + market_type = "spot" + } + network_interfaces = [{ + associate_public_ip_address = true + device_index = 0 + delete_on_termination = true + description = "interface1" + # security_groups = [var.security_group_id] + } + ] + instance_type = var.instance_type + #instance_requirements = var.instance_requirements description = "The attribute requirements for the type of instance. If present then `instance_type` cannot be present" + image_id = var.image_id + + create_iam_instance_profile = true + iam_role_name = "ssm-${var.name}" + iam_role_path = "/ec2/" + iam_role_description = "SSM IAM role for swarms" + iam_role_tags = { + CustomIamRole = "Yes" + } + + iam_role_policies = { + AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" + } + + # target_group_arn = + traffic_source_attachments = { + ex-alb = { + traffic_source_identifier = var.target_group_arn + traffic_source_type = "elbv2" # default + } + } + use_mixed_instances_policy = var.use_mixed_instances_policy + mixed_instances_policy = var.mixed_instances_policy + + # Target scaling policy schedule based on average CPU load + scaling_policies = { + avg-cpu-policy-greater-than-50 = { + policy_type = "TargetTrackingScaling" + estimated_instance_warmup = 1200 + target_tracking_configuration = { + predefined_metric_specification = { + predefined_metric_type = "ASGAverageCPUUtilization" + } + target_value = 50.0 + } + }, + predictive-scaling = { + policy_type = "PredictiveScaling" + predictive_scaling_configuration = { + mode = "ForecastAndScale" + scheduling_buffer_time = 10 + max_capacity_breach_behavior = "IncreaseMaxCapacity" + max_capacity_buffer = 10 + metric_specification = { + target_value = 32 + predefined_scaling_metric_specification = { + predefined_metric_type = "ASGAverageCPUUtilization" + resource_label = "testLabel" + } + predefined_load_metric_specification = { + predefined_metric_type = "ASGTotalCPUUtilization" + resource_label = "testLabel" + } + } + } + } + # request-count-per-target = { + # policy_type = "TargetTrackingScaling" + # estimated_instance_warmup = 120 + # target_tracking_configuration = { + # predefined_metric_specification = { + # predefined_metric_type = "ALBRequestCountPerTarget" + # resource_label = "swarms1" + # #"${module.alb.arn_suffix}/${module.alb.target_groups["ex_asg"].arn_suffix}" + # } + # target_value = 800 + # } + # } + + scale-out = { + name = "scale-out" + adjustment_type = "ExactCapacity" + policy_type = "StepScaling" + estimated_instance_warmup = 120 + step_adjustment = [ + { + scaling_adjustment = 1 + metric_interval_lower_bound = 0 + metric_interval_upper_bound = 10 + }, + { + scaling_adjustment = 2 + metric_interval_lower_bound = 10 + } + ] + } + } +} diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/resource_launchers/lambda/scale_lambda.py b/environments/swarms-aws-agent-api/dev/us-east-1/components/resource_launchers/lambda/scale_lambda.py new file mode 100644 index 0000000..67e270d --- /dev/null +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/resource_launchers/lambda/scale_lambda.py @@ -0,0 +1,19 @@ +#You'll need to create a file named `scale_lambda.py` in the `lambda/` directory with content like the following: + +#```python +import boto3 +import os + +def handler(event, context): + asg_name = os.environ['AUTO_SCALING_GROUP_NAME'] + client = boto3.client('autoscaling') + + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_name]) + desired_capacity = response['AutoScalingGroups'][0]['DesiredCapacity'] + + # Example scaling logic + client.set_desired_capacity( + AutoScalingGroupName=asg_name, + DesiredCapacity=desired_capacity + 1, # Increment desired instances + HonorCooldown=True + ) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index 625e6fd..53c3722 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -1,4 +1,4 @@ -variable "spot_max_price" { +variable spot_max_price { default = 0.0275 } variable "region" {} @@ -90,7 +90,7 @@ variable "test_instance_types" { # "t2.medium" # - + #"t3.medium" # no instances for now, this is commented out ] } @@ -236,11 +236,11 @@ module "asg_dynamic_new_ami" { module "asg_dynamic_new_ami_test" { # built with packer - for_each = toset(var.test_instance_types) - tags = merge(local.tags, local.dev_tags) - vpc_id = local.vpc_id - image_id = local.new_ami_id - ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 + for_each = toset(var.test_instance_types) + tags = merge(local.tags, local.dev_tags) + vpc_id = local.vpc_id + image_id = local.new_ami_id + ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn source = "./components/autoscaling_group/spot" @@ -253,7 +253,7 @@ module "asg_dynamic_new_ami_test" { module "asg_dynamic_new_ami_dev" { # built with packer - # count =0 +# count =0 tags = merge(local.tags, local.dev_tags) vpc_id = local.vpc_id image_id = local.new_ami_id @@ -261,7 +261,7 @@ module "asg_dynamic_new_ami_dev" { for_each = toset(var.dev_instance_types) aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn - source = "./components/autoscaling_group/spot" + source = "./components/autoscaling_group/spot" # security_group_id = module.security.internal_security_group_id instance_type = each.key name = "docker-swarms-ami-${each.key}" @@ -275,13 +275,13 @@ module "asg_dynamic_new_ami_dev" { on_demand_percentage_above_base_capacity = 0 spot_instance_pools = 2 spot_max_price = var.spot_max_price - # spot_allocation_strategy = "capacity-optimized" +# spot_allocation_strategy = "capacity-optimized" } override = [ { instance_requirements = { - cpu_manufacturers = ["amazon-web-services", "amd", "intel"] + cpu_manufacturers = ["amazon-web-services", "amd", "intel"] #cpu_manufacturers = ["amd"] #local_storage_types = ["ssd"] max_spot_price_as_percentage_of_optimal_on_demand_price = 60 From ecb23ac363977b89d46fdfbdc6c64c7195e502cc Mon Sep 17 00:00:00 2001 From: Mike DuPont Date: Thu, 26 Dec 2024 10:30:13 -0500 Subject: [PATCH 103/158] Delete environments/swarms-aws-agent-api/dev/us-east-1/Readme.md~ --- environments/swarms-aws-agent-api/dev/us-east-1/Readme.md~ | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/Readme.md~ diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/Readme.md~ b/environments/swarms-aws-agent-api/dev/us-east-1/Readme.md~ deleted file mode 100644 index a7b9d73..0000000 --- a/environments/swarms-aws-agent-api/dev/us-east-1/Readme.md~ +++ /dev/null @@ -1,2 +0,0 @@ -1. create vpc -2. create ami From ddb49a5cdf9b7fe73e15482f1f0eb6adc139f293 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Thu, 26 Dec 2024 10:51:15 -0500 Subject: [PATCH 104/158] moving traffic source attachments --- accounts/swarms/Readme.md | 5 +++++ .../components/autoscaling_group/spot/main.tf | 17 ++++++++++------- .../swarms-aws-agent-api/dev/us-east-1/main.tf | 8 ++++---- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/accounts/swarms/Readme.md b/accounts/swarms/Readme.md index 176495a..f5aff76 100644 --- a/accounts/swarms/Readme.md +++ b/accounts/swarms/Readme.md @@ -172,3 +172,8 @@ tofu apply -var "github_token=$(gh auth token)" see this branch for the pike generated policies https://github.com/jmikedupont2/terraform-aws-oidc-github/pull/1 + + +``` +tofu apply -target "module.swarms_api.module.asg_dynamic_new_ami_dev[\"t3.small\"].module.autoscaling.aws_autoscaling_group.this[0]" +``` diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/spot/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/spot/main.tf index 7adb4e9..bee397d 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/spot/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/spot/main.tf @@ -32,11 +32,19 @@ module "autoscaling" { version = "8.0.0" name = var.name + traffic_source_attachments = { + ex-alb = { + traffic_source_identifier = var.target_group_arn + traffic_source_type = "elbv2" # default + } + } + health_check_type = "EC2" desired_capacity = 1 max_size = 5 min_size = 1 - capacity_rebalance = true + create = true + #capacity_rebalance = true create_launch_template = false update_default_version = true @@ -73,12 +81,7 @@ module "autoscaling" { } # target_group_arn = - traffic_source_attachments = { - ex-alb = { - traffic_source_identifier = var.target_group_arn - traffic_source_type = "elbv2" # default - } - } + use_mixed_instances_policy = var.use_mixed_instances_policy mixed_instances_policy = var.mixed_instances_policy diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index 53c3722..f4dbcc6 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -1,5 +1,5 @@ variable spot_max_price { - default = 0.0275 + default = 0.028 } variable "region" {} variable "key_name" { @@ -273,7 +273,7 @@ module "asg_dynamic_new_ami_dev" { instances_distribution = { on_demand_base_capacity = 0 on_demand_percentage_above_base_capacity = 0 - spot_instance_pools = 2 + spot_instance_pools = 1 spot_max_price = var.spot_max_price # spot_allocation_strategy = "capacity-optimized" } @@ -287,14 +287,14 @@ module "asg_dynamic_new_ami_dev" { max_spot_price_as_percentage_of_optimal_on_demand_price = 60 memory_gib_per_vcpu = { min = 2 - max = 4 + max = 12 } memory_mib = { min = 2048 }, vcpu_count = { min = 2 - max = 4 + max = 12 } } } From 0357e1bcc7e2997f85fb0d99df422d605e044a03 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Thu, 26 Dec 2024 14:43:58 -0500 Subject: [PATCH 105/158] fixing size --- .../dev/us-east-1/components/autoscaling_group/spot/main.tf | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/spot/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/spot/main.tf index bee397d..a346db6 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/spot/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/spot/main.tf @@ -41,10 +41,12 @@ module "autoscaling" { health_check_type = "EC2" desired_capacity = 1 - max_size = 5 + desired_capacity_type = "units" + max_size = 2 min_size = 1 create = true - #capacity_rebalance = true + capacity_rebalance = false + create_launch_template = false update_default_version = true From 45768a5b220b50c29c68fc254cb6847f98fbd153 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Thu, 26 Dec 2024 18:18:47 -0500 Subject: [PATCH 106/158] increase size --- .../swarms-aws-agent-api/dev/us-east-1/main.tf | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index f4dbcc6..2909ace 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -167,7 +167,8 @@ variable "dev_instance_types" { } module "lt_dynamic_ami_docker" { - branch = "feature/squash2-docker" + #branch = "feature/squash2-docker" + branch = "feature/merge_latest_675" vpc_id = local.vpc_id for_each = toset(var.dev_instance_types) instance_type = each.key @@ -286,11 +287,11 @@ module "asg_dynamic_new_ami_dev" { #local_storage_types = ["ssd"] max_spot_price_as_percentage_of_optimal_on_demand_price = 60 memory_gib_per_vcpu = { - min = 2 + min = 4 max = 12 } memory_mib = { - min = 2048 + min = 4096 }, vcpu_count = { min = 2 @@ -316,3 +317,7 @@ output "vpc" { output "user_data_new" { value = module.lt_dynamic_ami_test["t3.medium"].user_data } +output "user_data_docker" { + value = module.lt_dynamic_ami_docker["t3.small"].user_data +} + From cc23a5aae484fcdd93e2b319031faf2806c5233f Mon Sep 17 00:00:00 2001 From: mike dupont Date: Thu, 26 Dec 2024 18:28:25 -0500 Subject: [PATCH 107/158] removing spot for now --- .../dev/us-east-1/main.tf | 60 ++++++++++++++++++- 1 file changed, 57 insertions(+), 3 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index 2909ace..1ea3739 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -162,7 +162,20 @@ variable "dev_instance_types" { #"t3.small", #"t2.small", not working # "t2.medium" # - "t3.small" +# "t3.small" + ] +} + +variable "dev2_instance_types" { + type = list(string) + default = [ + # "t4g.nano", "t3a.nano", "t3.nano", "t2.nano", + # "t4g.micro", "t3a.micro", "t3.micro", "t2.micro", "t1.micro", + #"t4g.small", "t3a.small", + #"t3.small", + #"t2.small", not working + # "t2.medium" # + "t3.medium" ] } @@ -186,6 +199,26 @@ module "lt_dynamic_ami_docker" { install_script = "/opt/swarms/api/rundocker.sh" } +module "lt_dynamic_ami_docker_normal" { + #branch = "feature/squash2-docker" + branch = "feature/merge_latest_675" + vpc_id = local.vpc_id + for_each = toset(var.dev2_instance_types) + instance_type = each.key + name = "swarms-docker-${each.key}" + security_group_id = module.security.internal_security_group_id + ami_id = local.new_ami_id + tags = merge(local.tags, { + environment = "test" + }) + source = "./components/launch_template_docker" + key_name = var.key_name #"mdupont-deployer-key" + ssm_parameter_name_cw_agent_config = "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/cloudwatch-agent/config/details" + iam_instance_profile_name = module.roles.ssm_profile_name + #install_script = "/opt/swarms/api/docker-boot.sh" this is called from ssm for a refresh + install_script = "/opt/swarms/api/rundocker.sh" +} + module "alb" { source = "./components/application_load_balancer" domain_name = local.domain @@ -236,6 +269,7 @@ module "asg_dynamic_new_ami" { } module "asg_dynamic_new_ami_test" { + # built with packer for_each = toset(var.test_instance_types) tags = merge(local.tags, local.dev_tags) @@ -252,7 +286,7 @@ module "asg_dynamic_new_ami_test" { target_group_arn = module.alb.test_alb_target_group_arn } -module "asg_dynamic_new_ami_dev" { +module "asg_dynamic_new_ami_dev_spot" { # built with packer # count =0 tags = merge(local.tags, local.dev_tags) @@ -303,6 +337,26 @@ module "asg_dynamic_new_ami_dev" { } instance_requirements = { } +} + +module "asg_dynamic_new_ami_dev_normal" { + # built with packer +# count =0 + + tags = merge(local.tags, local.dev_tags) + vpc_id = local.vpc_id + image_id = local.new_ami_id + ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 + for_each = toset(var.dev2_instance_types) + aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn + + source = "./components/autoscaling_group/spot" + # security_group_id = module.security.internal_security_group_id + instance_type = each.key + name = "docker-swarms-ami-${each.key}" + launch_template_id = module.lt_dynamic_ami_docker_normal[each.key].launch_template_id + target_group_arn = module.alb.dev_alb_target_group_arn + } @@ -318,6 +372,6 @@ output "user_data_new" { value = module.lt_dynamic_ami_test["t3.medium"].user_data } output "user_data_docker" { - value = module.lt_dynamic_ami_docker["t3.small"].user_data + value = module.lt_dynamic_ami_docker_normal["t3.medium"].user_data } From 00452be1a5340cd10b1073afd3e35498d279b2da Mon Sep 17 00:00:00 2001 From: mike dupont Date: Fri, 27 Dec 2024 08:39:52 -0500 Subject: [PATCH 108/158] fixing branch to be constant in user data --- .../dev/us-east-1/components/launch_template_docker/main.tf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template_docker/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template_docker/main.tf index 58fff78..6c9a46b 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template_docker/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template_docker/main.tf @@ -64,10 +64,9 @@ locals { git clone https://github.com/jmikedupont2/swarms "/opt/swarms/" fi cd "/opt/swarms/" || exit 1 - export BRANCH=${var.branch} git stash git fetch --all # get the latest version - git checkout --force $BRANCH + git checkout --track --force "origin/${var.branch}" bash -x ${var.install_script} EOF From a045f4bb6952edc505eb400bc96b01bc95c4957a Mon Sep 17 00:00:00 2001 From: mike dupont Date: Fri, 27 Dec 2024 08:39:38 -0500 Subject: [PATCH 109/158] creating cname --- .../components/application_load_balancer/route53/main.tf | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf index 98c3b96..77d7265 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf @@ -42,6 +42,15 @@ resource "aws_route53_record" "api-cname-dev" { } +resource "aws_route53_record" "api-cname-dev" { + zone_id = data.aws_route53_zone.primary.zone_id + name = "mcs.${var.domain_name}" + type = "CNAME" + records = [aws_route53_record.api-cname.fqdn] + ttl = 300 + +} + output "cname" { value = aws_route53_record.api-cname.fqdn } From aee30ade423ff38215d7b7de14fd29d49b75d443 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Fri, 27 Dec 2024 09:15:49 -0500 Subject: [PATCH 110/158] creating new mcs subcomponent --- .../application_load_balancer/https/main.tf | 17 +++ .../application_load_balancer/main.tf | 11 ++ .../application_load_balancer/route53/main.tf | 2 +- .../launch_template_docker_mcs/main.tf | 121 ++++++++++++++++++ .../dev/us-east-1/main.tf | 42 +++--- .../dev/us-east-1/mcs/main.tf | 65 ++++++++++ 6 files changed, 231 insertions(+), 27 deletions(-) create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template_docker_mcs/main.tf create mode 100644 environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf index 7767d3f..64e6210 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf @@ -4,6 +4,7 @@ variable "zone_id" {} variable "prod_target_group_arn" {} variable "test_target_group_arn" {} variable "dev_target_group_arn" {} +variable "mcs_target_group_arn" {} module "acm" { # count = 0 @@ -65,6 +66,22 @@ resource "aws_lb_listener_rule" "route_v1_api_dev" { } } +resource "aws_lb_listener_rule" "route_v1_api_mcs" { + listener_arn = aws_lb_listener.this.arn + priority = 101 # Set priority as needed, must be unique + + action { + type = "forward" + target_group_arn = var.mcs_target_group_arn # New target group's ARN + } + + condition { + host_header { + values = ["mcs.api.swarms.ai"] + } + } +} + resource "aws_lb_listener" "insecure" { port = 80 protocol = "HTTP" diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf index 9d5e4bc..3d4d905 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf @@ -63,6 +63,12 @@ module "tg_dev" { vpc_id = var.vpc_id # module.vpc.vpc_id } +module "tg_mcs" { + source = "./target_group/" + name_prefix = "mcs" + vpc_id = var.vpc_id # module.vpc.vpc_id +} + module "https" { source = "./https/" # vpc_id = var.vpc_id # module.vpc.vpc_id @@ -72,6 +78,7 @@ module "https" { prod_target_group_arn = module.tg_prod.alb_target_group_arn test_target_group_arn = module.tg_test.alb_target_group_arn dev_target_group_arn = module.tg_dev.alb_target_group_arn + mcs_target_group_arn = module.tg_mcs.alb_target_group_arn } @@ -86,3 +93,7 @@ output "test_alb_target_group_arn" { output "dev_alb_target_group_arn" { value = module.tg_dev.alb_target_group_arn } + +output "mcs_alb_target_group_arn" { + value = module.tg_mcs.alb_target_group_arn +} diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf index 77d7265..bb841b1 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf @@ -42,7 +42,7 @@ resource "aws_route53_record" "api-cname-dev" { } -resource "aws_route53_record" "api-cname-dev" { +resource "aws_route53_record" "api-cname-mcs" { zone_id = data.aws_route53_zone.primary.zone_id name = "mcs.${var.domain_name}" type = "CNAME" diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template_docker_mcs/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template_docker_mcs/main.tf new file mode 100644 index 0000000..db0c478 --- /dev/null +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template_docker_mcs/main.tf @@ -0,0 +1,121 @@ +# launch template for running agent api in docker +variable "git_repo" {} +variable "app_name" {} +variable "ssm_parameter_name_cw_agent_config" {} +variable "branch" {} +variable "install_script" {} +variable "iam_instance_profile_name" {} +variable "security_group_id" {} +variable "name" {} +variable "vpc_id" {} +variable "ami_id" {} +variable "tags" {} +variable "key_name" {} +variable "instance_type" {} + +locals { + tags = { + project = "swarms" + instance_type = var.instance_type + name = var.name + } + # FIXME refactor launch template to pass in user data as template parameter, + # split up user data into reusable chunks that we can use in different forms like docker files + user_data = <<-EOF + #!/bin/bash + export HOME=/root + apt update + apt-get install -y ec2-instance-connect git + + # Install docker + apt-get install -y cloud-utils apt-transport-https ca-certificates curl software-properties-common + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) \ + stable" + apt-get update + apt-get install -y docker-ce + usermod -aG docker ubuntu + + # Install docker-compose + curl -L https://github.com/docker/compose/releases/download/1.21.0/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose + chmod +x /usr/local/bin/docker-compose + + snap install amazon-ssm-agent --classic || echo oops1 + snap start amazon-ssm-agent || echo oops2 + apt-get install -y --no-install-recommends ca-certificates=20230311 curl=7.88.1-10+deb12u7 | echo oops + curl -O "https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/$(dpkg --print-architecture)/latest/amazon-cloudwatch-agent.deb" + dpkg -i -E amazon-cloudwatch-agent.deb + # Install prerequisite packages + apt-get install -y wget unzip systemd + # In case of missing dependencies + # apt-get install -f -y + # Configure and start the CloudWatch agent + /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c "ssm:${data.aws_ssm_parameter.cw_agent_config.name}" + # Enable and start the service using systemctl + systemctl enable amazon-cloudwatch-agent + systemctl start amazon-cloudwatch-agent + + # Clean up downloaded files + rm -f amazon-cloudwatch-agent.deb + # Verify installation + systemctl status amazon-cloudwatch-agent + + if [ ! -d "/opt/${var.app_name}/" ]; then + git clone ${var.git_repo} "/opt/${var.app_name}/" + fi + cd "/opt/swarms/" || exit 1 + git stash + git fetch --all # get the latest version + git checkout --track --force "origin/${var.branch}" + + bash -x ${var.install_script} + EOF + +} +data "aws_ssm_parameter" "cw_agent_config" { + name = var.ssm_parameter_name_cw_agent_config +} + +# defined +resource "aws_launch_template" "ec2_launch_template" { + name_prefix = "${var.name}-launch-template-" + image_id = var.ami_id + key_name = var.key_name + instance_type = var.instance_type + network_interfaces { + associate_public_ip_address = true + delete_on_termination = true + security_groups = [var.security_group_id] + } + + iam_instance_profile { + # iam_instance_profile_arn = aws_iam_instance_profile.ssm.arn + name = var.iam_instance_profile_name #aws_iam_instance_profile.ec2_instance_profile.name + } + lifecycle { + create_before_destroy = true + } + block_device_mappings { + device_name = "/dev/sda1" + ebs { + volume_size = 30 + volume_type = "gp3" + encrypted = true + } + } + user_data = base64encode(local.user_data) + tags = var.tags +} + + +output "lt" { + value = resource.aws_launch_template.ec2_launch_template +} +output "launch_template_id" { + value = resource.aws_launch_template.ec2_launch_template.id +} +output "user_data" { + value = local.user_data +} diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index 1ea3739..8221ba3 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -52,19 +52,7 @@ module "kp" { source = "./components/keypairs" } -# module "lt" { -# instance_type = local.instance_type -# security_group_id = module.security.security_group_id -# source = "./components/launch_template" -# } -# module "asg" { -# source = "./components/autoscaling_group" -# name="swarms" -# security_group_id = module.security.security_group_id -# instance_type = local.instance_type -# launch_template_id = module.lt.launch_template_id -# } variable "instance_types" { type = list(string) @@ -101,20 +89,6 @@ module "roles" { tags = local.tags } -# module "lt_dynamic" { -# vpc_id = local.vpc_id -# branch = "feature/ec2" -# for_each = toset(var.instance_types) -# instance_type = each.key -# name = "swarms-size-${each.key}" -# security_group_id = module.security.internal_security_group_id -# ami_id = var.ami_id -# tags= local.tags -# source = "./components/launch_template" -# iam_instance_profile_name = module.roles.ssm_profile_name -# #aws_iam_instance_profile.ssm.name -# install_script = "/opt/swarms/api/install.sh" -# } module "lt_dynamic_ami_prod" { vpc_id = local.vpc_id @@ -219,6 +193,7 @@ module "lt_dynamic_ami_docker_normal" { install_script = "/opt/swarms/api/rundocker.sh" } + module "alb" { source = "./components/application_load_balancer" domain_name = local.domain @@ -364,6 +339,21 @@ output "security_group_id" { value = module.security.security_group_id } +module mcs { + source = "./mcs" + alb_target_group_arn = module.alb.mcs_alb_target_group_arn + ssm_profile_arn = module.roles.ssm_profile_arn + ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 + iam_instance_profile_name = module.roles.ssm_profile_name + key_name = var.key_name + aws_account_id = var.aws_account_id + region = var.region + internal_security_group_id = module.security.internal_security_group_id + tags = local.tags + ami_id = local.new_ami_id + vpc_id = local.vpc_id +} + output "vpc" { value = module.vpc } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf new file mode 100644 index 0000000..69cc9c8 --- /dev/null +++ b/environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf @@ -0,0 +1,65 @@ +variable ec2_subnet_id{} +variable alb_target_group_arn {} +variable "aws_account_id" {} +variable "vpc_id" {} +# FIXME security, we use the ami of swarms for speed but want to split +variable "ami_id" {} +# FIXME security, we use the sg of swarms for speed but want to split +variable internal_security_group_id {} + +# FIXME security, we use the profile of swarms for speed but want to split +variable iam_instance_profile_name {} +variable ssm_profile_arn {} +variable "key_name" {} +variable "region" {} +variable "tags" {} + +variable "instance_types" { + type = list(string) + default = [ + #"t3.small", + # "t2.medium" # + "t3.medium" + ] +} + +module "lt_docker" { + source = "../components/launch_template_docker_mcs" + branch = "feature/mcs" + vpc_id = var.vpc_id + for_each = toset(var.instance_types) + instance_type = each.key + name = "mcs-docker-${each.key}" + security_group_id = var.internal_security_group_id + ami_id = var.ami_id + git_repo = "https://github.com/jmikedupont2/swarms-MedicalCoderSwarm-deployment.git" + app_name = "mcs" # used to construct /opt/mcs for where we install to + tags = merge(var.tags, { + environment = "mcs" + }) + + key_name = var.key_name #"mdupont-deployer-key" + ssm_parameter_name_cw_agent_config = "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/cloudwatch-agent/config/details" + iam_instance_profile_name = var.iam_instance_profile_name + #install_script = "/opt/swarms/api/docker-boot.sh" this is called from ssm for a refresh + install_script = "/opt/swarms/api/rundocker.sh" +} + + + +module "asg" { + source = "../components/autoscaling_group/spot" + tags = var.tags + vpc_id = var.vpc_id + image_id = var.ami_id + ec2_subnet_id = var.ec2_subnet_id + for_each = toset(var.instance_types) + aws_iam_instance_profile_ssm_arn = var.ssm_profile_arn + + + # security_group_id = module.security.internal_security_group_id + instance_type = each.key + name = "docker-mcs-ami-${each.key}" + launch_template_id = module.lt_docker[each.key].launch_template_id + target_group_arn = var.alb_target_group_arn +} From 3de651fd3e1b121d905a2130afa200c82bf9d3c5 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Fri, 27 Dec 2024 09:18:53 -0500 Subject: [PATCH 111/158] tofu fmt --- .../components/autoscaling_group/spot/main.tf | 18 +++--- .../dev/us-east-1/main.tf | 58 +++++++++---------- .../dev/us-east-1/mcs/main.tf | 16 ++--- 3 files changed, 46 insertions(+), 46 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/spot/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/spot/main.tf index a346db6..d8f6b7b 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/spot/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/spot/main.tf @@ -38,15 +38,15 @@ module "autoscaling" { traffic_source_type = "elbv2" # default } } - - health_check_type = "EC2" - desired_capacity = 1 - desired_capacity_type = "units" - max_size = 2 - min_size = 1 - create = true - capacity_rebalance = false - + + health_check_type = "EC2" + desired_capacity = 1 + desired_capacity_type = "units" + max_size = 2 + min_size = 1 + create = true + capacity_rebalance = false + create_launch_template = false update_default_version = true diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index 8221ba3..14447f0 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -1,4 +1,4 @@ -variable spot_max_price { +variable "spot_max_price" { default = 0.028 } variable "region" {} @@ -78,7 +78,7 @@ variable "test_instance_types" { # "t2.medium" # - + #"t3.medium" # no instances for now, this is commented out ] } @@ -136,7 +136,7 @@ variable "dev_instance_types" { #"t3.small", #"t2.small", not working # "t2.medium" # -# "t3.small" + # "t3.small" ] } @@ -148,8 +148,8 @@ variable "dev2_instance_types" { #"t4g.small", "t3a.small", #"t3.small", #"t2.small", not working - # "t2.medium" # - "t3.medium" + # "t2.medium" # + "t3.medium" ] } @@ -244,13 +244,13 @@ module "asg_dynamic_new_ami" { } module "asg_dynamic_new_ami_test" { - + # built with packer - for_each = toset(var.test_instance_types) - tags = merge(local.tags, local.dev_tags) - vpc_id = local.vpc_id - image_id = local.new_ami_id - ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 + for_each = toset(var.test_instance_types) + tags = merge(local.tags, local.dev_tags) + vpc_id = local.vpc_id + image_id = local.new_ami_id + ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn source = "./components/autoscaling_group/spot" @@ -263,7 +263,7 @@ module "asg_dynamic_new_ami_test" { module "asg_dynamic_new_ami_dev_spot" { # built with packer -# count =0 + # count =0 tags = merge(local.tags, local.dev_tags) vpc_id = local.vpc_id image_id = local.new_ami_id @@ -271,7 +271,7 @@ module "asg_dynamic_new_ami_dev_spot" { for_each = toset(var.dev_instance_types) aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn - source = "./components/autoscaling_group/spot" + source = "./components/autoscaling_group/spot" # security_group_id = module.security.internal_security_group_id instance_type = each.key name = "docker-swarms-ami-${each.key}" @@ -285,13 +285,13 @@ module "asg_dynamic_new_ami_dev_spot" { on_demand_percentage_above_base_capacity = 0 spot_instance_pools = 1 spot_max_price = var.spot_max_price -# spot_allocation_strategy = "capacity-optimized" + # spot_allocation_strategy = "capacity-optimized" } override = [ { instance_requirements = { - cpu_manufacturers = ["amazon-web-services", "amd", "intel"] + cpu_manufacturers = ["amazon-web-services", "amd", "intel"] #cpu_manufacturers = ["amd"] #local_storage_types = ["ssd"] max_spot_price_as_percentage_of_optimal_on_demand_price = 60 @@ -316,7 +316,7 @@ module "asg_dynamic_new_ami_dev_spot" { module "asg_dynamic_new_ami_dev_normal" { # built with packer -# count =0 + # count =0 tags = merge(local.tags, local.dev_tags) vpc_id = local.vpc_id @@ -325,7 +325,7 @@ module "asg_dynamic_new_ami_dev_normal" { for_each = toset(var.dev2_instance_types) aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn - source = "./components/autoscaling_group/spot" + source = "./components/autoscaling_group/spot" # security_group_id = module.security.internal_security_group_id instance_type = each.key name = "docker-swarms-ami-${each.key}" @@ -339,19 +339,19 @@ output "security_group_id" { value = module.security.security_group_id } -module mcs { - source = "./mcs" - alb_target_group_arn = module.alb.mcs_alb_target_group_arn - ssm_profile_arn = module.roles.ssm_profile_arn - ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 - iam_instance_profile_name = module.roles.ssm_profile_name - key_name = var.key_name - aws_account_id = var.aws_account_id - region = var.region +module "mcs" { + source = "./mcs" + alb_target_group_arn = module.alb.mcs_alb_target_group_arn + ssm_profile_arn = module.roles.ssm_profile_arn + ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 + iam_instance_profile_name = module.roles.ssm_profile_name + key_name = var.key_name + aws_account_id = var.aws_account_id + region = var.region internal_security_group_id = module.security.internal_security_group_id - tags = local.tags - ami_id = local.new_ami_id - vpc_id = local.vpc_id + tags = local.tags + ami_id = local.new_ami_id + vpc_id = local.vpc_id } output "vpc" { diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf index 69cc9c8..4c14e79 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf @@ -1,19 +1,19 @@ -variable ec2_subnet_id{} -variable alb_target_group_arn {} +variable "ec2_subnet_id" {} +variable "alb_target_group_arn" {} variable "aws_account_id" {} variable "vpc_id" {} # FIXME security, we use the ami of swarms for speed but want to split variable "ami_id" {} # FIXME security, we use the sg of swarms for speed but want to split -variable internal_security_group_id {} +variable "internal_security_group_id" {} # FIXME security, we use the profile of swarms for speed but want to split -variable iam_instance_profile_name {} -variable ssm_profile_arn {} +variable "iam_instance_profile_name" {} +variable "ssm_profile_arn" {} variable "key_name" {} variable "region" {} variable "tags" {} - + variable "instance_types" { type = list(string) default = [ @@ -32,8 +32,8 @@ module "lt_docker" { name = "mcs-docker-${each.key}" security_group_id = var.internal_security_group_id ami_id = var.ami_id - git_repo = "https://github.com/jmikedupont2/swarms-MedicalCoderSwarm-deployment.git" - app_name = "mcs" # used to construct /opt/mcs for where we install to + git_repo = "https://github.com/jmikedupont2/swarms-MedicalCoderSwarm-deployment.git" + app_name = "mcs" # used to construct /opt/mcs for where we install to tags = merge(var.tags, { environment = "mcs" }) From 41ef75c9a402162340247d135da43a6e21d08d08 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Fri, 27 Dec 2024 09:22:34 -0500 Subject: [PATCH 112/158] fixing path name --- .../us-east-1/components/launch_template_docker_mcs/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template_docker_mcs/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template_docker_mcs/main.tf index db0c478..b935bae 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template_docker_mcs/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template_docker_mcs/main.tf @@ -15,7 +15,7 @@ variable "instance_type" {} locals { tags = { - project = "swarms" + project = "mcs" instance_type = var.instance_type name = var.name } @@ -65,7 +65,7 @@ locals { if [ ! -d "/opt/${var.app_name}/" ]; then git clone ${var.git_repo} "/opt/${var.app_name}/" fi - cd "/opt/swarms/" || exit 1 + cd "/opt/${var.app_name}/" || exit 1 git stash git fetch --all # get the latest version git checkout --track --force "origin/${var.branch}" From a792f3181be1b63396581ed22ed146f58781eca9 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sat, 28 Dec 2024 06:25:08 -0500 Subject: [PATCH 113/158] fixing branch --- .../dev/us-east-1/components/roles/main.tf | 20 +++++++++++++++++++ .../dev/us-east-1/main.tf | 5 +++-- .../dev/us-east-1/mcs/main.tf | 4 ++-- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf index c320d8b..5dce95e 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf @@ -7,6 +7,26 @@ data "aws_iam_policy_document" "default" { effect = "Allow" } + + statement { + effect= "Allow" + actions= [ + "ecr:GetAuthorizationToken", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:GetRepositoryPolicy", + "ecr:DescribeRepositories", + "ecr:ListImages", + "ecr:DescribeImages", + "ecr:BatchGetImage", + "ecr:GetLifecyclePolicy", + "ecr:GetLifecyclePolicyPreview", + "ecr:ListTagsForResource", + "ecr:DescribeImageScanFindings" + ] + resources = ["*"] + } + statement { actions = ["kms:Decrypt"] resources = ["arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590"] diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index 14447f0..e13b6be 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -149,7 +149,7 @@ variable "dev2_instance_types" { #"t3.small", #"t2.small", not working # "t2.medium" # - "t3.medium" + #"t3.medium" ] } @@ -362,6 +362,7 @@ output "user_data_new" { value = module.lt_dynamic_ami_test["t3.medium"].user_data } output "user_data_docker" { - value = module.lt_dynamic_ami_docker_normal["t3.medium"].user_data + value = "" + #module.lt_dynamic_ami_docker_normal["t3.medium"].user_data } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf index 4c14e79..4247059 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf @@ -41,8 +41,8 @@ module "lt_docker" { key_name = var.key_name #"mdupont-deployer-key" ssm_parameter_name_cw_agent_config = "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/cloudwatch-agent/config/details" iam_instance_profile_name = var.iam_instance_profile_name - #install_script = "/opt/swarms/api/docker-boot.sh" this is called from ssm for a refresh - install_script = "/opt/swarms/api/rundocker.sh" + #install_script = "/opt/mcs/api/docker-boot.sh" this is called from ssm for a refresh + install_script = "/opt/mcs/api/rundocker.sh" } From 038b0b72565a58f004e5faf0b1a565b05bf61ded Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sat, 28 Dec 2024 06:36:01 -0500 Subject: [PATCH 114/158] changing health check path https://github.com/The-Swarm-Corporation/MedicalCoderSwarm/issues/8 --- .../us-east-1/components/application_load_balancer/main.tf | 1 + .../application_load_balancer/target_group/main.tf | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf index 3d4d905..05e1078 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf @@ -66,6 +66,7 @@ module "tg_dev" { module "tg_mcs" { source = "./target_group/" name_prefix = "mcs" + check_path = "/v1/medical-coder/patients" # this does not have a docs api yet vpc_id = var.vpc_id # module.vpc.vpc_id } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/target_group/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/target_group/main.tf index bba27f5..f0efe60 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/target_group/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/target_group/main.tf @@ -1,5 +1,8 @@ variable "vpc_id" {} variable "name_prefix" {} +variable "check_path" { +default = "/v1/docs" # the docs api +} resource "aws_lb_target_group" "this" { name_prefix = var.name_prefix protocol = "HTTP" @@ -13,7 +16,9 @@ resource "aws_lb_target_group" "this" { protocol_version = "HTTP1" # health_check { - path = "/v1/docs" # the docs api + path = var.check_path #"/v1/docs" # the docs api + #"/v1/medical-coder/patients" + enabled = true healthy_threshold = 10 interval = 130 From 73bb419cde8ff799f33979e3951101d3ae637db6 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sat, 28 Dec 2024 09:53:46 -0500 Subject: [PATCH 115/158] parser now producing better results --- logs/parse.py | 74 +++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 72 insertions(+), 2 deletions(-) diff --git a/logs/parse.py b/logs/parse.py index e508356..d5f3ef2 100644 --- a/logs/parse.py +++ b/logs/parse.py @@ -21,6 +21,47 @@ import glob log_files = glob.glob('logs/*.log') all_events = [] +seen = {} + +def process1(v,path): + + + if isinstance(v,list): + for i,j in enumerate(v): + path2= path.copy() + path2.extend(["array" + "["+str(i)+"]"]) + #v2 = v[i] + process1(j,path2) + + elif isinstance(v,dict): + # we are in the next level of substructrue + for k2 in v: + v2= v[k2] + path2= path.copy() + path2.extend([k2]) + qk2 = ".".join(path2) + vt2 = type(v2) + if qk2 not in seen: + + seen[qk2] =1 + else: + seen[qk2] = seen[qk2] + 1 + # some of these we can add directly + #print("DEBUG2",qk2,vt2) + process1(v2,path2) + else: + path2= path.copy() + path2.extend([str(v)]) + qk2 = ".".join(path2) + if qk2 not in seen: + seen[qk2] =1 + else: + seen[qk2] = seen[qk2] + 1 + # some of these we can add directly + #print("DEBUG3",qk2,v,seen[qk2]) + #process1("1",path2) + + for log_file in log_files: with open(log_file, 'r') as f: try: @@ -28,10 +69,39 @@ except Exception as e: print(log_file,e) e1 = event_data.get("Events", []) + for e in e1: - print(e1) + #print(e1) + target = "CloudTrailEvent" + if target in e: + e2 = json.loads(e[target]) # eval again + #print("DEBUG1",e2) + for k in e2: + #print("DEBUG2",k) + v= e2[k] + qualified_path = [target,k] + qk = ".".join(qualified_path) + vt = type(v) + if qk not in seen: + seen[qk] =1 + else: + seen[qk] = seen[qk] +1 + process1(v,qualified_path) + + + # DEBUG EventId 14ac0923-bb3d-4140-8c8e-3e0d493139fc + # DEBUG EventName LookupEvents + # DEBUG ReadOnly true + # DEBUG AccessKeyId AKIA5K4H36GTYFFWXUGH + # DEBUG EventTime 1734893533.0 + # DEBUG EventSource cloudtrail.amazonaws.com + # DEBUG Username dupont + # DEBUG Resources [] + # DEBUG CloudTrailEvent {"eventVersion":"1.11","userIdentity":{"type":"IAMUser","principalId":"AIDA5K4H36GT5MJLQHPRT","arn":"arn:aws:iam::916723593639:user/dupont","accountId":"916723593639","accessKeyId":"AKIA5K4H36GTYFFWXUGH","userName":"dupont"},"eventTime":"2024-12-22T18:52:13Z","eventSource":"cloudtrail.amazonaws.com","eventName":"LookupEvents","awsRegion":"us-east-2","sourceIPAddress":"98.110.51.114","userAgent":"aws-cli/1.32.85 md/Botocore#1.34.85 ua/2.0 os/linux#6.8.0-49-generic md/arch#x86_64 lang/python#3.10.12 md/pyimpl#CPython cfg/retry-mode#legacy botocore/1.34.85","requestParameters":{"startTime":"Dec 22, 2024, 8:52:29 AM","nextToken":"shqJhJryg7NPAzxDKddA7KOR5HR3qgzsO4Bskd7FRV/sRIOCCBsmD6H0dhqNTGZ0"},"responseElements":null,"requestID":"9c3ffb5a-05aa-4a31-b3e2-12914ed9c7b7","eventID":"14ac0923-bb3d-4140-8c8e-3e0d493139fc","readOnly":true,"eventType":"AwsApiCall","managementEvent":true,"recipientAccountId":"916723593639","eventCategory":"Management","tlsDetails":{"tlsVersion":"TLSv1.3","cipherSuite":"TLS_AES_128_GCM_SHA256","clientProvidedHostHeader":"cloudtrail.us-east-2.amazonaws.com"}} #all_events.extend(e1) - +for x in seen: + v = seen[x] + print("\t".join([str(v),x])) #print(all_events) # or process the events as needed # ### Explanation: From 6b959d06e00383a9ce48bef5310032c1e11ee7de Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sat, 28 Dec 2024 12:29:09 -0500 Subject: [PATCH 116/158] report --- logs/parse.py | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/logs/parse.py b/logs/parse.py index d5f3ef2..265d22c 100644 --- a/logs/parse.py +++ b/logs/parse.py @@ -1,3 +1,4 @@ +#from itertools import combinations # # in python # open up logs/*.log and read out json @@ -61,7 +62,7 @@ def process1(v,path): #print("DEBUG3",qk2,v,seen[qk2]) #process1("1",path2) - +report = {} for log_file in log_files: with open(log_file, 'r') as f: try: @@ -88,20 +89,24 @@ def process1(v,path): seen[qk] = seen[qk] +1 process1(v,qualified_path) - - # DEBUG EventId 14ac0923-bb3d-4140-8c8e-3e0d493139fc - # DEBUG EventName LookupEvents - # DEBUG ReadOnly true - # DEBUG AccessKeyId AKIA5K4H36GTYFFWXUGH - # DEBUG EventTime 1734893533.0 - # DEBUG EventSource cloudtrail.amazonaws.com - # DEBUG Username dupont - # DEBUG Resources [] - # DEBUG CloudTrailEvent {"eventVersion":"1.11","userIdentity":{"type":"IAMUser","principalId":"AIDA5K4H36GT5MJLQHPRT","arn":"arn:aws:iam::916723593639:user/dupont","accountId":"916723593639","accessKeyId":"AKIA5K4H36GTYFFWXUGH","userName":"dupont"},"eventTime":"2024-12-22T18:52:13Z","eventSource":"cloudtrail.amazonaws.com","eventName":"LookupEvents","awsRegion":"us-east-2","sourceIPAddress":"98.110.51.114","userAgent":"aws-cli/1.32.85 md/Botocore#1.34.85 ua/2.0 os/linux#6.8.0-49-generic md/arch#x86_64 lang/python#3.10.12 md/pyimpl#CPython cfg/retry-mode#legacy botocore/1.34.85","requestParameters":{"startTime":"Dec 22, 2024, 8:52:29 AM","nextToken":"shqJhJryg7NPAzxDKddA7KOR5HR3qgzsO4Bskd7FRV/sRIOCCBsmD6H0dhqNTGZ0"},"responseElements":null,"requestID":"9c3ffb5a-05aa-4a31-b3e2-12914ed9c7b7","eventID":"14ac0923-bb3d-4140-8c8e-3e0d493139fc","readOnly":true,"eventType":"AwsApiCall","managementEvent":true,"recipientAccountId":"916723593639","eventCategory":"Management","tlsDetails":{"tlsVersion":"TLSv1.3","cipherSuite":"TLS_AES_128_GCM_SHA256","clientProvidedHostHeader":"cloudtrail.us-east-2.amazonaws.com"}} - #all_events.extend(e1) -for x in seen: - v = seen[x] - print("\t".join([str(v),x])) + # now report on the event + facts = sorted(seen.keys()) + seen = {} + #for x in facts: + # v = seen[x] + # if (v>0): + #facts.append("\t".join([str(v),x])) + # seen[x] =0 # reset + #for p in combinations(facts,3): + print(facts) + for f in facts: + #k = ".".join(p) + if f not in report: + report[f] = 1 + else: + report[f] = report[f] + 1 +for k in report: + print(k,report[k]) #print(all_events) # or process the events as needed # ### Explanation: From 679e81ea478dc796749592824da337f2249aebd3 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sat, 28 Dec 2024 19:14:09 -0500 Subject: [PATCH 117/158] now generating new code --- logs/parse.py | 1272 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 1265 insertions(+), 7 deletions(-) diff --git a/logs/parse.py b/logs/parse.py index 265d22c..17cb4c7 100644 --- a/logs/parse.py +++ b/logs/parse.py @@ -1,3 +1,7 @@ +from datamodel_code_generator import ( DataModelType, + InputFileType, + generate) +SEP="|" #from itertools import combinations # # in python # open up logs/*.log and read out json @@ -24,23 +28,1245 @@ all_events = [] seen = {} +from pydantic import BaseModel, Field, create_model,ConfigDict +from datetime import datetime +from typing import List, Optional + +# from https://github.com/aws-samples/amazon-dynamodb-pitr-table-sync/blob/43b0f85f1a6bb07b17fe7f20437eb9834601661e/src/model/aws/dynamodb/dynamodb_pitr_notification.py#L111 + +class SessionIssuer (BaseModel): + _type:str + principal_id: str = Field(None, alias="principalId") + arn: str = Field(None) + account_id: str = Field(None, alias="accountId") + user_name: str = Field(None, alias="userName") + + +class SessionContext (BaseModel): + model_config = ConfigDict(extra="forbid") + # userIdentity|sessionContext|attributes + # userIdentity|sessionContext|attributes|creationDate + # userIdentity|sessionContext|attributes|mfaAuthenticated + # userIdentity|sessionContext|ec2RoleDelivery + # userIdentity|sessionContext|sessionIssuer + # userIdentity|sessionContext|sessionIssuer|accountId + # userIdentity|sessionContext|sessionIssuer|arn + # userIdentity|sessionContext|sessionIssuer|principalId + # userIdentity|sessionContext|sessionIssuer|type + # userIdentity|sessionContext|sessionIssuer|userName + # userIdentity|sessionContext|webIdFederationData + # userIdentity|sessionContext|webIdFederationData|federatedProvider + # "SessionContext", + # attributes=( + # create_model( + # "SessionContextAttributes", + # creation_date=(datetime, Field(None, alias="creationDate")), + # mfa_authenticated=(bool, Field(None, alias="mfaAuthenticated")), + # ) + # ), + session_issuer : Optional[SessionIssuer] +class UserIdentity(BaseModel): + model_config = ConfigDict(extra="forbid") + theType: str = Field(None, alias="type") + principal_id: str = Field(None, alias="principalId") + arn: str = Field(None) + account_id: str = Field(None, alias="accountId") + access_key_id: str = Field(None, alias="accessKeyId") + userName :Optional[str] = None + invokedBy :Optional[str] = None + identityProvider :Optional[str] = None + principalId :Optional[str] = None + sessionContext : Optional[SessionContext]=None + theType: Optional[str] = Field(None, alias="type") + + +class RequestParameters(BaseModel): + model_config = ConfigDict(extra="forbid") + startTime:Optional[str] =None + nextToken:Optional[str]= None + + # requestParameters + # accelerate + # accountId + # accountIdentifiers|array[0] + # accountIds|array[0] + # acl + # actionPrefix + # actionsEnabled + # activeOnly + # agentName + # agentStatus + # agentVersion + # alarmActions|array[0] + # alarmDescription + # alarmName + # alarmNames|array[0] + # alarmTypes|array[0] + # alarmTypes|array[1] + # allAvailabilityZones + # allowedPattern + # allowInvalidContent + # allRegions + # anomalyDetectorTypes|array[0] + # anomalyDetectorTypes|array[1] + # anomalyVisibilityTime + # associationId + # attribute + # autoScalingGroupName + # autoScalingGroupNames|array[0] + # availabilityZone + # availabilityZoneId + # awsAccountId + # baselineId + # bucketName + # byShared + # certificateArn + # clientId + # clientInstanceId + # clientName + # clientToken + # cloudWatchOutputConfig + # cloudWatchOutputConfig|cloudWatchLogGroupName + # cloudWatchOutputConfig|cloudWatchOutputEnabled + # commandId + # comment + # comparisonOperator + # complianceType + # computerName + # constraints + # constraints|encryptionContextSubset + # constraints|encryptionContextSubset|aws:ebs:id + # content + # cors + # CreateLaunchTemplateRequest + # CreateLaunchTemplateRequest|ClientToken + # CreateLaunchTemplateRequest|LaunchTemplateData + # CreateLaunchTemplateRequest|LaunchTemplateData|BlockDeviceMapping + # CreateLaunchTemplateRequest|LaunchTemplateData|BlockDeviceMapping|DeviceName + # CreateLaunchTemplateRequest|LaunchTemplateData|BlockDeviceMapping|Ebs + # CreateLaunchTemplateRequest|LaunchTemplateData|BlockDeviceMapping|Ebs|Encrypted + # CreateLaunchTemplateRequest|LaunchTemplateData|BlockDeviceMapping|Ebs|VolumeSize + # CreateLaunchTemplateRequest|LaunchTemplateData|BlockDeviceMapping|Ebs|VolumeType + # CreateLaunchTemplateRequest|LaunchTemplateData|BlockDeviceMapping|tag + # CreateLaunchTemplateRequest|LaunchTemplateData|IamInstanceProfile + # CreateLaunchTemplateRequest|LaunchTemplateData|IamInstanceProfile|Name + # CreateLaunchTemplateRequest|LaunchTemplateData|ImageId + # CreateLaunchTemplateRequest|LaunchTemplateData|InstanceType + # CreateLaunchTemplateRequest|LaunchTemplateData|KeyName + # CreateLaunchTemplateRequest|LaunchTemplateData|NetworkInterface + # CreateLaunchTemplateRequest|LaunchTemplateData|NetworkInterface|AssociatePublicIpAddress + # CreateLaunchTemplateRequest|LaunchTemplateData|NetworkInterface|DeleteOnTermination + # CreateLaunchTemplateRequest|LaunchTemplateData|NetworkInterface|DeviceIndex + # CreateLaunchTemplateRequest|LaunchTemplateData|NetworkInterface|NetworkCardIndex + # CreateLaunchTemplateRequest|LaunchTemplateData|NetworkInterface|SecurityGroupId + # CreateLaunchTemplateRequest|LaunchTemplateData|NetworkInterface|SecurityGroupId|content + # CreateLaunchTemplateRequest|LaunchTemplateData|NetworkInterface|SecurityGroupId|tag + # CreateLaunchTemplateRequest|LaunchTemplateData|NetworkInterface|tag + # CreateLaunchTemplateRequest|LaunchTemplateData|UserData + # CreateLaunchTemplateRequest|LaunchTemplateName + # CreateLaunchTemplateRequest|TagSpecification + # CreateLaunchTemplateRequest|TagSpecification|ResourceType + # CreateLaunchTemplateRequest|TagSpecification|tag + # CreateLaunchTemplateRequest|TagSpecification|Tag + # CreateLaunchTemplateRequest|TagSpecification|Tag|Key + # CreateLaunchTemplateRequest|TagSpecification|Tag|tag + # CreateLaunchTemplateRequest|TagSpecification|Tag|Value + # CreateLaunchTemplateVersionRequest + # CreateLaunchTemplateVersionRequest|ClientToken + # CreateLaunchTemplateVersionRequest|LaunchTemplateData + # CreateLaunchTemplateVersionRequest|LaunchTemplateData|BlockDeviceMapping + # CreateLaunchTemplateVersionRequest|LaunchTemplateData|BlockDeviceMapping|DeviceName + # CreateLaunchTemplateVersionRequest|LaunchTemplateData|BlockDeviceMapping|Ebs + # CreateLaunchTemplateVersionRequest|LaunchTemplateData|BlockDeviceMapping|Ebs|Encrypted + # CreateLaunchTemplateVersionRequest|LaunchTemplateData|BlockDeviceMapping|Ebs|VolumeSize + # CreateLaunchTemplateVersionRequest|LaunchTemplateData|BlockDeviceMapping|Ebs|VolumeType + # CreateLaunchTemplateVersionRequest|LaunchTemplateData|BlockDeviceMapping|tag + # CreateLaunchTemplateVersionRequest|LaunchTemplateData|IamInstanceProfile + # CreateLaunchTemplateVersionRequest|LaunchTemplateData|IamInstanceProfile|Name + # CreateLaunchTemplateVersionRequest|LaunchTemplateData|ImageId + # CreateLaunchTemplateVersionRequest|LaunchTemplateData|InstanceType + # CreateLaunchTemplateVersionRequest|LaunchTemplateData|KeyName + # CreateLaunchTemplateVersionRequest|LaunchTemplateData|NetworkInterface + # CreateLaunchTemplateVersionRequest|LaunchTemplateData|NetworkInterface|AssociatePublicIpAddress + # CreateLaunchTemplateVersionRequest|LaunchTemplateData|NetworkInterface|DeleteOnTermination + # CreateLaunchTemplateVersionRequest|LaunchTemplateData|NetworkInterface|DeviceIndex + # CreateLaunchTemplateVersionRequest|LaunchTemplateData|NetworkInterface|NetworkCardIndex + # CreateLaunchTemplateVersionRequest|LaunchTemplateData|NetworkInterface|SecurityGroupId + # CreateLaunchTemplateVersionRequest|LaunchTemplateData|NetworkInterface|SecurityGroupId|content + # CreateLaunchTemplateVersionRequest|LaunchTemplateData|NetworkInterface|SecurityGroupId|tag + # CreateLaunchTemplateVersionRequest|LaunchTemplateData|NetworkInterface|tag + # CreateLaunchTemplateVersionRequest|LaunchTemplateData|UserData + # CreateLaunchTemplateVersionRequest|LaunchTemplateId + # dashboardBody + # dashboardName + # DeleteLaunchTemplateRequest + # DeleteLaunchTemplateRequest|LaunchTemplateId + # descending + # DescribeAddressesAttributeRequest + # DescribeAddressesAttributeRequest|Attribute + # DescribeAddressesAttributeRequest|MaxResults + # DescribeAddressTransfersRequest + # DescribeAddressTransfersRequest|MaxResults + # DescribeCapacityReservationFleetsRequest + # DescribeCapacityReservationFleetsRequest|MaxResults + # DescribeCapacityReservationsRequest + # DescribeCapacityReservationsRequest|MaxResults + # DescribeFleetsRequest + # DescribeFleetsRequest|MaxResults + # DescribeFlowLogsRequest + # DescribeHostsRequest + # DescribeHostsRequest|MaxResults + # DescribeInstanceConnectEndpointsRequest + # DescribeInstanceConnectEndpointsRequest|MaxResults + # DescribeInstanceCreditSpecificationsRequest + # DescribeInstanceCreditSpecificationsRequest|InstanceId + # DescribeInstanceCreditSpecificationsRequest|InstanceId|content + # DescribeInstanceCreditSpecificationsRequest|InstanceId|tag + # DescribeInstanceImageMetadataRequest + # DescribeInstanceImageMetadataRequest|InstanceId + # DescribeInstanceImageMetadataRequest|InstanceId|content + # DescribeInstanceImageMetadataRequest|InstanceId|tag + # DescribeInstanceTypeOfferingsRequest + # DescribeInstanceTypeOfferingsRequest|LocationType + # DescribeInstanceTypeOfferingsRequest|NextToken + # DescribeInstanceTypesRequest + # DescribeInstanceTypesRequest|InstanceType + # DescribeInstanceTypesRequest|InstanceType|array[0] + # DescribeInstanceTypesRequest|InstanceType|array[0]|content + # DescribeInstanceTypesRequest|InstanceType|array[0]|tag + # DescribeInstanceTypesRequest|InstanceType|array[1] + # DescribeInstanceTypesRequest|InstanceType|array[10] + # DescribeInstanceTypesRequest|InstanceType|array[10]|content + # DescribeInstanceTypesRequest|InstanceType|array[10]|tag + # DescribeInstanceTypesRequest|InstanceType|array[11] + # DescribeInstanceTypesRequest|InstanceType|array[11]|content + # DescribeInstanceTypesRequest|InstanceType|array[11]|tag + # DescribeInstanceTypesRequest|InstanceType|array[12] + # DescribeInstanceTypesRequest|InstanceType|array[12]|content + # DescribeInstanceTypesRequest|InstanceType|array[12]|tag + # DescribeInstanceTypesRequest|InstanceType|array[13] + # DescribeInstanceTypesRequest|InstanceType|array[13]|content + # DescribeInstanceTypesRequest|InstanceType|array[13]|tag + # DescribeInstanceTypesRequest|InstanceType|array[14] + # DescribeInstanceTypesRequest|InstanceType|array[14]|content + # DescribeInstanceTypesRequest|InstanceType|array[14]|tag + # DescribeInstanceTypesRequest|InstanceType|array[15] + # DescribeInstanceTypesRequest|InstanceType|array[15]|content + # DescribeInstanceTypesRequest|InstanceType|array[15]|tag + # DescribeInstanceTypesRequest|InstanceType|array[16] + # DescribeInstanceTypesRequest|InstanceType|array[16]|content + # DescribeInstanceTypesRequest|InstanceType|array[16]|tag + # DescribeInstanceTypesRequest|InstanceType|array[17] + # DescribeInstanceTypesRequest|InstanceType|array[17]|content + # DescribeInstanceTypesRequest|InstanceType|array[17]|tag + # DescribeInstanceTypesRequest|InstanceType|array[18] + # DescribeInstanceTypesRequest|InstanceType|array[18]|content + # DescribeInstanceTypesRequest|InstanceType|array[18]|tag + # DescribeInstanceTypesRequest|InstanceType|array[19] + # DescribeInstanceTypesRequest|InstanceType|array[19]|content + # DescribeInstanceTypesRequest|InstanceType|array[19]|tag + # DescribeInstanceTypesRequest|InstanceType|array[1]|content + # DescribeInstanceTypesRequest|InstanceType|array[1]|tag + # DescribeInstanceTypesRequest|InstanceType|array[2] + # DescribeInstanceTypesRequest|InstanceType|array[20] + # DescribeInstanceTypesRequest|InstanceType|array[20]|content + # DescribeInstanceTypesRequest|InstanceType|array[20]|tag + # DescribeInstanceTypesRequest|InstanceType|array[2]|content + # DescribeInstanceTypesRequest|InstanceType|array[2]|tag + # DescribeInstanceTypesRequest|InstanceType|array[3] + # DescribeInstanceTypesRequest|InstanceType|array[3]|content + # DescribeInstanceTypesRequest|InstanceType|array[3]|tag + # DescribeInstanceTypesRequest|InstanceType|array[4] + # DescribeInstanceTypesRequest|InstanceType|array[4]|content + # DescribeInstanceTypesRequest|InstanceType|array[4]|tag + # DescribeInstanceTypesRequest|InstanceType|array[5] + # DescribeInstanceTypesRequest|InstanceType|array[5]|content + # DescribeInstanceTypesRequest|InstanceType|array[5]|tag + # DescribeInstanceTypesRequest|InstanceType|array[6] + # DescribeInstanceTypesRequest|InstanceType|array[6]|content + # DescribeInstanceTypesRequest|InstanceType|array[6]|tag + # DescribeInstanceTypesRequest|InstanceType|array[7] + # DescribeInstanceTypesRequest|InstanceType|array[7]|content + # DescribeInstanceTypesRequest|InstanceType|array[7]|tag + # DescribeInstanceTypesRequest|InstanceType|array[8] + # DescribeInstanceTypesRequest|InstanceType|array[8]|content + # DescribeInstanceTypesRequest|InstanceType|array[8]|tag + # DescribeInstanceTypesRequest|InstanceType|array[9] + # DescribeInstanceTypesRequest|InstanceType|array[9]|content + # DescribeInstanceTypesRequest|InstanceType|array[9]|tag + # DescribeInstanceTypesRequest|InstanceType|content + # DescribeInstanceTypesRequest|InstanceType|tag + # DescribeInstanceTypesRequest|MaxResults + # DescribeInstanceTypesRequest|NextToken + # DescribeLaunchTemplatesRequest + # DescribeLaunchTemplatesRequest|LaunchTemplateId + # DescribeLaunchTemplatesRequest|LaunchTemplateId|content + # DescribeLaunchTemplatesRequest|LaunchTemplateId|tag + # DescribeLaunchTemplatesRequest|LaunchTemplateName + # DescribeLaunchTemplatesRequest|LaunchTemplateName|content + # DescribeLaunchTemplatesRequest|LaunchTemplateName|tag + # DescribeLaunchTemplatesRequest|MaxResults + # DescribeLaunchTemplateVersionsRequest + # DescribeLaunchTemplateVersionsRequest|LaunchTemplateId + # DescribeLaunchTemplateVersionsRequest|LaunchTemplateVersion + # DescribeLaunchTemplateVersionsRequest|LaunchTemplateVersion|array[0] + # DescribeLaunchTemplateVersionsRequest|LaunchTemplateVersion|array[0]|content + # DescribeLaunchTemplateVersionsRequest|LaunchTemplateVersion|array[0]|tag + # DescribeLaunchTemplateVersionsRequest|LaunchTemplateVersion|array[1] + # DescribeLaunchTemplateVersionsRequest|LaunchTemplateVersion|array[1]|content + # DescribeLaunchTemplateVersionsRequest|LaunchTemplateVersion|array[1]|tag + # DescribeLaunchTemplateVersionsRequest|LaunchTemplateVersion|content + # DescribeLaunchTemplateVersionsRequest|LaunchTemplateVersion|tag + # DescribeLaunchTemplateVersionsRequest|MaxResults + # DescribeNatGatewaysRequest + # DescribeNatGatewaysRequest|MaxResults + # DescribeNetworkInsightsAccessScopesRequest + # DescribeNetworkInsightsAccessScopesRequest|MaxResults + # DescribeReplaceRootVolumeTasksRequest + # DescribeReplaceRootVolumeTasksRequest|Filter + # DescribeReplaceRootVolumeTasksRequest|Filter|Name + # DescribeReplaceRootVolumeTasksRequest|Filter|tag + # DescribeReplaceRootVolumeTasksRequest|Filter|Value + # DescribeReplaceRootVolumeTasksRequest|Filter|Value|content + # DescribeReplaceRootVolumeTasksRequest|Filter|Value|tag + # DescribeReplaceRootVolumeTasksRequest|MaxResults + # DescribeSecurityGroupRulesRequest + # DescribeSecurityGroupRulesRequest|Filter + # DescribeSecurityGroupRulesRequest|Filter|Name + # DescribeSecurityGroupRulesRequest|Filter|tag + # DescribeSecurityGroupRulesRequest|Filter|Value + # DescribeSecurityGroupRulesRequest|Filter|Value|content + # DescribeSecurityGroupRulesRequest|Filter|Value|tag + # DescribeSecurityGroupRulesRequest|MaxResults + # DescribeSpotFleetRequestsRequest + # DescribeSpotFleetRequestsRequest|MaxResults + # DescribeSpotFleetRequestsRequest|NextToken + # DescribeTrafficMirrorFiltersRequest + # DescribeTrafficMirrorFiltersRequest|MaxResults + # DescribeTransitGatewayAttachmentsRequest + # DescribeTransitGatewayAttachmentsRequest|Filter + # DescribeTransitGatewayAttachmentsRequest|Filter|Name + # DescribeTransitGatewayAttachmentsRequest|Filter|tag + # DescribeTransitGatewayAttachmentsRequest|Filter|Value|array[0] + # DescribeTransitGatewayAttachmentsRequest|Filter|Value|array[0]|content + # DescribeTransitGatewayAttachmentsRequest|Filter|Value|array[0]|tag + # DescribeTransitGatewayAttachmentsRequest|Filter|Value|array[1] + # DescribeTransitGatewayAttachmentsRequest|Filter|Value|array[10] + # DescribeTransitGatewayAttachmentsRequest|Filter|Value|array[10]|content + # DescribeTransitGatewayAttachmentsRequest|Filter|Value|array[10]|tag + # DescribeTransitGatewayAttachmentsRequest|Filter|Value|array[1]|content + # DescribeTransitGatewayAttachmentsRequest|Filter|Value|array[1]|tag + # DescribeTransitGatewayAttachmentsRequest|Filter|Value|array[2] + # DescribeTransitGatewayAttachmentsRequest|Filter|Value|array[2]|content + # DescribeTransitGatewayAttachmentsRequest|Filter|Value|array[2]|tag + # DescribeTransitGatewayAttachmentsRequest|Filter|Value|array[3] + # DescribeTransitGatewayAttachmentsRequest|Filter|Value|array[3]|content + # DescribeTransitGatewayAttachmentsRequest|Filter|Value|array[3]|tag + # DescribeTransitGatewayAttachmentsRequest|Filter|Value|array[4] + # DescribeTransitGatewayAttachmentsRequest|Filter|Value|array[4]|content + # DescribeTransitGatewayAttachmentsRequest|Filter|Value|array[4]|tag + # DescribeTransitGatewayAttachmentsRequest|Filter|Value|array[5] + # DescribeTransitGatewayAttachmentsRequest|Filter|Value|array[5]|content + # DescribeTransitGatewayAttachmentsRequest|Filter|Value|array[5]|tag + # DescribeTransitGatewayAttachmentsRequest|Filter|Value|array[6] + # DescribeTransitGatewayAttachmentsRequest|Filter|Value|array[6]|content + # DescribeTransitGatewayAttachmentsRequest|Filter|Value|array[6]|tag + # DescribeTransitGatewayAttachmentsRequest|Filter|Value|array[7] + # DescribeTransitGatewayAttachmentsRequest|Filter|Value|array[7]|content + # DescribeTransitGatewayAttachmentsRequest|Filter|Value|array[7]|tag + # DescribeTransitGatewayAttachmentsRequest|Filter|Value|array[8] + # DescribeTransitGatewayAttachmentsRequest|Filter|Value|array[8]|content + # DescribeTransitGatewayAttachmentsRequest|Filter|Value|array[8]|tag + # DescribeTransitGatewayAttachmentsRequest|Filter|Value|array[9] + # DescribeTransitGatewayAttachmentsRequest|Filter|Value|array[9]|content + # DescribeTransitGatewayAttachmentsRequest|Filter|Value|array[9]|tag + # DescribeTransitGatewayAttachmentsRequest|MaxResults + # DescribeTransitGatewayConnectPeersRequest + # DescribeTransitGatewayConnectPeersRequest|MaxResults + # DescribeTransitGatewayPolicyTablesRequest + # DescribeTransitGatewayPolicyTablesRequest|MaxResults + # DescribeVpcEndpointsRequest + # DescribeVpcEndpointsRequest|MaxResults + # details + # detectorName + # dimensions|array[0] + # dimensions|array[0]|name + # dimensions|array[0]|value + # dimensions|array[1] + # dimensions|array[1]|name + # dimensions|array[1]|value + # dimensions|array[2] + # dimensions|array[2]|name + # dimensions|array[2]|value + # dimensions|array[3] + # dimensions|array[3]|name + # dimensions|array[3]|value + # disableApiStop + # disableApiTermination + # documentFormat + # documentName + # documentVersion + # dryRun + durationSeconds :Optional[str]=None + # encryption + # encryptionAlgorithm + # encryptionContext + # encryptionContext|aws:ebs:id + # encryptionContext|aws:logs:arn + # encryptionContext|aws:s3:arn + # encryptionContext|aws:ssm:SessionId + # encryptionContext|aws:ssm:TargetId + # encryptionContext|PARAMETER_ARN + # endTime + # evaluationFrequency + # evaluationPeriods + # eventBusName + # excludeAutoscalingAlarms + # excludeManagedAlarms + # executionResult + # executionResult|errorCode + # executionResult|executionDate + # executionResult|executionSummary + # executionResult|status + # executionSummary + # executionSummary|executionId + # executionSummary|executionTime + # executionSummary|executionType + # expression + # filterExpiredLogStreams + # filterPattern + # filters|array[0] + # Filters|array[0] + # filters|array[0]|key + # Filters|array[0]|Key + # filters|array[0]|name + # filters|array[0]|value + # filters|array[0]|values|array[0] + # Filters|array[0]|Values|array[0] + # filters|array[1] + # filters|array[1]|key + # filters|array[1]|value + # filterSet + # filterSet|items|array[0] + # filterSet|items|array[0]|name + # filterSet|items|array[0]|valueSet + # filterSet|items|array[1] + # filterSet|items|array[1]|name + # filterSet|items|array[1]|valueSet + # filterSet|items|array[1]|valueSet|items|array[0] + # filterSet|items|array[1]|valueSet|items|array[0]|value + # filtersWithOperator|array[0] + # filtersWithOperator|array[0]|key + # filtersWithOperator|array[0]|operator + # filtersWithOperator|array[0]|values|array[0] + # fingerprint + # force + # GetAllowedImagesSettingsRequest + # GetInstanceTypesFromInstanceRequirementsRequest + # GetInstanceTypesFromInstanceRequirementsRequest|ArchitectureType + # GetInstanceTypesFromInstanceRequirementsRequest|ArchitectureType|content + # GetInstanceTypesFromInstanceRequirementsRequest|ArchitectureType|tag + # GetInstanceTypesFromInstanceRequirementsRequest|InstanceRequirements + # GetInstanceTypesFromInstanceRequirementsRequest|InstanceRequirements|MemoryMiB + # GetInstanceTypesFromInstanceRequirementsRequest|InstanceRequirements|MemoryMiB|Min + # GetInstanceTypesFromInstanceRequirementsRequest|InstanceRequirements|VCpuCount + # GetInstanceTypesFromInstanceRequirementsRequest|InstanceRequirements|VCpuCount|Min + # GetInstanceTypesFromInstanceRequirementsRequest|VirtualizationType + # GetInstanceTypesFromInstanceRequirementsRequest|VirtualizationType|content + # GetInstanceTypesFromInstanceRequirementsRequest|VirtualizationType|tag + # GetSecurityGroupsForVpcRequest + # GetSecurityGroupsForVpcRequest|Filter + # GetSecurityGroupsForVpcRequest|Filter|Name + # GetSecurityGroupsForVpcRequest|Filter|tag + # GetSecurityGroupsForVpcRequest|Filter|Value + # GetSecurityGroupsForVpcRequest|Filter|Value|content + # GetSecurityGroupsForVpcRequest|Filter|Value|tag + # GetSecurityGroupsForVpcRequest|MaxResults + # GetSecurityGroupsForVpcRequest|VpcId + # granteePrincipal + # Host + # imagesSet + # imagesSet|items|array[0] + # imagesSet|items|array[0]|imageId + # includeAllInstances + # include|array[0] + # includePublic + # includePublicKey + # includes + # includeShadowTrails + # includeShared + # includes|hasDnsFqdn + # includes|keyTypes|array[0] + # includes|keyTypes|array[1] + # includes|keyTypes|array[2] + # includes|keyTypes|array[3] + # includes|keyTypes|array[4] + # includes|keyTypes|array[5] + # includes|keyTypes|array[6] + # instanceArns|array[0] + # instanceId + # instanceIds|array[0] + # instancesSet + # instancesSet|items|array[0] + # instancesSet|items|array[0]|instanceId + # instancesSet|items|array[0]|maxCount + # instancesSet|items|array[0]|minCount + # instancesSet|items|array[1] + # instancesSet|items|array[1]|instanceId + # instancesSet|items|array[2] + # instancesSet|items|array[2]|instanceId + # instancesSet|items|array[3] + # instancesSet|items|array[3]|instanceId + # instanceType + # intelligent-tiering + # interactive + # internetGatewayIdSet + # internetGatewayIdSet|items|array[0] + # internetGatewayIdSet|items|array[0]|internetGatewayId + # iPAddress + # itemContentHash + # items|array[0] + # items|array[0]|captureTime + # items|array[0]|contentHash + # items|array[0]|details + # items|array[0]|details|DocumentName + # items|array[0]|details|DocumentVersion + # items|array[0]|id + # items|array[0]|schemaVersion + # items|array[0]|severity + # items|array[0]|status + # items|array[0]|title + # items|array[0]|typeName + # items|array[1] + # items|array[1]|captureTime + # items|array[1]|contentHash + # items|array[1]|schemaVersion + # items|array[1]|typeName + # items|array[2] + # items|array[2]|captureTime + # items|array[2]|contentHash + # items|array[2]|schemaVersion + # items|array[2]|typeName + # items|array[3] + # items|array[3]|captureTime + # items|array[3]|contentHash + # items|array[3]|schemaVersion + # items|array[3]|typeName + # items|array[4] + # items|array[4]|captureTime + # items|array[4]|contentHash + # items|array[4]|schemaVersion + # items|array[4]|typeName + # keyId + # keySet + # keySet|items|array[0] + # keySet|items|array[0]|keyName + # keySpec + # kmsKeyId + # launchTemplate + # launchTemplate|launchTemplateId + # launchTemplate|version + # lifecycle + # limit + # listenerArn + # listenerArns|array[0] + # loadBalancerArn + # loadBalancerArns|array[0] + # location + # logging + # logGroupArnList|array[0] + # logGroupClass + # logGroupIdentifier + # logGroupIdentifiers|array[0] + # logGroupIdentifiers|array[1] + # logGroupIdentifiers|array[2] + # logGroupIdentifiers|array[3] + # logGroupIdentifiers|array[4] + # logGroupIdentifiers|array[5] + # logGroupIdentifiers|array[6] + # logGroupName + # logGroupNamePrefix + # logGroupNames|array[0] + # logStreamName + # logStreamNamePrefix + # lookupAttributes|array[0] + # lookupAttributes|array[0]|attributeKey + # lookupAttributes|array[0]|attributeValue + # marker + # maxConcurrency + # maxErrors + # MaxItems + # maxRecords + # maxResults + # MaxResults + # messageSchemaVersion + # metricName + # monitoring + # monitoring|enabled + # name + # namespace + # networkAclIdSet + # networkAclIdSet|items|array[0] + # networkAclIdSet|items|array[0]|networkAclId + # networkInterfaceIdSet + # networkInterfaceIdSet|items|array[0] + # networkInterfaceIdSet|items|array[0]|networkInterfaceId + # networkInterfaceSet + # networkInterfaceSet|items|array[0] + # networkInterfaceSet|items|array[0]|deviceIndex + # networkInterfaceSet|items|array[0]|networkCardIndex + # networkInterfaceSet|items|array[0]|subnetId + # newInstancesProtectedFromScaleIn + # nextToken + # notification + # numberOfBytes + # object-lock + # operations|array[0] + # orderBy + # outputS3BucketName + # overwrite + # ownershipControls + # ownersSet + # ownersSet|items|array[0] + # ownersSet|items|array[0]|owner + # pageSize + # paginationToken + # parameterFilters|array[0] + # parameterFilters|array[0]|key + # parameterFilters|array[0]|option + # parameterFilters|array[0]|values|array[0] + # parameters + # period + # permissionType + # platformName + # platformType + # platformVersion + # pluginName + # policy + # policyName + # policyNames|array[0] + # policyType + # policyTypes|array[0] + # policyTypes|array[1] + # policyTypes|array[2] + # preferences + # preferences|autoRollback + # preferences|maxHealthyPercentage + # preferences|minHealthyPercentage + # preferences|scaleInProtectedInstances + # preferences|skipMatching + # preferences|standbyInstances + # publicAccessBlock + # publicKey + # publicKeyType + # queryId + # queryLanguage + # queryString + + + # queryString|fields @timestamp, @message, @logStream, @log + # replication + # requestId + # requestPayment + # requireAcknowledgement + # resourceArn + # resourceARN + # resourceArns|array[0] + # resourceId + # resourcesPerPage + # resourcesSet + # resourcesSet|items|array[0] + # resourcesSet|items|array[0]|resourceId + # resourceType + # resourceTypeFilters|array[0] + # resultAttributes|array[0] + # resultAttributes|array[0]|typeName + # retentionInDays + # retiringPrincipal + roleArn:Optional[str]=None + roleSessionName:Optional[str]=None + # routeTableIdSet + # routeTableIdSet|items|array[0] + # routeTableIdSet|items|array[0]|routeTableId + # rule + # ruleArns|array[0] + # securityGroupIdSet + # securityGroupIdSet|items|array[0] + # securityGroupIdSet|items|array[0]|groupId + # ServerSideEncryptionConfiguration + # ServerSideEncryptionConfiguration|Rule + # ServerSideEncryptionConfiguration|Rule|ApplyServerSideEncryptionByDefault + # ServerSideEncryptionConfiguration|Rule|ApplyServerSideEncryptionByDefault|KMSMasterKeyID + # ServerSideEncryptionConfiguration|Rule|ApplyServerSideEncryptionByDefault|SSEAlgorithm + # ServerSideEncryptionConfiguration|Rule|BucketKeyEnabled + # ServerSideEncryptionConfiguration|xmlns + # serviceNamespace + # sessionId + # showSubscriptionDestinations + # snapshotType + # sSMConnectionChannel + # stackStatusFilter|array[0] + # stackStatusFilter|array[1] + # stackStatusFilter|array[10] + # stackStatusFilter|array[11] + # stackStatusFilter|array[12] + # stackStatusFilter|array[13] + # stackStatusFilter|array[14] + # stackStatusFilter|array[15] + # stackStatusFilter|array[16] + # stackStatusFilter|array[17] + # stackStatusFilter|array[18] + # stackStatusFilter|array[19] + # stackStatusFilter|array[2] + # stackStatusFilter|array[3] + # stackStatusFilter|array[4] + # stackStatusFilter|array[5] + # stackStatusFilter|array[6] + # stackStatusFilter|array[7] + # stackStatusFilter|array[8] + # stackStatusFilter|array[9] + # startFromHead + # startTime + # state + # stateValue + # statistic + # status + # subnetSet + # subnetSet|items|array[0] + # subnetSet|items|array[0]|subnetId + # tagFilters|array[0] + # tagFilters|array[0]|key + # tagging + # Tagging + # Tagging|TagSet + # Tagging|TagSet|Tag + # Tagging|TagSet|Tag|Key + # Tagging|TagSet|Tag|Value + # Tagging|xmlns + # tags + # tags|array[0] + # tags|array[0]|key + # tags|array[0]|tagKey + # tags|array[0]|tagValue + # tags|array[0]|value + # tagSet + # tagSet|items|array[0] + # tagSet|items|array[0]|key + # tagSet|items|array[0]|value + # tagSpecificationSet + # tagSpecificationSet|items|array[0] + # tagSpecificationSet|items|array[0]|resourceType + # tagSpecificationSet|items|array[0]|tags|array[0] + # tagSpecificationSet|items|array[0]|tags|array[0]|key + # tagSpecificationSet|items|array[0]|tags|array[0]|value + # tagSpecificationSet|items|array[0]|tags|array[1] + # tagSpecificationSet|items|array[0]|tags|array[1]|key + # tagSpecificationSet|items|array[0]|tags|array[1]|value + # tags|project + # target + # targetGroupArn + # targetGroupArns|array[0] + # targets|array[0] + # targets|array[0]|id + # targets|array[0]|key + # targets|array[0]|values|array[0] + # template + # threshold + # tier + # timeoutSeconds + # topicArn + # trafficSourceType + # type + # Type + # unmask + # value + # versioning + # visibility + # volumeSet + # volumeSet|items|array[0] + # volumeSet|items|array[0]|volumeId + # vpcId + # vpcSet + # vpcSet|items|array[0] + # vpcSet|items|array[0]|vpcId + # website + # windowId + # windowTaskId + # withDecryption + +class Credentials(BaseModel): + model_config = ConfigDict(extra="forbid") + +class ResponseElements(BaseModel): + model_config = ConfigDict(extra="forbid") + nextToken : Optional[str] = None + credentials : Optional[Credentials]=None +class TLSDetails(BaseModel): + model_config = ConfigDict(extra="forbid") + tlsVersion: str + cipherSuite: str + clientProvidedHostHeader: str +class Resource(BaseModel): + model_config = ConfigDict(extra="forbid") + account_id: str #, Field(None, alias="account_id")), + type: str + arn: str #, Field(None, alias="ARN")), + + +class EventDetail(BaseModel): + model_config = ConfigDict(extra="forbid") + event_version: Optional[str] = Field(None, alias="eventVersion") + user_identity: UserIdentity = Field(None, alias="userIdentity") + event_time: datetime = Field(None, alias="eventTime") + event_source: str = Field(None, alias="eventSource") + event_name: str = Field(None, alias="eventName") + aws_region: str = Field(None, alias="awsRegion") + source_ip_address: str = Field(None, alias="sourceIPAddress") + user_agent: str = Field(None, alias="userAgent") + request_parameters: Optional[RequestParameters] = Field( + None, alias="requestParameters" + ) + response_elements: Optional[ ResponseElements] = Field(None, alias="responseElements") + request_id: str = Field(None, alias="requestID") + event_id: str = Field(None, alias="eventID") + read_only: bool = Field(None, alias="readOnly") + tlsDetails: Optional[TLSDetails] + resources: Optional[List[Resource]] = None + event_type: str = Field(None, alias="eventType") + api_version: str = Field(None, alias="apiVersion") + management_event: bool = Field(None, alias="managementEvent") + recipient_account_id: str = Field(None, alias="recipientAccountId") + event_category: str = Field(None, alias="eventCategory") + session_credential_from_console: bool = Field( + None, alias="sessionCredentialFromConsole" + ) + + errorCode: Optional[str] = None + errorMessage: Optional[str] = None + def process1(v,path): + skip =0 + #print(path,len(path)) + lpth = len(path) + + # eventType + # eventName + # eventVersion + # eventSource + # eventTime + + # apiVersion + # awsRegion + # errorCode + # errorMessage + # eventCategory + # eventID + # managementEvent + # readOnly + # recipientAccountId + # requestID + # sessionCredentialFromConsole + # sharedEventID + # sourceIPAddress + # tlsDetails + # tlsDetails|cipherSuite + # tlsDetails|clientProvidedHostHeader + # tlsDetails|tlsVersion + + # userAgent + + + # additionalEventData + # additionalEventData|AuthenticationMethod + # additionalEventData|bytesTransferredIn + # additionalEventData|bytesTransferredOut + # additionalEventData|CipherSuite + # additionalEventData|grantId + # additionalEventData|identityProviderConnectionVerificationMethod + # additionalEventData|LoginTo + # additionalEventData|MFAUsed + # additionalEventData|MobileVersion + # additionalEventData|service + # additionalEventData|SignatureVersion + # additionalEventData|x-amz-id-2 + + # resources|array[0] + # resources|array[0]|accountId + # resources|array[0]|ARN + # resources|array[0]|type + # resources|array[1] + # resources|array[10] + # resources|array[10]|accountId + # resources|array[10]|ARN + # resources|array[10]|type + # resources|array[11] + # resources|array[11]|accountId + # resources|array[11]|ARN + # resources|array[11]|type + # resources|array[1]|accountId + # resources|array[1]|ARN + # resources|array[1]|type + # resources|array[2] + # resources|array[2]|accountId + # resources|array[2]|ARN + # resources|array[2]|type + # resources|array[3] + # resources|array[3]|accountId + # resources|array[3]|ARN + # resources|array[3]|type + # resources|array[4] + # resources|array[4]|accountId + # resources|array[4]|ARN + # resources|array[4]|type + # resources|array[5] + # resources|array[5]|accountId + # resources|array[5]|ARN + # resources|array[5]|type + # resources|array[6] + # resources|array[6]|accountId + # resources|array[6]|ARN + # resources|array[6]|type + # resources|array[7] + # resources|array[7]|accountId + # resources|array[7]|ARN + # resources|array[7]|type + # resources|array[8] + # resources|array[8]|accountId + # resources|array[8]|ARN + # resources|array[8]|type + # resources|array[9] + # resources|array[9]|accountId + # resources|array[9]|ARN + # resources|array[9]|type + + + # responseElements + # responseElements|anomalyDetectorArn + # responseElements|assumedRoleUser + # responseElements|assumedRoleUser|arn + # responseElements|assumedRoleUser|assumedRoleId + # responseElements|audience + # responseElements|command + # responseElements|command|alarmConfiguration + # responseElements|command|alarmConfiguration|ignorePollAlarmFailure + # responseElements|command|clientName + # responseElements|command|clientSourceId + # responseElements|command|cloudWatchOutputConfig + # responseElements|command|cloudWatchOutputConfig|cloudWatchLogGroupName + # responseElements|command|cloudWatchOutputConfig|cloudWatchOutputEnabled + # responseElements|command|commandId + # responseElementso|command|comment + # responseElements|command|completedCount + # responseElements|command|deliveryTimedOutCount + # responseElements|command|documentName + # responseElements|command|documentVersion + # responseElements|command|errorCount + # responseElements|command|expiresAfter + # responseElements|command|hasCancelCommandSignature + # responseElements|command|hasSendCommandSignature + # responseElements|command|instanceIds|array[0] + # responseElements|command|interactive + # responseElements|command|maxConcurrency + # responseElements|command|maxErrors + # responseElements|command|notificationConfig + # responseElements|command|notificationConfig|notificationArn + # responseElements|command|notificationConfig|notificationType + # responseElements|command|outputS3BucketName + # responseElements|command|outputS3KeyPrefix + # responseElements|command|outputS3Region + # responseElements|command|parameters + # responseElements|command|requestedDateTime + # responseElements|command|serviceRole + # responseElements|command|status + # responseElements|command|statusDetails + # responseElements|command|targetCount + # responseElements|command|targets|array[0] + # responseElements|command|targets|array[0]|key + # responseElements|command|targets|array[0]|values|array[0] + # responseElements|command|timeoutSeconds + # responseElements|ConsoleLogin + # responseElements|CreateLaunchTemplateResponse + # responseElements|CreateLaunchTemplateResponse|launchTemplate + # responseElements|CreateLaunchTemplateResponse|launchTemplate|createdBy + # responseElements|CreateLaunchTemplateResponse|launchTemplate|createTime + # responseElements|CreateLaunchTemplateResponse|launchTemplate|defaultVersionNumber + # responseElements|CreateLaunchTemplateResponse|launchTemplate|latestVersionNumber + # responseElements|CreateLaunchTemplateResponse|launchTemplate|launchTemplateId + # responseElements|CreateLaunchTemplateResponse|launchTemplate|launchTemplateName + # responseElements|CreateLaunchTemplateResponse|launchTemplate|operator + # responseElements|CreateLaunchTemplateResponse|launchTemplate|operator|managed + # responseElements|CreateLaunchTemplateResponse|launchTemplate|tagSet + # responseElements|CreateLaunchTemplateResponse|launchTemplate|tagSet|item + # responseElements|CreateLaunchTemplateResponse|launchTemplate|tagSet|item|key + # responseElements|CreateLaunchTemplateResponse|launchTemplate|tagSet|item|value + # responseElements|CreateLaunchTemplateResponse|requestId + # responseElements|CreateLaunchTemplateResponse|xmlns + # responseElements|CreateLaunchTemplateVersionResponse + # responseElements|CreateLaunchTemplateVersionResponse|launchTemplateVersion + # responseElements|CreateLaunchTemplateVersionResponse|launchTemplateVersion|createdBy + # responseElements|CreateLaunchTemplateVersionResponse|launchTemplateVersion|createTime + # responseElements|CreateLaunchTemplateVersionResponse|launchTemplateVersion|defaultVersion + # responseElements|CreateLaunchTemplateVersionResponse|launchTemplateVersion|launchTemplateData + # responseElements|CreateLaunchTemplateVersionResponse|launchTemplateVersion|launchTemplateData|blockDeviceMappingSet + # responseElements|CreateLaunchTemplateVersionResponse|launchTemplateVersion|launchTemplateData|blockDeviceMappingSet|item + # responseElements|CreateLaunchTemplateVersionResponse|launchTemplateVersion|launchTemplateData|blockDeviceMappingSet|item|deviceName + # responseElements|CreateLaunchTemplateVersionResponse|launchTemplateVersion|launchTemplateData|blockDeviceMappingSet|item|ebs + # responseElements|CreateLaunchTemplateVersionResponse|launchTemplateVersion|launchTemplateData|blockDeviceMappingSet|item|ebs|encrypted + # responseElements|CreateLaunchTemplateVersionResponse|launchTemplateVersion|launchTemplateData|blockDeviceMappingSet|item|ebs|volumeSize + # responseElements|CreateLaunchTemplateVersionResponse|launchTemplateVersion|launchTemplateData|blockDeviceMappingSet|item|ebs|volumeType + # responseElements|CreateLaunchTemplateVersionResponse|launchTemplateVersion|launchTemplateData|iamInstanceProfile + # responseElements|CreateLaunchTemplateVersionResponse|launchTemplateVersion|launchTemplateData|iamInstanceProfile|name + # responseElements|CreateLaunchTemplateVersionResponse|launchTemplateVersion|launchTemplateData|imageId + # responseElements|CreateLaunchTemplateVersionResponse|launchTemplateVersion|launchTemplateData|instanceType + # responseElements|CreateLaunchTemplateVersionResponse|launchTemplateVersion|launchTemplateData|keyName + # responseElements|CreateLaunchTemplateVersionResponse|launchTemplateVersion|launchTemplateData|networkInterfaceSet + # responseElements|CreateLaunchTemplateVersionResponse|launchTemplateVersion|launchTemplateData|networkInterfaceSet|item + # responseElements|CreateLaunchTemplateVersionResponse|launchTemplateVersion|launchTemplateData|networkInterfaceSet|item|associatePublicIpAddress + # responseElements|CreateLaunchTemplateVersionResponse|launchTemplateVersion|launchTemplateData|networkInterfaceSet|item|deleteOnTermination + # responseElements|CreateLaunchTemplateVersionResponse|launchTemplateVersion|launchTemplateData|networkInterfaceSet|item|deviceIndex + # responseElements|CreateLaunchTemplateVersionResponse|launchTemplateVersion|launchTemplateData|networkInterfaceSet|item|groupSet + # responseElements|CreateLaunchTemplateVersionResponse|launchTemplateVersion|launchTemplateData|networkInterfaceSet|item|groupSet|groupId + # responseElements|CreateLaunchTemplateVersionResponse|launchTemplateVersion|launchTemplateData|networkInterfaceSet|item|networkCardIndex + # responseElements|CreateLaunchTemplateVersionResponse|launchTemplateVersion|launchTemplateData|userData + # responseElements|CreateLaunchTemplateVersionResponse|launchTemplateVersion|launchTemplateId + # responseElements|CreateLaunchTemplateVersionResponse|launchTemplateVersion|launchTemplateName + # responseElements|CreateLaunchTemplateVersionResponse|launchTemplateVersion|operator + # responseElements|CreateLaunchTemplateVersionResponse|launchTemplateVersion|operator|managed + # responseElements|CreateLaunchTemplateVersionResponse|launchTemplateVersion|versionNumber + # responseElements|CreateLaunchTemplateVersionResponse|requestId + # responseElements|CreateLaunchTemplateVersionResponse|xmlns + # responseElements|credentials + # responseElements|credentials|accessKeyId + # responseElements|credentials|expiration + # responseElements|credentials|sessionToken + # responseElements|DeleteLaunchTemplateResponse + # responseElements|DeleteLaunchTemplateResponse|launchTemplate + # responseElements|DeleteLaunchTemplateResponse|launchTemplate|createdBy + # responseElements|DeleteLaunchTemplateResponse|launchTemplate|createTime + # responseElements|DeleteLaunchTemplateResponse|launchTemplate|defaultVersionNumber + # responseElements|DeleteLaunchTemplateResponse|launchTemplate|latestVersionNumber + # responseElements|DeleteLaunchTemplateResponse|launchTemplate|launchTemplateId + # responseElements|DeleteLaunchTemplateResponse|launchTemplate|launchTemplateName + # responseElements|DeleteLaunchTemplateResponse|launchTemplate|operator + # responseElements|DeleteLaunchTemplateResponse|launchTemplate|operator|managed + # responseElements|DeleteLaunchTemplateResponse|requestId + # responseElements|DeleteLaunchTemplateResponse|xmlns + # responseElements|description + # responseElements|description|defaultVersion + # responseElements|description|name + # responseElements|documentDescription + # responseElements|documentDescription|createdDate + # responseElements|documentDescription|defaultVersion + # responseElements|documentDescription|documentFormat + # responseElements|documentDescription|documentId + # responseElements|documentDescription|documentType + # responseElements|documentDescription|documentVersion + # responseElements|documentDescription|hash + # responseElements|documentDescription|hashType + # responseElements|documentDescription|latestVersion + # responseElements|documentDescription|name + # responseElements|documentDescription|owner + # responseElements|documentDescription|platformTypes|array[0] + # responseElements|documentDescription|platformTypes|array[1] + # responseElements|documentDescription|platformTypes|array[2] + # responseElements|documentDescription|schemaVersion + # responseElements|documentDescription|status + # responseElements|documentDescription|tags|array[0] + # responseElements|documentDescription|tags|array[0]|key + # responseElements|documentDescription|tags|array[0]|value + # responseElements|grantId + # responseElements|instanceId + # responseElements|instanceRefreshId + # responseElements|instancesSet + # responseElements|instancesSet|items|array[0] + # responseElements|instancesSet|items|array[0]|amiLaunchIndex + # responseElements|instancesSet|items|array[0]|architecture + # responseElements|instancesSet|items|array[0]|bootMode + # responseElements|instancesSet|items|array[0]|capacityReservationSpecification + # responseElements|instancesSet|items|array[0]|capacityReservationSpecification|capacityReservationPreference + # responseElements|instancesSet|items|array[0]|clientToken + # responseElements|instancesSet|items|array[0]|cpuOptions + # responseElements|instancesSet|items|array[0]|cpuOptions|coreCount + # responseElements|instancesSet|items|array[0]|cpuOptions|threadsPerCore + # responseElements|instancesSet|items|array[0]|currentInstanceBootMode + # responseElements|instancesSet|items|array[0]|currentState + # responseElements|instancesSet|items|array[0]|currentState|code + # responseElements|instancesSet|items|array[0]|currentState|name + # responseElements|instancesSet|items|array[0]|ebsOptimized + # responseElements|instancesSet|items|array[0]|enaSupport + # responseElements|instancesSet|items|array[0]|enclaveOptions + # responseElements|instancesSet|items|array[0]|enclaveOptions|enabled + # responseElements|instancesSet|items|array[0]|groupSet + # responseElements|instancesSet|items|array[0]|groupSet|items|array[0] + # responseElements|instancesSet|items|array[0]|groupSet|items|array[0]|groupId + # responseElements|instancesSet|items|array[0]|groupSet|items|array[0]|groupName + # responseElements|instancesSet|items|array[0]|hypervisor + # responseElements|instancesSet|items|array[0]|iamInstanceProfile + # responseElements|instancesSet|items|array[0]|iamInstanceProfile|arn + # responseElements|instancesSet|items|array[0]|iamInstanceProfile|id + # responseElements|instancesSet|items|array[0]|imageId + # responseElements|instancesSet|items|array[0]|instanceId + # responseElements|instancesSet|items|array[0]|instanceState + # responseElements|instancesSet|items|array[0]|instanceState|code + # responseElements|instancesSet|items|array[0]|instanceState|name + # responseElements|instancesSet|items|array[0]|instanceType + # responseElements|instancesSet|items|array[0]|keyName + # responseElements|instancesSet|items|array[0]|launchTime + # responseElements|instancesSet|items|array[0]|maintenanceOptions + # responseElements|instancesSet|items|array[0]|maintenanceOptions|autoRecovery + # responseElements|instancesSet|items|array[0]|metadataOptions + # responseElements|instancesSet|items|array[0]|metadataOptions|httpEndpoint + # responseElements|instancesSet|items|array[0]|metadataOptions|httpProtocolIpv4 + # responseElements|instancesSet|items|array[0]|metadataOptions|httpProtocolIpv6 + # responseElements|instancesSet|items|array[0]|metadataOptions|httpPutResponseHopLimit + # responseElements|instancesSet|items|array[0]|metadataOptions|httpTokens + # responseElements|instancesSet|items|array[0]|metadataOptions|instanceMetadataTags + # responseElements|instancesSet|items|array[0]|metadataOptions|state + # responseElements|instancesSet|items|array[0]|monitoring + # responseElements|instancesSet|items|array[0]|monitoring|state + # responseElements|instancesSet|items|array[0]|networkInterfaceSet + # responseElements|instancesSet|items|array[0]|networkInterfaceSet|items|array[0] + # responseElements|instancesSet|items|array[0]|networkInterfaceSet|items|array[0]|attachment + # responseElements|instancesSet|items|array[0]|networkInterfaceSet|items|array[0]|attachment|attachmentId + # responseElements|instancesSet|items|array[0]|networkInterfaceSet|items|array[0]|attachment|attachTime + # responseElements|instancesSet|items|array[0]|networkInterfaceSet|items|array[0]|attachment|deleteOnTermination + # responseElements|instancesSet|items|array[0]|networkInterfaceSet|items|array[0]|attachment|deviceIndex + # responseElements|instancesSet|items|array[0]|networkInterfaceSet|items|array[0]|attachment|networkCardIndex + # responseElements|instancesSet|items|array[0]|networkInterfaceSet|items|array[0]|attachment|status + # responseElements|instancesSet|items|array[0]|networkInterfaceSet|items|array[0]|groupSet + # responseElements|instancesSet|items|array[0]|networkInterfaceSet|items|array[0]|groupSet|items|array[0] + # responseElements|instancesSet|items|array[0]|networkInterfaceSet|items|array[0]|groupSet|items|array[0]|groupId + # responseElements|instancesSet|items|array[0]|networkInterfaceSet|items|array[0]|groupSet|items|array[0]|groupName + # responseElements|instancesSet|items|array[0]|networkInterfaceSet|items|array[0]|interfaceType + # responseElements|instancesSet|items|array[0]|networkInterfaceSet|items|array[0]|macAddress + # responseElements|instancesSet|items|array[0]|networkInterfaceSet|items|array[0]|networkInterfaceId + # responseElements|instancesSet|items|array[0]|networkInterfaceSet|items|array[0]|operator + # responseElements|instancesSet|items|array[0]|networkInterfaceSet|items|array[0]|operator|managed + # responseElements|instancesSet|items|array[0]|networkInterfaceSet|items|array[0]|ownerId + # responseElements|instancesSet|items|array[0]|networkInterfaceSet|items|array[0]|privateDnsName + # responseElements|instancesSet|items|array[0]|networkInterfaceSet|items|array[0]|privateIpAddress + # responseElements|instancesSet|items|array[0]|networkInterfaceSet|items|array[0]|privateIpAddressesSet + # responseElements|instancesSet|items|array[0]|networkInterfaceSet|items|array[0]|privateIpAddressesSet|item|array[0] + # responseElements|instancesSet|items|array[0]|networkInterfaceSet|items|array[0]|privateIpAddressesSet|item|array[0]|primary + # responseElements|instancesSet|items|array[0]|networkInterfaceSet|items|array[0]|privateIpAddressesSet|item|array[0]|privateDnsName + # responseElements|instancesSet|items|array[0]|networkInterfaceSet|items|array[0]|privateIpAddressesSet|item|array[0]|privateIpAddress + # responseElements|instancesSet|items|array[0]|networkInterfaceSet|items|array[0]|sourceDestCheck + # responseElements|instancesSet|items|array[0]|networkInterfaceSet|items|array[0]|status + # responseElements|instancesSet|items|array[0]|networkInterfaceSet|items|array[0]|subnetId + # responseElements|instancesSet|items|array[0]|networkInterfaceSet|items|array[0]|vpcId + # responseElements|instancesSet|items|array[0]|operator + # responseElements|instancesSet|items|array[0]|operator|managed + # responseElements|instancesSet|items|array[0]|placement + # responseElements|instancesSet|items|array[0]|placement|availabilityZone + # responseElements|instancesSet|items|array[0]|placement|tenancy + # responseElements|instancesSet|items|array[0]|previousState + # responseElements|instancesSet|items|array[0]|previousState|code + # responseElements|instancesSet|items|array[0]|previousState|name + # responseElements|instancesSet|items|array[0]|privateDnsName + # responseElements|instancesSet|items|array[0]|privateDnsNameOptions + # responseElements|instancesSet|items|array[0]|privateDnsNameOptions|enableResourceNameDnsAAAARecord + # responseElements|instancesSet|items|array[0]|privateDnsNameOptions|enableResourceNameDnsARecord + # responseElements|instancesSet|items|array[0]|privateDnsNameOptions|hostnameType + # responseElements|instancesSet|items|array[0]|privateIpAddress + # responseElements|instancesSet|items|array[0]|rootDeviceName + # responseElements|instancesSet|items|array[0]|rootDeviceType + # responseElements|instancesSet|items|array[0]|sourceDestCheck + # responseElements|instancesSet|items|array[0]|stateReason + # responseElements|instancesSet|items|array[0]|stateReason|code + # responseElements|instancesSet|items|array[0]|stateReason|message + # responseElements|instancesSet|items|array[0]|subnetId + # responseElements|instancesSet|items|array[0]|tagSet + # responseElements|instancesSet|items|array[0]|tagSet|items|array[0] + # responseElements|instancesSet|items|array[0]|tagSet|items|array[0]|key + # responseElements|instancesSet|items|array[0]|tagSet|items|array[0]|value + # responseElements|instancesSet|items|array[0]|tagSet|items|array[1] + # responseElements|instancesSet|items|array[0]|tagSet|items|array[1]|key + # responseElements|instancesSet|items|array[0]|tagSet|items|array[1]|value + # responseElements|instancesSet|items|array[0]|tagSet|items|array[2] + # responseElements|instancesSet|items|array[0]|tagSet|items|array[2]|key + # responseElements|instancesSet|items|array[0]|tagSet|items|array[2]|value + # responseElements|instancesSet|items|array[0]|tagSet|items|array[3] + # responseElements|instancesSet|items|array[0]|tagSet|items|array[3]|key + # responseElements|instancesSet|items|array[0]|tagSet|items|array[3]|value + # responseElements|instancesSet|items|array[0]|virtualizationType + # responseElements|instancesSet|items|array[0]|vpcId + # responseElements|keyId + # responseElements|messageSchemaVersion + # responseElements|ownerId + # responseElements|packedPolicySize + # responseElements|provider + # responseElements|queryId + # responseElements|requesterId + # responseElements|requestId + # responseElements|reservationId + # responseElements|_return + # responseElements|sessionId + # responseElements|streamUrl + # responseElements|subjectFromWebIdentityToken + # responseElements|tier + # responseElements|tokenValue + # responseElements|version + + # serviceEventDetails + # serviceEventDetails|DocumentName + # serviceEventDetails|IdleSessionTimeout + # serviceEventDetails|IsCloudWatchEncryptionEnabled + # serviceEventDetails|IsKmsEncryptionEnabled + # serviceEventDetails|IsS3EncryptionEnabled + # serviceEventDetails|MaxSessionDuration + + + # userIdentity + + # vpcEndpointAccountId + # vpcEndpointId + + if (lpth == 6) and (path[0] in ["CloudTrailEvent"]) and (path[1] in ["userIdentity"]) and (path[2] in ["sessionContext"]) and (path[3] in ["sessionIssuer"]) and (path[4] in ["userName"]) : + if path[5] in ["aws:ec2-infrastructure", + "aws:ec2-instance", + "AWSServiceRoleForAmazonSSM" + "AWSServiceRoleForAutoScaling", + "AWSServiceRoleForCloudWatchApplicationSignals", + "AWSServiceRoleForConfig", + "AWSServiceRoleForResourceExplorer", + "github", + "ssm-swarms-role"]: + interesting =1 + elif (path[4] in ["accountId","arn","principalId","type","userName"]): + pass + +#if (lpth == 6) and (path[0] in ["CloudTrailEvent"]) and (path[1] in ["userIdentity"]) and (path[2] in ["sessionContext"]) and (path[3] in ["webIdFederationData"]) and (path[4] in ["federatedProvider"]) and (path[5] in ["arn:aws:iam::916723593639:oidc-provider/token.actions.githubusercontent.com"]): + + # if (lpth>2): + # if path[1] in ["userIdentity"]: + # if path[2] in ["accessKeyId"]: + # skip =1 + # if path[1] in [ "requestParameters"]: + + # if path[2] in [ "startTime", "nextToken"]: + # skip =1 + + #elif (lpth>1): + # if path[1] in ["eventID","requestID","eventTime"]: + # skip = 1 + if skip: + return if isinstance(v,list): for i,j in enumerate(v): path2= path.copy() path2.extend(["array" + "["+str(i)+"]"]) #v2 = v[i] process1(j,path2) - + elif isinstance(v,dict): # we are in the next level of substructrue for k2 in v: v2= v[k2] path2= path.copy() path2.extend([k2]) - qk2 = ".".join(path2) + qk2 = SEP.join(path2) vt2 = type(v2) if qk2 not in seen: @@ -53,7 +1279,7 @@ def process1(v,path): else: path2= path.copy() path2.extend([str(v)]) - qk2 = ".".join(path2) + qk2 = SEP.join(path2) if qk2 not in seen: seen[qk2] =1 else: @@ -63,6 +1289,7 @@ def process1(v,path): #process1("1",path2) report = {} +results = {} for log_file in log_files: with open(log_file, 'r') as f: try: @@ -77,11 +1304,26 @@ def process1(v,path): if target in e: e2 = json.loads(e[target]) # eval again #print("DEBUG1",e2) + #ct = EventDetail(**e2) + result = generate( e2, + disable_timestamp=True, + enable_version_header = False, + + input_file_type=InputFileType.Dict, + input_filename=None, + #output=output_file, + output_model_type=DataModelType.PydanticV2BaseModel, + snake_case_field=True + ) + + #print("DEBUG1",e2, result) + if result not in results: + results[result] =1 + print(result) for k in e2: - #print("DEBUG2",k) v= e2[k] qualified_path = [target,k] - qk = ".".join(qualified_path) + qk = SEP.join(qualified_path) vt = type(v) if qk not in seen: seen[qk] =1 @@ -98,7 +1340,7 @@ def process1(v,path): #facts.append("\t".join([str(v),x])) # seen[x] =0 # reset #for p in combinations(facts,3): - print(facts) + #print(facts) for f in facts: #k = ".".join(p) if f not in report: @@ -106,7 +1348,15 @@ def process1(v,path): else: report[f] = report[f] + 1 for k in report: - print(k,report[k]) + parts = k.split("|") + parts.pop() # remove the last one + #lpth = len(parts) + #parts2 = f"if (lpth == {lpth}) " + #for e,i in enumerate(parts): + # parts2 = parts2 + f" and (path[{e}] in [\"{i}\"])" + #print(parts2 + ":") + #print("""|".join(parts[1:])) + #print(k,report[k]) #print(all_events) # or process the events as needed # ### Explanation: @@ -180,3 +1430,11 @@ def process1(v,path): # print(f"last: {args.hours_ago}h") # print(f"service name filter: {args.service_name}") # print(json.dumps(policy, indent=4)) + +ideas = """ +1. some fields have unique values, count of one, all of them. +CloudTrailEvent.requestID +CloudTrailEvent.eventID + +2. some timestamps overlap, we can round up the time into chunks. +""" From 9784e4fd780bcc0d092b80ed50d4ce9638d6095d Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sun, 29 Dec 2024 07:20:32 -0500 Subject: [PATCH 118/158] generated first version --- logs/parse.py | 85 ++++++++++++++++++++++++++------------------------- 1 file changed, 43 insertions(+), 42 deletions(-) diff --git a/logs/parse.py b/logs/parse.py index 17cb4c7..58d5f17 100644 --- a/logs/parse.py +++ b/logs/parse.py @@ -1290,6 +1290,7 @@ def process1(v,path): report = {} results = {} +items = [] for log_file in log_files: with open(log_file, 'r') as f: try: @@ -1297,7 +1298,6 @@ def process1(v,path): except Exception as e: print(log_file,e) e1 = event_data.get("Events", []) - for e in e1: #print(e1) target = "CloudTrailEvent" @@ -1305,48 +1305,49 @@ def process1(v,path): e2 = json.loads(e[target]) # eval again #print("DEBUG1",e2) #ct = EventDetail(**e2) - result = generate( e2, - disable_timestamp=True, - enable_version_header = False, - - input_file_type=InputFileType.Dict, - input_filename=None, - #output=output_file, - output_model_type=DataModelType.PydanticV2BaseModel, - snake_case_field=True - ) - - #print("DEBUG1",e2, result) - if result not in results: - results[result] =1 - print(result) - for k in e2: - v= e2[k] - qualified_path = [target,k] - qk = SEP.join(qualified_path) - vt = type(v) - if qk not in seen: - seen[qk] =1 - else: - seen[qk] = seen[qk] +1 - process1(v,qualified_path) + items.append(e2) + +result = generate(items, + disable_timestamp=True, + enable_version_header = False, + input_file_type=InputFileType.Dict, + input_filename=None, + #output=output_file, + output_model_type=DataModelType.PydanticV2BaseModel, + snake_case_field=True + ) +#print("DEBUG1",e2, result) +if result not in results: + results[result] =1 + print(result) + for k in e2: + v= e2[k] + qualified_path = [target,k] + qk = SEP.join(qualified_path) + vt = type(v) + if qk not in seen: + seen[qk] =1 + else: + seen[qk] = seen[qk] +1 + process1(v,qualified_path) + + # now report on the event + facts = sorted(seen.keys()) + seen = {} + #for x in facts: + # v = seen[x] + # if (v>0): + #facts.append("\t".join([str(v),x])) + # seen[x] =0 # reset + #for p in combinations(facts,3): + #print(facts) + for f in facts: + #k = ".".join(p) + if f not in report: + report[f] = 1 + else: + report[f] = report[f] + 1 - # now report on the event - facts = sorted(seen.keys()) - seen = {} - #for x in facts: - # v = seen[x] - # if (v>0): - #facts.append("\t".join([str(v),x])) - # seen[x] =0 # reset - #for p in combinations(facts,3): - #print(facts) - for f in facts: - #k = ".".join(p) - if f not in report: - report[f] = 1 - else: - report[f] = report[f] + 1 for k in report: parts = k.split("|") parts.pop() # remove the last one From 4ab9ced01422c150d0f64f576d7874ab8ab4710f Mon Sep 17 00:00:00 2001 From: mike dupont Date: Mon, 30 Dec 2024 19:18:11 -0500 Subject: [PATCH 119/158] adding in documentation --- accounts/swarms/Readme.md | 14 ++++++++++++++ .../dev/us-east-1/components/roles/main.tf | 18 ++++++++++++++++++ modules/github/actions/aws/policy/main.json | 2 +- 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/accounts/swarms/Readme.md b/accounts/swarms/Readme.md index f5aff76..1612833 100644 --- a/accounts/swarms/Readme.md +++ b/accounts/swarms/Readme.md @@ -173,7 +173,21 @@ tofu apply -var "github_token=$(gh auth token)" see this branch for the pike generated policies https://github.com/jmikedupont2/terraform-aws-oidc-github/pull/1 +Latest repos added here: +https://github.com/unfunco/terraform-aws-oidc-github/commit/041576683835e6db4b2a62da7db93f15e52b29ee + +see usage here: +https://github.com/jmikedupont2/swarms-terraform/blob/feature/call_swarms/.github/workflows/call-swarms.yml ``` tofu apply -target "module.swarms_api.module.asg_dynamic_new_ami_dev[\"t3.small\"].module.autoscaling.aws_autoscaling_group.this[0]" ``` + +check the build +`aws ecr list-images --profile swarms --region us-east-2 --repository-name swarms/mcs` + + +# policy +see ~/2024/12/18/terraform-aws-oidc-github/examples/complete +arn:aws:iam::916723593639:policy/terraform_pike20241218205424118000000001 +iam_role_arn = "arn:aws:iam::916723593639:role/github diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf index 5dce95e..42fb875 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf @@ -48,6 +48,24 @@ data "aws_iam_policy_document" "default" { effect = "Allow" } + # statement { + # actions = [ + # "ecr:CompleteLayerUpload", + # "ecr:UploadLayerPart", + # "ecr:InitiateLayerUpload", + # "ecr:BatchCheckLayerAvailability", + # "ecr:PutImage", + # "ecr:BatchGetImage" + # ] + + # resources = [ + # "arn:aws:ecr:us-east-2:916723593639:swarms/mcs" + # ] + + # effect = "Allow" + # } + + statement { effect = "Allow" resources = ["arn:aws:s3:::swarms-session-logs*"] diff --git a/modules/github/actions/aws/policy/main.json b/modules/github/actions/aws/policy/main.json index cf5adc6..32aecec 100644 --- a/modules/github/actions/aws/policy/main.json +++ b/modules/github/actions/aws/policy/main.json @@ -62,7 +62,7 @@ "Action": [ "xray:PutTraceSegments", "xray:PutTelemetryRecords" - ], +m ], "Resource": "*" }, { From 1ff4b4e092ad2145efe282fc687c596da36d9dec Mon Sep 17 00:00:00 2001 From: mike dupont Date: Mon, 30 Dec 2024 19:54:31 -0500 Subject: [PATCH 120/158] getlogs --- logs/getlogs.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/logs/getlogs.sh b/logs/getlogs.sh index 62ff606..c5aec65 100644 --- a/logs/getlogs.sh +++ b/logs/getlogs.sh @@ -40,4 +40,5 @@ NEW_DATE=$(date -u -r ${FILENAME} +%FT%TZ) echo NEW_DATE $NEW_DATE # now look for the latest logs/$(date -d "today" +"%Y%m%d%H%M").log files and get the start time of them and use that -10 seconds to start the new one, we want all events newer than our latest -aws cloudtrail lookup-events --profile swarms --region us-east-2 --max-items 1000 --start-time $NEW_DATE --output json >> logs/$(date -d "today" +"%Y%m%d%H%M").log +aws cloudtrail lookup-events --profile swarms --region us-east-2 --start-time $NEW_DATE --output json >> logs/$(date -d "today" +"%Y%m%d%H%M").log +# --max-items 1000 From 67998a83331357c141bcdcba2e85ba7257486936 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Tue, 31 Dec 2024 10:07:11 -0500 Subject: [PATCH 121/158] creating mcs dev --- .../application_load_balancer/https/main.tf | 16 +++++++++++++++ .../application_load_balancer/main.tf | 12 +++++++++++ .../application_load_balancer/route53/main.tf | 7 +++++++ .../dev/us-east-1/main.tf | 20 +++++++++++++++++++ .../dev/us-east-1/mcs/main.tf | 11 ++++++++-- 5 files changed, 64 insertions(+), 2 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf index 64e6210..43074f0 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf @@ -82,6 +82,22 @@ resource "aws_lb_listener_rule" "route_v1_api_mcs" { } } +resource "aws_lb_listener_rule" "route_v1_api_mcs_dev" { + listener_arn = aws_lb_listener.this.arn + priority = 102 # Set priority as needed, must be unique + + action { + type = "forward" + target_group_arn = var.mcs_dev_target_group_arn # New target group's ARN + } + + condition { + host_header { + values = ["dev.mcs.api.swarms.ai"] + } + } +} + resource "aws_lb_listener" "insecure" { port = 80 protocol = "HTTP" diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf index 05e1078..8023ea2 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf @@ -70,6 +70,13 @@ module "tg_mcs" { vpc_id = var.vpc_id # module.vpc.vpc_id } +module "tg_mcs_dev" { + source = "./target_group/" + name_prefix = "mcs-dev" + check_path = "/v1/medical-coder/patients" # this does not have a docs api yet + vpc_id = var.vpc_id # module.vpc.vpc_id +} + module "https" { source = "./https/" # vpc_id = var.vpc_id # module.vpc.vpc_id @@ -80,6 +87,7 @@ module "https" { test_target_group_arn = module.tg_test.alb_target_group_arn dev_target_group_arn = module.tg_dev.alb_target_group_arn mcs_target_group_arn = module.tg_mcs.alb_target_group_arn + mcs_dev_target_group_arn = module.tg_mcs_dev.alb_target_group_arn } @@ -98,3 +106,7 @@ output "dev_alb_target_group_arn" { output "mcs_alb_target_group_arn" { value = module.tg_mcs.alb_target_group_arn } + +output "mcs_dev_alb_target_group_arn" { + value = module.tg_mcs_dev.alb_target_group_arn +} diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf index bb841b1..f374e73 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf @@ -48,7 +48,14 @@ resource "aws_route53_record" "api-cname-mcs" { type = "CNAME" records = [aws_route53_record.api-cname.fqdn] ttl = 300 +} +resource "aws_route53_record" "api-cname-mcs" { + zone_id = data.aws_route53_zone.primary.zone_id + name = "dev.mcs.${var.domain_name}" + type = "CNAME" + records = [aws_route53_record.api-cname.fqdn] + ttl = 300 } output "cname" { diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index e13b6be..5698bba 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -341,6 +341,8 @@ output "security_group_id" { module "mcs" { source = "./mcs" + branch = "feature/mcs" + git_repo = "https://github.com/jmikedupont2/swarms-MedicalCoderSwarm-deployment.git" alb_target_group_arn = module.alb.mcs_alb_target_group_arn ssm_profile_arn = module.roles.ssm_profile_arn ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 @@ -354,6 +356,24 @@ module "mcs" { vpc_id = local.vpc_id } +module "mcs_dev" { + source = "./mcs" + branch = "feature/mcs_dev" + git_repo = "https://github.com/jmikedupont2/swarms-MedicalCoderSwarm-deployment.git" + + alb_target_group_arn = module.alb.mcs_dev_alb_target_group_arn + ssm_profile_arn = module.roles.ssm_profile_arn + ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 + iam_instance_profile_name = module.roles.ssm_profile_name + key_name = var.key_name + aws_account_id = var.aws_account_id + region = var.region + internal_security_group_id = module.security.internal_security_group_id + tags = local.tags + ami_id = local.new_ami_id + vpc_id = local.vpc_id +} + output "vpc" { value = module.vpc } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf index 4247059..2b84432 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf @@ -1,3 +1,10 @@ +variable branch { + default = "feature/mcs" +} +variable git_repo { + default = "https://github.com/jmikedupont2/swarms-MedicalCoderSwarm-deployment.git" +} + variable "ec2_subnet_id" {} variable "alb_target_group_arn" {} variable "aws_account_id" {} @@ -25,14 +32,14 @@ variable "instance_types" { module "lt_docker" { source = "../components/launch_template_docker_mcs" - branch = "feature/mcs" + branch = var.branch vpc_id = var.vpc_id for_each = toset(var.instance_types) instance_type = each.key name = "mcs-docker-${each.key}" security_group_id = var.internal_security_group_id ami_id = var.ami_id - git_repo = "https://github.com/jmikedupont2/swarms-MedicalCoderSwarm-deployment.git" + git_repo = var.git_repo app_name = "mcs" # used to construct /opt/mcs for where we install to tags = merge(var.tags, { environment = "mcs" From bcc9ee1804edbc267d36215e2a0ff8b05c35b1d0 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Tue, 31 Dec 2024 10:12:38 -0500 Subject: [PATCH 122/158] adding mcs dev --- .../application_load_balancer/https/main.tf | 1 + .../application_load_balancer/main.tf | 2 +- .../application_load_balancer/route53/main.tf | 2 +- .../swarms-aws-agent-api/dev/us-east-1/main.tf | 17 ----------------- 4 files changed, 3 insertions(+), 19 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf index 43074f0..0b8f4d1 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf @@ -5,6 +5,7 @@ variable "prod_target_group_arn" {} variable "test_target_group_arn" {} variable "dev_target_group_arn" {} variable "mcs_target_group_arn" {} +variable "mcs_dev_target_group_arn" {} module "acm" { # count = 0 diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf index 8023ea2..d230c94 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf @@ -72,7 +72,7 @@ module "tg_mcs" { module "tg_mcs_dev" { source = "./target_group/" - name_prefix = "mcs-dev" + name_prefix = "mcsdev" check_path = "/v1/medical-coder/patients" # this does not have a docs api yet vpc_id = var.vpc_id # module.vpc.vpc_id } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf index f374e73..8627ddc 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf @@ -50,7 +50,7 @@ resource "aws_route53_record" "api-cname-mcs" { ttl = 300 } -resource "aws_route53_record" "api-cname-mcs" { +resource "aws_route53_record" "api-cname-mcs-dev" { zone_id = data.aws_route53_zone.primary.zone_id name = "dev.mcs.${var.domain_name}" type = "CNAME" diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index 5698bba..0cc2861 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -209,23 +209,6 @@ output "alb" { } -# this is the slow one, use the ami -# module "asg_dynamic" { -# tags = local.tags -# vpc_id = local.vpc_id -# image_id = local.ami_id -# ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 -# for_each = toset(var.instance_types) -# aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn -# #iam_instance_profile_name = module.roles.ssm_profile_name -# source = "./components/autoscaling_group" -# # security_group_id = module.security.internal_security_group_id -# instance_type = each.key -# name = "swarms-size-${each.key}" -# launch_template_id = module.lt_dynamic[each.key].launch_template_id -# target_group_arn = module.alb.alb_target_group_arn -# } - module "asg_dynamic_new_ami" { # built with packer #count =0 From e12be67719553e77619086ebf2e2df41a3d1facc Mon Sep 17 00:00:00 2001 From: mike dupont Date: Tue, 31 Dec 2024 10:39:35 -0500 Subject: [PATCH 123/158] update --- environments/swarms-aws-agent-api/dev/us-east-1/main.tf | 6 ++++++ environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index 0cc2861..478c6ae 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -337,10 +337,12 @@ module "mcs" { tags = local.tags ami_id = local.new_ami_id vpc_id = local.vpc_id + name = "docker-mcs-ami" } module "mcs_dev" { source = "./mcs" + name = "mcs-dev" branch = "feature/mcs_dev" git_repo = "https://github.com/jmikedupont2/swarms-MedicalCoderSwarm-deployment.git" @@ -355,6 +357,10 @@ module "mcs_dev" { tags = local.tags ami_id = local.new_ami_id vpc_id = local.vpc_id + + instance_types= [ + "t3.medium" + ] } output "vpc" { diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf index 2b84432..37ec454 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf @@ -1,6 +1,8 @@ +variable name {} variable branch { default = "feature/mcs" } + variable git_repo { default = "https://github.com/jmikedupont2/swarms-MedicalCoderSwarm-deployment.git" } @@ -66,7 +68,7 @@ module "asg" { # security_group_id = module.security.internal_security_group_id instance_type = each.key - name = "docker-mcs-ami-${each.key}" + name = "${var.name}-${each.key}" launch_template_id = module.lt_docker[each.key].launch_template_id target_group_arn = var.alb_target_group_arn } From d4b64fb1e31400f29dc2eb8e70e6af97337df01a Mon Sep 17 00:00:00 2001 From: mike dupont Date: Thu, 9 Jan 2025 14:16:21 -0500 Subject: [PATCH 124/158] applied --- accounts/mdupont/main.tf | 42 +++++++++---------- .../dev/us-east-1/components/ses/main.tf | 24 +++++------ 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/accounts/mdupont/main.tf b/accounts/mdupont/main.tf index f5408b4..1591e09 100644 --- a/accounts/mdupont/main.tf +++ b/accounts/mdupont/main.tf @@ -1,5 +1,5 @@ -variable "google_oauth_client_secret" {} -variable "google_oauth_client_id" {} +#variable "google_oauth_client_secret" {} +#variable "google_oauth_client_id" {} module ses { verify_dkim=true @@ -19,24 +19,24 @@ module ses { # source = "../../environments/swarms-aws-agent-api/dev/us-east-1/components/ses_verify" # } -module cognito { - aws_account =var.aws_account_id - myemail ="jmdupont" - mydomain ="introspector" - mydomain_suffix = "meme" - #../../../17/ - aws_region = var.aws_region - env={ - region = var.aws_region - profile = var.profile - } - source = "../../environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool" +#module cognito { +# aws_account =var.aws_account_id +# myemail ="jmdupont" +# mydomain ="introspector" +# mydomain_suffix = "meme" +# #../../../17/ +# aws_region = var.aws_region +# env={ +# region = var.aws_region +# profile = var.profile +# } +# source = "../../environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool" #source = "~/2024/12/17/cognito/terraform-aws-cognito-user-pool/examples/complete/" #source = "git::https://github.com/meta-introspector/terraform-aws-cognito-user-pool.git?ref=feature/meta-meme" - google_oauth_client_secret=var.google_oauth_client_secret - google_oauth_client_id=var.google_oauth_client_id -} -output cognito{ - value = module.cognito - sensitive = true -} +# google_oauth_client_secret=var.google_oauth_client_secret +# google_oauth_client_id=var.google_oauth_client_id +#} +#output cognito{ +# value = module.cognito +# sensitive = true +#} diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/ses/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/ses/main.tf index 27d1c2c..f529846 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/ses/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/ses/main.tf @@ -9,15 +9,15 @@ variable "domain" {} # tags = module.this.tags #} -module "ses" { - source = "git::https://github.com/cloudposse/terraform-aws-ses.git" - domain = var.domain - # zone_id = var.zone_id - zone_id = aws_route53_zone.private_dns_zone.zone_id - verify_dkim = var.verify_dkim - verify_domain = var.verify_domain - ses_group_name = var.group_name - context = module.this.context - #ses_group_enabled = false - ses_user_enabled = false -} +# module "ses" { +# source = "git::https://github.com/cloudposse/terraform-aws-ses.git" +# domain = var.domain +# # zone_id = var.zone_id +# # zone_id = aws_route53_zone.private_dns_zone.zone_id +# verify_dkim = var.verify_dkim +# verify_domain = var.verify_domain +# ses_group_name = var.group_name +# context = module.this.context +# #ses_group_enabled = false +# ses_user_enabled = false +# } From edce93d1ec0ef655d1ec9630d626f9b0aa522e10 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Thu, 9 Jan 2025 15:12:17 -0500 Subject: [PATCH 125/158] step one create ssm infra --- accounts/mdupont/data.tf | 3 - accounts/mdupont/main.tf | 57 +++- .../application_load_balancer/route53/main.tf | 28 +- .../components/autoscaling_group/spot/main.tf | 16 +- .../dev/us-east-1/main.tf | 305 +++++++++--------- .../dev/us-east-1/mcs/main.tf | 4 +- modules/aws/ssm/observability/main.tf | 40 +-- modules/aws/ssm/setup/main.tf | 72 +---- 8 files changed, 249 insertions(+), 276 deletions(-) diff --git a/accounts/mdupont/data.tf b/accounts/mdupont/data.tf index fbb2afa..d52e0cb 100644 --- a/accounts/mdupont/data.tf +++ b/accounts/mdupont/data.tf @@ -1,7 +1,4 @@ # task create an output file for this module to expose all interesting data, include the ids of each resource. -provider "aws" { - region = var.aws_region -} data "aws_iam_user" "example_user" { user_name = var.iam_user diff --git a/accounts/mdupont/main.tf b/accounts/mdupont/main.tf index 1591e09..0cc20b2 100644 --- a/accounts/mdupont/main.tf +++ b/accounts/mdupont/main.tf @@ -1,14 +1,25 @@ +provider "aws" { + region = "us-east-2" + profile = "mdupont" +} + #variable "google_oauth_client_secret" {} #variable "google_oauth_client_id" {} -module ses { - verify_dkim=true - domain="introspector.meme" # put the mail at the top level - #verify_domain =true - verify_domain =false # not on aws - group_name="introspector" - source = "../../environments/swarms-aws-agent-api/dev/us-east-1/components/ses" +locals { + ami_name = "ubuntu-minimal/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-minimal-*" # useast2id= + dns = "eliza.introspector.meme" + region = "us-east-2" } + +# module ses { +# verify_dkim=true +# domain="introspector.meme" # put the mail at the top level +# #verify_domain =true +# verify_domain =false # not on aws +# group_name="introspector" +# source = "../../environments/swarms-aws-agent-api/dev/us-east-1/components/ses" +# } # module ses_verification { # verify_dkim=true @@ -20,7 +31,7 @@ module ses { # } #module cognito { -# aws_account =var.aws_account_id + # myemail ="jmdupont" # mydomain ="introspector" # mydomain_suffix = "meme" @@ -40,3 +51,33 @@ module ses { # value = module.cognito # sensitive = true #} + + +data "aws_ami" "ami" { # slow + most_recent = true + name_regex = "^${local.ami_name}" + } + + module "eliza_server" { + count = 0 + #aws_account_id = local.account + aws_account_id =var.aws_account_id + region = local.region + source = "../../environments/swarms-aws-agent-api/dev/us-east-1" # FIXME rename + domain = local.dns + ami_id = data.aws_ami.ami.id + name = "eliza" + tags = { project = "eliza" } +} + +module "ssm_observer" { + source = "../../modules/aws/ssm/observability" + ami_id = data.aws_ami.ami.id +} + +module "ssm_setup" { + source = "../../modules/aws/ssm/setup" + bucket_name = "tine-session-logs" + access_log_bucket_name = "tine-session-access-logs" + project = "tine" + } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf index 8627ddc..ad074f9 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf @@ -42,21 +42,21 @@ resource "aws_route53_record" "api-cname-dev" { } -resource "aws_route53_record" "api-cname-mcs" { - zone_id = data.aws_route53_zone.primary.zone_id - name = "mcs.${var.domain_name}" - type = "CNAME" - records = [aws_route53_record.api-cname.fqdn] - ttl = 300 -} +# resource "aws_route53_record" "api-cname-mcs" { +# zone_id = data.aws_route53_zone.primary.zone_id +# name = "mcs.${var.domain_name}" +# type = "CNAME" +# records = [aws_route53_record.api-cname.fqdn] +# ttl = 300 +# } -resource "aws_route53_record" "api-cname-mcs-dev" { - zone_id = data.aws_route53_zone.primary.zone_id - name = "dev.mcs.${var.domain_name}" - type = "CNAME" - records = [aws_route53_record.api-cname.fqdn] - ttl = 300 -} +# resource "aws_route53_record" "api-cname-mcs-dev" { +# zone_id = data.aws_route53_zone.primary.zone_id +# name = "dev.mcs.${var.domain_name}" +# type = "CNAME" +# records = [aws_route53_record.api-cname.fqdn] +# ttl = 300 +# } output "cname" { value = aws_route53_record.api-cname.fqdn diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/spot/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/spot/main.tf index d8f6b7b..bdd2cea 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/spot/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/spot/main.tf @@ -13,7 +13,7 @@ variable "instance_requirements" { } variable "aws_iam_instance_profile_ssm_arn" {} -variable "target_group_arn" {} +#variable "target_group_arn" {} variable "name" {} variable "instance_type" {} variable "launch_template_id" {} @@ -32,12 +32,12 @@ module "autoscaling" { version = "8.0.0" name = var.name - traffic_source_attachments = { - ex-alb = { - traffic_source_identifier = var.target_group_arn - traffic_source_type = "elbv2" # default - } - } +# traffic_source_attachments = { +# ex-alb = { +# traffic_source_identifier = var.target_group_arn +# traffic_source_type = "elbv2" # default +# } +# } health_check_type = "EC2" desired_capacity = 1 @@ -82,8 +82,6 @@ module "autoscaling" { AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" } - # target_group_arn = - use_mixed_instances_policy = var.use_mixed_instances_policy mixed_instances_policy = var.mixed_instances_policy diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index 478c6ae..d34c7fc 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -9,10 +9,10 @@ locals { # instance_type = "t3.large" # instance_type = "t3.medium" ami_name = "ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*" - name = "swarms" + name = "eliza" domain = var.domain tags = { - project = "swarms" + project = "eliza" } dev_tags = { sandbox = "kye" @@ -27,7 +27,7 @@ variable "name" {} locals { ami_id = var.ami_id - new_ami_id = "ami-08093b6770af41b14" # environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/Readme.md +# new_ami_id = "ami-08093b6770af41b14" # environments/eliza-aws-agent-api/dev/us-east-1/components/machine_image/Readme.md } # SLOW @@ -52,8 +52,6 @@ module "kp" { source = "./components/keypairs" } - - variable "instance_types" { type = list(string) default = [ @@ -76,9 +74,6 @@ variable "test_instance_types" { #"t3.small", #"t2.small", not working # "t2.medium" # - - - #"t3.medium" # no instances for now, this is commented out ] } @@ -94,16 +89,16 @@ module "lt_dynamic_ami_prod" { vpc_id = local.vpc_id for_each = toset(var.instance_types) instance_type = each.key - name = "swarms-ami-${each.key}" + name = "eliza-ami-${each.key}" security_group_id = module.security.internal_security_group_id - ami_id = local.new_ami_id + ami_id = local.ami_id key_name = var.key_name tags = merge(local.tags, { environment = "production" }) source = "./components/launch_template" iam_instance_profile_name = module.roles.ssm_profile_name - install_script = "/opt/swarms/api/just_run.sh" + install_script = "/opt/eliza/api/just_run.sh" ssm_parameter_name_cw_agent_config = "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/cloudwatch-agent/config" branch = "feature/ec2" } @@ -113,9 +108,9 @@ module "lt_dynamic_ami_test" { vpc_id = local.vpc_id for_each = toset(var.instance_types) instance_type = each.key - name = "swarms-ami-${each.key}" + name = "eliza-ami-${each.key}" security_group_id = module.security.internal_security_group_id - ami_id = local.new_ami_id + ami_id = local.ami_id tags = merge(local.tags, { environment = "test" }) @@ -123,7 +118,7 @@ module "lt_dynamic_ami_test" { key_name = var.key_name #"mdupont-deployer-key" ssm_parameter_name_cw_agent_config = "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/cloudwatch-agent/config/details" iam_instance_profile_name = module.roles.ssm_profile_name - install_script = "/opt/swarms/api/just_run.sh" + install_script = "/opt/eliza/api/just_run.sh" } @@ -159,9 +154,9 @@ module "lt_dynamic_ami_docker" { vpc_id = local.vpc_id for_each = toset(var.dev_instance_types) instance_type = each.key - name = "swarms-docker-${each.key}" + name = "eliza-docker-${each.key}" security_group_id = module.security.internal_security_group_id - ami_id = local.new_ami_id + ami_id = local.ami_id tags = merge(local.tags, { environment = "test" }) @@ -169,8 +164,8 @@ module "lt_dynamic_ami_docker" { key_name = var.key_name #"mdupont-deployer-key" ssm_parameter_name_cw_agent_config = "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/cloudwatch-agent/config/details" iam_instance_profile_name = module.roles.ssm_profile_name - #install_script = "/opt/swarms/api/docker-boot.sh" this is called from ssm for a refresh - install_script = "/opt/swarms/api/rundocker.sh" + #install_script = "/opt/eliza/api/docker-boot.sh" this is called from ssm for a refresh + install_script = "/opt/eliza/api/rundocker.sh" } module "lt_dynamic_ami_docker_normal" { @@ -179,9 +174,9 @@ module "lt_dynamic_ami_docker_normal" { vpc_id = local.vpc_id for_each = toset(var.dev2_instance_types) instance_type = each.key - name = "swarms-docker-${each.key}" + name = "eliza-docker-${each.key}" security_group_id = module.security.internal_security_group_id - ami_id = local.new_ami_id + ami_id = local.ami_id tags = merge(local.tags, { environment = "test" }) @@ -189,113 +184,114 @@ module "lt_dynamic_ami_docker_normal" { key_name = var.key_name #"mdupont-deployer-key" ssm_parameter_name_cw_agent_config = "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/cloudwatch-agent/config/details" iam_instance_profile_name = module.roles.ssm_profile_name - #install_script = "/opt/swarms/api/docker-boot.sh" this is called from ssm for a refresh - install_script = "/opt/swarms/api/rundocker.sh" + #install_script = "/opt/eliza/api/docker-boot.sh" this is called from ssm for a refresh + install_script = "/opt/eliza/api/rundocker.sh" } -module "alb" { - source = "./components/application_load_balancer" - domain_name = local.domain - security_group_id = module.security.security_group_id # allowed to talk to internal - public_subnets = [ - local.ec2_public_subnet_id_1, - local.ec2_public_subnet_id_2] - vpc_id = local.vpc_id - name = local.name -} -output "alb" { - value = module.alb -} - - -module "asg_dynamic_new_ami" { - # built with packer - #count =0 - tags = local.tags - vpc_id = local.vpc_id - image_id = local.new_ami_id - ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 - for_each = toset(var.instance_types) - aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn - source = "./components/autoscaling_group" - # security_group_id = module.security.internal_security_group_id - instance_type = each.key - name = "swarms-ami-${each.key}" - launch_template_id = module.lt_dynamic_ami_prod[each.key].launch_template_id - target_group_arn = module.alb.prod_alb_target_group_arn -} - -module "asg_dynamic_new_ami_test" { - - # built with packer - for_each = toset(var.test_instance_types) - tags = merge(local.tags, local.dev_tags) - vpc_id = local.vpc_id - image_id = local.new_ami_id - ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 - - aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn - source = "./components/autoscaling_group/spot" - # security_group_id = module.security.internal_security_group_id - instance_type = each.key - name = "test-swarms-ami-${each.key}" - launch_template_id = module.lt_dynamic_ami_test[each.key].launch_template_id - target_group_arn = module.alb.test_alb_target_group_arn -} - -module "asg_dynamic_new_ami_dev_spot" { - # built with packer - # count =0 - tags = merge(local.tags, local.dev_tags) - vpc_id = local.vpc_id - image_id = local.new_ami_id - ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 - for_each = toset(var.dev_instance_types) - aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn - - source = "./components/autoscaling_group/spot" - # security_group_id = module.security.internal_security_group_id - instance_type = each.key - name = "docker-swarms-ami-${each.key}" - launch_template_id = module.lt_dynamic_ami_docker[each.key].launch_template_id - target_group_arn = module.alb.dev_alb_target_group_arn - - use_mixed_instances_policy = true - mixed_instances_policy = { - instances_distribution = { - on_demand_base_capacity = 0 - on_demand_percentage_above_base_capacity = 0 - spot_instance_pools = 1 - spot_max_price = var.spot_max_price - # spot_allocation_strategy = "capacity-optimized" - } - - override = [ - { - instance_requirements = { - cpu_manufacturers = ["amazon-web-services", "amd", "intel"] - #cpu_manufacturers = ["amd"] - #local_storage_types = ["ssd"] - max_spot_price_as_percentage_of_optimal_on_demand_price = 60 - memory_gib_per_vcpu = { - min = 4 - max = 12 - } - memory_mib = { - min = 4096 - }, - vcpu_count = { - min = 2 - max = 12 - } - } - } - ] - } - instance_requirements = { - } -} +# module "alb" { +# count = 0 # none for now +# source = "./components/application_load_balancer" +# domain_name = local.domain +# security_group_id = module.security.security_group_id # allowed to talk to internal +# public_subnets = [ +# local.ec2_public_subnet_id_1, +# local.ec2_public_subnet_id_2] +# vpc_id = local.vpc_id +# name = local.name +# } +# output "alb" { +# value = module.alb +# } + + +# module "asg_dynamic_new_ami" { +# # built with packer +# #count =0 +# tags = local.tags +# vpc_id = local.vpc_id +# image_id = local.new_ami_id +# ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 +# for_each = toset(var.instance_types) +# aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn +# source = "./components/autoscaling_group" +# # security_group_id = module.security.internal_security_group_id +# instance_type = each.key +# name = "eliza-ami-${each.key}" +# launch_template_id = module.lt_dynamic_ami_prod[each.key].launch_template_id +# target_group_arn = module.alb.prod_alb_target_group_arn +# } + +# module "asg_dynamic_new_ami_test" { + +# # built with packer +# for_each = toset(var.test_instance_types) +# tags = merge(local.tags, local.dev_tags) +# vpc_id = local.vpc_id +# image_id = local.new_ami_id +# ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 + +# aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn +# source = "./components/autoscaling_group/spot" +# # security_group_id = module.security.internal_security_group_id +# instance_type = each.key +# name = "test-eliza-ami-${each.key}" +# launch_template_id = module.lt_dynamic_ami_test[each.key].launch_template_id +# target_group_arn = module.alb.test_alb_target_group_arn +# } + +# module "asg_dynamic_new_ami_dev_spot" { +# # built with packer +# # count =0 +# tags = merge(local.tags, local.dev_tags) +# vpc_id = local.vpc_id +# image_id = local.new_ami_id +# ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 +# for_each = toset(var.dev_instance_types) +# aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn + +# source = "./components/autoscaling_group/spot" +# # security_group_id = module.security.internal_security_group_id +# instance_type = each.key +# name = "docker-eliza-ami-${each.key}" +# launch_template_id = module.lt_dynamic_ami_docker[each.key].launch_template_id +# target_group_arn = module.alb.dev_alb_target_group_arn + +# use_mixed_instances_policy = true +# mixed_instances_policy = { +# instances_distribution = { +# on_demand_base_capacity = 0 +# on_demand_percentage_above_base_capacity = 0 +# spot_instance_pools = 1 +# spot_max_price = var.spot_max_price +# # spot_allocation_strategy = "capacity-optimized" +# } + +# override = [ +# { +# instance_requirements = { +# cpu_manufacturers = ["amazon-web-services", "amd", "intel"] +# #cpu_manufacturers = ["amd"] +# #local_storage_types = ["ssd"] +# max_spot_price_as_percentage_of_optimal_on_demand_price = 60 +# memory_gib_per_vcpu = { +# min = 4 +# max = 12 +# } +# memory_mib = { +# min = 4096 +# }, +# vcpu_count = { +# min = 2 +# max = 12 +# } +# } +# } +# ] +# } +# instance_requirements = { +# } +# } module "asg_dynamic_new_ami_dev_normal" { # built with packer @@ -303,7 +299,7 @@ module "asg_dynamic_new_ami_dev_normal" { tags = merge(local.tags, local.dev_tags) vpc_id = local.vpc_id - image_id = local.new_ami_id + image_id = local.ami_id ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 for_each = toset(var.dev2_instance_types) aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn @@ -311,9 +307,9 @@ module "asg_dynamic_new_ami_dev_normal" { source = "./components/autoscaling_group/spot" # security_group_id = module.security.internal_security_group_id instance_type = each.key - name = "docker-swarms-ami-${each.key}" + name = "docker-eliza-ami-${each.key}" launch_template_id = module.lt_dynamic_ami_docker_normal[each.key].launch_template_id - target_group_arn = module.alb.dev_alb_target_group_arn +# target_group_arn = module.alb.dev_alb_target_group_arn } @@ -322,11 +318,12 @@ output "security_group_id" { value = module.security.security_group_id } -module "mcs" { - source = "./mcs" - branch = "feature/mcs" - git_repo = "https://github.com/jmikedupont2/swarms-MedicalCoderSwarm-deployment.git" - alb_target_group_arn = module.alb.mcs_alb_target_group_arn +module "eliza" { + source = "./mcs" # fixme rename to eliza + branch = "feature/reduce_modules_discord" + #git_repo = "https://github.com/meta-introspector/eliza-MedicalCoderSwarm-deployment.git" + git_repo = "https://github.com/meta-introspector/cloud-deployment-eliza/" +# alb_target_group_arn = module.alb.mcs_alb_target_group_arn ssm_profile_arn = module.roles.ssm_profile_arn ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 iam_instance_profile_name = module.roles.ssm_profile_name @@ -335,33 +332,33 @@ module "mcs" { region = var.region internal_security_group_id = module.security.internal_security_group_id tags = local.tags - ami_id = local.new_ami_id + ami_id = local.ami_id vpc_id = local.vpc_id name = "docker-mcs-ami" } -module "mcs_dev" { - source = "./mcs" - name = "mcs-dev" - branch = "feature/mcs_dev" - git_repo = "https://github.com/jmikedupont2/swarms-MedicalCoderSwarm-deployment.git" +# module "agent_dev" { +# source = "./agent" +# name = "agent-dev" +# branch = "feature/agent_dev" +# git_repo = "https://github.com/jmikedupont2/eliza-MedicalCoderSwarm-deployment.git" - alb_target_group_arn = module.alb.mcs_dev_alb_target_group_arn - ssm_profile_arn = module.roles.ssm_profile_arn - ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 - iam_instance_profile_name = module.roles.ssm_profile_name - key_name = var.key_name - aws_account_id = var.aws_account_id - region = var.region - internal_security_group_id = module.security.internal_security_group_id - tags = local.tags - ami_id = local.new_ami_id - vpc_id = local.vpc_id - - instance_types= [ - "t3.medium" - ] -} +# alb_target_group_arn = module.alb.agent_dev_alb_target_group_arn +# ssm_profile_arn = module.roles.ssm_profile_arn +# ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 +# iam_instance_profile_name = module.roles.ssm_profile_name +# key_name = var.key_name +# aws_account_id = var.aws_account_id +# region = var.region +# internal_security_group_id = module.security.internal_security_group_id +# tags = local.tags +# ami_id = local.ami_id +# vpc_id = local.vpc_id + +# instance_types= [ +# "t3.medium" +# ] +# } output "vpc" { value = module.vpc diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf index 37ec454..4a7d427 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf @@ -8,7 +8,7 @@ variable git_repo { } variable "ec2_subnet_id" {} -variable "alb_target_group_arn" {} +#variable "alb_target_group_arn" {} variable "aws_account_id" {} variable "vpc_id" {} # FIXME security, we use the ami of swarms for speed but want to split @@ -70,5 +70,5 @@ module "asg" { instance_type = each.key name = "${var.name}-${each.key}" launch_template_id = module.lt_docker[each.key].launch_template_id - target_group_arn = var.alb_target_group_arn +# target_group_arn = var.alb_target_group_arn } diff --git a/modules/aws/ssm/observability/main.tf b/modules/aws/ssm/observability/main.tf index a9fa252..f3f4f11 100755 --- a/modules/aws/ssm/observability/main.tf +++ b/modules/aws/ssm/observability/main.tf @@ -421,18 +421,18 @@ locals { }, # nginx - { - file_path = "/var/log/nginx/swarms/access.log" - log_group_name = "/swarms/ngnix_access" - log_stream_name = "{instance_id}" - timezone = "UTC" - }, - { - file_path = "/var/log/nginx/swarms/error.log" - log_group_name = "/swarms/nginx_error" - log_stream_name = "{instance_id}" - timezone = "UTC" - }, + # { + # file_path = "/var/log/nginx/swarms/access.log" + # log_group_name = "/swarms/ngnix_access" + # log_stream_name = "{instance_id}" + # timezone = "UTC" + # }, + # { + # file_path = "/var/log/nginx/swarms/error.log" + # log_group_name = "/swarms/nginx_error" + # log_stream_name = "{instance_id}" + # timezone = "UTC" + # }, { file_path = "/var/log/cloud-init-output.log" @@ -441,8 +441,8 @@ locals { timezone = "UTC" }, { - file_path = "/var/log/swarms_systemd.log" - log_group_name = "/swarms/systemd" + file_path = "/var/log/eliza_systemd.log" + log_group_name = "/eliza/systemd" log_stream_name = "{instance_id}" timezone = "UTC" }, @@ -483,25 +483,25 @@ resource "aws_ssm_association" "update_cloudwatch_config" { resource "aws_cloudwatch_log_group" "log_groups" { for_each = toset([ - "/swarms/ngnix_access", - "/swarms/nginx_error", - "/swarms/systemd", "/ec2/init"]) +# "/swarms/ngnix_access", +# "/swarms/nginx_error", + "/eliza/systemd", "/ec2/init"]) name = each.key retention_in_days = 30 - kms_key_id = "arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590" + #kms_key_id = "arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590" } # CloudWatch Log Groups for collected logs resource "aws_cloudwatch_log_group" "system_logs" { name = "/ec2/system" retention_in_days = 30 - kms_key_id = "arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590" + #kms_key_id = "arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590" } resource "aws_cloudwatch_log_group" "secure_logs" { name = "/ec2/secure" retention_in_days = 30 - kms_key_id = "arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590" + #kms_key_id = "arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590" } # SSM Document for CloudWatch agent troubleshooting diff --git a/modules/aws/ssm/setup/main.tf b/modules/aws/ssm/setup/main.tf index b5eb5f2..44ea9b7 100644 --- a/modules/aws/ssm/setup/main.tf +++ b/modules/aws/ssm/setup/main.tf @@ -1,12 +1,15 @@ +variable bucket_name{} +variable access_log_bucket_name{} +variable project{} module "ssm" { # source = "bridgecrewio/session-manager/aws" # version = "0.4.2" source = "git::https://github.com/jmikedupont2/terraform-aws-session-manager.git?ref=master" - bucket_name = "swarms-session-logs" - access_log_bucket_name = "swarms-session-access-logs" + bucket_name = var.bucket_name + access_log_bucket_name = var.access_log_bucket_name enable_log_to_s3 = true enable_log_to_cloudwatch = true - tags = { project = "swarms" } + tags = { project = var.project } #linux_shell_profile = "date" } @@ -27,66 +30,3 @@ resource "aws_cloudwatch_log_group" "app_signals2" { kms_key_id = "arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590" } -# module "session-manager-settings" { -# source = "gazoakley/session-manager-settings/aws" -# s3_bucket_name = "swarms-session-logs-bucket" -# cloudwatch_log_group_name = "/ssm/swarms-session-logs" -# #kms_key_id = arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590 -# kms_key_id = "cc8e1ee7-a05b-4642-bd81-ba5548635590" -# } -# Configure the AWS provider -# provider "aws" { -# region = "us-east-2" # Matching the region from your CloudTrail event -# } - -# # Create or update the S3 bucket with tags -# resource "aws_s3_bucket" "session_logs" { -# bucket = "swarms-session-logs-20241221151754799300000003" - -# # Force destroy can be set to true if you want to allow Terraform to delete the bucket even if it contains objects -# force_destroy = false - -# tags = { -# Environment = "Production" # Example tag -# Project = "Swarms" # Example tag -# Created = "2024-12-21" # Example tag -# } -# } - -# # Add bucket versioning (recommended for logging buckets) -# resource "aws_s3_bucket_versioning" "session_logs_versioning" { -# bucket = aws_s3_bucket.session_logs.id -# versioning_configuration { -# status = "Enabled" -# } -# } - -# # Add bucket encryption (recommended) -# resource "aws_s3_bucket_server_side_encryption_configuration" "session_logs_encryption" { -# bucket = aws_s3_bucket.session_logs.id - -# rule { -# apply_server_side_encryption_by_default { -# sse_algorithm = "AES256" -# } -# } -# } - -# # Add lifecycle rules (optional, but recommended for log buckets) -# resource "aws_s3_bucket_lifecycle_configuration" "session_logs_lifecycle" { -# bucket = aws_s3_bucket.session_logs.id - -# rule { -# id = "log_retention" -# status = "Enabled" - -# transition { -# days = 30 -# storage_class = "STANDARD_IA" -# } - -# expiration { -# days = 90 # Adjust retention period as needed -# } -# } -# } From faf3eb56f64a265f787245a6d27c108eb8ee40d8 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Thu, 9 Jan 2025 15:37:39 -0500 Subject: [PATCH 126/158] adding eliza --- accounts/mdupont/main.tf | 44 ++++++++++++------- .../dev/us-east-1/main.tf | 2 +- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/accounts/mdupont/main.tf b/accounts/mdupont/main.tf index 0cc20b2..68f2f30 100644 --- a/accounts/mdupont/main.tf +++ b/accounts/mdupont/main.tf @@ -7,7 +7,7 @@ provider "aws" { #variable "google_oauth_client_id" {} locals { - ami_name = "ubuntu-minimal/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-minimal-*" # useast2id= + dns = "eliza.introspector.meme" region = "us-east-2" } @@ -53,26 +53,21 @@ locals { #} -data "aws_ami" "ami" { # slow - most_recent = true - name_regex = "^${local.ami_name}" - } +#locals { ami_name = "ubuntu-minimal/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-minimal-*" } +# data "aws_ami" "ami" { # slow +# most_recent = true +# name_regex = "^${local.ami_name}" +# } - module "eliza_server" { - count = 0 - #aws_account_id = local.account - aws_account_id =var.aws_account_id - region = local.region - source = "../../environments/swarms-aws-agent-api/dev/us-east-1" # FIXME rename - domain = local.dns - ami_id = data.aws_ami.ami.id - name = "eliza" - tags = { project = "eliza" } +locals { + # hard coded to save time , fixme use a caching system + # we could generate a tf file + ami_id = "ami-0325b9a2dfb474b2d" } - module "ssm_observer" { source = "../../modules/aws/ssm/observability" - ami_id = data.aws_ami.ami.id + #ami_id = data.aws_ami.ami.id + ami_id = local.ami_id } module "ssm_setup" { @@ -81,3 +76,18 @@ module "ssm_setup" { access_log_bucket_name = "tine-session-access-logs" project = "tine" } + + + # now after we create the above resources, we can do the following, + # FIXME need to add dependencies + module "eliza_server" { + #count = 0 + #aws_account_id = local.account + aws_account_id =var.aws_account_id + region = local.region + source = "../../environments/swarms-aws-agent-api/dev/us-east-1" # FIXME rename + domain = local.dns + ami_id = local.ami_id #data.aws_ami.ami.id + name = "eliza" + tags = { project = "eliza" } +} diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index d34c7fc..8601a2f 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -293,7 +293,7 @@ module "lt_dynamic_ami_docker_normal" { # } # } -module "asg_dynamic_new_ami_dev_normal" { +module "asg" { # built with packer # count =0 From 0982e8810a4c55054c914cefad9021839949da44 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Thu, 9 Jan 2025 15:53:47 -0500 Subject: [PATCH 127/158] applied --- .../dev/us-east-1/components/keypairs/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/keypairs/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/keypairs/main.tf index 8a8040b..1cf1255 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/keypairs/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/keypairs/main.tf @@ -1,4 +1,4 @@ resource "aws_key_pair" "mdupont-deployer" { - key_name = "mdupont-deployer-key" + key_name = "mdupont-deployer-key-2" public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCYh8dzILXDkDpXDJ+wUiru8EPNPETWWHHFlnVI7Uc2Bb2f/yHJL9bV0QUQ+/hN2OGeN3r4z34vf444A7oMXT8T2bnMDzXqGfrqpQ6+Xs7Cu2F6wGik+KDBsu52vhaATVLOnDegbhpQW+IikSvxe4huKOvQpF+p7Ex80B+XpBPEV23DXapjZI+FIsbYoD4Mp5qY/PmXisNCByayhBG7WBhCEtHxkvpFntkz/9bwk2kC/z2W1SIHufN5TbrxKPKWY5iguW0Mn2e/rNvxnxFZaRx224rQnRFBMSq4Oi91MNdilwDHFzkv4oVBtpUmCet84np8+DxCfzphyIpo899dRV+/f7dwb6ZY3cvBkALcWahsscuE4ypbroXQ40UPAa3gW1PirTNdMEiX+Ie/IzEDWWCJKdDv4JaGtKAPORfC7bbXnBYn5RASglOjI24w974Llyj5TXXKexjxsjF3wlSS6pNHFlFJnQzVfemcY6AqSJ0Xr8dfFbxpSYH9OFkvBhzPaec= mdupont@mdupont-G470" } From 723bfaded8f91aeea46fa2417230fb96e84109ad Mon Sep 17 00:00:00 2001 From: mike dupont Date: Thu, 9 Jan 2025 16:52:32 -0500 Subject: [PATCH 128/158] fixing name of log file the current log file is tina-session-logs for my agent, this needs to be a variable for others to use. the aws buckets are unique globally so pick your names --- .../dev/us-east-1/components/roles/main.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf index 42fb875..a785380 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf @@ -68,7 +68,7 @@ data "aws_iam_policy_document" "default" { statement { effect = "Allow" - resources = ["arn:aws:s3:::swarms-session-logs*"] + resources = ["arn:aws:s3:::tine-session-logs*"] # FIXME must match actions = [ "s3:GetEncryptionConfiguration" ] @@ -145,7 +145,7 @@ data "aws_iam_policy" "AmazonSSMManagedInstanceCore" { resource "aws_iam_role" "ssm" { - name = "ssm-swarms-role" + name = "ssm-agent-role" tags = var.tags assume_role_policy = jsonencode({ @@ -164,7 +164,7 @@ resource "aws_iam_role" "ssm" { } resource "aws_iam_instance_profile" "ssm" { - name = "ssm-swarms-profile" + name = "ssm-agent-profile" role = aws_iam_role.ssm.name tags = var.tags } From d4a3eaa068e821ac08393776762c0700d0e920ff Mon Sep 17 00:00:00 2001 From: mike dupont Date: Thu, 9 Jan 2025 20:39:24 -0500 Subject: [PATCH 129/158] work in progress starting to boot --- README.md | 3 ++ accounts/mdupont/main.tf | 2 +- .../launch_template_docker_mcs/main.tf | 2 +- .../dev/us-east-1/main.tf | 38 +++++++++---------- .../dev/us-east-1/mcs/main.tf | 14 +++---- 5 files changed, 30 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index bdbcb19..5fe28cb 100644 --- a/README.md +++ b/README.md @@ -439,3 +439,6 @@ This structure facilitates a modular, maintainable, and automated approach to in It seems you might want to continue with a task or scenario related to the previous content but haven't specified what you'd like next. If you need more details or a specific aspect of the Terraform setup, architecture diagram, or other inquiries, please let me know, and I'll be glad to assist! + + +aws ssm start-session --target i-0e156165e86473c93 --profile mdupont --region us-east-2 diff --git a/accounts/mdupont/main.tf b/accounts/mdupont/main.tf index 68f2f30..b5f64cf 100644 --- a/accounts/mdupont/main.tf +++ b/accounts/mdupont/main.tf @@ -61,7 +61,7 @@ locals { locals { # hard coded to save time , fixme use a caching system - # we could generate a tf file + ami_id = "ami-0325b9a2dfb474b2d" } module "ssm_observer" { diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template_docker_mcs/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template_docker_mcs/main.tf index b935bae..ffdc0f3 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template_docker_mcs/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template_docker_mcs/main.tf @@ -15,7 +15,7 @@ variable "instance_type" {} locals { tags = { - project = "mcs" + project = "agent" instance_type = var.instance_type name = var.name } diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index 8601a2f..7ef4340 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -85,23 +85,23 @@ module "roles" { } -module "lt_dynamic_ami_prod" { - vpc_id = local.vpc_id - for_each = toset(var.instance_types) - instance_type = each.key - name = "eliza-ami-${each.key}" - security_group_id = module.security.internal_security_group_id - ami_id = local.ami_id - key_name = var.key_name - tags = merge(local.tags, { - environment = "production" - }) - source = "./components/launch_template" - iam_instance_profile_name = module.roles.ssm_profile_name - install_script = "/opt/eliza/api/just_run.sh" - ssm_parameter_name_cw_agent_config = "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/cloudwatch-agent/config" - branch = "feature/ec2" -} +# module "lt_dynamic_ami_prod" { +# vpc_id = local.vpc_id +# for_each = toset(var.instance_types) +# instance_type = each.key +# name = "eliza-ami-${each.key}" +# security_group_id = module.security.internal_security_group_id +# ami_id = local.ami_id +# key_name = var.key_name +# tags = merge(local.tags, { +# environment = "production" +# }) +# source = "./components/launch_template" +# iam_instance_profile_name = module.roles.ssm_profile_name +# install_script = "/opt/eliza/api/just_run.sh" +# ssm_parameter_name_cw_agent_config = "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/cloudwatch-agent/config" +# branch = "feature/ec2" +# } module "lt_dynamic_ami_test" { branch = "feature/cloudwatch" @@ -320,7 +320,7 @@ output "security_group_id" { module "eliza" { source = "./mcs" # fixme rename to eliza - branch = "feature/reduce_modules_discord" + branch = "feature/aws" #git_repo = "https://github.com/meta-introspector/eliza-MedicalCoderSwarm-deployment.git" git_repo = "https://github.com/meta-introspector/cloud-deployment-eliza/" # alb_target_group_arn = module.alb.mcs_alb_target_group_arn @@ -334,7 +334,7 @@ module "eliza" { tags = local.tags ami_id = local.ami_id vpc_id = local.vpc_id - name = "docker-mcs-ami" + name = "docker-agent-ami" } # module "agent_dev" { diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf index 4a7d427..7e2cf92 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf @@ -1,7 +1,5 @@ variable name {} -variable branch { - default = "feature/mcs" -} +variable branch {} variable git_repo { default = "https://github.com/jmikedupont2/swarms-MedicalCoderSwarm-deployment.git" @@ -38,20 +36,20 @@ module "lt_docker" { vpc_id = var.vpc_id for_each = toset(var.instance_types) instance_type = each.key - name = "mcs-docker-${each.key}" + name = "agent-docker-${each.key}" security_group_id = var.internal_security_group_id ami_id = var.ami_id git_repo = var.git_repo - app_name = "mcs" # used to construct /opt/mcs for where we install to + app_name = "agent" # used to construct /opt/agent for where we install to tags = merge(var.tags, { - environment = "mcs" + environment = "agent" }) key_name = var.key_name #"mdupont-deployer-key" ssm_parameter_name_cw_agent_config = "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/cloudwatch-agent/config/details" iam_instance_profile_name = var.iam_instance_profile_name - #install_script = "/opt/mcs/api/docker-boot.sh" this is called from ssm for a refresh - install_script = "/opt/mcs/api/rundocker.sh" + #install_script = "/opt/agent/api/docker-boot.sh" this is called from ssm for a refresh + install_script = "/opt/agent/rundocker.sh" } From 27ee585112ac094b2b093bcbfcc0978d177d8c29 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Fri, 10 Jan 2025 07:45:25 -0500 Subject: [PATCH 130/158] applied --- .../swarms-aws-agent-api/dev/us-east-1/main.tf | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf index 7ef4340..8d74de6 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/main.tf @@ -55,13 +55,14 @@ module "kp" { variable "instance_types" { type = list(string) default = [ - # "t4g.nano", "t3a.nano", "t3.nano", "t2.nano", - # "t4g.micro", "t3a.micro", "t3.micro", "t2.micro", "t1.micro", - #"t4g.small", "t3a.small", - #"t3.small", - #"t2.small", not working - # "t2.medium" # - "t3.medium" + "t4g.nano", "t3a.nano", "t3.nano", "t2.nano", + "t4g.micro", "t3a.micro", "t3.micro", "t2.micro", "t1.micro", + "t4g.small", + "t3a.small", + "t3.small", + "t2.small", + "t2.medium" , + "t3.medium" # works for sure ] } From 97d36b06f17f20bf6287b73f0c60bf820bb2e62f Mon Sep 17 00:00:00 2001 From: mike dupont Date: Fri, 10 Jan 2025 07:59:32 -0500 Subject: [PATCH 131/158] new sizes --- .../swarms-aws-agent-api/dev/us-east-1/mcs/main.tf | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf b/environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf index 7e2cf92..dde5c5d 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf +++ b/environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf @@ -26,7 +26,18 @@ variable "instance_types" { default = [ #"t3.small", # "t2.medium" # - "t3.medium" + #"t3.medium" + # "t4g.nano", + "t3a.nano", "t3.nano", "t2.nano", + # "t4g.micro", + "t3a.micro", "t3.micro", "t2.micro", + #"t1.micro", +# "t4g.small", + "t3a.small", + "t3.small", + "t2.small", + "t3.medium" # works for sure + # fixme pass this list in ] } From c9a23d4af53dd080cba59368d3ade5da70955989 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sun, 12 Jan 2025 06:45:39 -0500 Subject: [PATCH 132/158] renaming --- accounts/AI_Token_Team/main.tf | 42 +++++++++++++++++++ .../us-east-1 => eliza-agent-api}/Makefile | 0 .../us-east-1 => eliza-agent-api}/Readme.md | 0 .../cloudflare.tf | 0 .../components/Readme.md | 0 .../application_load_balancer/Readme.md | 0 .../application_load_balancer/cloudflare.tf | 0 .../application_load_balancer/https/main.tf | 0 .../application_load_balancer/main.tf | 0 .../application_load_balancer/notes.prg | 0 .../application_load_balancer/route53/main.tf | 0 .../target_group/main.tf | 0 .../components/autoscaling_group/Makefile | 0 .../components/autoscaling_group/Readme.md | 0 .../components/autoscaling_group/main.tf | 0 .../components/autoscaling_group/spot/main.tf | 0 .../components/cognito_user_pool/README.md | 0 .../components/cognito_user_pool/Readme.md | 0 .../components/cognito_user_pool/main.tf | 0 .../components/cognito_user_pool/provider.tf | 0 .../components/cognito_user_pool/variables.tf | 0 .../components/keypairs/main.tf | 0 .../components/lambda_workers/Readme.md | 0 .../launch_template/.terraform.lock.hcl | 0 .../components/launch_template/Makefile | 0 .../components/launch_template/main.tf | 0 .../components/launch_template_docker/main.tf | 0 .../launch_template_docker_mcs/main.tf | 0 .../machine_image/.terraform.lock.hcl | 0 .../components/machine_image/Makefile | 0 .../components/machine_image/Readme.md | 0 .../components/machine_image/main.tf | 0 .../components/machine_image/main.tf.asg | 0 .../machine_image/main.tf.consts.org | 0 .../machine_image/ubuntu-fastapi.pkr.hcl | 0 .../components/machine_image/variables.tf | 0 .../components/machine_image/versions.tf | 0 .../components/resource_launchers/Readme.md | 0 .../resource_launchers/lambda/scale_lambda.py | 0 .../components/resource_launchers/main.tf | 0 .../components/roles/main.tf | 0 .../components/security/main.tf | 0 .../components/ses/context.tf | 0 .../components/ses/main.tf | 0 .../components/vpc/.terraform.lock.hcl | 0 .../components/vpc/main.tf | 0 .../components/vpc/outputs.tf | 0 .../components/vpc/versions.tf | 0 .../components/work_queue/Readme.md | 0 .../dev/us-east-1/.terraform.lock.hcl | 0 .../mcs => eliza-agent-api/eliza}/main.tf | 6 ++- .../us-east-1 => eliza-agent-api}/main-1.tf | 0 .../dev/us-east-1 => eliza-agent-api}/main.tf | 23 +++++----- .../us-east-1 => eliza-agent-api}/refresh.sh | 0 .../us-east-1 => eliza-agent-api}/rewrite.txt | 0 .../dev/us-east-1 => eliza-agent-api}/seed.md | 0 56 files changed, 57 insertions(+), 14 deletions(-) create mode 100644 accounts/AI_Token_Team/main.tf rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/Makefile (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/Readme.md (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/cloudflare.tf (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/Readme.md (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/application_load_balancer/Readme.md (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/application_load_balancer/cloudflare.tf (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/application_load_balancer/https/main.tf (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/application_load_balancer/main.tf (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/application_load_balancer/notes.prg (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/application_load_balancer/route53/main.tf (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/application_load_balancer/target_group/main.tf (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/autoscaling_group/Makefile (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/autoscaling_group/Readme.md (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/autoscaling_group/main.tf (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/autoscaling_group/spot/main.tf (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/cognito_user_pool/README.md (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/cognito_user_pool/Readme.md (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/cognito_user_pool/main.tf (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/cognito_user_pool/provider.tf (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/cognito_user_pool/variables.tf (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/keypairs/main.tf (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/lambda_workers/Readme.md (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/launch_template/.terraform.lock.hcl (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/launch_template/Makefile (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/launch_template/main.tf (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/launch_template_docker/main.tf (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/launch_template_docker_mcs/main.tf (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/machine_image/.terraform.lock.hcl (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/machine_image/Makefile (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/machine_image/Readme.md (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/machine_image/main.tf (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/machine_image/main.tf.asg (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/machine_image/main.tf.consts.org (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/machine_image/ubuntu-fastapi.pkr.hcl (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/machine_image/variables.tf (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/machine_image/versions.tf (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/resource_launchers/Readme.md (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/resource_launchers/lambda/scale_lambda.py (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/resource_launchers/main.tf (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/roles/main.tf (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/security/main.tf (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/ses/context.tf (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/ses/main.tf (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/vpc/.terraform.lock.hcl (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/vpc/main.tf (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/vpc/outputs.tf (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/vpc/versions.tf (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/components/work_queue/Readme.md (100%) rename environments/{swarms-aws-agent-api => eliza-agent-api}/dev/us-east-1/.terraform.lock.hcl (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1/mcs => eliza-agent-api/eliza}/main.tf (95%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/main-1.tf (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/main.tf (98%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/refresh.sh (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/rewrite.txt (100%) rename environments/{swarms-aws-agent-api/dev/us-east-1 => eliza-agent-api}/seed.md (100%) diff --git a/accounts/AI_Token_Team/main.tf b/accounts/AI_Token_Team/main.tf new file mode 100644 index 0000000..0506a09 --- /dev/null +++ b/accounts/AI_Token_Team/main.tf @@ -0,0 +1,42 @@ +provider "aws" { + region = "us-west-1" + profile = "ai-token-team" +} + +locals { + dns = "aitokenteam.com" # fixme + region = "us-west-1" + project = "ai-token-team" # dont change +} + +locals { + # hard coded to save time , fixme use a caching system +# ami_id = "ami-0325b9a2dfb474b2d" +} +module "ssm_observer" { + source = "../../modules/aws/ssm/observability" + #ami_id = data.aws_ami.ami.id + ami_id = local.ami_id +} + +module "ssm_setup" { + source = "../../modules/aws/ssm/setup" + bucket_name = "${local.project}session-logs" + access_log_bucket_name = "${local.project}-session-access-logs" + project = local.project + } + + + module "eliza_server" { + #count = 0 + #aws_account_id = local.account + aws_account_id =var.aws_account_id + region = local.region + source = "../../environments/swarms-aws-agent-api/dev/us-east-1" # FIXME rename + domain = local.dns + ami_id = local.ami_id #data.aws_ami.ami.id + name = local.project + key_name= = "ai-token-deployer-key" + tags = { project = local.project } + ami_name = "ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*" +} diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/Makefile b/environments/eliza-agent-api/Makefile similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/Makefile rename to environments/eliza-agent-api/Makefile diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/Readme.md b/environments/eliza-agent-api/Readme.md similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/Readme.md rename to environments/eliza-agent-api/Readme.md diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/cloudflare.tf b/environments/eliza-agent-api/cloudflare.tf similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/cloudflare.tf rename to environments/eliza-agent-api/cloudflare.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md b/environments/eliza-agent-api/components/Readme.md similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/Readme.md rename to environments/eliza-agent-api/components/Readme.md diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/Readme.md b/environments/eliza-agent-api/components/application_load_balancer/Readme.md similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/Readme.md rename to environments/eliza-agent-api/components/application_load_balancer/Readme.md diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/cloudflare.tf b/environments/eliza-agent-api/components/application_load_balancer/cloudflare.tf similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/cloudflare.tf rename to environments/eliza-agent-api/components/application_load_balancer/cloudflare.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf b/environments/eliza-agent-api/components/application_load_balancer/https/main.tf similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/https/main.tf rename to environments/eliza-agent-api/components/application_load_balancer/https/main.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf b/environments/eliza-agent-api/components/application_load_balancer/main.tf similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/main.tf rename to environments/eliza-agent-api/components/application_load_balancer/main.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/notes.prg b/environments/eliza-agent-api/components/application_load_balancer/notes.prg similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/notes.prg rename to environments/eliza-agent-api/components/application_load_balancer/notes.prg diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf b/environments/eliza-agent-api/components/application_load_balancer/route53/main.tf similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/route53/main.tf rename to environments/eliza-agent-api/components/application_load_balancer/route53/main.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/target_group/main.tf b/environments/eliza-agent-api/components/application_load_balancer/target_group/main.tf similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/application_load_balancer/target_group/main.tf rename to environments/eliza-agent-api/components/application_load_balancer/target_group/main.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/Makefile b/environments/eliza-agent-api/components/autoscaling_group/Makefile similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/Makefile rename to environments/eliza-agent-api/components/autoscaling_group/Makefile diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/Readme.md b/environments/eliza-agent-api/components/autoscaling_group/Readme.md similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/Readme.md rename to environments/eliza-agent-api/components/autoscaling_group/Readme.md diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf b/environments/eliza-agent-api/components/autoscaling_group/main.tf similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/main.tf rename to environments/eliza-agent-api/components/autoscaling_group/main.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/spot/main.tf b/environments/eliza-agent-api/components/autoscaling_group/spot/main.tf similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/autoscaling_group/spot/main.tf rename to environments/eliza-agent-api/components/autoscaling_group/spot/main.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/README.md b/environments/eliza-agent-api/components/cognito_user_pool/README.md similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/README.md rename to environments/eliza-agent-api/components/cognito_user_pool/README.md diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/Readme.md b/environments/eliza-agent-api/components/cognito_user_pool/Readme.md similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/Readme.md rename to environments/eliza-agent-api/components/cognito_user_pool/Readme.md diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/main.tf b/environments/eliza-agent-api/components/cognito_user_pool/main.tf similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/main.tf rename to environments/eliza-agent-api/components/cognito_user_pool/main.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/provider.tf b/environments/eliza-agent-api/components/cognito_user_pool/provider.tf similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/provider.tf rename to environments/eliza-agent-api/components/cognito_user_pool/provider.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/variables.tf b/environments/eliza-agent-api/components/cognito_user_pool/variables.tf similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/cognito_user_pool/variables.tf rename to environments/eliza-agent-api/components/cognito_user_pool/variables.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/keypairs/main.tf b/environments/eliza-agent-api/components/keypairs/main.tf similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/keypairs/main.tf rename to environments/eliza-agent-api/components/keypairs/main.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/lambda_workers/Readme.md b/environments/eliza-agent-api/components/lambda_workers/Readme.md similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/lambda_workers/Readme.md rename to environments/eliza-agent-api/components/lambda_workers/Readme.md diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/.terraform.lock.hcl b/environments/eliza-agent-api/components/launch_template/.terraform.lock.hcl similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/.terraform.lock.hcl rename to environments/eliza-agent-api/components/launch_template/.terraform.lock.hcl diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/Makefile b/environments/eliza-agent-api/components/launch_template/Makefile similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/Makefile rename to environments/eliza-agent-api/components/launch_template/Makefile diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf b/environments/eliza-agent-api/components/launch_template/main.tf similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template/main.tf rename to environments/eliza-agent-api/components/launch_template/main.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template_docker/main.tf b/environments/eliza-agent-api/components/launch_template_docker/main.tf similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template_docker/main.tf rename to environments/eliza-agent-api/components/launch_template_docker/main.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template_docker_mcs/main.tf b/environments/eliza-agent-api/components/launch_template_docker_mcs/main.tf similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/launch_template_docker_mcs/main.tf rename to environments/eliza-agent-api/components/launch_template_docker_mcs/main.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/.terraform.lock.hcl b/environments/eliza-agent-api/components/machine_image/.terraform.lock.hcl similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/.terraform.lock.hcl rename to environments/eliza-agent-api/components/machine_image/.terraform.lock.hcl diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/Makefile b/environments/eliza-agent-api/components/machine_image/Makefile similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/Makefile rename to environments/eliza-agent-api/components/machine_image/Makefile diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/Readme.md b/environments/eliza-agent-api/components/machine_image/Readme.md similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/Readme.md rename to environments/eliza-agent-api/components/machine_image/Readme.md diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf b/environments/eliza-agent-api/components/machine_image/main.tf similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf rename to environments/eliza-agent-api/components/machine_image/main.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf.asg b/environments/eliza-agent-api/components/machine_image/main.tf.asg similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf.asg rename to environments/eliza-agent-api/components/machine_image/main.tf.asg diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf.consts.org b/environments/eliza-agent-api/components/machine_image/main.tf.consts.org similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/main.tf.consts.org rename to environments/eliza-agent-api/components/machine_image/main.tf.consts.org diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/ubuntu-fastapi.pkr.hcl b/environments/eliza-agent-api/components/machine_image/ubuntu-fastapi.pkr.hcl similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/ubuntu-fastapi.pkr.hcl rename to environments/eliza-agent-api/components/machine_image/ubuntu-fastapi.pkr.hcl diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/variables.tf b/environments/eliza-agent-api/components/machine_image/variables.tf similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/variables.tf rename to environments/eliza-agent-api/components/machine_image/variables.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/versions.tf b/environments/eliza-agent-api/components/machine_image/versions.tf similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/machine_image/versions.tf rename to environments/eliza-agent-api/components/machine_image/versions.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/resource_launchers/Readme.md b/environments/eliza-agent-api/components/resource_launchers/Readme.md similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/resource_launchers/Readme.md rename to environments/eliza-agent-api/components/resource_launchers/Readme.md diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/resource_launchers/lambda/scale_lambda.py b/environments/eliza-agent-api/components/resource_launchers/lambda/scale_lambda.py similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/resource_launchers/lambda/scale_lambda.py rename to environments/eliza-agent-api/components/resource_launchers/lambda/scale_lambda.py diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/resource_launchers/main.tf b/environments/eliza-agent-api/components/resource_launchers/main.tf similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/resource_launchers/main.tf rename to environments/eliza-agent-api/components/resource_launchers/main.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf b/environments/eliza-agent-api/components/roles/main.tf similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/roles/main.tf rename to environments/eliza-agent-api/components/roles/main.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/security/main.tf b/environments/eliza-agent-api/components/security/main.tf similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/security/main.tf rename to environments/eliza-agent-api/components/security/main.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/ses/context.tf b/environments/eliza-agent-api/components/ses/context.tf similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/ses/context.tf rename to environments/eliza-agent-api/components/ses/context.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/ses/main.tf b/environments/eliza-agent-api/components/ses/main.tf similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/ses/main.tf rename to environments/eliza-agent-api/components/ses/main.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/.terraform.lock.hcl b/environments/eliza-agent-api/components/vpc/.terraform.lock.hcl similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/.terraform.lock.hcl rename to environments/eliza-agent-api/components/vpc/.terraform.lock.hcl diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/main.tf b/environments/eliza-agent-api/components/vpc/main.tf similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/main.tf rename to environments/eliza-agent-api/components/vpc/main.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/outputs.tf b/environments/eliza-agent-api/components/vpc/outputs.tf similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/outputs.tf rename to environments/eliza-agent-api/components/vpc/outputs.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/versions.tf b/environments/eliza-agent-api/components/vpc/versions.tf similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/vpc/versions.tf rename to environments/eliza-agent-api/components/vpc/versions.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/components/work_queue/Readme.md b/environments/eliza-agent-api/components/work_queue/Readme.md similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/components/work_queue/Readme.md rename to environments/eliza-agent-api/components/work_queue/Readme.md diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/.terraform.lock.hcl b/environments/eliza-agent-api/dev/us-east-1/.terraform.lock.hcl similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/.terraform.lock.hcl rename to environments/eliza-agent-api/dev/us-east-1/.terraform.lock.hcl diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf b/environments/eliza-agent-api/eliza/main.tf similarity index 95% rename from environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf rename to environments/eliza-agent-api/eliza/main.tf index dde5c5d..914d6f9 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/mcs/main.tf +++ b/environments/eliza-agent-api/eliza/main.tf @@ -28,9 +28,11 @@ variable "instance_types" { # "t2.medium" # #"t3.medium" # "t4g.nano", - "t3a.nano", "t3.nano", "t2.nano", + #"t3a.nano", "t3.nano", "t2.nano", connection refused # "t4g.micro", - "t3a.micro", "t3.micro", "t2.micro", + #"t3a.micro", + #"t3.micro", + #"t2.micro", failed to boot docker #"t1.micro", # "t4g.small", "t3a.small", diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main-1.tf b/environments/eliza-agent-api/main-1.tf similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/main-1.tf rename to environments/eliza-agent-api/main-1.tf diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf b/environments/eliza-agent-api/main.tf similarity index 98% rename from environments/swarms-aws-agent-api/dev/us-east-1/main.tf rename to environments/eliza-agent-api/main.tf index 8d74de6..87cb9f2 100644 --- a/environments/swarms-aws-agent-api/dev/us-east-1/main.tf +++ b/environments/eliza-agent-api/main.tf @@ -1,28 +1,27 @@ -variable "spot_max_price" { +vasriable "spot_max_price" { default = 0.028 } variable "region" {} +variable "domain" {} +variable "aws_account_id" {} +variable "ami_id" {} +variable "tags" {} +variable "name" {} + variable "key_name" { - default = "mdupont-deployer-key" # FIXME: move to settings +# default = "mdupont-deployer-key" # FIXME: move to settings } locals { # instance_type = "t3.large" # instance_type = "t3.medium" ami_name = "ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*" - name = "eliza" + name = var.project domain = var.domain tags = { - project = "eliza" - } - dev_tags = { - sandbox = "kye" + project = var.project } + dev_tags = { } } -variable "domain" {} -variable "aws_account_id" {} -variable "ami_id" {} -variable "tags" {} -variable "name" {} locals { diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/refresh.sh b/environments/eliza-agent-api/refresh.sh similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/refresh.sh rename to environments/eliza-agent-api/refresh.sh diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/rewrite.txt b/environments/eliza-agent-api/rewrite.txt similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/rewrite.txt rename to environments/eliza-agent-api/rewrite.txt diff --git a/environments/swarms-aws-agent-api/dev/us-east-1/seed.md b/environments/eliza-agent-api/seed.md similarity index 100% rename from environments/swarms-aws-agent-api/dev/us-east-1/seed.md rename to environments/eliza-agent-api/seed.md From 8db2f0fe44f662781a8273d5e78574946a5815a6 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sun, 12 Jan 2025 07:29:07 -0500 Subject: [PATCH 133/158] wip --- accounts/AI_Token_Team/main.tf | 35 +- accounts/AI_Token_Team/variables.tf | 54 +++ .../eliza-agent-api/components/vpc/main.tf | 176 +--------- environments/eliza-agent-api/eliza/main.tf | 28 +- environments/eliza-agent-api/main.tf | 320 +----------------- 5 files changed, 105 insertions(+), 508 deletions(-) create mode 100644 accounts/AI_Token_Team/variables.tf diff --git a/accounts/AI_Token_Team/main.tf b/accounts/AI_Token_Team/main.tf index 0506a09..54f9b60 100644 --- a/accounts/AI_Token_Team/main.tf +++ b/accounts/AI_Token_Team/main.tf @@ -9,14 +9,10 @@ locals { project = "ai-token-team" # dont change } -locals { - # hard coded to save time , fixme use a caching system -# ami_id = "ami-0325b9a2dfb474b2d" -} module "ssm_observer" { source = "../../modules/aws/ssm/observability" - #ami_id = data.aws_ami.ami.id - ami_id = local.ami_id + ami_id = data.aws_ami.ami.id + } module "ssm_setup" { @@ -26,17 +22,34 @@ module "ssm_setup" { project = local.project } +locals { ami_name = "ubuntu-minimal/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-minimal-*" } + data "aws_ami" "ami" { # slow + most_recent = true + name_regex = "^${local.ami_name}" + } module "eliza_server" { #count = 0 #aws_account_id = local.account aws_account_id =var.aws_account_id region = local.region - source = "../../environments/swarms-aws-agent-api/dev/us-east-1" # FIXME rename + source = "../../environments/eliza-agent-api/" # FIXME rename domain = local.dns - ami_id = local.ami_id #data.aws_ami.ami.id + ami_id = data.aws_ami.ami.id name = local.project - key_name= = "ai-token-deployer-key" + project = local.project + key_name = "ai-token-deployer-key" tags = { project = local.project } - ami_name = "ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*" -} + + branch = "feature/AI_Token_Team" + spot_max_price = 0.028 + instance_types = [ + "t3a.small", +# "t3.small", +# "t2.small", +# "t3.medium" # works for sure + + ] + + aws_availability_zones = ["us-west-1a","us-west-1b","us-west-1c"] + } diff --git a/accounts/AI_Token_Team/variables.tf b/accounts/AI_Token_Team/variables.tf new file mode 100644 index 0000000..33de27a --- /dev/null +++ b/accounts/AI_Token_Team/variables.tf @@ -0,0 +1,54 @@ +variable "project_name" { + type = string + default = "ai-token-team" +} + +variable "profile" { + type = string + default = "ai-token-team" +} + +variable "aws_region" { + type = string + default = "us-west-1" +} + +variable "aws_account_id" { + type = string + default = "699475930706" +} + +variable "iam_user" { + type = string + default = "aitokenteam" +} + +variable "lock_resource" { + type = string + default = "terraform/state/lock" +} + +variable "partition" { + type = string + default = "aws" +} + +variable "logs_resource" { + type = string + default = "aws_logs" +} + +variable "permissions_check" { + type = string + default = "config-permissions-check" +} + +variable "delivery_service" { + type = string + default = "delivery.logs.amazonaws.com" +} + +variable "logging_service" { + type = string + default = "logging.s3.amazonaws.com" +} diff --git a/environments/eliza-agent-api/components/vpc/main.tf b/environments/eliza-agent-api/components/vpc/main.tf index daba127..4374c88 100644 --- a/environments/eliza-agent-api/components/vpc/main.tf +++ b/environments/eliza-agent-api/components/vpc/main.tf @@ -1,12 +1,14 @@ #from https://github.com/terraform-aws-modules/terraform-aws-vpc data "aws_availability_zones" "available" {} - +variable name {} +variable aws_availability_zones {} locals { - name = "swarms" - + name = var.name vpc_cidr = "10.0.0.0/16" - azs = slice(data.aws_availability_zones.available.names, 0, 3) + + #azs = slice(data.aws_availability_zones.available.names, 0, 3) + azs = var.aws_availability_zones tags = { project = local.name @@ -15,31 +17,6 @@ locals { } } -# resource "vpc" "swarms" { -# source = "terraform-aws-modules/vpc/aws" -# #source = "https://github.com/terraform-aws-modules/terraform-aws-vpc.git" -# name = "swarms" -# cidr = "10.0.0.0/16" -# private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] -# public_subnets = ["10.0.101.0/24", "10.0.102.0/24", "10.0.103.0/24"] - -# enable_nat_gateway = true -# enable_vpn_gateway = true - -# tags = { -# Terraform = "true" -# Environment = "dev" -# } -# } - -# provider_name = "aws" - - - -################################################################################ -# VPC Module -################################################################################ - module "vpc" { source = "terraform-aws-modules/vpc/aws" name = local.name @@ -92,143 +69,10 @@ module "vpc" { tags = local.tags } -# ################################################################################ -# # VPC Endpoints Module -# ################################################################################ - -# module "vpc_endpoints" { -# source = "../../modules/vpc-endpoints" - -# vpc_id = module.vpc.vpc_id - -# create_security_group = true -# security_group_name_prefix = "${local.name}-vpc-endpoints-" -# security_group_description = "VPC endpoint security group" -# security_group_rules = { -# ingress_https = { -# description = "HTTPS from VPC" -# cidr_blocks = [module.vpc.vpc_cidr_block] -# } -# } - -# endpoints = { -# s3 = { -# service = "s3" -# private_dns_enabled = true -# dns_options = { -# private_dns_only_for_inbound_resolver_endpoint = false -# } -# tags = { Name = "s3-vpc-endpoint" } -# }, -# dynamodb = { -# service = "dynamodb" -# service_type = "Gateway" -# route_table_ids = flatten([module.vpc.intra_route_table_ids, module.vpc.private_route_table_ids, module.vpc.public_route_table_ids]) -# policy = data.aws_iam_policy_document.dynamodb_endpoint_policy.json -# tags = { Name = "dynamodb-vpc-endpoint" } -# }, -# ecs = { -# service = "ecs" -# private_dns_enabled = true -# subnet_ids = module.vpc.private_subnets -# }, -# ecs_telemetry = { -# create = false -# service = "ecs-telemetry" -# private_dns_enabled = true -# subnet_ids = module.vpc.private_subnets -# }, -# ecr_api = { -# service = "ecr.api" -# private_dns_enabled = true -# subnet_ids = module.vpc.private_subnets -# policy = data.aws_iam_policy_document.generic_endpoint_policy.json -# }, -# ecr_dkr = { -# service = "ecr.dkr" -# private_dns_enabled = true -# subnet_ids = module.vpc.private_subnets -# policy = data.aws_iam_policy_document.generic_endpoint_policy.json -# }, -# rds = { -# service = "rds" -# private_dns_enabled = true -# subnet_ids = module.vpc.private_subnets -# security_group_ids = [aws_security_group.rds.id] -# }, -# } - -# tags = merge(local.tags, { -# Project = "Secret" -# Endpoint = "true" -# }) -# } - -# module "vpc_endpoints_nocreate" { -# source = "../../modules/vpc-endpoints" - -# create = false -# } - -# ################################################################################ -# # Supporting Resources -# ################################################################################ - -# data "aws_iam_policy_document" "dynamodb_endpoint_policy" { -# statement { -# effect = "Deny" -# actions = ["dynamodb:*"] -# resources = ["*"] - -# principals { -# type = "*" -# identifiers = ["*"] -# } - -# condition { -# test = "StringNotEquals" -# variable = "aws:sourceVpc" - -# values = [module.vpc.vpc_id] -# } -# } -# } - -# data "aws_iam_policy_document" "generic_endpoint_policy" { -# statement { -# effect = "Deny" -# actions = ["*"] -# resources = ["*"] - -# principals { -# type = "*" -# identifiers = ["*"] -# } - -# condition { -# test = "StringNotEquals" -# variable = "aws:SourceVpc" - -# values = [module.vpc.vpc_id] -# } -# } -# } - -# resource "aws_security_group" "rds" { -# name_prefix = "${local.name}-rds" -# description = "Allow PostgreSQL inbound traffic" -# vpc_id = module.vpc.vpc_id - -# ingress { -# description = "TLS from VPC" -# from_port = 5432 -# to_port = 5432 -# protocol = "tcp" -# cidr_blocks = [module.vpc.vpc_cidr_block] -# } - -# tags = local.tags -# } output "vpc" { value = module.vpc } + +output "azs" { + value = data.aws_availability_zones.available.names +} diff --git a/environments/eliza-agent-api/eliza/main.tf b/environments/eliza-agent-api/eliza/main.tf index 914d6f9..3b75b13 100644 --- a/environments/eliza-agent-api/eliza/main.tf +++ b/environments/eliza-agent-api/eliza/main.tf @@ -1,20 +1,14 @@ variable name {} +#variable ami_name {} variable branch {} - -variable git_repo { - default = "https://github.com/jmikedupont2/swarms-MedicalCoderSwarm-deployment.git" -} +variable git_repo {} variable "ec2_subnet_id" {} #variable "alb_target_group_arn" {} variable "aws_account_id" {} variable "vpc_id" {} -# FIXME security, we use the ami of swarms for speed but want to split variable "ami_id" {} -# FIXME security, we use the sg of swarms for speed but want to split variable "internal_security_group_id" {} - -# FIXME security, we use the profile of swarms for speed but want to split variable "iam_instance_profile_name" {} variable "ssm_profile_arn" {} variable "key_name" {} @@ -23,24 +17,6 @@ variable "tags" {} variable "instance_types" { type = list(string) - default = [ - #"t3.small", - # "t2.medium" # - #"t3.medium" - # "t4g.nano", - #"t3a.nano", "t3.nano", "t2.nano", connection refused - # "t4g.micro", - #"t3a.micro", - #"t3.micro", - #"t2.micro", failed to boot docker - #"t1.micro", -# "t4g.small", - "t3a.small", - "t3.small", - "t2.small", - "t3.medium" # works for sure - # fixme pass this list in - ] } module "lt_docker" { diff --git a/environments/eliza-agent-api/main.tf b/environments/eliza-agent-api/main.tf index 87cb9f2..9c627f3 100644 --- a/environments/eliza-agent-api/main.tf +++ b/environments/eliza-agent-api/main.tf @@ -1,20 +1,19 @@ -vasriable "spot_max_price" { - default = 0.028 -} +variable instance_types {} +variable "spot_max_price" {} +variable "branch" {} +variable "project" {} + variable "region" {} variable "domain" {} variable "aws_account_id" {} variable "ami_id" {} +#variable "ami_name" {} + variable "tags" {} variable "name" {} +variable "key_name" {} -variable "key_name" { -# default = "mdupont-deployer-key" # FIXME: move to settings -} locals { - # instance_type = "t3.large" - # instance_type = "t3.medium" - ami_name = "ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*" name = var.project domain = var.domain tags = { @@ -23,15 +22,15 @@ locals { dev_tags = { } } - locals { ami_id = var.ami_id -# new_ami_id = "ami-08093b6770af41b14" # environments/eliza-aws-agent-api/dev/us-east-1/components/machine_image/Readme.md } -# SLOW module "vpc" { source = "./components/vpc" + name = var.project + aws_availability_zones = var.aws_availability_zones + } locals { @@ -51,283 +50,26 @@ module "kp" { source = "./components/keypairs" } -variable "instance_types" { - type = list(string) - default = [ - "t4g.nano", "t3a.nano", "t3.nano", "t2.nano", - "t4g.micro", "t3a.micro", "t3.micro", "t2.micro", "t1.micro", - "t4g.small", - "t3a.small", - "t3.small", - "t2.small", - "t2.medium" , - "t3.medium" # works for sure - ] -} - -variable "test_instance_types" { - type = list(string) - default = [ - # "t4g.nano", "t3a.nano", "t3.nano", "t2.nano", - # "t4g.micro", "t3a.micro", "t3.micro", "t2.micro", "t1.micro", - #"t4g.small", "t3a.small", - #"t3.small", - #"t2.small", not working - # "t2.medium" # - #"t3.medium" # no instances for now, this is commented out - ] -} - - module "roles" { source = "./components/roles" tags = local.tags } -# module "lt_dynamic_ami_prod" { -# vpc_id = local.vpc_id -# for_each = toset(var.instance_types) -# instance_type = each.key -# name = "eliza-ami-${each.key}" -# security_group_id = module.security.internal_security_group_id -# ami_id = local.ami_id -# key_name = var.key_name -# tags = merge(local.tags, { -# environment = "production" -# }) -# source = "./components/launch_template" -# iam_instance_profile_name = module.roles.ssm_profile_name -# install_script = "/opt/eliza/api/just_run.sh" -# ssm_parameter_name_cw_agent_config = "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/cloudwatch-agent/config" -# branch = "feature/ec2" -# } - -module "lt_dynamic_ami_test" { - branch = "feature/cloudwatch" - vpc_id = local.vpc_id - for_each = toset(var.instance_types) - instance_type = each.key - name = "eliza-ami-${each.key}" - security_group_id = module.security.internal_security_group_id - ami_id = local.ami_id - tags = merge(local.tags, { - environment = "test" - }) - source = "./components/launch_template" - key_name = var.key_name #"mdupont-deployer-key" - ssm_parameter_name_cw_agent_config = "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/cloudwatch-agent/config/details" - iam_instance_profile_name = module.roles.ssm_profile_name - install_script = "/opt/eliza/api/just_run.sh" -} - - -variable "dev_instance_types" { - type = list(string) - default = [ - # "t4g.nano", "t3a.nano", "t3.nano", "t2.nano", - # "t4g.micro", "t3a.micro", "t3.micro", "t2.micro", "t1.micro", - #"t4g.small", "t3a.small", - #"t3.small", - #"t2.small", not working - # "t2.medium" # - # "t3.small" - ] -} - -variable "dev2_instance_types" { - type = list(string) - default = [ - # "t4g.nano", "t3a.nano", "t3.nano", "t2.nano", - # "t4g.micro", "t3a.micro", "t3.micro", "t2.micro", "t1.micro", - #"t4g.small", "t3a.small", - #"t3.small", - #"t2.small", not working - # "t2.medium" # - #"t3.medium" - ] -} - -module "lt_dynamic_ami_docker" { - #branch = "feature/squash2-docker" - branch = "feature/merge_latest_675" - vpc_id = local.vpc_id - for_each = toset(var.dev_instance_types) - instance_type = each.key - name = "eliza-docker-${each.key}" - security_group_id = module.security.internal_security_group_id - ami_id = local.ami_id - tags = merge(local.tags, { - environment = "test" - }) - source = "./components/launch_template_docker" - key_name = var.key_name #"mdupont-deployer-key" - ssm_parameter_name_cw_agent_config = "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/cloudwatch-agent/config/details" - iam_instance_profile_name = module.roles.ssm_profile_name - #install_script = "/opt/eliza/api/docker-boot.sh" this is called from ssm for a refresh - install_script = "/opt/eliza/api/rundocker.sh" -} - -module "lt_dynamic_ami_docker_normal" { - #branch = "feature/squash2-docker" - branch = "feature/merge_latest_675" - vpc_id = local.vpc_id - for_each = toset(var.dev2_instance_types) - instance_type = each.key - name = "eliza-docker-${each.key}" - security_group_id = module.security.internal_security_group_id - ami_id = local.ami_id - tags = merge(local.tags, { - environment = "test" - }) - source = "./components/launch_template_docker" - key_name = var.key_name #"mdupont-deployer-key" - ssm_parameter_name_cw_agent_config = "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/cloudwatch-agent/config/details" - iam_instance_profile_name = module.roles.ssm_profile_name - #install_script = "/opt/eliza/api/docker-boot.sh" this is called from ssm for a refresh - install_script = "/opt/eliza/api/rundocker.sh" -} - - -# module "alb" { -# count = 0 # none for now -# source = "./components/application_load_balancer" -# domain_name = local.domain -# security_group_id = module.security.security_group_id # allowed to talk to internal -# public_subnets = [ -# local.ec2_public_subnet_id_1, -# local.ec2_public_subnet_id_2] -# vpc_id = local.vpc_id -# name = local.name -# } -# output "alb" { -# value = module.alb -# } - - -# module "asg_dynamic_new_ami" { -# # built with packer -# #count =0 -# tags = local.tags -# vpc_id = local.vpc_id -# image_id = local.new_ami_id -# ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 -# for_each = toset(var.instance_types) -# aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn -# source = "./components/autoscaling_group" -# # security_group_id = module.security.internal_security_group_id -# instance_type = each.key -# name = "eliza-ami-${each.key}" -# launch_template_id = module.lt_dynamic_ami_prod[each.key].launch_template_id -# target_group_arn = module.alb.prod_alb_target_group_arn -# } - -# module "asg_dynamic_new_ami_test" { - -# # built with packer -# for_each = toset(var.test_instance_types) -# tags = merge(local.tags, local.dev_tags) -# vpc_id = local.vpc_id -# image_id = local.new_ami_id -# ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 - -# aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn -# source = "./components/autoscaling_group/spot" -# # security_group_id = module.security.internal_security_group_id -# instance_type = each.key -# name = "test-eliza-ami-${each.key}" -# launch_template_id = module.lt_dynamic_ami_test[each.key].launch_template_id -# target_group_arn = module.alb.test_alb_target_group_arn -# } - -# module "asg_dynamic_new_ami_dev_spot" { -# # built with packer -# # count =0 -# tags = merge(local.tags, local.dev_tags) -# vpc_id = local.vpc_id -# image_id = local.new_ami_id -# ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 -# for_each = toset(var.dev_instance_types) -# aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn - -# source = "./components/autoscaling_group/spot" -# # security_group_id = module.security.internal_security_group_id -# instance_type = each.key -# name = "docker-eliza-ami-${each.key}" -# launch_template_id = module.lt_dynamic_ami_docker[each.key].launch_template_id -# target_group_arn = module.alb.dev_alb_target_group_arn - -# use_mixed_instances_policy = true -# mixed_instances_policy = { -# instances_distribution = { -# on_demand_base_capacity = 0 -# on_demand_percentage_above_base_capacity = 0 -# spot_instance_pools = 1 -# spot_max_price = var.spot_max_price -# # spot_allocation_strategy = "capacity-optimized" -# } - -# override = [ -# { -# instance_requirements = { -# cpu_manufacturers = ["amazon-web-services", "amd", "intel"] -# #cpu_manufacturers = ["amd"] -# #local_storage_types = ["ssd"] -# max_spot_price_as_percentage_of_optimal_on_demand_price = 60 -# memory_gib_per_vcpu = { -# min = 4 -# max = 12 -# } -# memory_mib = { -# min = 4096 -# }, -# vcpu_count = { -# min = 2 -# max = 12 -# } -# } -# } -# ] -# } -# instance_requirements = { -# } -# } - -module "asg" { - # built with packer - # count =0 - - tags = merge(local.tags, local.dev_tags) - vpc_id = local.vpc_id - image_id = local.ami_id - ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 - for_each = toset(var.dev2_instance_types) - aws_iam_instance_profile_ssm_arn = module.roles.ssm_profile_arn - - source = "./components/autoscaling_group/spot" - # security_group_id = module.security.internal_security_group_id - instance_type = each.key - name = "docker-eliza-ami-${each.key}" - launch_template_id = module.lt_dynamic_ami_docker_normal[each.key].launch_template_id -# target_group_arn = module.alb.dev_alb_target_group_arn - - -} - output "security_group_id" { value = module.security.security_group_id } module "eliza" { - source = "./mcs" # fixme rename to eliza - branch = "feature/aws" - #git_repo = "https://github.com/meta-introspector/eliza-MedicalCoderSwarm-deployment.git" + source = "./eliza" # fixme rename to eliza +# ami_name = var.ami_name + branch = var.branch git_repo = "https://github.com/meta-introspector/cloud-deployment-eliza/" -# alb_target_group_arn = module.alb.mcs_alb_target_group_arn ssm_profile_arn = module.roles.ssm_profile_arn ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 iam_instance_profile_name = module.roles.ssm_profile_name key_name = var.key_name + instance_types = var.instance_types aws_account_id = var.aws_account_id region = var.region internal_security_group_id = module.security.internal_security_group_id @@ -337,38 +79,6 @@ module "eliza" { name = "docker-agent-ami" } -# module "agent_dev" { -# source = "./agent" -# name = "agent-dev" -# branch = "feature/agent_dev" -# git_repo = "https://github.com/jmikedupont2/eliza-MedicalCoderSwarm-deployment.git" - -# alb_target_group_arn = module.alb.agent_dev_alb_target_group_arn -# ssm_profile_arn = module.roles.ssm_profile_arn -# ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 -# iam_instance_profile_name = module.roles.ssm_profile_name -# key_name = var.key_name -# aws_account_id = var.aws_account_id -# region = var.region -# internal_security_group_id = module.security.internal_security_group_id -# tags = local.tags -# ami_id = local.ami_id -# vpc_id = local.vpc_id - -# instance_types= [ -# "t3.medium" -# ] -# } - output "vpc" { value = module.vpc } - -output "user_data_new" { - value = module.lt_dynamic_ami_test["t3.medium"].user_data -} -output "user_data_docker" { - value = "" - #module.lt_dynamic_ami_docker_normal["t3.medium"].user_data -} - From 50ed993fedf626d296127c9ae64d01086063c3cd Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sun, 12 Jan 2025 07:32:01 -0500 Subject: [PATCH 134/158] adding in bogus azs --- environments/eliza-agent-api/main.tf | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/environments/eliza-agent-api/main.tf b/environments/eliza-agent-api/main.tf index 9c627f3..e383f08 100644 --- a/environments/eliza-agent-api/main.tf +++ b/environments/eliza-agent-api/main.tf @@ -1,14 +1,12 @@ +variable aws_availability_zones {} variable instance_types {} variable "spot_max_price" {} variable "branch" {} variable "project" {} - variable "region" {} variable "domain" {} variable "aws_account_id" {} variable "ami_id" {} -#variable "ami_name" {} - variable "tags" {} variable "name" {} variable "key_name" {} From 03d3b0dcf0d62b1071312d384cf5e498405cdbeb Mon Sep 17 00:00:00 2001 From: mike dupont Date: Mon, 13 Jan 2025 15:36:26 -0500 Subject: [PATCH 135/158] adding owners --- README.md | 39 ++++++++++++++++++++++++++++++++++ accounts/AI_Token_Team/main.tf | 10 +++++---- accounts/mdupont/main.tf | 19 +++++++++++++---- 3 files changed, 60 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 5fe28cb..34e541f 100644 --- a/README.md +++ b/README.md @@ -442,3 +442,42 @@ It seems you might want to continue with a task or scenario related to the previ aws ssm start-session --target i-0e156165e86473c93 --profile mdupont --region us-east-2 + +aws ssm put-parameter --name "agent_openai_key" --value "${OPENAI_API_KEY}" --type String + +│ Error: reading SSM Parameter (arn:aws:ssm:us-east-1:AKIA4SYAMCQ5MMLC6NU3:parameter/cloudwatch-agent/config/details): operation error SSM: GetParameter, https response error StatusCode: 400, RequestID: 159177cb-91f0-4c2e-a354-07cdc2e64041, api error ValidationException: Invalid Account Id in: arn:aws:ssm:us-east-1:AKIA4SYAMCQ5MMLC6NU3:parameter/cloudwatch-agent/config/details +│ +│ with module.eliza_server.module.eliza.module.lt_docker["t3a.small"].data.aws_ssm_parameter.cw_agent_config, +│ on ../../environments/eliza-agent-api/components/launch_template_docker_mcs/main.tf line 77, in data "aws_ssm_parameter" "cw_agent_config": +│ 77: data "aws_ssm_parameter" "cw_agent_config" { + +tofu apply --target module.ssm_observer.aws_ssm_parameter.cw_agent_config --target module.ssm_observer.aws_ssm_parameter.cw_agent_config_details + ++`aws ecr list-images --profile swarms --region us-east-2 --repository-name swarms/mcs` + +docker login -u AWS -p $(aws ecr get-login-password --region us-east-2) 767503528736.dkr.ecr.us-east-2.amazonaws.com' 767503528736.dkr.ecr.us-east-2.amazonaws.com/agent/eliza:latest + +# setup +we dont use terraform for the ecr or the secrets because they will change so often here are the simple aws scripts for them. + +`aws ecr create-repository --repository-name agent/eliza` + +`set_secrets.sh` calls +`aws ssm put-parameter --name "agent_openai_key" --value "${OPENAI_API_KEY}" --type String` + +## connecting with server + +`ssh-ssm.py` to find the server + +for example: +`aws ssm start-session --target i-0e156165e86473c93 --profile mdupont --region us-east-2` + +## terraform-aws-oidc-github +https://github.com/jmikedupont2/terraform-aws-oidc-github + +in my time 2024/12/18/terraform-aws-oidc-github on branch +run the tofu apply in the example after editing the variables and files. + +# debug +`pnpm start:debug --characters=./characters/eliza.character.json` +start direct client here diff --git a/accounts/AI_Token_Team/main.tf b/accounts/AI_Token_Team/main.tf index 54f9b60..166fea5 100644 --- a/accounts/AI_Token_Team/main.tf +++ b/accounts/AI_Token_Team/main.tf @@ -12,7 +12,6 @@ locals { module "ssm_observer" { source = "../../modules/aws/ssm/observability" ami_id = data.aws_ami.ami.id - } module "ssm_setup" { @@ -22,10 +21,13 @@ module "ssm_setup" { project = local.project } -locals { ami_name = "ubuntu-minimal/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-minimal-*" } + locals { + ami_name = "ubuntu-minimal/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-minimal-*" + } data "aws_ami" "ami" { # slow - most_recent = true - name_regex = "^${local.ami_name}" + most_recent = true + owners = ["679593333241"] + name_regex = "^${local.ami_name}" } module "eliza_server" { diff --git a/accounts/mdupont/main.tf b/accounts/mdupont/main.tf index b5f64cf..c62596a 100644 --- a/accounts/mdupont/main.tf +++ b/accounts/mdupont/main.tf @@ -61,9 +61,10 @@ locals { locals { # hard coded to save time , fixme use a caching system - - ami_id = "ami-0325b9a2dfb474b2d" + # ami_id = "ami-0325b9a2dfb474b2d" for ami_name = "ubuntu-minimal/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-minimal-*" } + ami_id = "ami-0e44962f5c9a2baab" } + module "ssm_observer" { source = "../../modules/aws/ssm/observability" #ami_id = data.aws_ami.ami.id @@ -85,8 +86,18 @@ module "ssm_setup" { #aws_account_id = local.account aws_account_id =var.aws_account_id region = local.region - source = "../../environments/swarms-aws-agent-api/dev/us-east-1" # FIXME rename - domain = local.dns + source = "../../environments/eliza-agent-api" + domain = local.dns + key_name = "mdupont-deployer-key" + branch = "feature/micro" + project = "tine" + # instance_types = ["t4g.small"] # not big enough for building + instance_types = ["t4g.medium"] + aws_availability_zones =["us-east-2a", + "us-east-2b", + "us-east-2c" + ] + spot_max_price= 0.01 ami_id = local.ami_id #data.aws_ami.ami.id name = "eliza" tags = { project = "eliza" } From cc47c0981e31e0d878827fd52523e256ac1a0568 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Mon, 13 Jan 2025 15:40:43 -0500 Subject: [PATCH 136/158] removing my key --- environments/eliza-agent-api/components/keypairs/main.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/environments/eliza-agent-api/components/keypairs/main.tf b/environments/eliza-agent-api/components/keypairs/main.tf index 1cf1255..cb563d8 100644 --- a/environments/eliza-agent-api/components/keypairs/main.tf +++ b/environments/eliza-agent-api/components/keypairs/main.tf @@ -1,4 +1,4 @@ -resource "aws_key_pair" "mdupont-deployer" { - key_name = "mdupont-deployer-key-2" - public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCYh8dzILXDkDpXDJ+wUiru8EPNPETWWHHFlnVI7Uc2Bb2f/yHJL9bV0QUQ+/hN2OGeN3r4z34vf444A7oMXT8T2bnMDzXqGfrqpQ6+Xs7Cu2F6wGik+KDBsu52vhaATVLOnDegbhpQW+IikSvxe4huKOvQpF+p7Ex80B+XpBPEV23DXapjZI+FIsbYoD4Mp5qY/PmXisNCByayhBG7WBhCEtHxkvpFntkz/9bwk2kC/z2W1SIHufN5TbrxKPKWY5iguW0Mn2e/rNvxnxFZaRx224rQnRFBMSq4Oi91MNdilwDHFzkv4oVBtpUmCet84np8+DxCfzphyIpo899dRV+/f7dwb6ZY3cvBkALcWahsscuE4ypbroXQ40UPAa3gW1PirTNdMEiX+Ie/IzEDWWCJKdDv4JaGtKAPORfC7bbXnBYn5RASglOjI24w974Llyj5TXXKexjxsjF3wlSS6pNHFlFJnQzVfemcY6AqSJ0Xr8dfFbxpSYH9OFkvBhzPaec= mdupont@mdupont-G470" -} +# resource "aws_key_pair" "mdupont-deployer" { +# key_name = "mdupont-deployer-key-2" +# public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCYh8dzILXDkDpXDJ+wUiru8EPNPETWWHHFlnVI7Uc2Bb2f/yHJL9bV0QUQ+/hN2OGeN3r4z34vf444A7oMXT8T2bnMDzXqGfrqpQ6+Xs7Cu2F6wGik+KDBsu52vhaATVLOnDegbhpQW+IikSvxe4huKOvQpF+p7Ex80B+XpBPEV23DXapjZI+FIsbYoD4Mp5qY/PmXisNCByayhBG7WBhCEtHxkvpFntkz/9bwk2kC/z2W1SIHufN5TbrxKPKWY5iguW0Mn2e/rNvxnxFZaRx224rQnRFBMSq4Oi91MNdilwDHFzkv4oVBtpUmCet84np8+DxCfzphyIpo899dRV+/f7dwb6ZY3cvBkALcWahsscuE4ypbroXQ40UPAa3gW1PirTNdMEiX+Ie/IzEDWWCJKdDv4JaGtKAPORfC7bbXnBYn5RASglOjI24w974Llyj5TXXKexjxsjF3wlSS6pNHFlFJnQzVfemcY6AqSJ0Xr8dfFbxpSYH9OFkvBhzPaec= mdupont@mdupont-G470" +# } From 2fef929b60e20a13a895a2b583a93ce4818a5c53 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Mon, 13 Jan 2025 15:43:36 -0500 Subject: [PATCH 137/158] update readme --- README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.md b/README.md index 34e541f..cbc3670 100644 --- a/README.md +++ b/README.md @@ -481,3 +481,19 @@ run the tofu apply in the example after editing the variables and files. # debug `pnpm start:debug --characters=./characters/eliza.character.json` start direct client here + +# set the az + #~/terraform/accounts/AI_Token_Team/main.tf +edit + aws_availability_zones = ["us-west-1a","us-west-1b","us-west-1c"] + +if you ge the errro : +00000003] +module.ssm_setup.module.ssm.aws_s3_bucket_lifecycle_configuration.access_log_bucket: Creation complete after 32s [id=ai-token-team-session-access-logs-20250113203757298300000002] +╷ +│ Error: creating EC2 Subnet: operation error EC2: CreateSubnet, https response error StatusCode: 400, RequestID: 05eeaa25-0855-4d71-8945-04b0d6233520, api error InvalidParameterValue: Value (us-west-1b) for parameter availabilityZone is invalid. Subnets can currently only be created in the following availability zones: us-east-1a, us-east-1b, us-east-1c, us-east-1d, us-east-1e, us-east-1f. +│ +│ with module.eliza_server.module.vpc.module.vpc.aws_subnet.public[1], +│ on .terraform/modules/eliza_server.vpc.vpc/main.tf line 98, in resource "aws_subnet" "public": +│ 98: resource "aws_subnet" "public" { +│ From f09fb87ada4e2258f5cf09c276c34d36b5cc6d97 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Mon, 13 Jan 2025 15:45:11 -0500 Subject: [PATCH 138/158] comment out kms id --- modules/aws/ssm/setup/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/aws/ssm/setup/main.tf b/modules/aws/ssm/setup/main.tf index 44ea9b7..a9f322b 100644 --- a/modules/aws/ssm/setup/main.tf +++ b/modules/aws/ssm/setup/main.tf @@ -20,13 +20,13 @@ resource "aws_cloudwatch_log_group" "app_signals" { for_each = toset(["ec2", "eks", "generic", "k8s"]) name = "/aws/appsignals/${each.key}" retention_in_days = 30 - kms_key_id = "arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590" +# kms_key_id = "arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590" } resource "aws_cloudwatch_log_group" "app_signals2" { for_each = toset(["data"]) name = "/aws/application-signals/${each.key}" retention_in_days = 30 - kms_key_id = "arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590" +# kms_key_id = "arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590" } From 8b46a9a243270136939e3541e48bb76e987af6aa Mon Sep 17 00:00:00 2001 From: mike dupont Date: Mon, 13 Jan 2025 16:00:45 -0500 Subject: [PATCH 139/158] keypairs --- README.md | 21 +++++++++++++++++++ .../components/keypairs/main.tf | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cbc3670..ac8c220 100644 --- a/README.md +++ b/README.md @@ -497,3 +497,24 @@ module.ssm_setup.module.ssm.aws_s3_bucket_lifecycle_configuration.access_log_buc │ on .terraform/modules/eliza_server.vpc.vpc/main.tf line 98, in resource "aws_subnet" "public": │ 98: resource "aws_subnet" "public" { │ + + +nter a value: yes + +# key pair +module.eliza_server.module.eliza.module.asg["t3a.small"].module.autoscaling.aws_autoscaling_group.this[0]: Creating... +╷ +│ Error: creating Auto Scaling Group (docker-agent-ami-t3a.small-20250113205357496400000001): operation error Auto Scaling: CreateAutoScalingGroup, https response error StatusCode: 400, RequestID: 380d0f57-bf60-40fd-ace6-d86563107c85, api error ValidationError: You must use a valid fully-formed launch template. The key pair does not exist +│ +│ with module.eliza_server.module.eliza.module.asg["t3a.small"].module.autoscaling.aws_autoscaling_group.this[0], +│ on .terraform/modules/eliza_server.eliza.asg.autoscaling/main.tf line 347, in resource "aws_autoscaling_group" "this": +│ 347: resource "aws_autoscaling_group" "this" { +│ +edit +~/terraform/environments/eliza-agent-api/components/keypairs/main.tf + +resource "aws_key_pair" "deployer" { + key_name = "ai-token-deployer-key" + public_key = " Date: Mon, 13 Jan 2025 16:26:27 -0500 Subject: [PATCH 140/158] fixing --- .../eliza-agent-api/components/keypairs/main.tf | 1 - .../eliza-agent-api/components/roles/main.tf | 12 ++++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/environments/eliza-agent-api/components/keypairs/main.tf b/environments/eliza-agent-api/components/keypairs/main.tf index 22cbb1e..2ce57c4 100644 --- a/environments/eliza-agent-api/components/keypairs/main.tf +++ b/environments/eliza-agent-api/components/keypairs/main.tf @@ -1,4 +1,3 @@ # resource "aws_key_pair" "mdupont-deployer" { # key_name = "mdupont-deployer-key-2" - # } diff --git a/environments/eliza-agent-api/components/roles/main.tf b/environments/eliza-agent-api/components/roles/main.tf index a785380..3e96a35 100644 --- a/environments/eliza-agent-api/components/roles/main.tf +++ b/environments/eliza-agent-api/components/roles/main.tf @@ -27,11 +27,11 @@ data "aws_iam_policy_document" "default" { resources = ["*"] } - statement { - actions = ["kms:Decrypt"] - resources = ["arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590"] - effect = "Allow" - } +# statement { +# actions = ["kms:Decrypt"] +# resources = ["arn:aws:kms:us-east-2:916723593639:key/cc8e1ee7-a05b-4642-bd81-ba5548635590"] +# effect = "Allow" +# } statement { actions = [ @@ -68,7 +68,7 @@ data "aws_iam_policy_document" "default" { statement { effect = "Allow" - resources = ["arn:aws:s3:::tine-session-logs*"] # FIXME must match + resources = ["arn:aws:s3:::*session-logs*"] # FIXME must match actions = [ "s3:GetEncryptionConfiguration" ] From c5be84114ce12d01e19b1d2de8e92bf54d194b8d Mon Sep 17 00:00:00 2001 From: mike dupont Date: Mon, 13 Jan 2025 21:09:47 -0500 Subject: [PATCH 141/158] move to branch --- accounts/mdupont/main.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/accounts/mdupont/main.tf b/accounts/mdupont/main.tf index c62596a..8011971 100644 --- a/accounts/mdupont/main.tf +++ b/accounts/mdupont/main.tf @@ -89,10 +89,10 @@ module "ssm_setup" { source = "../../environments/eliza-agent-api" domain = local.dns key_name = "mdupont-deployer-key" - branch = "feature/micro" + branch = "feature/arm64_fastembed" project = "tine" - # instance_types = ["t4g.small"] # not big enough for building - instance_types = ["t4g.medium"] + instance_types = ["t4g.small"] # not big enough for building + #instance_types = ["t4g.medium"] aws_availability_zones =["us-east-2a", "us-east-2b", "us-east-2c" From 4eb3ee845560fa9ba95df0237e324dc3c95cb4b4 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Tue, 14 Jan 2025 08:42:51 -0500 Subject: [PATCH 142/158] allow github to create repositories --- accounts/mdupont/main.tf | 3 ++- .../components/launch_template_docker_mcs/main.tf | 9 +-------- environments/eliza-agent-api/components/roles/main.tf | 1 + environments/eliza-agent-api/main.tf | 3 ++- 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/accounts/mdupont/main.tf b/accounts/mdupont/main.tf index 8011971..38db3d9 100644 --- a/accounts/mdupont/main.tf +++ b/accounts/mdupont/main.tf @@ -92,7 +92,8 @@ module "ssm_setup" { branch = "feature/arm64_fastembed" project = "tine" instance_types = ["t4g.small"] # not big enough for building - #instance_types = ["t4g.medium"] + #instance_types = ["t4g.medium"] + repo = "https://github.com/meta-introspector/cloud-deployment-eliza/" aws_availability_zones =["us-east-2a", "us-east-2b", "us-east-2c" diff --git a/environments/eliza-agent-api/components/launch_template_docker_mcs/main.tf b/environments/eliza-agent-api/components/launch_template_docker_mcs/main.tf index ffdc0f3..c104c6b 100644 --- a/environments/eliza-agent-api/components/launch_template_docker_mcs/main.tf +++ b/environments/eliza-agent-api/components/launch_template_docker_mcs/main.tf @@ -26,7 +26,6 @@ locals { export HOME=/root apt update apt-get install -y ec2-instance-connect git - # Install docker apt-get install -y cloud-utils apt-transport-https ca-certificates curl software-properties-common curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - @@ -37,11 +36,9 @@ locals { apt-get update apt-get install -y docker-ce usermod -aG docker ubuntu - - # Install docker-compose + # Install docker-compose FIXME remove this as not needed? curl -L https://github.com/docker/compose/releases/download/1.21.0/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose chmod +x /usr/local/bin/docker-compose - snap install amazon-ssm-agent --classic || echo oops1 snap start amazon-ssm-agent || echo oops2 apt-get install -y --no-install-recommends ca-certificates=20230311 curl=7.88.1-10+deb12u7 | echo oops @@ -56,12 +53,10 @@ locals { # Enable and start the service using systemctl systemctl enable amazon-cloudwatch-agent systemctl start amazon-cloudwatch-agent - # Clean up downloaded files rm -f amazon-cloudwatch-agent.deb # Verify installation systemctl status amazon-cloudwatch-agent - if [ ! -d "/opt/${var.app_name}/" ]; then git clone ${var.git_repo} "/opt/${var.app_name}/" fi @@ -69,10 +64,8 @@ locals { git stash git fetch --all # get the latest version git checkout --track --force "origin/${var.branch}" - bash -x ${var.install_script} EOF - } data "aws_ssm_parameter" "cw_agent_config" { name = var.ssm_parameter_name_cw_agent_config diff --git a/environments/eliza-agent-api/components/roles/main.tf b/environments/eliza-agent-api/components/roles/main.tf index 3e96a35..40fe881 100644 --- a/environments/eliza-agent-api/components/roles/main.tf +++ b/environments/eliza-agent-api/components/roles/main.tf @@ -12,6 +12,7 @@ data "aws_iam_policy_document" "default" { effect= "Allow" actions= [ "ecr:GetAuthorizationToken", + "ecr:CreateRepository", "ecr:BatchCheckLayerAvailability", "ecr:GetDownloadUrlForLayer", "ecr:GetRepositoryPolicy", diff --git a/environments/eliza-agent-api/main.tf b/environments/eliza-agent-api/main.tf index e383f08..cf6643b 100644 --- a/environments/eliza-agent-api/main.tf +++ b/environments/eliza-agent-api/main.tf @@ -1,3 +1,4 @@ +variable repo{} variable aws_availability_zones {} variable instance_types {} variable "spot_max_price" {} @@ -62,7 +63,7 @@ module "eliza" { source = "./eliza" # fixme rename to eliza # ami_name = var.ami_name branch = var.branch - git_repo = "https://github.com/meta-introspector/cloud-deployment-eliza/" + git_repo = var.repo ssm_profile_arn = module.roles.ssm_profile_arn ec2_subnet_id = module.vpc.ec2_public_subnet_id_1 iam_instance_profile_name = module.roles.ssm_profile_name From a62af865e3a07493c309c5fb6a8190a1f4c899ba Mon Sep 17 00:00:00 2001 From: mike dupont Date: Tue, 14 Jan 2025 17:22:31 -0500 Subject: [PATCH 143/158] wip --- accounts/mdupont/.terraform.lock.hcl | 18 ++++++++++++++++++ accounts/mdupont/main.tf | 5 +++++ 2 files changed, 23 insertions(+) diff --git a/accounts/mdupont/.terraform.lock.hcl b/accounts/mdupont/.terraform.lock.hcl index d73d409..ff8924c 100644 --- a/accounts/mdupont/.terraform.lock.hcl +++ b/accounts/mdupont/.terraform.lock.hcl @@ -17,3 +17,21 @@ provider "registry.opentofu.org/hashicorp/aws" { "zh:cc2d7985df86e52fe3fb40f75d59aab2736541b675d78cbe1280066fd15e0443", ] } + +provider "registry.opentofu.org/hashicorp/random" { + version = "3.6.3" + constraints = ">= 2.1.0" + hashes = [ + "h1:Ry0Lr0zaoicslZlcUR4rAySPpl/a7QupfMfuAxhW3fw=", + "zh:1bfd2e54b4eee8c761a40b6d99d45880b3a71abc18a9a7a5319204da9c8363b2", + "zh:21a15ac74adb8ba499aab989a4248321b51946e5431219b56fc827e565776714", + "zh:221acfac3f7a5bcd6cb49f79a1fca99da7679bde01017334bad1f951a12d85ba", + "zh:3026fcdc0c1258e32ab519df878579160b1050b141d6f7883b39438244e08954", + "zh:50d07a7066ea46873b289548000229556908c3be746059969ab0d694e053ee4c", + "zh:54280cdac041f2c2986a585f62e102bc59ef412cad5f4ebf7387c2b3a357f6c0", + "zh:632adf40f1f63b0c5707182853c10ae23124c00869ffff05f310aef2ed26fcf3", + "zh:b8c2876cce9a38501d14880a47e59a5182ee98732ad7e576e9a9ce686a46d8f5", + "zh:f27e6995e1e9fe3914a2654791fc8d67cdce44f17bf06e614ead7dfd2b13d3ae", + "zh:f423f2b7e5c814799ad7580b5c8ae23359d8d342264902f821c357ff2b3c6d3d", + ] +} diff --git a/accounts/mdupont/main.tf b/accounts/mdupont/main.tf index 38db3d9..23d938b 100644 --- a/accounts/mdupont/main.tf +++ b/accounts/mdupont/main.tf @@ -103,3 +103,8 @@ module "ssm_setup" { name = "eliza" tags = { project = "eliza" } } + + +module "codebuild" { + source = "./codebuild" +} From ecd59508ddaf11f490455c32f88808c3f221f3b1 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Tue, 14 Jan 2025 17:49:02 -0500 Subject: [PATCH 144/158] adding more detailed instructions moving ai code out --- README.md | 479 ++++----------------------------- accounts/AI_Token_Team/main.tf | 7 +- 2 files changed, 58 insertions(+), 428 deletions(-) diff --git a/README.md b/README.md index ac8c220..3242179 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,49 @@ # terraform-template +``` +git clone https://github.com/aitokenteam/ai-agent-terraform.git +cd ai-agent-terraform/accounts/AI_Token_Team/ +tofu init + +aws ecr create-repository --repository-name agent/eliza + +aws ssm put-parameter --name "agent_openai_key" --value "${OPENAI_API_KEY}" --type String + +tofu apply --target module.ssm_observer.aws_ssm_parameter.cw_agent_config --target module.ssm_observer.aws_ssm_parameter.cw_agent_config_details +tofu apply +``` + Terraform project template for deploying infrastructure across multiple environments and regions, following best practices with modular structure and automated syntax checks (GitHub Actions) +## Installation + +1. setup aws account +2. create user with administrator privledges for terraform/tofu + +Directly attach Policy name `AdministratorAccess` to user +replace in the following url +`https://us-east-1.console.aws.amazon.com/iam/home?region=us-east-2#/users/details/?section=permissions` + +create access key +https://us-east-1.console.aws.amazon.com/iam/home?region=us-east-2#/users/details/mdupont/create-access-key + +~/.aws/credentials + +aws cli install +https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html + +`aws configure` + +opentofu install +https://opentofu.org/docs/intro/install/ + +Go to ami catalog in your region: +https://us-west-1.console.aws.amazon.com/ec2/home?region=us-west-1#AMICatalog: + +Search for "ubuntu-noble-24.04-amd64-minimal" in the third tab "aws marketplace amis" + +Accept license. + ## Status report Currently we are building out a dev infrastructure using our existing code @@ -23,439 +65,27 @@ terraform-template/ # Root directory of the Terraform template │ ├── swarms ``` +to ss to the server +`aws ssm start-session --target i-0e156165e86473c93 --profile mdupont --region us-east-2` -``` -help me create a c4/semantic web/mermaid plantuml deployment diagram -for a self service cognito enabled, smart agent server hosting, -with many different deployment options ranging from t2-small to any larger size. -using terraform, aws, ec2, alb,diagram -accounts/swarms - -> environments/swarms-aws-agent-api/dev/us-east-1 - godaddy -> ns api.swarms.ai -> - route53 -> cname ->application load balancer - -> autoscaling group -> launch template -> ami - AMI : components/machine_image/ubuntu-fastapi.pkr.hcl - - swarms -> ec2, user data - github clone branch run api/install.sh, swarms user, app/boot.sh as swarms user (virtual env) - instance profile (our server, customer server) -> - todo :read access to specific ssm secrets -> openai key - servers tagged as user X can access secrets tagged Y. - outgoing ip address : (can talk to other servers) (allow certain ip ranges) - , nginx, systemd (isolate user process, write access to home), uvicorn -> swarms 4 processes fast api. - - githubactions -> oidc connection -> aws -> ssm document -> update your part of the server. - githubactions -> oidc connection -> aws -> ssm document -> terraform -> create new clusters - -shared hosted aws account : - profiles for each customer, - tags for customer resources - which api calls they can use - tags for cost tracking - custom vpc, subsets network resources. - - on servers, the agents will have own aws role/profiles (like lamda) - either each customer/agent gets own unix user, homedir, storage, email , backup. - containerization, k8s, docker. -``` - -# diagram1 - -```plantuml -@startuml - -!define RECTANGLE class - -RECTANGLE "Self-Service Cognito-Enabled Smart Agent Server" { - (User) -} - -rectangle "Accounts/Swarm" { - usecase "Create Environments" as U1 - usecase "Deploy EC2 Instances" as U2 - usecase "Manage Load Balancer" as U3 - usecase "Configure Auto-Scaling" as U4 - usecase "Provision AMI" as U5 - usecase "Access SSM Secrets" as U6 - usecase "Update Server Configuration" as U7 - usecase "Track Customer API Calls" as U8 - usecase "Provision Shared Resources" as U9 -} - -User --> U1 : "Creates / Manages" -User --> U2 : "Deploys Instances" -User --> U3 : "Configures Load Balancer" -User --> U4 : "Sets Auto-Scaling Parameters" -User --> U5 : "Uses AMI for Deployment" -User --> U6 : "Requests Secrets for Access" -User --> U7 : "Updates Configuration" -User --> U8 : "Monitors API Usage" -User --> U9 : "Shares Resources among Customers" - -U1 --> "Route53" : "Configures CNAME" -U2 --> "EC2" : "Launches Instances" -U3 --> "ALB" : "Routes Traffic" -U4 --> "AutoScaling Group" : "Manages Scalability" -U5 --> "AMI" : "Provides Machine Image" -U7 --> "GitHub Actions" : "Automates Deploy" -U6 --> "Instance Profile" : "Manages Permissions" -U8 --> "Tags" : "Tracks Costs" - -note right of U5 : Component: \ncomponents/machine_image/ubuntu-fastapi.pkr.hcl -note right of U8 : Custom VPC \n\n Customer Profiles - -@enduml -``` -![1000019322](https://github.com/user-attachments/assets/7f2c8f90-8222-4344-a007-59ea518b0f00) -https://www.planttext.com/?text=RPJVRzem4CVVyrVSUDeUA1gK_dgOccQeGu8AaRAdIfCnbs0riPalRbTD-zztjfCre3q6V_pr-NE7RwaHjd5rbOYgCg-r-gLqcIGVTbWeZR2UPO_fm_rY1h8IH4do_iCWnweOvcXVb4J8JAbLOuOpBROLxY2lEHRI4dK3da4xWDy9mCSDeVsK_4aIYx8HkgmG1gcKnkc6Hlchi7K47Q4Kn8kPHT4WpFIBiaRN74W340IRpttgAnugymQpR0nppNVI4bjitCUMGeiIOM743kv4vJ4RiSdHcKONgdGM8NLygLoAYhL4yBmFhgnvKQICXdGvZ-lJ_ZhV38aWpvTiGLhiQh_eGvl3pjymQeBkRD5mRAGl-_IZ5V8PCaUDgHbFLtF8k7TjyDN_gyltmd9NraZ6sNzghdqVV4DWEFp2Gk6cqqqmWkYA1ZrYx8cecdgUUyGaP7JwwCHq3pptOCv2ZXJ3IbXHOuFsA7NgOVuahnaAOo4MuQUXprruReq_7H87jyO37nlHPoz3Pb1F9z9xr9MdbiRFjgKWUqDifOzSXvjumVGk64Hj-3fTc_ZaXFrwVh0shi7fP71YAdj8bpFE7KUA9paG2-6qtEEnxabaqiLTO48Y0Kz6KIZ9w-VjEfljnDvRumw1RliAVaLiLQMQDzuoRRVuCRvfHsV7r5B8lNy9vdMyzcNaxbNppMrPgnT6OOi_hGusOtCOleXmLpfmQg4gF2fZXRPMH24cOTzXnwCeOsYOTjyVRYWxr8R_A5QL-mPC4USYC_N1Q1w46tZIilkWKHqB_w5yuIEtvGV63Gl1bHpKsU7PdkrbTPhdknrR1jzN6USzwVT_MbiyFuAtg7VyPFu5 - -### Explanation: -- This diagram represents the primary use cases associated with your smart agent server project. -- Each use case such as creating environments, deploying EC2 instances, and accessing SSM secrets represents functionalities your system supports. -- Annotations explain the components (like AMI) and setups (like custom VPC) tied to specific activities. - - -```plantuml -@startuml - -!define RECTANGLE(x) rectangle x as x -!define DIAMOND(x) diamond x as x - -package "Accounts/Swarm" { - RECTANGLE(swarms-aws-agent-api) { - package "Environments" { - RECTANGLE(dev) { - RECTANGLE(us_east_1) { - RECTANGLE(Godaddy) - RECTANGLE(Route53) - RECTANGLE(ALB) as alb - RECTANGLE(AutoScalingGroup) as asg - RECTANGLE(LaunchTemplate) as lt - RECTANGLE(AMI) as ami - } - } - } - } - Godaddy -> Route53: ns api.swarms.ai - Route53 -> alb: CNAME - alb -> asg - asg -> lt - lt -> ami: "components/machine_image/ubuntu-fastapi.pkr.hcl" - - package "EC2 Instance" { - RECTANGLE(EC2) { - RECTANGLE(UserData) - RECTANGLE(InstanceProfile) - RECTANGLE(Nginx) - RECTANGLE(Systemd) - RECTANGLE(Uvicorn) - RECTANGLE(FastAPI) - - EC2 -> UserData: github clone branch run - EC2 -> InstanceProfile: read access to SSM secrets - EC2 -> Nginx: isolates user process - EC2 -> Systemd: manage services - EC2 -> Uvicorn: "4 instances of FastAPI" - } - } - - UserData -> EC2: "run api/install.sh" - InstanceProfile -> EC2: "access tagged secrets" - Nginx -> Systemd: "manage processes" -} - -package "GitHub Actions" { - RECTANGLE(OIDC) { - RECTANGLE(SSM_Document) - RECTANGLE(Terraform) - } - OIDC -> AWS - AWS -> SSM_Document: "update server" - OIDC -> AWS: "create new clusters" - AWS -> Terraform -} - -package "Shared Hosted AWS Account" { - RECTANGLE(Profiles) { - RECTANGLE(Tags) - RECTANGLE(VPC) - } - Profiles -> Tags: "customer resources" - Profiles -> VPC: "subnet resources" - Profiles -> EC2: "instance roles" -} - -@enduml - - -Here's an updated PlantUML deployment diagram reflecting your architecture with key components: - -```plantuml -@startuml - -!define RECTANGLE(x) rectangle x as x -!define DIAMOND(x) diamond x as x - -package "Accounts/Swarm" { - RECTANGLE(swarms-aws-agent-api) { - package "Environments" { - RECTANGLE(dev) { - RECTANGLE(us_east_1) { - RECTANGLE(Godaddy) - RECTANGLE(Route53) - RECTANGLE(ALB) as alb - RECTANGLE(AutoScalingGroup) as asg - RECTANGLE(LaunchTemplate) as lt - RECTANGLE(AMI) as ami - } - } - } - } - Godaddy -> Route53: ns api.swarms.ai - Route53 -> alb: CNAME - alb -> asg - asg -> lt - lt -> ami: "components/machine_image/ubuntu-fastapi.pkr.hcl" - - package "EC2 Instance" { - RECTANGLE(EC2) { - RECTANGLE(UserData) - RECTANGLE(InstanceProfile) - RECTANGLE(Nginx) - RECTANGLE(Systemd) - RECTANGLE(Uvicorn) - RECTANGLE(FastAPI) - - EC2 -> UserData: github clone branch run - EC2 -> InstanceProfile: read access to SSM secrets - EC2 -> Nginx: isolates user process - EC2 -> Systemd: manage services - EC2 -> Uvicorn: "4 instances of FastAPI" - } - } - - UserData -> EC2: "run api/install.sh" - InstanceProfile -> EC2: "access tagged secrets" - Nginx -> Systemd: "manage processes" -} - -package "GitHub Actions" { - RECTANGLE(OIDC) { - RECTANGLE(SSM_Document) - RECTANGLE(Terraform) - } - OIDC -> AWS - AWS -> SSM_Document: "update server" - OIDC -> AWS: "create new clusters" - AWS -> Terraform -} - -package "Shared Hosted AWS Account" { - RECTANGLE(Profiles) { - RECTANGLE(Tags) - RECTANGLE(VPC) - } - Profiles -> Tags: "customer resources" - Profiles -> VPC: "subnet resources" - Profiles -> EC2: "instance roles" -} - -@enduml -``` - - -It seems there was some confusion. Here’s a refined version of the complete PlantUML deployment diagram for your smart agent server setup: - -```plantuml -@startuml - -!define RECTANGLE(x) rectangle x as x -!define DIAMOND(x) diamond x as x - -package "Accounts/Swarms" { - RECTANGLE("swarms-aws-agent-api") { - package "Environments" { - RECTANGLE("dev") { - RECTANGLE("us-east-1") { - RECTANGLE(Godaddy) - RECTANGLE(Route53) - RECTANGLE(ALB) as alb - RECTANGLE("AutoScalingGroup") as asg - RECTANGLE("LaunchTemplate") as lt - RECTANGLE("AMI") as ami - } - } - } - } - Godaddy -> Route53: ns api.swarms.ai - Route53 -> alb: CNAME - alb -> asg - asg -> lt - lt -> ami: "components/machine_image/ubuntu-fastapi.pkr.hcl" - - package "EC2 Instance" { - RECTANGLE("EC2") { - RECTANGLE("UserData") as userdata - RECTANGLE("InstanceProfile") as ip - RECTANGLE("Nginx") as nginx - RECTANGLE("Systemd") as systemd - RECTANGLE("Uvicorn") as uvicorn - RECTANGLE("FastAPI") as fastapi - - EC2 -> userdata: "github clone branch run api/install.sh" - EC2 -> ip: "read access to SSM secrets" - EC2 -> nginx: "isolates user process" - EC2 -> systemd: "manages services" - EC2 -> uvicorn: "4 instances of FastAPI" - } - } - - userdata -> EC2: "execute boot.sh" - ip -> EC2: "access tagged secrets" - nginx -> systemd: "manage processes" -} - -package "GitHub Actions" { - RECTANGLE("OIDC") { - RECTANGLE("AWS") - RECTANGLE("SSM Document") - RECTANGLE("Terraform") - } - OIDC -> AWS - AWS -> SSM Document: "update server" - OIDC -> AWS: "create new clusters" -} - -package "Shared Hosted AWS Account" { - RECTANGLE("Profiles") { - RECTANGLE("Tags") - RECTANGLE("VPC") - } - Profiles -> Tags: "customer resources" - Profiles -> VPC: "subnet resources" - Profiles -> EC2: "instance roles" -} +to install secret +`aws ssm put-parameter --name "agent_openai_key" --value "${OPENAI_API_KEY}" --type String` -note right of ami: "Designed for\nFastAPI deployments" -@enduml +If you get this error: ``` - -### Key Components: -1. **Accounts/Swarms**: Represents the main framework for your smart agent server. -2. **Environments**: Contains different environments such as `dev` in `us-east-1`. -3. **EC2 Instance**: Details the EC2 setup, including user data and necessary components (Nginx, Systemd, Uvicorn). -4. **GitHub Actions**: Integrates CI/CD using OIDC, SSM, and Terraform. -5. **Shared Hosted AWS Account**: Manages profiles, tags, and VPC configurations for customer resources. - -You can copy this code into a PlantUML tool to visualize the deployment diagram. -### Explanation: -- **Packages and Rectangles**: The diagram encapsulates components like Route 53, ALB, and Auto Scaling Group within a structured format. -- **Interactions**: Arrows show the relationships and interactions among components, such as the CNAME setup with Route 53 and the EC2 instances launching user data scripts. -- **GitHub Actions**: Represents automation activities and connections to AWS services. -- **Shared AWS Account**: Displays customer management and tagging for resources. - -This structure captures your specifications for the deployment of a self-service Cognito-enabled smart agent server using Terraform and AWS components effectively. Adjust as necessary for your specific documentation needs!``` - -### Explanation: -- The main structure includes the accounts, environments, EC2 instances, GitHub Actions, and shared AWS resources. -- The individual components and their interactions are represented using rectangles for entities and arrows for relationships. -- Customize further by adding details and refining the relationships based on your specific needs.``` - -To manage a shared hosted AWS account for multiple customers with the requirements you've laid out, you can consider the following architecture: - -### AWS Account Structure: -1. **Profiles**: Utilize IAM roles and users for each customer. Create separate profiles in the AWS CLI for easy access. - -2. **Resource Tagging**: Implement a consistent tagging strategy for all resources (e.g., `CustomerName`, `Project`, `Environment`, `CostCenter`) to facilitate cost management and resource identification. - -3. **API Call Restrictions**: Use IAM policies to specify which API calls each customer's role can perform based on their needs. - -4. **Cost Tracking**: Leverage AWS Cost Explorer and AWS Budgets, filtering by tags to track costs per customer. - -5. **Networking**: Set up custom VPCs for customers if isolation is required. Use subnets and security groups to segment resources. - -### Server and Agent Management: -1. **AWS Roles for Agents**: Each agent (server) should have an IAM role assigned that grants necessary permissions. If using Lambda, this can be done via execution roles. - -2. **Unix User Management**: Depending on your requirements: - - One option is to create a Unix user for each customer/agent. - - Assign unique home directories, storage solutions (e.g., EBS, S3), and ensure proper permissions. - -3. **Containerization**: Use Docker and Kubernetes (EKS) to manage applications. This allows for easier resource allocation and isolation between different customers. - -4. **Email & Backup**: Consider using Amazon SES for email needs and AWS Backup/CloudFormation for automated backups of customer data. - -### Best Practices: -- **Security**: Regularly audit IAM roles and policies for least privilege access. -- **Monitoring**: Use CloudWatch for logging and monitoring resource utilization. -- **Cost Management**: Use AWS Cost Allocation Reports to analyze customer costs effectively. - -This architecture ensures streamlined operations, cost tracking, and enhanced security across customer resources in a shared AWS account. - - - -### Repo Structure Overview - -- **Root Directory**: The main container of your Terraform project (`terraform-template/`). - -- **README.md**: Provides an overview and documentation for the project, helping users understand its purpose and how to use it. - -- **Environments**: Contains specific configurations for different environments (e.g., development, staging). - - **swarms-aws-agent-api**: An example service with various setups. - - **dev/us-east-1**: The development environment tailored for the US East (N. Virginia) region. - -- **Modules**: Encapsulates reusable Terraform code to ensure DRY (Don't Repeat Yourself) principles. - - **swarms**: A collection of Terraform modules related to the swarm infrastructure. - -### Components Explained - -- **DNS Management**: - - Using GoDaddy for domain management and AWS Route 53 for DNS routing (e.g., `api.swarms.ai`). - -- **Application Layer**: - - Includes an Application Load Balancer directing traffic to an Auto Scaling Group, which manages instances defined by a launch template. - -- **Instance Management**: - - AMIs (Amazon Machine Images) created using Packer (e.g., `ubuntu-fastapi.pkr.hcl`). - -- **Settings for EC2 Instances**: - - User data scripts to set up necessary configurations, such as cloning from GitHub and running scripts as the `swarms` user to set up the FastAPI application. - -- **Access Management**: - - IAM roles and instance profiles are created to manage permissions. SSM (AWS Systems Manager) secrets are tagged, allowing specific servers to access configurations, like OpenAI keys. - -- **Networking**: - - Outbound traffic settings defined to control which IP ranges can communicate with the infrastructure. - -- **Application Deployment**: - - Utilizes Nginx and Systemd to manage FastAPI processes, ensuring multiple instances run smoothly. - -### CI/CD with GitHub Actions -- Implements OIDC connections to AWS for secure deployments and updates using GitHub Actions. -- SSM documents are utilized for executing specific management tasks, like creating new clusters and updating configurations seamlessly. - -This structure facilitates a modular, maintainable, and automated approach to infrastructure as code using Terraform, alongside solid CI/CD practices. - - - -It seems you might want to continue with a task or scenario related to the previous content but haven't specified what you'd like next. If you need more details or a specific aspect of the Terraform setup, architecture diagram, or other inquiries, please let me know, and I'll be glad to assist! - - -aws ssm start-session --target i-0e156165e86473c93 --profile mdupont --region us-east-2 - -aws ssm put-parameter --name "agent_openai_key" --value "${OPENAI_API_KEY}" --type String - │ Error: reading SSM Parameter (arn:aws:ssm:us-east-1:AKIA4SYAMCQ5MMLC6NU3:parameter/cloudwatch-agent/config/details): operation error SSM: GetParameter, https response error StatusCode: 400, RequestID: 159177cb-91f0-4c2e-a354-07cdc2e64041, api error ValidationException: Invalid Account Id in: arn:aws:ssm:us-east-1:AKIA4SYAMCQ5MMLC6NU3:parameter/cloudwatch-agent/config/details -│ │ with module.eliza_server.module.eliza.module.lt_docker["t3a.small"].data.aws_ssm_parameter.cw_agent_config, │ on ../../environments/eliza-agent-api/components/launch_template_docker_mcs/main.tf line 77, in data "aws_ssm_parameter" "cw_agent_config": │ 77: data "aws_ssm_parameter" "cw_agent_config" { +``` +we need to apply this first: +`tofu apply --target module.ssm_observer.aws_ssm_parameter.cw_agent_config --target module.ssm_observer.aws_ssm_parameter.cw_agent_config_details` -tofu apply --target module.ssm_observer.aws_ssm_parameter.cw_agent_config --target module.ssm_observer.aws_ssm_parameter.cw_agent_config_details - -+`aws ecr list-images --profile swarms --region us-east-2 --repository-name swarms/mcs` +check the ECR images +`aws ecr list-images --region us-east-2 --repository-name swarms/mcs` -docker login -u AWS -p $(aws ecr get-login-password --region us-east-2) 767503528736.dkr.ecr.us-east-2.amazonaws.com' 767503528736.dkr.ecr.us-east-2.amazonaws.com/agent/eliza:latest +To login from docker to ecr +`docker login -u AWS -p $(aws ecr get-login-password --region us-east-2) 767503528736.dkr.ecr.us-east-2.amazonaws.com' 767503528736.dkr.ecr.us-east-2.amazonaws.com/agent/eliza:latest` # setup we dont use terraform for the ecr or the secrets because they will change so often here are the simple aws scripts for them. @@ -466,7 +96,6 @@ we dont use terraform for the ecr or the secrets because they will change so oft `aws ssm put-parameter --name "agent_openai_key" --value "${OPENAI_API_KEY}" --type String` ## connecting with server - `ssh-ssm.py` to find the server for example: diff --git a/accounts/AI_Token_Team/main.tf b/accounts/AI_Token_Team/main.tf index 166fea5..3afbf7e 100644 --- a/accounts/AI_Token_Team/main.tf +++ b/accounts/AI_Token_Team/main.tf @@ -1,6 +1,7 @@ provider "aws" { region = "us-west-1" - profile = "ai-token-team" + #profile = "ai-token-team" + profile = "default" } locals { @@ -16,8 +17,8 @@ module "ssm_observer" { module "ssm_setup" { source = "../../modules/aws/ssm/setup" - bucket_name = "${local.project}session-logs" - access_log_bucket_name = "${local.project}-session-access-logs" + bucket_name = "${local.project}-session-logs" + access_log_bucket_name = "${local.project}-ssm-access-logs" project = local.project } From 316a26eb3b1deb56b649ddf878770477b0ea2a86 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Tue, 14 Jan 2025 17:49:22 -0500 Subject: [PATCH 145/158] adding ai notes back in --- ai-notes.md | 415 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 415 insertions(+) create mode 100644 ai-notes.md diff --git a/ai-notes.md b/ai-notes.md new file mode 100644 index 0000000..668d832 --- /dev/null +++ b/ai-notes.md @@ -0,0 +1,415 @@ +``` +help me create a c4/semantic web/mermaid plantuml deployment diagram +for a self service cognito enabled, smart agent server hosting, +with many different deployment options ranging from t2-small to any larger size. +using terraform, aws, ec2, alb,diagram +accounts/swarms + -> environments/swarms-aws-agent-api/dev/us-east-1 + godaddy -> ns api.swarms.ai -> + route53 -> cname ->application load balancer + -> autoscaling group -> launch template -> ami + AMI : components/machine_image/ubuntu-fastapi.pkr.hcl + + swarms -> ec2, user data + github clone branch run api/install.sh, swarms user, app/boot.sh as swarms user (virtual env) + instance profile (our server, customer server) -> + todo :read access to specific ssm secrets -> openai key + servers tagged as user X can access secrets tagged Y. + outgoing ip address : (can talk to other servers) (allow certain ip ranges) + , nginx, systemd (isolate user process, write access to home), uvicorn -> swarms 4 processes fast api. + + githubactions -> oidc connection -> aws -> ssm document -> update your part of the server. + githubactions -> oidc connection -> aws -> ssm document -> terraform -> create new clusters + +shared hosted aws account : + profiles for each customer, + tags for customer resources + which api calls they can use + tags for cost tracking + custom vpc, subsets network resources. + + on servers, the agents will have own aws role/profiles (like lamda) + either each customer/agent gets own unix user, homedir, storage, email , backup. + containerization, k8s, docker. +``` + +# diagram1 + +```plantuml +@startuml + +!define RECTANGLE class + +RECTANGLE "Self-Service Cognito-Enabled Smart Agent Server" { + (User) +} + +rectangle "Accounts/Swarm" { + usecase "Create Environments" as U1 + usecase "Deploy EC2 Instances" as U2 + usecase "Manage Load Balancer" as U3 + usecase "Configure Auto-Scaling" as U4 + usecase "Provision AMI" as U5 + usecase "Access SSM Secrets" as U6 + usecase "Update Server Configuration" as U7 + usecase "Track Customer API Calls" as U8 + usecase "Provision Shared Resources" as U9 +} + +User --> U1 : "Creates / Manages" +User --> U2 : "Deploys Instances" +User --> U3 : "Configures Load Balancer" +User --> U4 : "Sets Auto-Scaling Parameters" +User --> U5 : "Uses AMI for Deployment" +User --> U6 : "Requests Secrets for Access" +User --> U7 : "Updates Configuration" +User --> U8 : "Monitors API Usage" +User --> U9 : "Shares Resources among Customers" + +U1 --> "Route53" : "Configures CNAME" +U2 --> "EC2" : "Launches Instances" +U3 --> "ALB" : "Routes Traffic" +U4 --> "AutoScaling Group" : "Manages Scalability" +U5 --> "AMI" : "Provides Machine Image" +U7 --> "GitHub Actions" : "Automates Deploy" +U6 --> "Instance Profile" : "Manages Permissions" +U8 --> "Tags" : "Tracks Costs" + +note right of U5 : Component: \ncomponents/machine_image/ubuntu-fastapi.pkr.hcl +note right of U8 : Custom VPC \n\n Customer Profiles + +@enduml +``` +![1000019322](https://github.com/user-attachments/assets/7f2c8f90-8222-4344-a007-59ea518b0f00) +https://www.planttext.com/?text=RPJVRzem4CVVyrVSUDeUA1gK_dgOccQeGu8AaRAdIfCnbs0riPalRbTD-zztjfCre3q6V_pr-NE7RwaHjd5rbOYgCg-r-gLqcIGVTbWeZR2UPO_fm_rY1h8IH4do_iCWnweOvcXVb4J8JAbLOuOpBROLxY2lEHRI4dK3da4xWDy9mCSDeVsK_4aIYx8HkgmG1gcKnkc6Hlchi7K47Q4Kn8kPHT4WpFIBiaRN74W340IRpttgAnugymQpR0nppNVI4bjitCUMGeiIOM743kv4vJ4RiSdHcKONgdGM8NLygLoAYhL4yBmFhgnvKQICXdGvZ-lJ_ZhV38aWpvTiGLhiQh_eGvl3pjymQeBkRD5mRAGl-_IZ5V8PCaUDgHbFLtF8k7TjyDN_gyltmd9NraZ6sNzghdqVV4DWEFp2Gk6cqqqmWkYA1ZrYx8cecdgUUyGaP7JwwCHq3pptOCv2ZXJ3IbXHOuFsA7NgOVuahnaAOo4MuQUXprruReq_7H87jyO37nlHPoz3Pb1F9z9xr9MdbiRFjgKWUqDifOzSXvjumVGk64Hj-3fTc_ZaXFrwVh0shi7fP71YAdj8bpFE7KUA9paG2-6qtEEnxabaqiLTO48Y0Kz6KIZ9w-VjEfljnDvRumw1RliAVaLiLQMQDzuoRRVuCRvfHsV7r5B8lNy9vdMyzcNaxbNppMrPgnT6OOi_hGusOtCOleXmLpfmQg4gF2fZXRPMH24cOTzXnwCeOsYOTjyVRYWxr8R_A5QL-mPC4USYC_N1Q1w46tZIilkWKHqB_w5yuIEtvGV63Gl1bHpKsU7PdkrbTPhdknrR1jzN6USzwVT_MbiyFuAtg7VyPFu5 + +### Explanation: +- This diagram represents the primary use cases associated with your smart agent server project. +- Each use case such as creating environments, deploying EC2 instances, and accessing SSM secrets represents functionalities your system supports. +- Annotations explain the components (like AMI) and setups (like custom VPC) tied to specific activities. + + +```plantuml +@startuml + +!define RECTANGLE(x) rectangle x as x +!define DIAMOND(x) diamond x as x + +package "Accounts/Swarm" { + RECTANGLE(swarms-aws-agent-api) { + package "Environments" { + RECTANGLE(dev) { + RECTANGLE(us_east_1) { + RECTANGLE(Godaddy) + RECTANGLE(Route53) + RECTANGLE(ALB) as alb + RECTANGLE(AutoScalingGroup) as asg + RECTANGLE(LaunchTemplate) as lt + RECTANGLE(AMI) as ami + } + } + } + } + Godaddy -> Route53: ns api.swarms.ai + Route53 -> alb: CNAME + alb -> asg + asg -> lt + lt -> ami: "components/machine_image/ubuntu-fastapi.pkr.hcl" + + package "EC2 Instance" { + RECTANGLE(EC2) { + RECTANGLE(UserData) + RECTANGLE(InstanceProfile) + RECTANGLE(Nginx) + RECTANGLE(Systemd) + RECTANGLE(Uvicorn) + RECTANGLE(FastAPI) + + EC2 -> UserData: github clone branch run + EC2 -> InstanceProfile: read access to SSM secrets + EC2 -> Nginx: isolates user process + EC2 -> Systemd: manage services + EC2 -> Uvicorn: "4 instances of FastAPI" + } + } + + UserData -> EC2: "run api/install.sh" + InstanceProfile -> EC2: "access tagged secrets" + Nginx -> Systemd: "manage processes" +} + +package "GitHub Actions" { + RECTANGLE(OIDC) { + RECTANGLE(SSM_Document) + RECTANGLE(Terraform) + } + OIDC -> AWS + AWS -> SSM_Document: "update server" + OIDC -> AWS: "create new clusters" + AWS -> Terraform +} + +package "Shared Hosted AWS Account" { + RECTANGLE(Profiles) { + RECTANGLE(Tags) + RECTANGLE(VPC) + } + Profiles -> Tags: "customer resources" + Profiles -> VPC: "subnet resources" + Profiles -> EC2: "instance roles" +} + +@enduml + + +Here's an updated PlantUML deployment diagram reflecting your architecture with key components: + +```plantuml +@startuml + +!define RECTANGLE(x) rectangle x as x +!define DIAMOND(x) diamond x as x + +package "Accounts/Swarm" { + RECTANGLE(swarms-aws-agent-api) { + package "Environments" { + RECTANGLE(dev) { + RECTANGLE(us_east_1) { + RECTANGLE(Godaddy) + RECTANGLE(Route53) + RECTANGLE(ALB) as alb + RECTANGLE(AutoScalingGroup) as asg + RECTANGLE(LaunchTemplate) as lt + RECTANGLE(AMI) as ami + } + } + } + } + Godaddy -> Route53: ns api.swarms.ai + Route53 -> alb: CNAME + alb -> asg + asg -> lt + lt -> ami: "components/machine_image/ubuntu-fastapi.pkr.hcl" + + package "EC2 Instance" { + RECTANGLE(EC2) { + RECTANGLE(UserData) + RECTANGLE(InstanceProfile) + RECTANGLE(Nginx) + RECTANGLE(Systemd) + RECTANGLE(Uvicorn) + RECTANGLE(FastAPI) + + EC2 -> UserData: github clone branch run + EC2 -> InstanceProfile: read access to SSM secrets + EC2 -> Nginx: isolates user process + EC2 -> Systemd: manage services + EC2 -> Uvicorn: "4 instances of FastAPI" + } + } + + UserData -> EC2: "run api/install.sh" + InstanceProfile -> EC2: "access tagged secrets" + Nginx -> Systemd: "manage processes" +} + +package "GitHub Actions" { + RECTANGLE(OIDC) { + RECTANGLE(SSM_Document) + RECTANGLE(Terraform) + } + OIDC -> AWS + AWS -> SSM_Document: "update server" + OIDC -> AWS: "create new clusters" + AWS -> Terraform +} + +package "Shared Hosted AWS Account" { + RECTANGLE(Profiles) { + RECTANGLE(Tags) + RECTANGLE(VPC) + } + Profiles -> Tags: "customer resources" + Profiles -> VPC: "subnet resources" + Profiles -> EC2: "instance roles" +} + +@enduml +``` + + +It seems there was some confusion. Here’s a refined version of the complete PlantUML deployment diagram for your smart agent server setup: + +```plantuml +@startuml + +!define RECTANGLE(x) rectangle x as x +!define DIAMOND(x) diamond x as x + +package "Accounts/Swarms" { + RECTANGLE("swarms-aws-agent-api") { + package "Environments" { + RECTANGLE("dev") { + RECTANGLE("us-east-1") { + RECTANGLE(Godaddy) + RECTANGLE(Route53) + RECTANGLE(ALB) as alb + RECTANGLE("AutoScalingGroup") as asg + RECTANGLE("LaunchTemplate") as lt + RECTANGLE("AMI") as ami + } + } + } + } + Godaddy -> Route53: ns api.swarms.ai + Route53 -> alb: CNAME + alb -> asg + asg -> lt + lt -> ami: "components/machine_image/ubuntu-fastapi.pkr.hcl" + + package "EC2 Instance" { + RECTANGLE("EC2") { + RECTANGLE("UserData") as userdata + RECTANGLE("InstanceProfile") as ip + RECTANGLE("Nginx") as nginx + RECTANGLE("Systemd") as systemd + RECTANGLE("Uvicorn") as uvicorn + RECTANGLE("FastAPI") as fastapi + + EC2 -> userdata: "github clone branch run api/install.sh" + EC2 -> ip: "read access to SSM secrets" + EC2 -> nginx: "isolates user process" + EC2 -> systemd: "manages services" + EC2 -> uvicorn: "4 instances of FastAPI" + } + } + + userdata -> EC2: "execute boot.sh" + ip -> EC2: "access tagged secrets" + nginx -> systemd: "manage processes" +} + +package "GitHub Actions" { + RECTANGLE("OIDC") { + RECTANGLE("AWS") + RECTANGLE("SSM Document") + RECTANGLE("Terraform") + } + OIDC -> AWS + AWS -> SSM Document: "update server" + OIDC -> AWS: "create new clusters" +} + +package "Shared Hosted AWS Account" { + RECTANGLE("Profiles") { + RECTANGLE("Tags") + RECTANGLE("VPC") + } + Profiles -> Tags: "customer resources" + Profiles -> VPC: "subnet resources" + Profiles -> EC2: "instance roles" +} + +note right of ami: "Designed for\nFastAPI deployments" +@enduml +``` + +### Key Components: +1. **Accounts/Swarms**: Represents the main framework for your smart agent server. +2. **Environments**: Contains different environments such as `dev` in `us-east-1`. +3. **EC2 Instance**: Details the EC2 setup, including user data and necessary components (Nginx, Systemd, Uvicorn). +4. **GitHub Actions**: Integrates CI/CD using OIDC, SSM, and Terraform. +5. **Shared Hosted AWS Account**: Manages profiles, tags, and VPC configurations for customer resources. + +You can copy this code into a PlantUML tool to visualize the deployment diagram. +### Explanation: +- **Packages and Rectangles**: The diagram encapsulates components like Route 53, ALB, and Auto Scaling Group within a structured format. +- **Interactions**: Arrows show the relationships and interactions among components, such as the CNAME setup with Route 53 and the EC2 instances launching user data scripts. +- **GitHub Actions**: Represents automation activities and connections to AWS services. +- **Shared AWS Account**: Displays customer management and tagging for resources. + +This structure captures your specifications for the deployment of a self-service Cognito-enabled smart agent server using Terraform and AWS components effectively. Adjust as necessary for your specific documentation needs!``` + +### Explanation: +- The main structure includes the accounts, environments, EC2 instances, GitHub Actions, and shared AWS resources. +- The individual components and their interactions are represented using rectangles for entities and arrows for relationships. +- Customize further by adding details and refining the relationships based on your specific needs.``` + +To manage a shared hosted AWS account for multiple customers with the requirements you've laid out, you can consider the following architecture: + +### AWS Account Structure: +1. **Profiles**: Utilize IAM roles and users for each customer. Create separate profiles in the AWS CLI for easy access. + +2. **Resource Tagging**: Implement a consistent tagging strategy for all resources (e.g., `CustomerName`, `Project`, `Environment`, `CostCenter`) to facilitate cost management and resource identification. + +3. **API Call Restrictions**: Use IAM policies to specify which API calls each customer's role can perform based on their needs. + +4. **Cost Tracking**: Leverage AWS Cost Explorer and AWS Budgets, filtering by tags to track costs per customer. + +5. **Networking**: Set up custom VPCs for customers if isolation is required. Use subnets and security groups to segment resources. + +### Server and Agent Management: +1. **AWS Roles for Agents**: Each agent (server) should have an IAM role assigned that grants necessary permissions. If using Lambda, this can be done via execution roles. + +2. **Unix User Management**: Depending on your requirements: + - One option is to create a Unix user for each customer/agent. + - Assign unique home directories, storage solutions (e.g., EBS, S3), and ensure proper permissions. + +3. **Containerization**: Use Docker and Kubernetes (EKS) to manage applications. This allows for easier resource allocation and isolation between different customers. + +4. **Email & Backup**: Consider using Amazon SES for email needs and AWS Backup/CloudFormation for automated backups of customer data. + +### Best Practices: +- **Security**: Regularly audit IAM roles and policies for least privilege access. +- **Monitoring**: Use CloudWatch for logging and monitoring resource utilization. +- **Cost Management**: Use AWS Cost Allocation Reports to analyze customer costs effectively. + +This architecture ensures streamlined operations, cost tracking, and enhanced security across customer resources in a shared AWS account. + + + +### Repo Structure Overview + +- **Root Directory**: The main container of your Terraform project (`terraform-template/`). + +- **README.md**: Provides an overview and documentation for the project, helping users understand its purpose and how to use it. + +- **Environments**: Contains specific configurations for different environments (e.g., development, staging). + - **swarms-aws-agent-api**: An example service with various setups. + - **dev/us-east-1**: The development environment tailored for the US East (N. Virginia) region. + +- **Modules**: Encapsulates reusable Terraform code to ensure DRY (Don't Repeat Yourself) principles. + - **swarms**: A collection of Terraform modules related to the swarm infrastructure. + +### Components Explained + +- **DNS Management**: + - Using GoDaddy for domain management and AWS Route 53 for DNS routing (e.g., `api.swarms.ai`). + +- **Application Layer**: + - Includes an Application Load Balancer directing traffic to an Auto Scaling Group, which manages instances defined by a launch template. + +- **Instance Management**: + - AMIs (Amazon Machine Images) created using Packer (e.g., `ubuntu-fastapi.pkr.hcl`). + +- **Settings for EC2 Instances**: + - User data scripts to set up necessary configurations, such as cloning from GitHub and running scripts as the `swarms` user to set up the FastAPI application. + +- **Access Management**: + - IAM roles and instance profiles are created to manage permissions. SSM (AWS Systems Manager) secrets are tagged, allowing specific servers to access configurations, like OpenAI keys. + +- **Networking**: + - Outbound traffic settings defined to control which IP ranges can communicate with the infrastructure. + +- **Application Deployment**: + - Utilizes Nginx and Systemd to manage FastAPI processes, ensuring multiple instances run smoothly. + +### CI/CD with GitHub Actions +- Implements OIDC connections to AWS for secure deployments and updates using GitHub Actions. +- SSM documents are utilized for executing specific management tasks, like creating new clusters and updating configurations seamlessly. + +This structure facilitates a modular, maintainable, and automated approach to infrastructure as code using Terraform, alongside solid CI/CD practices. + + + + From 88ef79a5ddc0e54292aa3961226b197b601b30d1 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Tue, 14 Jan 2025 17:50:24 -0500 Subject: [PATCH 146/158] tofu --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 3242179..9c9d5fd 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,8 @@ ``` git clone https://github.com/aitokenteam/ai-agent-terraform.git cd ai-agent-terraform/accounts/AI_Token_Team/ +git checkout feature/aitokenteam + tofu init aws ecr create-repository --repository-name agent/eliza From 24b2c909b8c953144f37cf6f5d8f82fdcc7330a6 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Tue, 14 Jan 2025 17:52:27 -0500 Subject: [PATCH 147/158] adding codebuild --- accounts/mdupont/codebuild/main.tf | 42 ++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 accounts/mdupont/codebuild/main.tf diff --git a/accounts/mdupont/codebuild/main.tf b/accounts/mdupont/codebuild/main.tf new file mode 100644 index 0000000..624fe7d --- /dev/null +++ b/accounts/mdupont/codebuild/main.tf @@ -0,0 +1,42 @@ +# https://github.com/cloudposse/terraform-aws-codebuild.githttps://github.com/cloudposse/terraform-aws-codebuild.git +# module "build" { +# source = "cloudposse/codebuild/aws" +# version = "2.0.2" +# # Optional extra environment variables +# environment_variables = [ + +# ] +# } + + +module "build" { + source = "cloudposse/cicd/aws" + # Cloud Posse recommends pinning every module to a specific version + version = "0.22.0" + namespace = "main" + stage = "staging" + name = "arm64-tokenizers" + enabled = true + # Application repository on GitHub +# github_oauth_token = "(Required) " +# repo_owner = "" +# repo_name = "" +# branch = "" + + # http://docs.aws.amazon.com/codebuild/latest/userguide/build-env-ref.html + # http://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html + build_compute_type = "BUILD_GENERAL1_SMALL" + privileged_mode = true + region = "us-east-2" + image_repo_name = "nodemodules/tokenizer" + image_tag = "latest" + + build_image = "aws/codebuild/amazonlinux2-aarch64-standard:3.0" + build_type = "ARM_CONTAINER" + build_timeout = 60 + +# image_repo_name = "nodemodules/tokenizer" +# image_tag = "latest" + + +} From 7c751db4f978e1b766b12686edffbd7f2128502f Mon Sep 17 00:00:00 2001 From: mike dupont Date: Tue, 14 Jan 2025 22:08:34 -0500 Subject: [PATCH 148/158] notes --- README.md | 28 ++++++++++++++++++++++++++++ accounts/mdupont/codebuild/main.tf | 2 +- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9c9d5fd..7d26879 100644 --- a/README.md +++ b/README.md @@ -149,3 +149,31 @@ resource "aws_key_pair" "deployer" { public_key = " Date: Tue, 14 Jan 2025 22:26:29 -0500 Subject: [PATCH 149/158] bugfix --- accounts/AI_Token_Team/main.tf | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/accounts/AI_Token_Team/main.tf b/accounts/AI_Token_Team/main.tf index 3afbf7e..e34822a 100644 --- a/accounts/AI_Token_Team/main.tf +++ b/accounts/AI_Token_Team/main.tf @@ -53,6 +53,8 @@ module "ssm_setup" { # "t3.medium" # works for sure ] - - aws_availability_zones = ["us-west-1a","us-west-1b","us-west-1c"] + repo = "https://github.com/meta-introspector/cloud-deployment-eliza/" + aws_availability_zones = ["us-west-1a", + #"us-west-1b", + "us-west-1c"] } From 02933b3a4f5214eb2b95f7e0704fdc032e33d850 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sun, 19 Jan 2025 19:08:34 -0500 Subject: [PATCH 150/158] v1 of solfnum meme infra plan --- README.md | 54 ++++++++++++++++++ accounts/mdupont/.terraform.lock.hcl | 83 +++++++++++++++++++++++++++- 2 files changed, 136 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7d26879..f3aaa15 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,57 @@ +# solfunmeme branch of ai-agent terraform + +This will deploy the : +1. solfunmeme dao agent to work for the solfunmeme dao +1.1. (eliza) + +2. solfunmeme web server(done) +2.1 to vercel +2.1.1. (done, manually not terraformed) +2.2 to aws (TBD) +https://codeberg.org/introspector/SOLFUNMEME/issues/25 + +stakeholders will login via the phantom wallet compatible web app with wallet and be able to vote on important decisions : + +3. terraform to admin discord : +who is admin in + telegram + discord + +what versions of what bots with what permissions do we have in telegram +what software projects are important to work on +what marketing and listing should we persue +etc. +This will be the core of the dao, I will use this as my oracle for every day decisions and try and act in the best interest of the team and hope all of us will. + +We are going to build something amazing, I have tons of work ready, many ideas ready to launch when we have this running. + +3. the dns records to namecheap. +3.1 done manually +3.2 add terraform + +4. cognito with web3 Integration +https://codeberg.org/introspector/SOLFUNMEME/issues/5 + +## agent spec : + +What the agent will do + +resolve tickets : + in codeberg and + in github and + later in gitlab, jira + +but talk in: + discord, + telegram, + twitter it should + +the agent will update the main web page as well. +we will deploy the main page to terraform as well. + +LANGUAGE API: +Deploy open lite llm server, connect to multiple backends. + # terraform-template ``` diff --git a/accounts/mdupont/.terraform.lock.hcl b/accounts/mdupont/.terraform.lock.hcl index ff8924c..04b866b 100644 --- a/accounts/mdupont/.terraform.lock.hcl +++ b/accounts/mdupont/.terraform.lock.hcl @@ -1,4 +1,4 @@ -# This file is maintained automatically by "tofu init". +# This file is maintained automatically by "terraform init". # Manual edits may be lost in future updates. provider "registry.opentofu.org/hashicorp/aws" { @@ -35,3 +35,84 @@ provider "registry.opentofu.org/hashicorp/random" { "zh:f423f2b7e5c814799ad7580b5c8ae23359d8d342264902f821c357ff2b3c6d3d", ] } + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.83.1" + constraints = ">= 2.0.0, >= 3.29.0, >= 4.6.0, >= 5.46.0, >= 5.55.0" + hashes = [ + "h1:vInFMDq9oMs53/i+7IU8hZgmTLhFfng8L8kbuALZxSI=", + "zh:0313253c78f195973752c4d1f62bfdd345a9c99c1bc7a612a8c1f1e27d51e49e", + "zh:108523f3e9ebc93f7d900c51681f6edbd3f3a56b8a62b0afc31d8214892f91e0", + "zh:175b9bf2a00bea6ac1c73796ad77b0e00dcbbde166235017c49377d7763861d8", + "zh:1c8bf55b8548bbad683cd6d7bdb03e8840a00b2422dc1529ffb9892820657130", + "zh:22338f09bae62d5ff646de00182417f992548da534fee7d98c5d0136d4bd5d7a", + "zh:92de1107ec43de60612be5f6255616f16a9cf82d88df1af1c0471b81f3a82c16", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9c7bfb7afea330e6d90e1466125a8cba3db1ed4043c5da52f737459c89290a6e", + "zh:ba59b374d477e5610674b70f5abfe0408e8f809390347372751384151440d3d0", + "zh:bd1c433966002f586d63cb1e3e16326991f238bc6beeb2352be36ec651917b0b", + "zh:ca2b4d1d02651c15261fffa4b142e45def9a22c6069353f0f663fd2046e268f8", + "zh:d8ed98c748f7a3f1a72277cfee9afe346aca39ab319d17402277852551d8f14a", + "zh:ed3d8bc89de5f35f3c5f4802ff7c749fda2e2be267f9af4a850694f099960a72", + "zh:f698732a4391c3f4d7079b4aaa52389da2a460cac5eed438ed688f147d603689", + "zh:f9f51b17f2978394954e9f6ab9ef293b8e11f1443117294ccf87f7f8212b3439", + ] +} + +provider "registry.terraform.io/hashicorp/github" { + version = "3.0.0" + constraints = "3.0.0" + hashes = [ + "h1:1uhEr3As7wjphmSyInm3W2eF3WrEYwncp133C3cx14s=", + "zh:043081c4c2f8fdeb3d694eb6cd81225c55747ac8b6adec11968aba68dc97c026", + "zh:09c6aab8b2b4c7017b8c2d8e1ef0939cc7e7db81e1cca961291251d0bfeb7050", + "zh:28109dca3cb62347dffbe03fc923d7db0989f0e0dca3e3f20b8eb2883f21617f", + "zh:4af0a15bbc996abc3c20978d600af53143e4f8343b81b76f543c0a926d3a6e82", + "zh:876416f2e1445c8e53a84c723d9c2051115a142efa6082d5939ca6371531919b", + "zh:9aad918f0295adde2f4752361c93522a975a095b95ec6a331d97587bb586e061", + "zh:a977cc85757a8953de32d792993ca38f562edd0d79b3c4bd93021a68f20adc77", + "zh:ba384ba916dd8bc0a2c854e16a31d2075f62018a73a60917c89945da07a13fdd", + "zh:e870f594225491d6b05b6db05a064e391b42fd8e43576f3254a3f9dbb012b685", + "zh:f722f3ca830d59dc30ba25b8b4ab9e77ed229385584f2501275a4321189da54d", + ] +} + +provider "registry.terraform.io/hashicorp/local" { + version = "2.5.2" + constraints = ">= 1.2.0" + hashes = [ + "h1:JlMZD6nYqJ8sSrFfEAH0Vk/SL8WLZRmFaMUF9PJK5wM=", + "zh:136299545178ce281c56f36965bf91c35407c11897f7082b3b983d86cb79b511", + "zh:3b4486858aa9cb8163378722b642c57c529b6c64bfbfc9461d940a84cd66ebea", + "zh:4855ee628ead847741aa4f4fc9bed50cfdbf197f2912775dd9fe7bc43fa077c0", + "zh:4b8cd2583d1edcac4011caafe8afb7a95e8110a607a1d5fb87d921178074a69b", + "zh:52084ddaff8c8cd3f9e7bcb7ce4dc1eab00602912c96da43c29b4762dc376038", + "zh:71562d330d3f92d79b2952ffdda0dad167e952e46200c767dd30c6af8d7c0ed3", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:805f81ade06ff68fa8b908d31892eaed5c180ae031c77ad35f82cb7a74b97cf4", + "zh:8b6b3ebeaaa8e38dd04e56996abe80db9be6f4c1df75ac3cccc77642899bd464", + "zh:ad07750576b99248037b897de71113cc19b1a8d0bc235eb99173cc83d0de3b1b", + "zh:b9f1c3bfadb74068f5c205292badb0661e17ac05eb23bfe8bd809691e4583d0e", + "zh:cc4cbcd67414fefb111c1bf7ab0bc4beb8c0b553d01719ad17de9a047adff4d1", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.6.3" + constraints = ">= 2.1.0" + hashes = [ + "h1:Fnaec9vA8sZ8BXVlN3Xn9Jz3zghSETIKg7ch8oXhxno=", + "zh:04ceb65210251339f07cd4611885d242cd4d0c7306e86dda9785396807c00451", + "zh:448f56199f3e99ff75d5c0afacae867ee795e4dfda6cb5f8e3b2a72ec3583dd8", + "zh:4b4c11ccfba7319e901df2dac836b1ae8f12185e37249e8d870ee10bb87a13fe", + "zh:4fa45c44c0de582c2edb8a2e054f55124520c16a39b2dfc0355929063b6395b1", + "zh:588508280501a06259e023b0695f6a18149a3816d259655c424d068982cbdd36", + "zh:737c4d99a87d2a4d1ac0a54a73d2cb62974ccb2edbd234f333abd079a32ebc9e", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:a357ab512e5ebc6d1fda1382503109766e21bbfdfaa9ccda43d313c122069b30", + "zh:c51bfb15e7d52cc1a2eaec2a903ac2aff15d162c172b1b4c17675190e8147615", + "zh:e0951ee6fa9df90433728b96381fb867e3db98f66f735e0c3e24f8f16903f0ad", + "zh:e3cdcb4e73740621dabd82ee6a37d6cfce7fee2a03d8074df65086760f5cf556", + "zh:eff58323099f1bd9a0bec7cb04f717e7f1b2774c7d612bf7581797e1622613a0", + ] +} From 9c707b96ce74d2578198943847adf95cab290a38 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sun, 19 Jan 2025 20:25:07 -0500 Subject: [PATCH 151/158] plan with one click --- README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f3aaa15..ccec41a 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,15 @@ +# one click installer plan + +Using cloudformation +https://github.com/meta-introspector/cfn-tf-meta-introspector/issues/1 still need someone to try this + +we want a step by step instructions to setup your aws +and then they click on this +https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/new?stackName=tfcfn-custom-type-resources&templateURL=https://s3.amazonaws.com/ianmckay-us-east-1/cfn-tf-custom-types/template.yml + +and then we port our repo https://github.com/meta-introspector/ai-agent-terraform/tree/feature/aitokenteam into running inside of cloudformation + + # solfunmeme branch of ai-agent terraform This will deploy the : @@ -172,7 +184,7 @@ start direct client here edit aws_availability_zones = ["us-west-1a","us-west-1b","us-west-1c"] -if you ge the errro : +if you ge the errror: 00000003] module.ssm_setup.module.ssm.aws_s3_bucket_lifecycle_configuration.access_log_bucket: Creation complete after 32s [id=ai-token-team-session-access-logs-20250113203757298300000002] ╷ From 1de1e7fd66d01755b7dfbb8d67486b21ddd0d50c Mon Sep 17 00:00:00 2001 From: mike dupont Date: Mon, 20 Jan 2025 13:40:25 -0500 Subject: [PATCH 152/158] parking for jump --- README.md | 16 ++++++++++++---- accounts/mdupont/codebuild/main.tf | 12 ++++++------ 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index ccec41a..3572948 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,16 @@ https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/new? and then we port our repo https://github.com/meta-introspector/ai-agent-terraform/tree/feature/aitokenteam into running inside of cloudformation +TODO : Great asset + transform our terraform resources into cloudformation yaml to use tf provider + we could deploy azure or gcp from aws in terraform. + +For the mean time, we will deploy terraform. + +Another alternative is +https://runatlantis.io/ -# solfunmeme branch of ai-agent terraform +# solfun meme branch of ai-agent terraform This will deploy the : 1. solfunmeme dao agent to work for the solfunmeme dao @@ -71,14 +79,14 @@ git clone https://github.com/aitokenteam/ai-agent-terraform.git cd ai-agent-terraform/accounts/AI_Token_Team/ git checkout feature/aitokenteam -tofu init +terraform init aws ecr create-repository --repository-name agent/eliza aws ssm put-parameter --name "agent_openai_key" --value "${OPENAI_API_KEY}" --type String -tofu apply --target module.ssm_observer.aws_ssm_parameter.cw_agent_config --target module.ssm_observer.aws_ssm_parameter.cw_agent_config_details -tofu apply +terraform apply --target module.ssm_observer.aws_ssm_parameter.cw_agent_config --target module.ssm_observer.aws_ssm_parameter.cw_agent_config_details +terraform apply ``` Terraform project template for deploying infrastructure across multiple environments and regions, following best practices with modular structure and automated syntax checks (GitHub Actions) diff --git a/accounts/mdupont/codebuild/main.tf b/accounts/mdupont/codebuild/main.tf index ab49e10..93c6793 100644 --- a/accounts/mdupont/codebuild/main.tf +++ b/accounts/mdupont/codebuild/main.tf @@ -14,14 +14,14 @@ module "build" { # Cloud Posse recommends pinning every module to a specific version version = "0.20.0" namespace = "main" + repo_owner = "meta-introspector" stage = "staging" name = "arm64-tokenizers" enabled = true # Application repository on GitHub -# github_oauth_token = "(Required) " -# repo_owner = "" -# repo_name = "" -# branch = "" + github_oauth_token = "(Required) " + repo_name = "test" + branch = "test" # http://docs.aws.amazon.com/codebuild/latest/userguide/build-env-ref.html # http://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html @@ -32,8 +32,8 @@ module "build" { image_tag = "latest" build_image = "aws/codebuild/amazonlinux2-aarch64-standard:3.0" - build_type = "ARM_CONTAINER" - build_timeout = 60 + # build_type = "ARM_CONTAINER" +# build_timeout = 60 # image_repo_name = "nodemodules/tokenizer" # image_tag = "latest" From 66e00d4e8b25901a8f6bafba899a426ec9f12069 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Tue, 21 Jan 2025 10:17:57 -0500 Subject: [PATCH 153/158] now applied with terraform adding readme to remove driftwood --- accounts/mdupont/Readme.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 accounts/mdupont/Readme.md diff --git a/accounts/mdupont/Readme.md b/accounts/mdupont/Readme.md new file mode 100644 index 0000000..d215e72 --- /dev/null +++ b/accounts/mdupont/Readme.md @@ -0,0 +1,7 @@ +removed +``` +terraform state rm "module.codebuild.module.build.random_string.bucket_prefix[0]" +terraform state rm "module.codebuild.module.build.aws_iam_role_policy_attachment.default[0]" +terraform state rm "module.codebuild.module.build.aws_iam_role.default[0]" +terraform state rm "module.codebuild.module.build.aws_iam_policy.default[0]" +``` From 6b012ca455b067be1bb32dd425df90084e30a3d1 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Tue, 21 Jan 2025 11:03:09 -0500 Subject: [PATCH 154/158] lagging changes --- accounts/mdupont/codebuild/main.tf | 3 ++- accounts/mdupont/main.tf | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/accounts/mdupont/codebuild/main.tf b/accounts/mdupont/codebuild/main.tf index 93c6793..a72a27f 100644 --- a/accounts/mdupont/codebuild/main.tf +++ b/accounts/mdupont/codebuild/main.tf @@ -11,10 +11,11 @@ module "build" { source = "cloudposse/cicd/aws" + # 2025/01/14/terraform-aws-cicd # Cloud Posse recommends pinning every module to a specific version version = "0.20.0" namespace = "main" - repo_owner = "meta-introspector" + repo_owner = "meta-introspector" stage = "staging" name = "arm64-tokenizers" enabled = true diff --git a/accounts/mdupont/main.tf b/accounts/mdupont/main.tf index 23d938b..6c484dd 100644 --- a/accounts/mdupont/main.tf +++ b/accounts/mdupont/main.tf @@ -105,6 +105,6 @@ module "ssm_setup" { } -module "codebuild" { - source = "./codebuild" -} +#module "codebuild" { +# source = "./codebuild" +#} From db8077a4daf0727c0960fd3c00893cd76fe69863 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sun, 26 Jan 2025 08:10:36 -0500 Subject: [PATCH 155/158] update readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 3572948..e7df890 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,7 @@ git checkout feature/aitokenteam terraform init aws ecr create-repository --repository-name agent/eliza +aws ecr create-repository --repository-name nodemodules/sql-lite-vec aws ssm put-parameter --name "agent_openai_key" --value "${OPENAI_API_KEY}" --type String From 92398e9c89e54390fdabb5d3b8199bbf2bd6669a Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sun, 26 Jan 2025 09:45:08 -0500 Subject: [PATCH 156/158] refactor split into main.tf, variables and constants only the variables need to be edited by users --- accounts/np-introspector/constants.tf | 30 ++++++++++++++++ accounts/np-introspector/main.tf | 52 +++++++++++++++++++++++++++ accounts/np-introspector/variables.tf | 37 +++++++++++++++++++ 3 files changed, 119 insertions(+) create mode 100644 accounts/np-introspector/constants.tf create mode 100644 accounts/np-introspector/main.tf create mode 100644 accounts/np-introspector/variables.tf diff --git a/accounts/np-introspector/constants.tf b/accounts/np-introspector/constants.tf new file mode 100644 index 0000000..822ee80 --- /dev/null +++ b/accounts/np-introspector/constants.tf @@ -0,0 +1,30 @@ + +variable "lock_resource" { + type = string + default = "terraform/state/lock" +} + +variable "partition" { + type = string + default = "aws" +} + +variable "logs_resource" { + type = string + default = "aws_logs" +} + +variable "permissions_check" { + type = string + default = "config-permissions-check" +} + +variable "delivery_service" { + type = string + default = "delivery.logs.amazonaws.com" +} + +variable "logging_service" { + type = string + default = "logging.s3.amazonaws.com" +} diff --git a/accounts/np-introspector/main.tf b/accounts/np-introspector/main.tf new file mode 100644 index 0000000..75f66bb --- /dev/null +++ b/accounts/np-introspector/main.tf @@ -0,0 +1,52 @@ +provider "aws" { + region = "ap-south-1" + profile = "np-introspector" +} + +locals { + # hard coded to save time , fixme use a caching system + # ami_id = "ami-0325b9a2dfb474b2d" for ami_name = "ubuntu-minimal/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-minimal-*" } + ami_id = "ami-0e44962f5c9a2baab" +} + +module "ssm_observer" { + source = "../../modules/aws/ssm/observability" + ami_id = local.ami_id +} + +module "ssm_setup" { + source = "../../modules/aws/ssm/setup" + bucket_name = "${var.codename}-session-logs" + access_log_bucket_name = "${var.codename}-session-access-logs" + project = "${var.codename}" + } + + + # now after we create the above resources, we can do the following, + # FIXME need to add dependencies + module "eliza_server" { + #count = 0 + #aws_account_id = var.account + aws_account_id =var.aws_account_id + region = var.region + source = "../../environments/eliza-agent-api" + domain = var.dns + key_name = "mdupont-deployer-key" + branch = "feature/arm64_fastembed" + project = var.codename + instance_types = ["t4g.small"] # not big enough for building + + repo = "https://github.com/meta-introspector/cloud-deployment-eliza/" + aws_availability_zones =[ + "${var.region}a", + "${var.region}b", + "${var.region}c" + ] + +# FIXME not used right now + spot_max_price= 0.01 + ami_id = local.ami_id #data.aws_ami.ami.id + name = "${var.codename}" + tags = { project = "${var.codename}" } +} + diff --git a/accounts/np-introspector/variables.tf b/accounts/np-introspector/variables.tf new file mode 100644 index 0000000..faf567f --- /dev/null +++ b/accounts/np-introspector/variables.tf @@ -0,0 +1,37 @@ +variable "project_name" { + type = string + default = "np.introspector.meme" +} + +variable "codename" { + type = string + default = "hanuman" +} + +variable "project_name" { + type = string + default = "np-introspector" +} + +variable "profile" { + type = string + default = "np-introspector" +} + +variable "aws_region" { + type = string + default = "ap-south-2" +} + +variable "aws_account_id" { + type = string + default = "084375543224" +} + +variable "iam_user" { + type = string + default = "np_introspector" +} + + + From 730e400e0467fca4899a0c777f248c7e5a31d46b Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sun, 26 Jan 2025 10:06:08 -0500 Subject: [PATCH 157/158] fixed deps --- accounts/np-introspector/main.tf | 31 ++++++++++++++++++++------- accounts/np-introspector/variables.tf | 2 +- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/accounts/np-introspector/main.tf b/accounts/np-introspector/main.tf index 75f66bb..e9ec708 100644 --- a/accounts/np-introspector/main.tf +++ b/accounts/np-introspector/main.tf @@ -3,10 +3,19 @@ provider "aws" { profile = "np-introspector" } +locals { ami_name = "ubuntu-minimal/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-minimal-*" } + +data "aws_ami" "ami" { # slow + most_recent = true + name_regex = "^${local.ami_name}" + } + + locals { # hard coded to save time , fixme use a caching system # ami_id = "ami-0325b9a2dfb474b2d" for ami_name = "ubuntu-minimal/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-minimal-*" } - ami_id = "ami-0e44962f5c9a2baab" + # ami_id = "ami-0e44962f5c9a2baab" + ami_id = data.aws_ami.ami.id } module "ssm_observer" { @@ -25,12 +34,18 @@ module "ssm_setup" { # now after we create the above resources, we can do the following, # FIXME need to add dependencies module "eliza_server" { - #count = 0 + depends_on = [ + # module.ssm_setup. + #│ arn:aws:ssm:ap-south-2:084375543224:parameter/cloudwatch-agent/config/details + module.ssm_observer #.aws_ssm_parameter.cw_agent_config, + # module.ssm_observer.aws_ssm_parameter.cw_agent_config_details + ] +#count = 0 #aws_account_id = var.account aws_account_id =var.aws_account_id - region = var.region + region = var.aws_region source = "../../environments/eliza-agent-api" - domain = var.dns + domain = var.dns_name key_name = "mdupont-deployer-key" branch = "feature/arm64_fastembed" project = var.codename @@ -38,14 +53,14 @@ module "ssm_setup" { repo = "https://github.com/meta-introspector/cloud-deployment-eliza/" aws_availability_zones =[ - "${var.region}a", - "${var.region}b", - "${var.region}c" + "${var.aws_region}a", + "${var.aws_region}b", + "${var.aws_region}c" # FIXME ] # FIXME not used right now spot_max_price= 0.01 - ami_id = local.ami_id #data.aws_ami.ami.id + ami_id = local.ami_id # name = "${var.codename}" tags = { project = "${var.codename}" } } diff --git a/accounts/np-introspector/variables.tf b/accounts/np-introspector/variables.tf index faf567f..20a990a 100644 --- a/accounts/np-introspector/variables.tf +++ b/accounts/np-introspector/variables.tf @@ -1,4 +1,4 @@ -variable "project_name" { +variable "dns_name" { type = string default = "np.introspector.meme" } From 87b281261958468f26b9e22542a2461680cfc5c7 Mon Sep 17 00:00:00 2001 From: Deadsg <144394753+Deadsg@users.noreply.github.com> Date: Sun, 26 Jan 2025 17:45:53 -0500 Subject: [PATCH 158/158] Update README.md --- .../components/cognito_user_pool/README.md | 276 +----------------- 1 file changed, 1 insertion(+), 275 deletions(-) diff --git a/environments/eliza-agent-api/components/cognito_user_pool/README.md b/environments/eliza-agent-api/components/cognito_user_pool/README.md index 505f95a..1665bd2 100644 --- a/environments/eliza-agent-api/components/cognito_user_pool/README.md +++ b/environments/eliza-agent-api/components/cognito_user_pool/README.md @@ -1,275 +1 @@ -# manual setup - -1. Created auth client here -https://console.cloud.google.com/auth/clients/ - -2. stored password in -``` -export TF_VAR_google_oauth_client_id=DDDDD -export TF_VAR_google_oauth_client_secret=XXXX -``` - -# This is a complete example - -``` -module "aws_cognito_user_pool_complete_example" { - - source = "lgallard/cognito-user-pool/aws" - - user_pool_name = "mypool_complete" - alias_attributes = ["email", "phone_number"] - auto_verified_attributes = ["email"] - sms_authentication_message = "Your username is {username} and temporary password is {####}." - sms_verification_message = "This is the verification message {####}." - - deletion_protection = "ACTIVE" - - mfa_configuration = "OPTIONAL" - software_token_mfa_configuration = { - enabled = true - } - - admin_create_user_config = { - email_message = "Dear {username}, your verification code is {####}." - email_subject = "Here, your verification code baby" - sms_message = "Your username is {username} and temporary password is {####}." - } - - device_configuration = { - challenge_required_on_new_device = true - device_only_remembered_on_user_prompt = true - } - - email_configuration = { - email_sending_account = "DEVELOPER" - reply_to_email_address = "email@mydomain.com" - source_arn = "arn:aws:ses:us-east-1:123456789012:identity/myemail@mydomain.com" - } - - lambda_config = { - create_auth_challenge = "arn:aws:lambda:us-east-1:123456789012:function:create_auth_challenge" - custom_message = "arn:aws:lambda:us-east-1:123456789012:function:custom_message" - define_auth_challenge = "arn:aws:lambda:us-east-1:123456789012:function:define_auth_challenge" - post_authentication = "arn:aws:lambda:us-east-1:123456789012:function:post_authentication" - post_confirmation = "arn:aws:lambda:us-east-1:123456789012:function:post_confirmation" - pre_authentication = "arn:aws:lambda:us-east-1:123456789012:function:pre_authentication" - pre_sign_up = "arn:aws:lambda:us-east-1:123456789012:function:pre_sign_up" - pre_token_generation = "arn:aws:lambda:us-east-1:123456789012:function:pre_token_generation" - user_migration = "arn:aws:lambda:us-east-1:123456789012:function:user_migration" - verify_auth_challenge_response = "arn:aws:lambda:us-east-1:123456789012:function:verify_auth_challenge_response" - } - - password_policy = { - minimum_length = 10 - require_lowercase = false - require_numbers = true - require_symbols = true - require_uppercase = true - temporary_password_validity_days = 120 - - } - - user_pool_add_ons = { - advanced_security_mode = "ENFORCED" - } - - verification_message_template = { - default_email_option = "CONFIRM_WITH_CODE" - } - - schemas = [ - { - attribute_data_type = "Boolean" - developer_only_attribute = false - mutable = true - name = "available" - required = false - }, - { - attribute_data_type = "Boolean" - developer_only_attribute = true - mutable = true - name = "registered" - required = false - } - ] - - string_schemas = [ - { - attribute_data_type = "String" - developer_only_attribute = false - mutable = false - name = "email" - required = true - - string_attribute_constraints = { - min_length = 7 - max_length = 15 - } - }, - { - attribute_data_type = "String" - developer_only_attribute = false - mutable = false - name = "gender" - required = true - - string_attribute_constraints = { - min_length = 7 - max_length = 15 - } - }, - ] - - number_schemas = [ - { - attribute_data_type = "Number" - developer_only_attribute = true - mutable = true - name = "mynumber1" - required = false - - number_attribute_constraints = { - min_value = 2 - max_value = 6 - } - }, - { - attribute_data_type = "Number" - developer_only_attribute = true - mutable = true - name = "mynumber2" - required = false - - number_attribute_constraints = { - min_value = 2 - max_value = 6 - } - }, - ] - - # user_pool_domain - domain = "mydomain-com" - - # clients - clients = [ - { - allowed_oauth_flows = [] - allowed_oauth_flows_user_pool_client = false - allowed_oauth_scopes = [] - callback_urls = ["https://mydomain.com/callback"] - default_redirect_uri = "https://mydomain.com/callback" - explicit_auth_flows = [] - generate_secret = true - logout_urls = [] - name = "test1" - read_attributes = ["email"] - supported_identity_providers = [] - write_attributes = [] - access_token_validity = 1 - id_token_validity = 1 - refresh_token_validity = 60 - token_validity_units = { - access_token = "hours" - id_token = "hours" - refresh_token = "days" - } - }, - { - allowed_oauth_flows = [] - allowed_oauth_flows_user_pool_client = false - allowed_oauth_scopes = [] - callback_urls = ["https://mydomain.com/callback"] - default_redirect_uri = "https://mydomain.com/callback" - explicit_auth_flows = [] - generate_secret = false - logout_urls = [] - name = "test2" - read_attributes = [] - supported_identity_providers = [] - write_attributes = [] - refresh_token_validity = 30 - }, - { - allowed_oauth_flows = ["code", "implicit"] - allowed_oauth_flows_user_pool_client = true - allowed_oauth_scopes = ["email", "openid"] - callback_urls = ["https://mydomain.com/callback"] - default_redirect_uri = "https://mydomain.com/callback" - explicit_auth_flows = ["CUSTOM_AUTH_FLOW_ONLY", "ADMIN_NO_SRP_AUTH"] - generate_secret = false - logout_urls = ["https://mydomain.com/logout"] - name = "test3" - read_attributes = ["email", "phone_number"] - supported_identity_providers = [] - write_attributes = ["email", "gender", "locale", ] - refresh_token_validity = 30 - } - ] - - # user_group - user_groups = [ - { name = "mygroup1" - description = "My group 1" - }, - { name = "mygroup2" - description = "My group 2" - }, - ] - - # resource_servers - resource_servers = [ - { - identifier = "https://mydomain.com" - name = "mydomain" - scope = [ - { - scope_name = "sample-scope-1" - scope_description = "A sample Scope Description for mydomain.com" - }, - { - scope_name = "sample-scope-2" - scope_description = "Another sample Scope Description for mydomain.com" - }, - ] - }, - { - identifier = "https://weather-read-app.com" - name = "weather-read" - scope = [ - { - scope_name = "weather.read" - scope_description = "Read weather forecasts" - } - ] - } - ] - - # identity_providers - identity_providers = [ - { - provider_name = "Google" - provider_type = "Google" - - provider_details = { - authorize_scopes = "email" - client_id = "your client_id" - client_secret = "your client_secret" - } - - attribute_mapping = { - email = "email" - username = "sub" - gender = "gender" - } - } - ] - - # tags - tags = { - Owner = "infra" - Environment = "production" - Terraform = true - } -} -``` +cognito_user_pool