Self hosted PostgresDB + Timescale extension + Regular data backups + Destroy github actions

.github/workflows/destroy_aws_deployment.yml

@@ -0,0 +1,50 @@
+name: Destroy HydroServer AWS Cloud Deployment
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: 'Enter a deployment environment name.'
+        required: true
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  destroy-deployment:
+    runs-on: ubuntu-20.04
+    environment: ${{ github.event.inputs.environment }}
+    defaults:
+      run:
+        working-directory: ./terraform/aws
+    steps:
+      - name: configureawscredentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.AWS_IAM_ROLE }}
+          role-session-name: create-hydroserver-resources
+          aws-region: ${{ vars.AWS_REGION }}
+
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+        with:
+          ref: main
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v1
+
+      - name: Terraform Init
+        run: terraform init -backend-config="bucket=${{ vars.TERRAFORM_BUCKET }}" -backend-config="region=${{ vars.AWS_REGION }}" -backend-config="key=state/aws_application_${{ github.event.inputs.environment }}"
+
+      - name: Terraform Plan
+        id: plan
+        run: terraform plan -destroy -no-color -var instance="${{ github.event.inputs.environment }}" -var region="${{ vars.AWS_REGION }}"
+        continue-on-error: true
+
+      - name: Terraform Plan Status
+        if: steps.plan.outcome == 'failure'
+        run: exit 1
+
+      - name: Terraform Apply
+        run: terraform apply -destroy -auto-approve -var instance="${{ github.event.inputs.environment }}" -var region="${{ vars.AWS_REGION }}"

.github/workflows/destroy_self_hosted_timescale.yml

@@ -0,0 +1,70 @@
+name: Destroy Self Hosted TimescaleScale Database Cluster with Backup
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: 'Enter a deployment environment name.'
+        required: true
+      db-user:
+        description: 'Enter a username for the timescale db'
+        required: true
+      db-password:
+        description: 'Enter a password for the timescale db'
+        required: true
+      aws-instance-type:
+        description: 'Enter aws ec2 isntance type, please use the same instance type as the one used to create the database'
+        required: false
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  destroy-deployment:
+    runs-on: ubuntu-20.04
+    environment: ${{ github.event.inputs.environment }}
+    steps:
+      - name: configureawscredentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.AWS_IAM_ROLE }}
+          role-session-name: create-hydroserver-resources
+          aws-region: ${{ vars.AWS_REGION }}
+
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+        with:
+          ref: main
+          path: ops
+
+      - name: Get EC2 Instance Type
+        id: get_instance_type
+        run: echo "instance_type=${{ github.event.inputs.aws-instance-type || 't2.micro' }}" >> $GITHUB_OUTPUT
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v3
+
+      - name: Terraform Init
+        working-directory: ./ops/terraform/self_hosted_timescale
+        run: terraform init -backend-config="bucket=${{ vars.TERRAFORM_BUCKET }}" -backend-config="region=${{ vars.AWS_REGION }}" -backend-config="key=state/timescale_self_hosted_database_${{ github.event.inputs.environment }}"
+
+      - name: Terraform Plan
+        id: plan
+        working-directory: ./ops/terraform/self_hosted_timescale
+        run: terraform plan -destroy -no-color -var instance="${{ github.event.inputs.environment }}" -var region="${{ vars.AWS_REGION }}" -var db_user="${{ github.event.inputs.db-user }}" -var db_password="${{ github.event.inputs.db-password }}" -var access_key="${{ secrets.USER_ACCESS_KEY_ID }}" -var secret_key="${{ secrets.USER_SECRET_ACCESS_KEY }}" -var private_key="${{ secrets.SELF_HOSTED_TIMESCALE_PRIVATE_KEY }}" -var public_key="${{ secrets.SELF_HOSTED_TIMESCALE_PUBLIC_KEY }}" -var aws_type="${{ github.event.inputs.aws-instance-type || steps.get_instance_type.outputs.instance_type }}"
+        continue-on-error: true
+
+      - name: Terraform Plan Status
+        if: steps.plan.outcome == 'failure'
+        run: exit 1
+
+      - name: Terraform Apply
+        working-directory: ./ops/terraform/self_hosted_timescale
+        run: |
+          terraform apply -destroy -auto-approve -var instance="${{ github.event.inputs.environment }}" -var region="${{ vars.AWS_REGION }}" -var db_user="${{ github.event.inputs.db-user }}" -var db_password="${{ github.event.inputs.db-password }}" -var access_key="${{ secrets.TIMESCALE_ACCESS_KEY }}" -var secret_key="${{ secrets.TIMESCALE_SECRET_KEY }}" -var private_key="${{ secrets.SELF_HOSTED_TIMESCALE_PRIVATE_KEY }}" -var public_key="${{ secrets.SELF_HOSTED_TIMESCALE_PUBLIC_KEY }}" -var aws_type="${{ github.event.inputs.aws-instance-type || steps.get_instance_type.outputs.instance_type }}"
+
+      - name: Delete the Connection Details in S3
+        working-directory: ./ops/terraform/self_hosted_timescale
+        run: |
+          aws s3 rm s3://${{ vars.TERRAFORM_BUCKET }}/output/timescale_${{ github.event.inputs.environment }}_connection.txt

.github/workflows/self_hosted_timescale_ create_database.yml

@@ -0,0 +1,123 @@
+name: Create Self Hosted TimescaleScale Database Cluster with Backup For HydroServer
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: 'Enter a deployment environment name.'
+        required: true
+      superuser-email:
+        description: 'Enter the email for the Django superuser.'
+        required: true
+      superuser-password:
+        description: 'Enter the password for the Django superuser.'
+        required: true
+      partition-interval:
+        description: 'Enter a partition interval in days.'
+        default: '365'
+        required: true
+      db-user:
+        description: 'Enter a username for the timescale db'
+        required: true
+      db-password:
+        description: 'Enter a password for the timescale db'
+        required: true
+      aws-instance-type:
+        description: 'Enter aws ec2 isntance type (default is t2.micro)'
+        required: false
+      hydroserver-version:
+        description: 'Enter a version of HydroServer to use. Leave blank to use the latest version.'
+        required: false
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  setup-deployment:
+    runs-on: ubuntu-20.04
+    environment: ${{ github.event.inputs.environment }}
+    steps:
+      - name: configureawscredentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.AWS_IAM_ROLE }}
+          role-session-name: create-hydroserver-resources
+          aws-region: ${{ vars.AWS_REGION }}
+
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+        with:
+          ref: main
+          path: ops
+
+      - name: Get Latest HydroServer Version
+        id: get_latest_tag
+        run: echo "tag=$(curl -sL https://api.github.com/repos/hydroserver2/hydroserver-api-services/releases/latest | jq -r '.tag_name')" >> $GITHUB_OUTPUT
+
+      - name: Get EC2 Instance Type
+        id: get_instance_type
+        run: echo "instance_type=${{ github.event.inputs.aws-instance-type || 't2.micro' }}" >> $GITHUB_OUTPUT
+
+      - name: Checkout Backend Repo
+        uses: actions/checkout@v4
+        with:
+          repository: hydroserver2/hydroserver-api-services
+          ref: refs/tags/${{ github.event.inputs.hydroserver-version || steps.get_latest_tag.outputs.tag }}
+          path: backend
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v3
+
+      - name: Terraform Init
+        working-directory: ./ops/terraform/self_hosted_timescale
+        run: terraform init -backend-config="bucket=${{ vars.TERRAFORM_BUCKET }}" -backend-config="region=${{ vars.AWS_REGION }}" -backend-config="key=state/timescale_self_hosted_database_${{ github.event.inputs.environment }}"
+
+      - name: Terraform Plan
+        id: plan
+        working-directory: ./ops/terraform/self_hosted_timescale
+        run: terraform plan -no-color -var instance="${{ github.event.inputs.environment }}" -var region="${{ vars.AWS_REGION }}" -var db_user="${{ github.event.inputs.db-user }}" -var db_password="${{ github.event.inputs.db-password }}" -var access_key="${{ secrets.USER_ACCESS_KEY_ID }}" -var secret_key="${{ secrets.USER_SECRET_ACCESS_KEY }}" -var private_key="${{ secrets.SELF_HOSTED_TIMESCALE_PRIVATE_KEY }}" -var public_key="${{ secrets.SELF_HOSTED_TIMESCALE_PUBLIC_KEY }}" -var aws_type="${{ github.event.inputs.aws-instance-type || steps.get_instance_type.outputs.instance_type }}"
+        continue-on-error: true
+
+      - name: Terraform Plan Status
+        if: steps.plan.outcome == 'failure'
+        run: exit 1
+
+      - name: Terraform Apply
+        working-directory: ./ops/terraform/self_hosted_timescale
+        run: |
+          terraform apply -auto-approve -var instance="${{ github.event.inputs.environment }}" -var region="${{ vars.AWS_REGION }}" -var db_user="${{ github.event.inputs.db-user }}" -var db_password="${{ github.event.inputs.db-password }}" -var access_key="${{ secrets.USER_ACCESS_KEY_ID }}" -var secret_key="${{ secrets.USER_SECRET_ACCESS_KEY }}" -var private_key="${{ secrets.SELF_HOSTED_TIMESCALE_PRIVATE_KEY }}" -var public_key="${{ secrets.SELF_HOSTED_TIMESCALE_PUBLIC_KEY }}" -var aws_type="${{ github.event.inputs.aws-instance-type || steps.get_instance_type.outputs.instance_type }}"
+          echo "HOSTNAME=$(terraform output -json | jq -r '.self_hosted_tsdb_hostname.value')" > timescale_${{ github.event.inputs.environment }}_connection.txt
+          echo "PORT=5432" >> timescale_${{ github.event.inputs.environment }}_connection.txt
+          echo "PASSWORD=${{ github.event.inputs.db-password }}" >> timescale_${{ github.event.inputs.environment }}_connection.txt
+          echo "CONNECTION_STRING=postgresql://${{ github.event.inputs.db-user }}:${{ github.event.inputs.db-password }}@$(terraform output -json | jq -r '.self_hosted_tsdb_hostname.value'):5432/tsdb" >> timescale_${{ github.event.inputs.environment }}_connection.txt
+          cat << EOF > ../../../backend/.env
+          PROXY_BASE_URL=http://127.0.0.1:8000
+          DATABASE_URL=postgresql://${{ github.event.inputs.db-user }}:${{ github.event.inputs.db-password }}@$(terraform output -json | jq -r '.self_hosted_tsdb_hostname.value'):5432/tsdb
+          DEPLOYED=True
+          EOF
+
+      - name: Upload Connection Details to S3
+        working-directory: ./ops/terraform/self_hosted_timescale
+        run: |
+          aws s3 cp timescale_${{ github.event.inputs.environment }}_connection.txt s3://${{ vars.TERRAFORM_BUCKET }}/output/timescale_${{ github.event.inputs.environment }}_connection.txt
+
+      - name: Install Django Dependencies
+        working-directory: ./backend
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          pip install pyopenssl --upgrade
+
+      - name: Run Database Setup Commands
+        working-directory: ./backend
+        env:
+          DJANGO_SETTINGS_MODULE: hydroserver.settings
+          DJANGO_SUPERUSER_EMAIL: ${{ github.event.inputs.superuser-email }}
+          DJANGO_SUPERUSER_PASSWORD: ${{ github.event.inputs.superuser-password }}
+          DJANGO_SUPERUSER_FIRST_NAME: ADMIN
+          DJANGO_SUPERUSER_LAST_NAME: ADMIN
+        run: |
+          python manage.py migrate
+          python manage.py configure_timescaledb --partition-interval-days ${{ github.event.inputs.partition-interval }}
+          python manage.py createsuperuser --noinput

.github/workflows/timescale_create_database.yml

@@ -16,7 +16,7 @@ on:
         description: 'Enter a partition interval in days.'
         default: '365'
         required: true
-      hydroserver-version: 
+      hydroserver-version:
         description: 'Enter a version of HydroServer to use. Leave blank to use the latest version.'
         required: false
 
@@ -35,7 +35,7 @@ jobs:
           role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.AWS_IAM_ROLE }}
           role-session-name: create-hydroserver-resources
           aws-region: ${{ vars.AWS_REGION }}
-    
+
       - name: Checkout Repo
         uses: actions/checkout@v3
         with:
@@ -45,7 +45,7 @@ jobs:
       - name: Get Latest HydroServer Version
         id: get_latest_tag
         run: echo "tag=$(curl -sL https://api.github.com/repos/hydroserver2/hydroserver-api-services/releases/latest | jq -r '.tag_name')" >> $GITHUB_OUTPUT
-      
+
       - name: Checkout Backend Repo
         uses: actions/checkout@v4
         with:
@@ -88,14 +88,14 @@ jobs:
         working-directory: ./ops/terraform/timescale
         run: |
           aws s3 cp timescale_${{ github.event.inputs.environment }}_connection.txt s3://${{ vars.TERRAFORM_BUCKET }}/output/timescale_${{ github.event.inputs.environment }}_connection.txt
-    
+
       - name: Install Django Dependencies
         working-directory: ./backend
         run: |
           python -m pip install --upgrade pip
           pip install -r requirements.txt
           pip install pyopenssl --upgrade
-          
+
       - name: Run Database Setup Commands
         working-directory: ./backend
         env:
@@ -107,4 +107,4 @@ jobs:
         run: |
           python manage.py migrate
           python manage.py configure_timescaledb --partition-interval-days ${{ github.event.inputs.partition-interval }}
-          python manage.py createsuperuser --noinput          
+          python manage.py createsuperuser --noinput

terraform/aws/elasticbeanstalk.tf

@@ -14,7 +14,7 @@ resource "aws_elastic_beanstalk_application" "hydroserver_django_app" {
 resource "aws_elastic_beanstalk_environment" "hydroserver_django_env" {
   name                = "hydroserver-${var.instance}-env"
   application         = aws_elastic_beanstalk_application.hydroserver_django_app.name
-  solution_stack_name = "64bit Amazon Linux 2 v3.5.12 running Python 3.8"
+  solution_stack_name = "64bit Amazon Linux 2 v3.6.0 running Python 3.8"
 
   setting {
     namespace = "aws:elasticbeanstalk:environment"

terraform/self_hosted_timescale/app-instances.tf

@@ -0,0 +1,62 @@
+resource "aws_instance" "primary_1" {
+  ami             = var.aws_ami
+  instance_type   = var.aws_type
+  security_groups = ["${aws_security_group.swarm.name}"]
+  key_name        = aws_key_pair.deployer.key_name
+  connection {
+    host        = self.public_ip
+    user        = "ec2-user"
+    private_key = var.private_key
+  }
+  provisioner "remote-exec" {
+    inline = [
+      "sudo yum update -y",
+      "sudo yum install git -y",
+      "sudo yum install -y docker",
+      "sudo service docker start",
+      "sudo curl -L https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose",
+      "sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose",
+      "sudo chmod +x /usr/local/bin/docker-compose;",
+      "sudo docker network create tsdb",
+      "sudo docker run --restart=unless-stopped --name=tsdb_db -d -p 5432:5432 --network tsdb -e POSTGRES_DB=tsdb -e POSTGRES_USER=${var.db_user} -e POSTGRES_PASSWORD=${var.db_password} -v $(pwd)/data:/var/lib/postgresql/data timescale/timescaledb:latest-pg13",
+      "sudo docker run -d --restart=unless-stopped --name=postgres_backup --network tsdb -e SCHEDULE='@daily' -e S3_REGION=${var.region}  -e S3_ACCESS_KEY_ID=${var.access_key} -e S3_SECRET_ACCESS_KEY=${var.secret_key} -e S3_BUCKET=timescale-backup-${var.instance}-${data.aws_caller_identity.current.account_id} -e POSTGRES_DATABASE=tsdb -e POSTGRES_USER=${var.db_user}  -e POSTGRES_HOST=tsdb_db -e POSTGRES_PASSWORD=${var.db_password} -e S3_PREFIX=backup -e POSTGRES_EXTRA_OPTS='--format=plain --quote-all-identifiers --no-tablespaces --no-owner --no-privileges' schickling/postgres-backup-s3"
+    ]
+  }
+  tags = {
+    Name = "tsdb-primary-${var.instance}-${data.aws_caller_identity.current.account_id}"
+  }
+  depends_on = [
+    aws_s3_bucket.timescale_backup_bucket
+  ]
+}
+resource "aws_instance" "replica_1" {
+  ami             = var.aws_ami
+  instance_type   = var.aws_type
+  security_groups = ["${aws_security_group.swarm.name}"]
+  key_name        = aws_key_pair.deployer.key_name
+  connection {
+    host        = self.public_ip
+    user        = "ec2-user"
+    private_key = var.private_key
+  }
+  provisioner "remote-exec" {
+    inline = [
+      "sudo yum update -y",
+      "sudo yum install git -y",
+      "sudo yum install -y docker",
+      "sudo service docker start",
+      "sudo curl -L https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose",
+      "sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose",
+      "sudo chmod +x /usr/local/bin/docker-compose;",
+      "sudo docker network create tsdb",
+      "sudo docker run --restart=unless-stopped --name=tsdb_db -d -p 5432:5432 --network tsdb -e POSTGRES_DB=tsdb -e POSTGRES_USER=${var.db_user} -e POSTGRES_PASSWORD=${var.db_password} -v $(pwd)/data:/var/lib/postgresql/data timescale/timescaledb:latest-pg13",
+      "sudo docker run -d --restart=unless-stopped --name=postgres_backup --network tsdb -e SCHEDULE='@daily' -e S3_REGION=${var.region}  -e S3_ACCESS_KEY_ID=${var.access_key} -e S3_SECRET_ACCESS_KEY=${var.secret_key} -e S3_BUCKET=timescale-backup-${var.instance}-${data.aws_caller_identity.current.account_id} -e POSTGRES_DATABASE=tsdb -e POSTGRES_USER=${var.db_user}  -e POSTGRES_HOST=tsdb_db -e POSTGRES_PASSWORD=${var.db_password} -e S3_PREFIX=backup -e POSTGRES_EXTRA_OPTS='--format=plain --quote-all-identifiers --no-tablespaces --no-owner --no-privileges' schickling/postgres-backup-s3"
+    ]
+  }
+  tags = {
+    Name = "tsdb-replica-${var.instance}-${data.aws_caller_identity.current.account_id}"
+  }
+  depends_on = [
+    aws_s3_bucket.timescale_backup_bucket
+  ]
+}

terraform/self_hosted_timescale/key-pairs.tf

@@ -0,0 +1,4 @@
+resource "aws_key_pair" "deployer" {
+  key_name   = "deploy-tsdb-${var.instance}-${data.aws_caller_identity.current.account_id}"
+  public_key = var.public_key
+}

terraform/self_hosted_timescale/main.tf

@@ -0,0 +1,16 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 4.16"
+    }
+  }
+  backend "s3" {}
+  required_version = ">= 1.2.0"
+}
+
+provider "aws" {
+  region = var.region
+}
+
+data "aws_caller_identity" "current" {}

terraform/self_hosted_timescale/outputs.tf

@@ -0,0 +1,3 @@
+output "self_hosted_tsdb_hostname" {
+  value = aws_instance.primary_1.public_ip
+}