From 549f9eb4e5eca6465f6cb2695b644a50d4ff6955 Mon Sep 17 00:00:00 2001 From: Bobbie Soedirgo Date: Thu, 26 Sep 2024 13:53:44 +0100 Subject: [PATCH 01/94] fix: account for `public` grantee --- ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh index e978b2a54..4c776844b 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh @@ -356,6 +356,7 @@ begin end , case when rec.grantee = 'postgres'::regrole then 'supabase_admin' when rec.grantee = 'supabase_admin'::regrole then 'postgres' + when rec.grantee = 0 then 'public' else rec.grantee::regrole end )); @@ -382,7 +383,7 @@ begin when obj->>'objtype' = 'T' then 'types' when obj->>'objtype' = 'n' then 'schemas' end - , rec.grantee::regrole + , case when rec.grantee = 0 then 'public' else rec.grantee::regrole end , case when rec.is_grantable then 'with grant option' else '' end )); end if; From c83b1c6eb4d3bdf2af9e5b5477b56a924a9eae84 Mon Sep 17 00:00:00 2001 From: Bobbie Soedirgo <31685197+soedirgo@users.noreply.github.com> Date: Thu, 26 Sep 2024 14:19:38 +0100 Subject: [PATCH 02/94] fix(ci): respect postgresVersion input (#1237) --- .github/workflows/publish-nix-pgupgrade-scripts.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/publish-nix-pgupgrade-scripts.yml b/.github/workflows/publish-nix-pgupgrade-scripts.yml index 5d373ad8f..eb5f7a755 100644 --- a/.github/workflows/publish-nix-pgupgrade-scripts.yml +++ b/.github/workflows/publish-nix-pgupgrade-scripts.yml @@ -72,6 +72,9 @@ jobs: id: process_release_version run: | VERSION=$(grep 'postgres-version' common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g') + if [[ "${{ inputs.postgresVersion }}" != "" ]]; then + VERSION=${{ inputs.postgresVersion }} + fi echo "version=$VERSION" >> "$GITHUB_OUTPUT" - name: Create a tarball containing pg_upgrade scripts From 5671a3996e90ce9fcf23881ac9de28f88b78c3ba Mon Sep 17 00:00:00 2001 From: Kang Ming Date: Sat, 28 Sep 2024 06:33:03 -0700 Subject: [PATCH 03/94] feat: bump gotrue version to v2.162.0 (#1241) --- ansible/vars.yml | 4 ++-- common-nix.vars.pkr.hcl | 2 +- common.vars.pkr.hcl | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index e76e99cb0..471b073c6 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -17,8 +17,8 @@ postgrest_release: "12.2.3" postgrest_arm_release_checksum: sha1:fbfd6613d711ce1afa25c42d5df8f1b017f396f9 postgrest_x86_release_checksum: sha1:61c513f91a8931be4062587b9d4a18b42acf5c05 -gotrue_release: 2.161.0 -gotrue_release_checksum: sha1:8e45f3511fee8f99a0b1567c73673991a0a5986c +gotrue_release: 2.162.0 +gotrue_release_checksum: sha1:855b23bd002577290c7d42d7042ac0f5316984b1 aws_cli_release: "2.2.7" diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index bacdd2c63..b7089a21b 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.124" +postgres-version = "15.6.1.125" diff --git a/common.vars.pkr.hcl b/common.vars.pkr.hcl index 3c999dccd..fa1387690 100644 --- a/common.vars.pkr.hcl +++ b/common.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.1.1.93" +postgres-version = "15.1.1.94" From a2f1ff813807601ba8e22a921c5358aa2f5916a7 Mon Sep 17 00:00:00 2001 From: Bobbie Soedirgo <31685197+soedirgo@users.noreply.github.com> Date: Mon, 30 Sep 2024 16:32:13 +0100 Subject: [PATCH 04/94] fix: only grant pg_read_all_data if it exists (#1242) * fix: only grant pg_read_all_data if it exists * fix: prevent `public` from being casted into `regrole` --- .../admin_api_scripts/pg_upgrade_scripts/common.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh index 4c776844b..a11f8ff32 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh @@ -357,7 +357,7 @@ begin , case when rec.grantee = 'postgres'::regrole then 'supabase_admin' when rec.grantee = 'supabase_admin'::regrole then 'postgres' when rec.grantee = 0 then 'public' - else rec.grantee::regrole + else rec.grantee::regrole::text end )); end if; @@ -383,7 +383,7 @@ begin when obj->>'objtype' = 'T' then 'types' when obj->>'objtype' = 'n' then 'schemas' end - , case when rec.grantee = 0 then 'public' else rec.grantee::regrole end + , case when rec.grantee = 0 then 'public' else rec.grantee::regrole::text end , case when rec.is_grantable then 'with grant option' else '' end )); end if; @@ -530,7 +530,14 @@ $$; alter database postgres connection limit -1; -- #incident-2024-09-12-project-upgrades-are-temporarily-disabled -grant pg_read_all_data, pg_signal_backend to postgres; +do $$ +begin + if exists (select from pg_authid where rolname = 'pg_read_all_data') then + execute('grant pg_read_all_data to postgres'); + end if; +end +$$; +grant pg_signal_backend to postgres; set session authorization supabase_admin; drop role supabase_tmp; From d71ec5f7f2465a35654a1070d02ea377026a6b5f Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Wed, 2 Oct 2024 12:58:08 +0300 Subject: [PATCH 05/94] fix(15.6): account for pg_stat_monitor major version upgrade (#1247) --- ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 9443d2acb..c0a213d02 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -11,6 +11,7 @@ # them depending on regtypes referencing system OIDs or outdated library files. EXTENSIONS_TO_DISABLE=( "pg_graphql" + "pg_stat_monitor" ) PG14_EXTENSIONS_TO_DISABLE=( From 8706e3adf1ee060f77e7a1bdeb0e1d4bbfbb684b Mon Sep 17 00:00:00 2001 From: angelico Date: Wed, 2 Oct 2024 18:37:29 +0800 Subject: [PATCH 06/94] chore: release updates to run physical backup as a service to 15.6 image (#1248) * chore: updates to run physical backups as a service (#1235) * chore: init commence-backup service * chore: bump adminapi and adminmgr * chore: bump version * fix: provide correct filename * chore: bump postgres version --- ansible/files/adminapi.sudoers.conf | 2 ++ ansible/files/commence-backup.service.j2 | 12 ++++++++++++ ansible/tasks/internal/admin-api.yml | 5 +++++ ansible/vars.yml | 4 ++-- common-nix.vars.pkr.hcl | 2 +- 5 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 ansible/files/commence-backup.service.j2 diff --git a/ansible/files/adminapi.sudoers.conf b/ansible/files/adminapi.sudoers.conf index eada0a94b..ae5537715 100644 --- a/ansible/files/adminapi.sudoers.conf +++ b/ansible/files/adminapi.sudoers.conf @@ -17,6 +17,8 @@ Cmnd_Alias PGBOUNCER = /bin/systemctl start pgbouncer.service, /bin/systemctl st %adminapi ALL= NOPASSWD: /usr/bin/systemctl restart postgresql.service %adminapi ALL= NOPASSWD: /usr/bin/systemctl show -p NRestarts postgresql.service %adminapi ALL= NOPASSWD: /usr/bin/systemctl restart adminapi.service +%adminapi ALL= NOPASSWD: /usr/bin/systemctl is-active commence-backup.service +%adminapi ALL= NOPASSWD: /usr/bin/systemctl start commence-backup.service %adminapi ALL= NOPASSWD: /bin/systemctl daemon-reload %adminapi ALL= NOPASSWD: /bin/systemctl restart services.slice %adminapi ALL= NOPASSWD: /usr/sbin/nft -f /etc/nftables/supabase_managed.conf diff --git a/ansible/files/commence-backup.service.j2 b/ansible/files/commence-backup.service.j2 new file mode 100644 index 000000000..9d4ad0c68 --- /dev/null +++ b/ansible/files/commence-backup.service.j2 @@ -0,0 +1,12 @@ +[Unit] +Description=Async commence physical backup + +[Service] +Type=simple +User=adminapi +ExecStart=/usr/bin/admin-mgr commence-backup --run-as-service true +Restart=no +OOMScoreAdjust=-1000 + +[Install] +WantedBy=multi-user.target diff --git a/ansible/tasks/internal/admin-api.yml b/ansible/tasks/internal/admin-api.yml index cea0109fd..da93fef62 100644 --- a/ansible/tasks/internal/admin-api.yml +++ b/ansible/tasks/internal/admin-api.yml @@ -79,6 +79,11 @@ src: files/adminapi.service.j2 dest: /etc/systemd/system/adminapi.service +- name: adminapi - create service file for commence backup process + template: + src: files/commence-backup.service.j2 + dest: /etc/systemd/system/commence-backup.service + - name: UFW - Allow connections to adminapi ports ufw: rule: allow diff --git a/ansible/vars.yml b/ansible/vars.yml index 471b073c6..3c5ef2172 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -50,8 +50,8 @@ postgres_exporter_release_checksum: arm64: sha256:29ba62d538b92d39952afe12ee2e1f4401250d678ff4b354ff2752f4321c87a0 amd64: sha256:cb89fc5bf4485fb554e0d640d9684fae143a4b2d5fa443009bd29c59f9129e84 -adminapi_release: 0.68.0 -adminmgr_release: 0.22.1 +adminapi_release: 0.71.1 +adminmgr_release: 0.24.0 # Postgres Extensions postgis_release: "3.3.2" diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index b7089a21b..3a55dd70e 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.125" +postgres-version = "15.6.1.126" From 1d1396e434000431e09cd723c9268afb64611b9b Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Wed, 2 Oct 2024 14:57:18 +0300 Subject: [PATCH 07/94] fix(15.6/pg_upgrade): retry commands within the cleanup step; wait until PG is ready to accept connections (#1250) --- .../admin_api_scripts/pg_upgrade_scripts/initiate.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index c0a213d02..31a7cd5b3 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -120,20 +120,22 @@ cleanup() { CI_start_postgres fi + retry 8 pg_isready -h localhost -U supabase_admin + echo "Re-enabling extensions" if [ -f $POST_UPGRADE_EXTENSION_SCRIPT ]; then - run_sql -f $POST_UPGRADE_EXTENSION_SCRIPT + retry 5 run_sql -f $POST_UPGRADE_EXTENSION_SCRIPT fi echo "Removing SUPERUSER grant from postgres" - run_sql -c "ALTER USER postgres WITH NOSUPERUSER;" + retry 5 run_sql -c "ALTER USER postgres WITH NOSUPERUSER;" echo "Resetting postgres database connection limit" - run_sql -c "ALTER DATABASE postgres CONNECTION LIMIT -1;" + retry 5 run_sql -c "ALTER DATABASE postgres CONNECTION LIMIT -1;" if [ -z "$IS_CI" ] && [ -z "$IS_LOCAL_UPGRADE" ]; then echo "Unmounting data disk from ${MOUNT_POINT}" - umount $MOUNT_POINT + retry 3 umount $MOUNT_POINT fi echo "$UPGRADE_STATUS" > /tmp/pg-upgrade-status From 320305e9721b3055c91560187cd6a7f52cd56280 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Wed, 2 Oct 2024 20:37:59 +0300 Subject: [PATCH 08/94] fix(15.6/upgrades): collision when patching wrappers lib locations for upgrades (#1252) --- .../files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 31a7cd5b3..235b4f4c1 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -211,7 +211,7 @@ function patch_wrappers { WRAPPERS_LIB_PATH_DIR=$(dirname "$WRAPPERS_LIB_PATH") if [ "$WRAPPERS_LIB_PATH" != "$WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" ]; then echo "Copying $WRAPPERS_LIB_PATH to $WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" - cp "$WRAPPERS_LIB_PATH" "$WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" + cp "$WRAPPERS_LIB_PATH" "$WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" || true fi fi done @@ -225,7 +225,7 @@ function patch_wrappers { LIB_FILE_NAME=$(basename "$OLD_WRAPPER_LIB_PATH") if [ "$WRAPPERS_LIB_PATH" != "$PGLIBNEW/${LIB_FILE_NAME}" ]; then echo "Copying $WRAPPERS_LIB_PATH to $PGLIBNEW/${LIB_FILE_NAME}" - cp "$WRAPPERS_LIB_PATH" "$PGLIBNEW/${LIB_FILE_NAME}" + cp "$WRAPPERS_LIB_PATH" "$PGLIBNEW/${LIB_FILE_NAME}" || true fi fi fi From d264b9dae820baa123c955c7dbd7220f114f3e27 Mon Sep 17 00:00:00 2001 From: Stojan Dimitrovski Date: Thu, 3 Oct 2024 11:57:23 +0200 Subject: [PATCH 09/94] feat: bump auth 2.162.1 on 15.6 (#1256) --- ansible/vars.yml | 4 ++-- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 3c5ef2172..ed746051d 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -17,8 +17,8 @@ postgrest_release: "12.2.3" postgrest_arm_release_checksum: sha1:fbfd6613d711ce1afa25c42d5df8f1b017f396f9 postgrest_x86_release_checksum: sha1:61c513f91a8931be4062587b9d4a18b42acf5c05 -gotrue_release: 2.162.0 -gotrue_release_checksum: sha1:855b23bd002577290c7d42d7042ac0f5316984b1 +gotrue_release: 2.162.1 +gotrue_release_checksum: sha1:a8b248521f000e027feea2b44e8d9dfb6b054b2e aws_cli_release: "2.2.7" diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 3a55dd70e..89abf35c5 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.126" +postgres-version = "15.6.1.127" From 5177fa3822b752d19ff840154c012b3eb324d93f Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Fri, 4 Oct 2024 20:05:59 +0300 Subject: [PATCH 10/94] fix(15.6): disable pg_stat_monitor (#1260) * fix: disable pg_stat_monitor * chore: bump version --- common-nix.vars.pkr.hcl | 2 +- nix/tests/postgresql.conf.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 89abf35c5..d988536f3 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.127" +postgres-version = "15.6.1.128" diff --git a/nix/tests/postgresql.conf.in b/nix/tests/postgresql.conf.in index d2d667b9a..ef860afcb 100644 --- a/nix/tests/postgresql.conf.in +++ b/nix/tests/postgresql.conf.in @@ -718,7 +718,7 @@ default_text_search_config = 'pg_catalog.english' #local_preload_libraries = '' #session_preload_libraries = '' -shared_preload_libraries = 'pg_stat_statements, pg_stat_monitor, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter, pg_backtrace' # (change requires restart) +shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter, pg_backtrace' # (change requires restart) jit_provider = 'llvmjit' # JIT library to use From ce2eb0c55030770d56c944ee479274903bcefdfe Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Fri, 4 Oct 2024 22:50:38 +0300 Subject: [PATCH 11/94] fix(15.6): disable pg_stat_monitor (#1262) * fix: disable pg_stat_monitor * chore: bump version --- ansible/files/postgresql_config/postgresql.conf.j2 | 2 +- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/files/postgresql_config/postgresql.conf.j2 b/ansible/files/postgresql_config/postgresql.conf.j2 index ee16715a6..1604d94f9 100644 --- a/ansible/files/postgresql_config/postgresql.conf.j2 +++ b/ansible/files/postgresql_config/postgresql.conf.j2 @@ -688,7 +688,7 @@ default_text_search_config = 'pg_catalog.english' #local_preload_libraries = '' #session_preload_libraries = '' -shared_preload_libraries = 'pg_stat_statements, pg_stat_monitor, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter' # (change requires restart) +shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter' # (change requires restart) jit_provider = 'llvmjit' # JIT library to use # - Other Defaults - diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index d988536f3..dd5d407f5 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.128" +postgres-version = "15.6.1.129" From 9062da41f618509f60f5f3eaa005e8ea9b9b6ae0 Mon Sep 17 00:00:00 2001 From: Kang Ming Date: Mon, 7 Oct 2024 00:07:08 -0700 Subject: [PATCH 12/94] feat: bump gotrue to v2.162.2 (#1264) --- ansible/vars.yml | 4 ++-- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index ed746051d..c4c9e52be 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -17,8 +17,8 @@ postgrest_release: "12.2.3" postgrest_arm_release_checksum: sha1:fbfd6613d711ce1afa25c42d5df8f1b017f396f9 postgrest_x86_release_checksum: sha1:61c513f91a8931be4062587b9d4a18b42acf5c05 -gotrue_release: 2.162.1 -gotrue_release_checksum: sha1:a8b248521f000e027feea2b44e8d9dfb6b054b2e +gotrue_release: 2.162.2 +gotrue_release_checksum: sha1:283c30b68b61332a05e6368aa59ece34cd519fe0 aws_cli_release: "2.2.7" diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index dd5d407f5..e5e2f4712 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.129" +postgres-version = "15.6.1.130" From 0c52b1f2b4c8ba671d9247fe610ca94a30348bdb Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Tue, 15 Oct 2024 19:45:41 +0300 Subject: [PATCH 13/94] chore: add timescaledb 2.9.1; wrappers upgrade fix; wrappers & plv8 naming fix (#1259) Co-authored-by: Bobbie Soedirgo Co-authored-by: Bobbie Soedirgo <31685197+soedirgo@users.noreply.github.com> Co-authored-by: Kang Ming Co-authored-by: Stojan Dimitrovski Co-authored-by: Sam Rose fix(ci): respect postgresVersion input (#1237) fix: only grant pg_read_all_data if it exists (#1242) fix(15.6): disable pg_stat_monitor (#1260) --- .../pg_upgrade_scripts/initiate.sh | 55 ---------------- flake.nix | 1 + nix/ext/plv8.nix | 16 +++-- nix/ext/timescaledb-2.9.1.nix | 51 +++++++++++++++ nix/ext/wrappers/default.nix | 64 +++++++++++++------ 5 files changed, 108 insertions(+), 79 deletions(-) create mode 100644 nix/ext/timescaledb-2.9.1.nix diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 235b4f4c1..ed9a7ed9b 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -47,7 +47,6 @@ SERVER_ENCODING=$(run_sql -A -t -c "SHOW server_encoding;") POSTGRES_CONFIG_PATH="/etc/postgresql/postgresql.conf" PGBINOLD="/usr/lib/postgresql/bin" -PGLIBOLD="/usr/lib/postgresql/lib" PG_UPGRADE_BIN_DIR="/tmp/pg_upgrade_bin/$PGVERSION" NIX_INSTALLER_PATH="/tmp/persistent/nix-installer" @@ -181,58 +180,6 @@ EOF done } -function patch_wrappers { - local IS_NIX_UPGRADE=$1 - - WRAPPERS_ENABLED=$(run_sql -A -t -c "SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'wrappers');") - if [ "$WRAPPERS_ENABLED" = "f" ]; then - echo "Wrappers extension not enabled. Skipping." - return - fi - - # This is a workaround for older versions of wrappers which don't have the expected - # naming scheme, containing the version in their library's file name - # e.g. wrappers-0.1.16.so, rather than wrappers.so - # pg_upgrade errors out when it doesn't find an equivalent file in the new PG version's - # library directory, so we're making sure the new version has the expected (old version's) - # file name. - # After the upgrade completes, the new version's library file is used. - # i.e. - # - old version: wrappers-0.1.16.so - # - new version: wrappers-0.1.18.so - # - workaround to make pg_upgrade happy: copy wrappers-0.1.18.so to wrappers-0.1.16.so - if [ "$IS_NIX_UPGRADE" = "true" ]; then - if [ -d "$PGLIBOLD" ]; then - OLD_WRAPPER_LIB_PATH=$(find "$PGLIBOLD" -name "wrappers*so" -print -quit) - OLD_LIB_FILE_NAME=$(basename "$OLD_WRAPPER_LIB_PATH") - - find /nix/store/ -name "wrappers*so" -print0 | while read -r -d $'\0' WRAPPERS_LIB_PATH; do - if [ -f "$WRAPPERS_LIB_PATH" ]; then - WRAPPERS_LIB_PATH_DIR=$(dirname "$WRAPPERS_LIB_PATH") - if [ "$WRAPPERS_LIB_PATH" != "$WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" ]; then - echo "Copying $WRAPPERS_LIB_PATH to $WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" - cp "$WRAPPERS_LIB_PATH" "$WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" || true - fi - fi - done - fi - else - if [ -d "$PGLIBOLD" ]; then - WRAPPERS_LIB_PATH=$(find "$PGLIBNEW" -name "wrappers*so" -print -quit) - if [ -f "$WRAPPERS_LIB_PATH" ]; then - OLD_WRAPPER_LIB_PATH=$(find "$PGLIBOLD" -name "wrappers*so" -print -quit) - if [ -f "$OLD_WRAPPER_LIB_PATH" ]; then - LIB_FILE_NAME=$(basename "$OLD_WRAPPER_LIB_PATH") - if [ "$WRAPPERS_LIB_PATH" != "$PGLIBNEW/${LIB_FILE_NAME}" ]; then - echo "Copying $WRAPPERS_LIB_PATH to $PGLIBNEW/${LIB_FILE_NAME}" - cp "$WRAPPERS_LIB_PATH" "$PGLIBNEW/${LIB_FILE_NAME}" || true - fi - fi - fi - fi - fi -} - function initiate_upgrade { mkdir -p "$MOUNT_POINT" SHARED_PRELOAD_LIBRARIES=$(cat "$POSTGRES_CONFIG_PATH" | grep shared_preload_libraries | sed "s/shared_preload_libraries =\s\{0,1\}'\(.*\)'.*/\1/") @@ -409,8 +356,6 @@ function initiate_upgrade { export LD_LIBRARY_PATH="${PGLIBNEW}" fi - patch_wrappers "$IS_NIX_UPGRADE" - echo "9. Creating new data directory, initializing database" chown -R postgres:postgres "$MOUNT_POINT/" rm -rf "${PGDATANEW:?}/" diff --git a/flake.nix b/flake.nix index 1f9d5f64d..512d32d31 100644 --- a/flake.nix +++ b/flake.nix @@ -107,6 +107,7 @@ ourExtensions = [ ./nix/ext/rum.nix ./nix/ext/timescaledb.nix + ./nix/ext/timescaledb-2.9.1.nix ./nix/ext/pgroonga.nix ./nix/ext/index_advisor.nix ./nix/ext/wal2json.nix diff --git a/nix/ext/plv8.nix b/nix/ext/plv8.nix index 5cb4fb67e..338bba62e 100644 --- a/nix/ext/plv8.nix +++ b/nix/ext/plv8.nix @@ -103,16 +103,20 @@ stdenv.mkDerivation (finalAttrs: { # so changing them does not cause issues. mv "$out/nix/store"/*/* "$out" rmdir "$out/nix/store"/* "$out/nix/store" "$out/nix" - + mv "$out/lib/plv8-${finalAttrs.version}.so" "$out/lib/plv8.so" + ln -s "$out/lib/plv8.so" "$out/lib/plv8-${finalAttrs.version}.so" + sed -i 's|module_pathname = '"'"'$libdir/plv8-[0-9.]*'"'"'|module_pathname = '"'"'$libdir/plv8'"'"'|' "$out/share/postgresql/extension/plv8.control" + sed -i 's|module_pathname = '"'"'$libdir/plv8-[0-9.]*'"'"'|module_pathname = '"'"'$libdir/plv8'"'"'|' "$out/share/postgresql/extension/plcoffee.control" + sed -i 's|module_pathname = '"'"'$libdir/plv8-[0-9.]*'"'"'|module_pathname = '"'"'$libdir/plv8'"'"'|' "$out/share/postgresql/extension/plls.control" ${lib.optionalString stdenv.isDarwin '' - install_name_tool -add_rpath "${v8}/lib" $out/lib/plv8-${finalAttrs.version}.so - install_name_tool -add_rpath "${postgresql}/lib" $out/lib/plv8-${finalAttrs.version}.so - install_name_tool -add_rpath "${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}.so - install_name_tool -change @rpath/libv8_monolith.dylib ${v8}/lib/libv8_monolith.dylib $out/lib/plv8-${finalAttrs.version}.so + install_name_tool -add_rpath "${v8}/lib" $out/lib/plv8.so + install_name_tool -add_rpath "${postgresql}/lib" $out/lib/plv8.so + install_name_tool -add_rpath "${stdenv.cc.cc.lib}/lib" $out/lib/plv8.so + install_name_tool -change @rpath/libv8_monolith.dylib ${v8}/lib/libv8_monolith.dylib $out/lib/plv8.so ''} ${lib.optionalString (!stdenv.isDarwin) '' - ${patchelf}/bin/patchelf --set-rpath "${v8}/lib:${postgresql}/lib:${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}.so + ${patchelf}/bin/patchelf --set-rpath "${v8}/lib:${postgresql}/lib:${stdenv.cc.cc.lib}/lib" $out/lib/plv8.so ''} ''; diff --git a/nix/ext/timescaledb-2.9.1.nix b/nix/ext/timescaledb-2.9.1.nix new file mode 100644 index 000000000..ad955e86f --- /dev/null +++ b/nix/ext/timescaledb-2.9.1.nix @@ -0,0 +1,51 @@ +{ lib, stdenv, fetchFromGitHub, cmake, postgresql, openssl, libkrb5 }: + +stdenv.mkDerivation rec { + pname = "timescaledb-apache"; + version = "2.9.1"; + + nativeBuildInputs = [ cmake ]; + buildInputs = [ postgresql openssl libkrb5 ]; + + src = fetchFromGitHub { + owner = "timescale"; + repo = "timescaledb"; + rev = version; + hash = "sha256-fvVSxDiGZAewyuQ2vZDb0I6tmlDXl6trjZp8+qDBtb8="; + }; + + cmakeFlags = [ "-DSEND_TELEMETRY_DEFAULT=OFF" "-DREGRESS_CHECKS=OFF" "-DTAP_CHECKS=OFF" "-DAPACHE_ONLY=1" ] + ++ lib.optionals stdenv.isDarwin [ "-DLINTER=OFF" ]; + + # Fix the install phase which tries to install into the pgsql extension dir, + # and cannot be manually overridden. This is rather fragile but works OK. + postPatch = '' + for x in CMakeLists.txt sql/CMakeLists.txt; do + substituteInPlace "$x" \ + --replace 'DESTINATION "''${PG_SHAREDIR}/extension"' "DESTINATION \"$out/share/postgresql/extension\"" + done + + for x in src/CMakeLists.txt src/loader/CMakeLists.txt tsl/src/CMakeLists.txt; do + substituteInPlace "$x" \ + --replace 'DESTINATION ''${PG_PKGLIBDIR}' "DESTINATION \"$out/lib\"" + done + ''; + + + # timescaledb-2.9.1.so already exists in the lib directory + # we have no need for the timescaledb.so or control file + postInstall = '' + rm $out/lib/timescaledb.so + rm $out/share/postgresql/extension/timescaledb.control + ''; + + meta = with lib; { + description = "Scales PostgreSQL for time-series data via automatic partitioning across time and space"; + homepage = "https://www.timescale.com/"; + changelog = "https://github.com/timescale/timescaledb/blob/${version}/CHANGELOG.md"; + maintainers = with maintainers; [ samrose ]; + platforms = postgresql.meta.platforms; + license = licenses.asl20; + broken = versionOlder postgresql.version "13"; + }; +} diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index 98d6a7cbc..808946d3a 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -63,24 +63,52 @@ buildPgrxExtension_0_11_3 rec { echo '${builtins.concatStringsSep "," previousVersions}' | sed 's/,/\n/g' > git_tags.txt ''; - postInstall = '' - echo "Creating SQL files for previous versions..." - current_version="${version}" - sql_file="$out/share/postgresql/extension/wrappers--$current_version.sql" - - if [ -f "$sql_file" ]; then - while read -r previous_version; do - if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then - new_file="$out/share/postgresql/extension/wrappers--$previous_version--$current_version.sql" - echo "Creating $new_file" - cp "$sql_file" "$new_file" - fi - done < git_tags.txt - else - echo "Warning: $sql_file not found" - fi - rm git_tags.txt - ''; + postInstall = '' + echo "Modifying main SQL file to use unversioned library name..." + current_version="${version}" + main_sql_file="$out/share/postgresql/extension/wrappers--$current_version.sql" + if [ -f "$main_sql_file" ]; then + sed -i 's|$libdir/wrappers-[0-9.]*|$libdir/wrappers|g' "$main_sql_file" + echo "Modified $main_sql_file" + else + echo "Warning: $main_sql_file not found" + fi + echo "Creating and modifying SQL files for previous versions..." + + if [ -f "$main_sql_file" ]; then + while read -r previous_version; do + if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then + new_file="$out/share/postgresql/extension/wrappers--$previous_version--$current_version.sql" + echo "Creating $new_file" + cp "$main_sql_file" "$new_file" + sed -i 's|$libdir/wrappers-[0-9.]*|$libdir/wrappers|g' "$new_file" + echo "Modified $new_file" + fi + done < git_tags.txt + else + echo "Warning: $main_sql_file not found" + fi + mv $out/lib/wrappers-${version}.so $out/lib/wrappers.so + + echo "Creating wrappers.so symlinks to support pg_upgrade..." + if [ -f "$out/lib/wrappers.so" ]; then + while read -r previous_version; do + if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then + new_file="$out/lib/wrappers-$previous_version.so" + echo "Creating $new_file" + ln -s "$out/lib/wrappers.so" "$new_file" + fi + done < git_tags.txt + else + echo "Warning: $out/lib/wrappers.so not found" + fi + + rm git_tags.txt + echo "Contents of updated wrappers.control:" + cat "$out/share/postgresql/extension/wrappers.control" + echo "List of generated SQL files:" + ls -l $out/share/postgresql/extension/wrappers--*.sql + ''; meta = with lib; { description = "Various Foreign Data Wrappers (FDWs) for PostreSQL"; From ba417af1242b6ce179b36b8036fb44f8414d6efd Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Tue, 15 Oct 2024 20:13:28 +0300 Subject: [PATCH 14/94] chore(15.6): bump pg version (#1273) --- common-nix.vars.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index e5e2f4712..beeddf2c3 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.130" +postgres-version = "15.6.1.131" From 82152fad96d18a9b170b0752ec097bc98156c691 Mon Sep 17 00:00:00 2001 From: Stojan Dimitrovski Date: Tue, 15 Oct 2024 21:04:09 +0200 Subject: [PATCH 15/94] feat: bump auth to v2.163.0 on 15.6 (#1275) --- ansible/vars.yml | 4 ++-- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index c4c9e52be..29bef7162 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -17,8 +17,8 @@ postgrest_release: "12.2.3" postgrest_arm_release_checksum: sha1:fbfd6613d711ce1afa25c42d5df8f1b017f396f9 postgrest_x86_release_checksum: sha1:61c513f91a8931be4062587b9d4a18b42acf5c05 -gotrue_release: 2.162.2 -gotrue_release_checksum: sha1:283c30b68b61332a05e6368aa59ece34cd519fe0 +gotrue_release: 2.163.0 +gotrue_release_checksum: sha1:e4005515371c41cb04f6dee863ad1bfbcd706286 aws_cli_release: "2.2.7" diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index beeddf2c3..2b2aa48c5 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.131" +postgres-version = "15.6.1.132" From 1912ed8f2c7fad38be6973cdf1269ac100c0bebb Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Wed, 16 Oct 2024 10:54:40 +0300 Subject: [PATCH 16/94] fix: restart PG during pre-upgrade steps to shed hanging connections (#1271) --- .../files/admin_api_scripts/pg_upgrade_scripts/common.sh | 9 +++++++++ .../admin_api_scripts/pg_upgrade_scripts/initiate.sh | 8 ++++++++ 2 files changed, 17 insertions(+) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh index a11f8ff32..cc87fadd0 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh @@ -91,6 +91,15 @@ swap_postgres_and_supabase_admin() { alter database postgres connection limit 0; select pg_terminate_backend(pid) from pg_stat_activity where backend_type = 'client backend' and pid != pg_backend_pid(); EOSQL + + if [ -z "$IS_CI" ]; then + retry 5 systemctl restart postgresql + else + CI_start_postgres "" + fi + + retry 8 pg_isready -h localhost -U supabase_admin + run_sql <<'EOSQL' set statement_timeout = '600s'; begin; diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index ed9a7ed9b..deb462012 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -147,6 +147,14 @@ cleanup() { } function handle_extensions { + if [ -z "$IS_CI" ]; then + retry 5 systemctl restart postgresql + else + CI_start_postgres + fi + + retry 8 pg_isready -h localhost -U supabase_admin + rm -f $POST_UPGRADE_EXTENSION_SCRIPT touch $POST_UPGRADE_EXTENSION_SCRIPT From 3cf6055e8e850db799f509b57a22eca629939ab1 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Thu, 17 Oct 2024 15:33:54 +0300 Subject: [PATCH 17/94] fix(upgrades): wrappers 4.2.0 -> wrappers 4.2.0 (#1278) --- nix/ext/wrappers/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index 808946d3a..1a4360044 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -89,6 +89,7 @@ buildPgrxExtension_0_11_3 rec { echo "Warning: $main_sql_file not found" fi mv $out/lib/wrappers-${version}.so $out/lib/wrappers.so + ln -s $out/lib/wrappers.so $out/lib/wrappers-${version}.so echo "Creating wrappers.so symlinks to support pg_upgrade..." if [ -f "$out/lib/wrappers.so" ]; then From 054dbc16c1e705581925b907d3e2282f8727de19 Mon Sep 17 00:00:00 2001 From: Bobbie Soedirgo Date: Mon, 21 Oct 2024 22:59:37 +0800 Subject: [PATCH 18/94] fix: handle supabase_admin authenticator membership snowflake --- ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh index cc87fadd0..ea7217fcc 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh @@ -106,6 +106,9 @@ begin; create role supabase_tmp superuser; set session authorization supabase_tmp; +-- to handle snowflakes that happened in the past +revoke supabase_admin from authenticator; + do $$ begin if exists (select from pg_extension where extname = 'timescaledb') then From fb024e822a974ec3daf00ba505bb2f60af511202 Mon Sep 17 00:00:00 2001 From: Stojan Dimitrovski Date: Tue, 22 Oct 2024 12:17:40 +0200 Subject: [PATCH 19/94] feat: add auth v2.163.1 to 15.6 (#1283) --- ansible/vars.yml | 4 ++-- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 29bef7162..a863fc27d 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -17,8 +17,8 @@ postgrest_release: "12.2.3" postgrest_arm_release_checksum: sha1:fbfd6613d711ce1afa25c42d5df8f1b017f396f9 postgrest_x86_release_checksum: sha1:61c513f91a8931be4062587b9d4a18b42acf5c05 -gotrue_release: 2.163.0 -gotrue_release_checksum: sha1:e4005515371c41cb04f6dee863ad1bfbcd706286 +gotrue_release: 2.163.1 +gotrue_release_checksum: sha1:f4f3fee930ac72cadccbbcef1ff076d72e1c31c0 aws_cli_release: "2.2.7" diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 2b2aa48c5..b86205058 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.132" +postgres-version = "15.6.1.133" From 223952baaaabaaf4a22a2e1ffcf57852b137d7e7 Mon Sep 17 00:00:00 2001 From: Kang Ming Date: Wed, 23 Oct 2024 11:57:06 +0800 Subject: [PATCH 20/94] feat: bump gotrue to v2.163.2 (#1287) --- ansible/vars.yml | 4 ++-- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index a863fc27d..28a499a7f 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -17,8 +17,8 @@ postgrest_release: "12.2.3" postgrest_arm_release_checksum: sha1:fbfd6613d711ce1afa25c42d5df8f1b017f396f9 postgrest_x86_release_checksum: sha1:61c513f91a8931be4062587b9d4a18b42acf5c05 -gotrue_release: 2.163.1 -gotrue_release_checksum: sha1:f4f3fee930ac72cadccbbcef1ff076d72e1c31c0 +gotrue_release: 2.163.2 +gotrue_release_checksum: sha1:31889bc8c498b924c2cb3b6c4084ef6e57ed97c0 aws_cli_release: "2.2.7" diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index b86205058..a6e4461e0 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.133" +postgres-version = "15.6.1.134" From 2d658fc9fe74a386b9427fd0a614820a658e0cf0 Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Thu, 24 Oct 2024 10:24:33 -0500 Subject: [PATCH 21/94] pg_net 0.11 on 15.6 release branch (#1290) * bump pg_net to 0.11.0 * bump image to 15.6.1.135 --- common-nix.vars.pkr.hcl | 2 +- nix/ext/pg_net.nix | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index a6e4461e0..eafb1e602 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.134" +postgres-version = "15.6.1.135" diff --git a/nix/ext/pg_net.nix b/nix/ext/pg_net.nix index fe4404cdf..dc923b9a8 100644 --- a/nix/ext/pg_net.nix +++ b/nix/ext/pg_net.nix @@ -2,7 +2,7 @@ stdenv.mkDerivation rec { pname = "pg_net"; - version = "0.10.0"; + version = "0.11.0"; buildInputs = [ curl postgresql ]; @@ -10,7 +10,7 @@ stdenv.mkDerivation rec { owner = "supabase"; repo = pname; rev = "refs/tags/v${version}"; - hash = "sha256-R9Mzw5gvV7b2R59LTOzuOc0AI99+3ncFNzijI4mySUg="; + hash = "sha256-XN441jXK1q+I/LZRNwvzbSsebXHgZ8iYsslZvcPFlAs="; }; env.NIX_CFLAGS_COMPILE = "-Wno-error"; @@ -18,7 +18,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp sql/*.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension ''; From a5e5919b701c6e28b272f956c022a24719874c37 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Fri, 25 Oct 2024 11:14:45 +0300 Subject: [PATCH 22/94] fix: don't copy custom extension scripts during pg_upgrade (#1291) --- ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index deb462012..fd1466c07 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -426,6 +426,7 @@ EOF cp -R /etc/postgresql-custom/* "$MOUNT_POINT/conf/" # removing supautils config as to allow the latest one provided by the latest image to be used rm -f "$MOUNT_POINT/conf/supautils.conf" || true + rm -f "$MOUNT_POINT/conf/extension-custom-scripts" || true # removing wal-g config as to allow it to be explicitly enabled on the new instance rm -f "$MOUNT_POINT/conf/wal-g.conf" From 1211077fa28f668f70908eadc6b57787bad150c2 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Fri, 25 Oct 2024 11:20:23 +0300 Subject: [PATCH 23/94] fix: add recursive flag to custom extension script directory delete (#1292) --- ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index fd1466c07..a301bcdbd 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -426,7 +426,7 @@ EOF cp -R /etc/postgresql-custom/* "$MOUNT_POINT/conf/" # removing supautils config as to allow the latest one provided by the latest image to be used rm -f "$MOUNT_POINT/conf/supautils.conf" || true - rm -f "$MOUNT_POINT/conf/extension-custom-scripts" || true + rm -rf "$MOUNT_POINT/conf/extension-custom-scripts" || true # removing wal-g config as to allow it to be explicitly enabled on the new instance rm -f "$MOUNT_POINT/conf/wal-g.conf" From 407d41aeab2d6d7bc3241804c8b13497173f0228 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Wed, 30 Oct 2024 15:35:59 +0200 Subject: [PATCH 24/94] [GEN-11027] chore: reserve a fixed amount of blocks for the data volume; remove ansible pkg + ppa (#1295) --- common-nix.vars.pkr.hcl | 2 +- ebssurrogate/scripts/surrogate-bootstrap-nix.sh | 5 +++++ scripts/90-cleanup.sh | 5 ++++- scripts/nix-provision.sh | 6 +++++- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index eafb1e602..adcb6c8f3 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.135" +postgres-version = "15.6.1.136" diff --git a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh index 45cbc8ac6..3b0bb93c8 100755 --- a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh +++ b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh @@ -110,6 +110,11 @@ function format_and_mount_rootfs { fi mkfs.ext4 /dev/xvdh + + # Explicitly reserving 100MiB worth of blocks for the data volume + RESERVED_DATA_VOLUME_BLOCK_COUNT=$((100 * 1024 * 1024 / 4096)) + tune2fs -r $RESERVED_DATA_VOLUME_BLOCK_COUNT /dev/xvdh + mkdir -p /mnt/data mount -o defaults,discard /dev/xvdh /mnt/data } diff --git a/scripts/90-cleanup.sh b/scripts/90-cleanup.sh index ebbf49080..f2e19686d 100644 --- a/scripts/90-cleanup.sh +++ b/scripts/90-cleanup.sh @@ -36,7 +36,10 @@ elif [ -n "$(command -v apt-get)" ]; then libicu-dev \ libcgal-dev \ libgcc-9-dev \ - libgcc-8-dev + libgcc-8-dev \ + ansible + + add-apt-repository --yes --remove ppa:ansible/ansible source /etc/os-release apt-get -y remove --purge linux-headers-5.11.0-1021-aws diff --git a/scripts/nix-provision.sh b/scripts/nix-provision.sh index 223e84926..1c7acbd58 100644 --- a/scripts/nix-provision.sh +++ b/scripts/nix-provision.sh @@ -38,8 +38,12 @@ EOF $ARGS } +function cleanup_packages { + apt-get -y remove --purge ansible + add-apt-repository --yes --remove ppa:ansible/ansible +} install_packages install_nix execute_stage2_playbook - +cleanup_packages From 19ac49312771398e41256a0b83def82daa659091 Mon Sep 17 00:00:00 2001 From: Bobbie Soedirgo Date: Thu, 31 Oct 2024 13:49:30 +0700 Subject: [PATCH 25/94] fix: grant predefined roles post-upgrade --- .../admin_api_scripts/pg_upgrade_scripts/common.sh | 10 ---------- .../admin_api_scripts/pg_upgrade_scripts/complete.sh | 3 +++ 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh index ea7217fcc..1b46d1712 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh @@ -541,16 +541,6 @@ $$; alter database postgres connection limit -1; --- #incident-2024-09-12-project-upgrades-are-temporarily-disabled -do $$ -begin - if exists (select from pg_authid where rolname = 'pg_read_all_data') then - execute('grant pg_read_all_data to postgres'); - end if; -end -$$; -grant pg_signal_backend to postgres; - set session authorization supabase_admin; drop role supabase_tmp; commit; diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh index d19b72422..c8542c685 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh @@ -78,6 +78,9 @@ EOF run_sql -c "$RECREATE_PG_CRON_QUERY" fi + + # #incident-2024-09-12-project-upgrades-are-temporarily-disabled + run_sql -c "grant pg_read_all_data, pg_signal_backend to postgres" } function complete_pg_upgrade { From 21d3823ec4641a5156842d59389bfc3e0511968e Mon Sep 17 00:00:00 2001 From: Bobbie Soedirgo <31685197+soedirgo@users.noreply.github.com> Date: Thu, 31 Oct 2024 14:15:36 +0700 Subject: [PATCH 26/94] fix: add more roles to reserved_roles & reserved_memberships (#1303) * fix: add more roles to reserved_roles & reserved_memberships * Update common-nix.vars.pkr.hcl --- ansible/files/postgresql_config/supautils.conf.j2 | 4 ++-- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/files/postgresql_config/supautils.conf.j2 b/ansible/files/postgresql_config/supautils.conf.j2 index 2ab4257fd..b2a259693 100644 --- a/ansible/files/postgresql_config/supautils.conf.j2 +++ b/ansible/files/postgresql_config/supautils.conf.j2 @@ -8,5 +8,5 @@ supautils.privileged_extensions_custom_scripts_path = '/etc/postgresql-custom/ex supautils.privileged_extensions_superuser = 'supabase_admin' supautils.privileged_role = 'postgres' supautils.privileged_role_allowed_configs = 'auto_explain.log_min_duration, auto_explain.log_nested_statements, log_lock_waits, log_min_messages, pg_net.batch_size, pg_net.ttl, pgaudit.log, pgaudit.log_catalog, pgaudit.log_client, pgaudit.log_level, pgaudit.log_relation, pgaudit.log_rows, pgaudit.log_statement, pgaudit.log_statement_once, pgaudit.role, pgrst.*, plan_filter.*, safeupdate.enabled, session_replication_role, track_io_timing' -supautils.reserved_memberships = 'pg_read_server_files, pg_write_server_files, pg_execute_server_program, authenticator' -supautils.reserved_roles = 'supabase_admin, supabase_auth_admin, supabase_storage_admin, supabase_read_only_user, supabase_replication_admin, dashboard_user, pgbouncer, service_role*, authenticator*, authenticated*, anon*' +supautils.reserved_memberships = 'pg_read_server_files, pg_write_server_files, pg_execute_server_program, supabase_admin, supabase_auth_admin, supabase_storage_admin, supabase_read_only_user, supabase_realtime_admin, supabase_replication_admin, dashboard_user, pgbouncer, authenticator' +supautils.reserved_roles = 'supabase_admin, supabase_auth_admin, supabase_storage_admin, supabase_read_only_user, supabase_realtime_admin, supabase_replication_admin, dashboard_user, pgbouncer, service_role*, authenticator*, authenticated*, anon*' diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index adcb6c8f3..3b06b818f 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.136" +postgres-version = "15.6.1.137" From 1273c413e3f977df54c2b67bf3b8a49ceecca7c9 Mon Sep 17 00:00:00 2001 From: Stojan Dimitrovski Date: Fri, 1 Nov 2024 13:14:29 +0100 Subject: [PATCH 27/94] feat: update envoy lds config with auth jwks, oidc URLs, strip `sb-opk` header (#1296) --- ansible/files/envoy_config/lds.yaml | 23 ++++++++++++++++++++++- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/ansible/files/envoy_config/lds.yaml b/ansible/files/envoy_config/lds.yaml index 60a38f65d..2fc7cae13 100644 --- a/ansible/files/envoy_config/lds.yaml +++ b/ansible/files/envoy_config/lds.yaml @@ -254,8 +254,13 @@ resources: type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute - match: safe_regex: + google_re2: + max_program_size: 150 regex: >- - /auth/v1/(verify|callback|authorize|sso/saml/(acs|metadata|slo)) + /auth/v1/(verify|callback|authorize|sso/saml/(acs|metadata|slo)|\.well-known/(openid-configuration|jwks\.json)) + request_headers_to_remove: + - apikey + - sb-opk route: cluster: gotrue regex_rewrite: @@ -269,6 +274,9 @@ resources: typed_per_filter_config: *ref_0 - match: prefix: /auth/v1/ + request_headers_to_remove: + - apikey + - sb-opk route: cluster: gotrue prefix_rewrite: / @@ -280,6 +288,7 @@ resources: present_match: true request_headers_to_remove: - apikey + - sb-opk route: cluster: postgrest prefix_rewrite: / @@ -293,6 +302,7 @@ resources: prefix: /rest/v1/ request_headers_to_remove: - apikey + - sb-opk route: cluster: postgrest prefix_rewrite: / @@ -309,6 +319,7 @@ resources: present_match: true request_headers_to_remove: - apikey + - sb-opk route: cluster: postgrest_admin prefix_rewrite: / @@ -321,6 +332,7 @@ resources: prefix: /rest-admin/v1/ request_headers_to_remove: - apikey + - sb-opk route: cluster: postgrest_admin prefix_rewrite: / @@ -330,18 +342,25 @@ resources: header: key: Content-Profile value: graphql_public + request_headers_to_remove: + - apikey + - sb-opk route: cluster: postgrest prefix_rewrite: /rpc/graphql timeout: 125s - match: prefix: /admin/v1/ + request_headers_to_remove: + - sb-opk route: cluster: admin_api prefix_rewrite: / timeout: 600s - match: prefix: /customer/v1/privileged/ + request_headers_to_remove: + - sb-opk route: cluster: admin_api prefix_rewrite: /privileged/ @@ -365,6 +384,8 @@ resources: treat_missing_header_as_empty: true - match: prefix: /metrics/aggregated + request_headers_to_remove: + - sb-opk route: cluster: admin_api prefix_rewrite: /supabase-internal/metrics diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 3b06b818f..c4285caa1 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.137" +postgres-version = "15.6.1.138" From 936713c36ae34b1bf3370f04dc423ee1b8820863 Mon Sep 17 00:00:00 2001 From: angelico Date: Mon, 4 Nov 2024 17:07:32 +0800 Subject: [PATCH 28/94] chore: udpate package repo for salt (#1307) --- ansible/tasks/internal/install-salt.yml | 34 +++++++++++++++++++------ 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/ansible/tasks/internal/install-salt.yml b/ansible/tasks/internal/install-salt.yml index 281e370bb..73cd6ee85 100644 --- a/ansible/tasks/internal/install-salt.yml +++ b/ansible/tasks/internal/install-salt.yml @@ -1,29 +1,47 @@ - name: Add apt repository for Saltstack (arm) block: + - name: Ensure /etc/apt/keyrings directory exists + file: + path: /etc/apt/keyrings + state: directory + mode: '0755' + - name: salt gpg key - ansible.builtin.apt_key: - url: https://repo.saltproject.io/salt/py3/ubuntu/20.04/arm64/SALT-PROJECT-GPG-PUBKEY-2023.gpg - keyring: /etc/apt/trusted.gpg.d/salt-archive-keyring-2023-arm.gpg + ansible.builtin.get_url: + url: https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public + dest: /etc/apt/keyrings/salt-archive-keyring-2023.pgp + mode: '0644' - name: salt apt repo ansible.builtin.apt_repository: - repo: deb [signed-by=/etc/apt/trusted.gpg.d/salt-archive-keyring-2023-arm.gpg arch=arm64] https://repo.saltproject.io/salt/py3/ubuntu/20.04/arm64/{{ salt_minion_version }} focal main + repo: "deb [signed-by=/etc/apt/keyrings/salt-archive-keyring-2023.pgp arch=arm64] https://packages.broadcom.com/artifactory/saltproject-deb/ stable main" + filename: 'salt.list' state: present when: platform == "arm64" - name: Add apt repository for Saltstack (amd) block: + - name: Ensure /etc/apt/keyrings directory exists + file: + path: /etc/apt/keyrings + state: directory + mode: '0755' + - name: salt gpg key - ansible.builtin.apt_key: - url: https://repo.saltproject.io/salt/py3/ubuntu/20.04/amd64/SALT-PROJECT-GPG-PUBKEY-2023.gpg - keyring: /etc/apt/trusted.gpg.d/salt-archive-keyring-2023-amd.gpg + ansible.builtin.get_url: + url: https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public + dest: /etc/apt/keyrings/salt-archive-keyring-2023.pgp + mode: '0644' - name: salt apt repo ansible.builtin.apt_repository: - repo: deb [signed-by=/etc/apt/trusted.gpg.d/salt-archive-keyring-2023-amd.gpg arch=amd64] https://repo.saltproject.io/salt/py3/ubuntu/20.04/amd64/{{ salt_minon_version }} focal main + repo: "deb [signed-by=/etc/apt/keyrings/salt-archive-keyring-2023.pgp arch=amd64] https://packages.broadcom.com/artifactory/saltproject-deb/ stable main" + filename: 'salt.list' state: present when: platform == "amd64" - name: Salt minion install apt: name: salt-minion + state: present + update_cache: yes From 535042092c656c11376c91e58806d9b60f67b27b Mon Sep 17 00:00:00 2001 From: Bobbie Soedirgo Date: Wed, 6 Nov 2024 15:17:57 +0800 Subject: [PATCH 29/94] fix: use sudo for apt-get commands --- scripts/nix-provision.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/nix-provision.sh b/scripts/nix-provision.sh index 1c7acbd58..5ed678d7a 100644 --- a/scripts/nix-provision.sh +++ b/scripts/nix-provision.sh @@ -39,8 +39,8 @@ EOF } function cleanup_packages { - apt-get -y remove --purge ansible - add-apt-repository --yes --remove ppa:ansible/ansible + sudo apt-get -y remove --purge ansible + sudo add-apt-repository --yes --remove ppa:ansible/ansible } install_packages From 9e285dd9843ca32592ed23a035489d5324f8768e Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Fri, 8 Nov 2024 06:11:52 -0600 Subject: [PATCH 30/94] Update pg net to 0.13.0 on pg 15.6 branch (#1315) * upgrade pg_net to 0.13.0 on 15.6 * bump postgres-version --- common-nix.vars.pkr.hcl | 2 +- nix/ext/pg_net.nix | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index c4285caa1..797387418 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.138" +postgres-version = "15.6.1.139" diff --git a/nix/ext/pg_net.nix b/nix/ext/pg_net.nix index dc923b9a8..3bf04ff94 100644 --- a/nix/ext/pg_net.nix +++ b/nix/ext/pg_net.nix @@ -2,7 +2,7 @@ stdenv.mkDerivation rec { pname = "pg_net"; - version = "0.11.0"; + version = "0.13.0"; buildInputs = [ curl postgresql ]; @@ -10,7 +10,7 @@ stdenv.mkDerivation rec { owner = "supabase"; repo = pname; rev = "refs/tags/v${version}"; - hash = "sha256-XN441jXK1q+I/LZRNwvzbSsebXHgZ8iYsslZvcPFlAs="; + hash = "sha256-FRaTZPCJQPYAFmsJg22hYJJ0+gH1tMdDQoCQgiqEnaA="; }; env.NIX_CFLAGS_COMPILE = "-Wno-error"; From a627e176c1e8816bc73f5e49ccb7042e5647fbca Mon Sep 17 00:00:00 2001 From: Angelico Date: Wed, 6 Nov 2024 15:18:41 +0800 Subject: [PATCH 31/94] chore: cleanup pgbouncer.get_auth ownership accordingly --- .../pg_upgrade_scripts/initiate.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index a301bcdbd..e28eda38e 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -39,6 +39,7 @@ MOUNT_POINT="/data_migration" LOG_FILE="/var/log/pg-upgrade-initiate.log" POST_UPGRADE_EXTENSION_SCRIPT="/tmp/pg_upgrade/pg_upgrade_extensions.sql" +POST_UPGRADE_PGBOUNCER_CLEANUP_SCRIPT="/tmp/pg_upgrade/pg_upgrade_pgbouncer_cleanup.sql" OLD_PGVERSION=$(run_sql -A -t -c "SHOW server_version;") SERVER_LC_COLLATE=$(run_sql -A -t -c "SHOW lc_collate;") @@ -132,6 +133,22 @@ cleanup() { echo "Resetting postgres database connection limit" retry 5 run_sql -c "ALTER DATABASE postgres CONNECTION LIMIT -1;" + echo "Making sure postgres still has access to pg_shadow" + cat << EOF >> $POST_UPGRADE_PGBOUNCER_CLEANUP_SCRIPT +DO \$\$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_views WHERE viewname = 'pg_shadow' AND viewowner = 'supabase_admin') THEN + ALTER function pgbouncer.get_auth owner to supabase_admin; + GRANT EXECUTE ON FUNCTION pgbouncer.get_auth(p_usename TEXT) TO postgres; + END IF; +END; +\$\$; +EOF + + if [ -f $POST_UPGRADE_PGBOUNCER_CLEANUP_SCRIPT ]; then + retry 5 run_sql -f $POST_UPGRADE_PGBOUNCER_CLEANUP_SCRIPT + fi + if [ -z "$IS_CI" ] && [ -z "$IS_LOCAL_UPGRADE" ]; then echo "Unmounting data disk from ${MOUNT_POINT}" retry 3 umount $MOUNT_POINT From 4b0e66101ae45032803894f3ab5c90ca6093f7b9 Mon Sep 17 00:00:00 2001 From: Angelico Date: Wed, 6 Nov 2024 15:48:34 +0800 Subject: [PATCH 32/94] chore: standardize --- .../pg_upgrade_scripts/initiate.sh | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index e28eda38e..88518766b 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -39,7 +39,7 @@ MOUNT_POINT="/data_migration" LOG_FILE="/var/log/pg-upgrade-initiate.log" POST_UPGRADE_EXTENSION_SCRIPT="/tmp/pg_upgrade/pg_upgrade_extensions.sql" -POST_UPGRADE_PGBOUNCER_CLEANUP_SCRIPT="/tmp/pg_upgrade/pg_upgrade_pgbouncer_cleanup.sql" +POST_UPGRADE_POSTGRES_PERMS_SCRIPT="/tmp/pg_upgrade/pg_upgrade_postgres_perms.sql" OLD_PGVERSION=$(run_sql -A -t -c "SHOW server_version;") SERVER_LC_COLLATE=$(run_sql -A -t -c "SHOW lc_collate;") @@ -134,19 +134,19 @@ cleanup() { retry 5 run_sql -c "ALTER DATABASE postgres CONNECTION LIMIT -1;" echo "Making sure postgres still has access to pg_shadow" - cat << EOF >> $POST_UPGRADE_PGBOUNCER_CLEANUP_SCRIPT + cat << EOF >> $POST_UPGRADE_POSTGRES_PERMS_SCRIPT DO \$\$ -BEGIN - IF EXISTS (SELECT 1 FROM pg_views WHERE viewname = 'pg_shadow' AND viewowner = 'supabase_admin') THEN - ALTER function pgbouncer.get_auth owner to supabase_admin; - GRANT EXECUTE ON FUNCTION pgbouncer.get_auth(p_usename TEXT) TO postgres; - END IF; -END; +begin + if exists (select from pg_authid where rolname = 'pg_read_all_data') then + execute('grant pg_read_all_data to postgres'); + end if; +end \$\$; +grant pg_signal_backend to postgres; EOF - if [ -f $POST_UPGRADE_PGBOUNCER_CLEANUP_SCRIPT ]; then - retry 5 run_sql -f $POST_UPGRADE_PGBOUNCER_CLEANUP_SCRIPT + if [ -f $POST_UPGRADE_POSTGRES_PERMS_SCRIPT ]; then + retry 5 run_sql -f $POST_UPGRADE_POSTGRES_PERMS_SCRIPT fi if [ -z "$IS_CI" ] && [ -z "$IS_LOCAL_UPGRADE" ]; then From a3cce049849db6cd69f2e697eb5a990d5ae4417e Mon Sep 17 00:00:00 2001 From: Div Arora Date: Thu, 7 Nov 2024 11:56:54 +0800 Subject: [PATCH 33/94] chore: try a qemu build --- Dockerfile-kubevirt | 2 + Makefile | 24 ++ amazon-arm64-nix.pkr.hcl | 212 ++++------- ansible/files/permission_check.py | 72 ++-- ansible/playbook.yml | 18 +- ansible/tasks/internal/admin-api.yml | 2 +- ansible/tasks/setup-fail2ban.yml | 9 +- ansible/tasks/setup-postgres.yml | 22 +- ansible/tasks/stage2-setup-postgres.yml | 4 +- ansible/vars.yml | 2 +- .../scripts/surrogate-bootstrap-nix.sh | 342 ++++-------------- http/.gitkeep | 0 meta-data | 2 + scripts/90-cleanup.sh | 36 +- scripts/nix-provision.sh | 8 +- stage2-nix-psql.pkr.hcl | 87 +++-- user-data-cloudimg | 16 + 17 files changed, 320 insertions(+), 538 deletions(-) create mode 100644 Dockerfile-kubevirt create mode 100644 Makefile create mode 100644 http/.gitkeep create mode 100644 meta-data create mode 100644 user-data-cloudimg diff --git a/Dockerfile-kubevirt b/Dockerfile-kubevirt new file mode 100644 index 000000000..c58e3dbb3 --- /dev/null +++ b/Dockerfile-kubevirt @@ -0,0 +1,2 @@ +FROM scratch +ADD --chown=107:107 focal.img /disk/focal.qcow2 diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..386388385 --- /dev/null +++ b/Makefile @@ -0,0 +1,24 @@ +# TODO (darora): we can get rid of this once we're actually building nix things on this +UPSTREAM_NIX_GIT_SHA := $(shell git rev-parse origin/release/15.6) +GIT_SHA := $(shell git describe --tags --always --dirty) + +init: amazon-arm64-nix.pkr.hcl + packer init amazon-arm64-nix.pkr.hcl + +output-cloudimg/packer-cloudimg: ansible amazon-arm64-nix.pkr.hcl + packer build -var "git_sha=$(UPSTREAM_NIX_GIT_SHA)" amazon-arm64-nix.pkr.hcl + +disk/focal-raw.img: output-cloudimg/packer-cloudimg + mkdir -p disk + sudo qemu-img convert -O raw output-cloudimg/packer-cloudimg disk/focal-raw.img + +container-disk-image: disk/focal-raw.img + sudo nerdctl build . -t supabase-postgres-test:$(GIT_SHA) --namespace k8s.io -f ./Dockerfile-kubevirt + +host-disk: disk/focal-raw.img + sudo chown 107 -R disk + +clean: + rm -rf output-cloudimg + +.PHONY: container-disk-image host-disk init clean diff --git a/amazon-arm64-nix.pkr.hcl b/amazon-arm64-nix.pkr.hcl index 118196473..8afc87d23 100644 --- a/amazon-arm64-nix.pkr.hcl +++ b/amazon-arm64-nix.pkr.hcl @@ -38,10 +38,14 @@ variable "environment" { default = "prod" } -variable "region" { +variable "git_sha" { type = string } +# variable "region" { +# type = string +# } + variable "build-vol" { type = string default = "xvdc" @@ -98,140 +102,69 @@ packer { source = "github.com/hashicorp/amazon" version = "~> 1" } - } -} - -# source block -source "amazon-ebssurrogate" "source" { - profile = "${var.profile}" - #access_key = "${var.aws_access_key}" - #ami_name = "${var.ami_name}-arm64-${formatdate("YYYY-MM-DD-hhmm", timestamp())}" - ami_name = "${var.ami_name}-${var.postgres-version}-stage-1" - ami_virtualization_type = "hvm" - ami_architecture = "arm64" - ami_regions = "${var.ami_regions}" - instance_type = "c6g.4xlarge" - region = "${var.region}" - #secret_key = "${var.aws_secret_key}" - force_deregister = var.force-deregister - - # Use latest official ubuntu focal ami owned by Canonical. - source_ami_filter { - filters = { - virtualization-type = "hvm" - name = "${var.ami}" - root-device-type = "ebs" + qemu = { + version = "~> 1.0" + source = "github.com/hashicorp/qemu" } - owners = [ "099720109477" ] - most_recent = true - } - ena_support = true - launch_block_device_mappings { - device_name = "/dev/xvdf" - delete_on_termination = true - volume_size = 10 - volume_type = "gp3" - } - - launch_block_device_mappings { - device_name = "/dev/xvdh" - delete_on_termination = true - volume_size = 8 - volume_type = "gp3" - } - - launch_block_device_mappings { - device_name = "/dev/${var.build-vol}" - delete_on_termination = true - volume_size = 16 - volume_type = "gp2" - omit_from_artifact = true - } - - run_tags = { - creator = "packer" - appType = "postgres" - packerExecutionId = "${var.packer-execution-id}" - } - run_volume_tags = { - creator = "packer" - appType = "postgres" - } - snapshot_tags = { - creator = "packer" - appType = "postgres" - } - tags = { - creator = "packer" - appType = "postgres" - postgresVersion = "${var.postgres-version}-stage1" - sourceSha = "${var.git-head-version}" - } - - communicator = "ssh" - ssh_pty = true - ssh_username = "ubuntu" - ssh_timeout = "5m" - - ami_root_device { - source_device_name = "/dev/xvdf" - device_name = "/dev/xvda" - delete_on_termination = true - volume_size = 10 - volume_type = "gp2" } +} - associate_public_ip_address = true +source "null" "dependencies" { + communicator = "none" } -# a build block invokes sources and runs provisioning steps on them. build { - sources = ["source.amazon-ebssurrogate.source"] - - provisioner "file" { - source = "ebssurrogate/files/sources-arm64.cfg" - destination = "/tmp/sources.list" - } + name = "cloudimg.deps" + sources = ["source.null.dependencies"] - provisioner "file" { - source = "ebssurrogate/files/ebsnvme-id" - destination = "/tmp/ebsnvme-id" - } - - provisioner "file" { - source = "ebssurrogate/files/70-ec2-nvme-devices.rules" - destination = "/tmp/70-ec2-nvme-devices.rules" - } - - provisioner "file" { - source = "ebssurrogate/scripts/chroot-bootstrap-nix.sh" - destination = "/tmp/chroot-bootstrap-nix.sh" - } - - provisioner "file" { - source = "ebssurrogate/files/cloud.cfg" - destination = "/tmp/cloud.cfg" - } - - provisioner "file" { - source = "ebssurrogate/files/vector.timer" - destination = "/tmp/vector.timer" - } - - provisioner "file" { - source = "ebssurrogate/files/apparmor_profiles" - destination = "/tmp" - } - - provisioner "file" { - source = "migrations" - destination = "/tmp" - } + provisioner "shell-local" { + inline = [ + "cp /usr/share/AAVMF/AAVMF_VARS.fd AAVMF_VARS.fd", + "cloud-localds seeds-cloudimg.iso user-data-cloudimg meta-data" + ] + inline_shebang = "/bin/bash -e" + } +} + +source "qemu" "cloudimg" { + boot_wait = "2s" + cpus = 12 + disk_image = true + disk_size = "15G" + format = "qcow2" + # TODO (darora): disable backing image for qcow2 + headless = true + http_directory = "http" + iso_checksum = "file:https://cloud-images.ubuntu.com/focal/current/SHA256SUMS" + iso_url = "https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-arm64.img" + memory = 20000 + qemu_binary = "qemu-system-aarch64" + qemu_img_args { + create = ["-F", "qcow2"] + } + qemuargs = [ + ["-machine", "virt"], + ["-cpu", "host"], + ["-device", "virtio-gpu-pci"], + ["-drive", "if=pflash,format=raw,id=ovmf_code,readonly=on,file=/usr/share/AAVMF/AAVMF_CODE.fd"], + ["-drive", "if=pflash,format=raw,id=ovmf_vars,file=AAVMF_VARS.fd"], + ["-drive", "file=output-cloudimg/packer-cloudimg,format=qcow2"], + ["-drive", "file=seeds-cloudimg.iso,format=raw"], + ["--enable-kvm"] + ] + shutdown_command = "sudo -S shutdown -P now" + ssh_handshake_attempts = 500 + ssh_password = "ubuntu" + ssh_timeout = "1h" + ssh_username = "ubuntu" + ssh_wait_timeout = "1h" + use_backing_file = true + accelerator = "kvm" +} - provisioner "file" { - source = "ebssurrogate/files/unit-tests" - destination = "/tmp" - } +build { + name = "cloudimg.image" + sources = ["source.qemu.cloudimg"] # Copy ansible playbook provisioner "shell" { @@ -249,29 +182,24 @@ build { } provisioner "file" { - source = "ansible/vars.yml" - destination = "/tmp/ansible-playbook/vars.yml" + source = "migrations" + destination = "/tmp" + } + + provisioner "file" { + source = "ebssurrogate/files/unit-tests" + destination = "/tmp" } provisioner "shell" { environment_vars = [ - "ARGS=${var.ansible_arguments}", - "DOCKER_USER=${var.docker_user}", - "DOCKER_PASSWD=${var.docker_passwd}", - "DOCKER_IMAGE=${var.docker_image}", - "DOCKER_IMAGE_TAG=${var.docker_image_tag}", - "POSTGRES_SUPABASE_VERSION=${var.postgres-version}" + "POSTGRES_SUPABASE_VERSION=${var.postgres-version}", + "GIT_SHA=${var.git_sha}" ] use_env_var_file = true script = "ebssurrogate/scripts/surrogate-bootstrap-nix.sh" - execute_command = "sudo -S sh -c '. {{.EnvVarFile}} && {{.Path}}'" + execute_command = "sudo -S sh -c '. {{.EnvVarFile}} && cd /tmp/ansible-playbook && {{.Path}}'" start_retry_timeout = "5m" skip_clean = true } - - provisioner "file" { - source = "/tmp/ansible.log" - destination = "/tmp/ansible.log" - direction = "download" - } } diff --git a/ansible/files/permission_check.py b/ansible/files/permission_check.py index 724acb10a..e980e6e66 100644 --- a/ansible/files/permission_check.py +++ b/ansible/files/permission_check.py @@ -9,18 +9,18 @@ {"groupname": "ssl-cert", "username": "postgres"} ], "ubuntu": [ - {"groupname":"ubuntu","username":"ubuntu"}, - {"groupname":"adm","username":"ubuntu"}, - {"groupname":"dialout","username":"ubuntu"}, - {"groupname":"cdrom","username":"ubuntu"}, - {"groupname":"floppy","username":"ubuntu"}, - {"groupname":"sudo","username":"ubuntu"}, - {"groupname":"audio","username":"ubuntu"}, - {"groupname":"dip","username":"ubuntu"}, - {"groupname":"video","username":"ubuntu"}, - {"groupname":"plugdev","username":"ubuntu"}, - {"groupname":"lxd","username":"ubuntu"}, - {"groupname":"netdev","username":"ubuntu"} + {'groupname': 'adm', 'username': 'ubuntu'}, + {'groupname': 'audio', 'username': 'ubuntu'}, + {'groupname': 'cdrom', 'username': 'ubuntu'}, + {'groupname': 'dialout', 'username': 'ubuntu'}, + {'groupname': 'dip', 'username': 'ubuntu'}, + {'groupname': 'floppy', 'username': 'ubuntu'}, + {'groupname': 'lxd', 'username': 'ubuntu'}, + {'groupname': 'netdev', 'username': 'ubuntu'}, + {'groupname': 'plugdev', 'username': 'ubuntu'}, + {'groupname': 'sudo', 'username': 'ubuntu'}, + {'groupname': 'ubuntu', 'username': 'ubuntu'}, + {'groupname': 'video', 'username': 'ubuntu'} ], "root": [ {"groupname":"root","username":"root"} @@ -88,20 +88,17 @@ "messagebus": [ {"groupname":"messagebus","username":"messagebus"} ], - "ec2-instance-connect": [ - {"groupname":"nogroup","username":"ec2-instance-connect"} - ], "sshd": [ {"groupname":"nogroup","username":"sshd"} ], "wal-g": [ - {"groupname":"wal-g","username":"wal-g"}, - {"groupname":"postgres","username":"wal-g"} + {"groupname":"postgres","username":"wal-g"}, + {"groupname":"wal-g","username":"wal-g"} ], "pgbouncer": [ {"groupname":"pgbouncer","username":"pgbouncer"}, - {"groupname":"ssl-cert","username":"pgbouncer"}, - {"groupname":"postgres","username":"pgbouncer"} + {"groupname":"postgres","username":"pgbouncer"}, + {"groupname":"ssl-cert","username":"pgbouncer"} ], "gotrue": [ {"groupname":"gotrue","username":"gotrue"} @@ -116,23 +113,23 @@ {"groupname":"nginx","username":"nginx"} ], "vector": [ - {"groupname":"vector","username":"vector"}, {"groupname":"adm","username":"vector"}, + {"groupname":"postgres","username":"vector"}, {"groupname":"systemd-journal","username":"vector"}, - {"groupname":"postgres","username":"vector"} + {"groupname":"vector","username":"vector"} ], "adminapi": [ - {"groupname":"adminapi","username":"adminapi"}, - {"groupname":"root","username":"adminapi"}, - {"groupname":"systemd-journal","username":"adminapi"}, {"groupname":"admin","username":"adminapi"}, - {"groupname":"postgres","username":"adminapi"}, - {"groupname":"pgbouncer","username":"adminapi"}, - {"groupname":"wal-g","username":"adminapi"}, - {"groupname":"postgrest","username":"adminapi"}, + {"groupname":"adminapi","username":"adminapi"}, {"groupname":"envoy","username":"adminapi"}, {"groupname":"kong","username":"adminapi"}, - {"groupname":"vector","username":"adminapi"} + {"groupname":"pgbouncer","username":"adminapi"}, + {"groupname":"postgres","username":"adminapi"}, + {"groupname":"postgrest","username":"adminapi"}, + {"groupname":"root","username":"adminapi"}, + {"groupname":"systemd-journal","username":"adminapi"}, + {"groupname":"vector","username":"adminapi"}, + {"groupname":"wal-g","username":"adminapi"} ], "postgrest": [ {"groupname":"postgrest","username":"postgrest"} @@ -144,6 +141,7 @@ {"groupname":"systemd-coredump","username":"systemd-coredump"} ] } + # This program depends on osquery being installed on the system # Function to run osquery def run_osquery(query): @@ -174,28 +172,28 @@ def compare_results(username, query_result): def check_nixbld_users(): query = """ - SELECT u.username, g.groupname - FROM users u - JOIN user_groups ug ON u.uid = ug.uid - JOIN groups g ON ug.gid = g.gid + SELECT u.username, g.groupname + FROM users u + JOIN user_groups ug ON u.uid = ug.uid + JOIN groups g ON ug.gid = g.gid WHERE u.username LIKE 'nixbld%'; """ query_result = run_osquery(query) parsed_result = parse_json(query_result) - + for user in parsed_result: if user['groupname'] != 'nixbld': print(f"User '{user['username']}' is in group '{user['groupname']}' instead of 'nixbld'.") sys.exit(1) - + print("All nixbld users are in the 'nixbld' group.") # Define usernames for which you want to compare results -usernames = ["postgres", "ubuntu", "root", "daemon", "bin", "sys", "sync", "games","man","lp","mail","news","uucp","proxy","www-data","backup","list","irc","gnats","nobody","systemd-network","systemd-resolve","systemd-timesync","messagebus","ec2-instance-connect","sshd","wal-g","pgbouncer","gotrue","envoy","kong","nginx","vector","adminapi","postgrest","tcpdump","systemd-coredump"] +usernames = ["postgres", "ubuntu", "root", "daemon", "bin", "sys", "sync", "games","man","lp","mail","news","uucp","proxy","www-data","backup","list","irc","gnats","nobody","systemd-network","systemd-resolve","systemd-timesync","messagebus","sshd","wal-g","pgbouncer","gotrue","envoy","kong","nginx","vector","adminapi","postgrest","tcpdump","systemd-coredump"] # Iterate over usernames, run the query, and compare results for username in usernames: - query = f"SELECT u.username, g.groupname FROM users u JOIN user_groups ug ON u.uid = ug.uid JOIN groups g ON ug.gid = g.gid WHERE u.username = '{username}';" + query = f"SELECT u.username, g.groupname FROM users u JOIN user_groups ug ON u.uid = ug.uid JOIN groups g ON ug.gid = g.gid WHERE u.username = '{username}' ORDER BY g.groupname;" query_result = run_osquery(query) parsed_result = parse_json(query_result) compare_results(username, parsed_result) diff --git a/ansible/playbook.yml b/ansible/playbook.yml index a6bea19e9..06d30a93d 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -1,4 +1,4 @@ -- hosts: all +- hosts: localhost become: yes pre_tasks: @@ -142,16 +142,6 @@ import_tasks: tasks/setup-fail2ban.yml when: debpkg_mode or nixpkg_mode - # Install EC2 instance connect - # Only for AWS images - - name: install EC2 instance connect - become: yes - apt: - pkg: - - ec2-instance-connect - tags: - - aws-only - # Install this at the end to prevent it from kicking in during the apt process, causing conflicts - name: Install security tools become: yes @@ -196,6 +186,12 @@ - collect-binaries when: debpkg_mode + - name: Install osquery from nixpkgs binary cache + become: yes + shell: | + apt autoremove -y --purge snapd + when: stage2_nix + - name: Install osquery from nixpkgs binary cache become: yes shell: | diff --git a/ansible/tasks/internal/admin-api.yml b/ansible/tasks/internal/admin-api.yml index da93fef62..f23849a58 100644 --- a/ansible/tasks/internal/admin-api.yml +++ b/ansible/tasks/internal/admin-api.yml @@ -37,7 +37,7 @@ - name: Download adminapi archive get_url: - url: "https://supabase-public-artifacts-bucket.s3.amazonaws.com/supabase-admin-api/v{{ adminapi_release }}/supabase-admin-api_{{ adminapi_release }}_linux_{{ arch }}.tar.gz" + url: "https://supabase-public-artifacts-bucket.s3.us-east-1.amazonaws.com/supabase-admin-api/v3-temp/supabase-admin-api-v{{ adminapi_release }}.tar.gz" dest: "/tmp/adminapi.tar.gz" timeout: 90 diff --git a/ansible/tasks/setup-fail2ban.yml b/ansible/tasks/setup-fail2ban.yml index 1f6065d32..7d9088d46 100644 --- a/ansible/tasks/setup-fail2ban.yml +++ b/ansible/tasks/setup-fail2ban.yml @@ -65,16 +65,9 @@ - install-supabase-internal when: debpkg_mode or nixpkg_mode -# Restart -- name: fail2ban - restart - systemd: - name: fail2ban - state: restarted - when: debpkg_mode or nixpkg_mode - - name: fail2ban - disable service systemd: name: fail2ban enabled: no daemon_reload: yes - when: debpkg_mode or nixpkg_mode \ No newline at end of file + when: debpkg_mode or nixpkg_mode diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index 0038e3afc..75806995f 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -204,6 +204,14 @@ ansible_command_timeout: 60 when: debpkg_mode +- name: Make sure .bashrc exists + file: + path: /var/lib/postgresql/.bashrc + state: touch + owner: postgres + group: postgres + when: nixpkg_mode + - name: Initialize the database stage2_nix become: yes become_user: postgres @@ -234,6 +242,13 @@ dest: /etc/systemd/system/database-optimizations.service when: debpkg_mode or stage2_nix +- name: initialize pg required state + become: yes + shell: | + mkdir -p /run/postgresql + chown -R postgres:postgres /run/postgresql + when: stage2_nix + - name: Restart Postgres Database without Systemd become: yes become_user: postgres @@ -257,13 +272,6 @@ daemon_reload: yes when: debpkg_mode or stage2_nix -- name: Make sure .bashrc exists - file: - path: /var/lib/postgresql/.bashrc - state: touch - owner: postgres - group: postgres - when: nixpkg_mode - name: Add LOCALE_ARCHIVE to .bashrc lineinfile: diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index 20ad069e2..711bea0b5 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -99,7 +99,7 @@ - name: import pgsodium_getkey script template: - src: /tmp/ansible-playbook/ansible/files/pgsodium_getkey_readonly.sh.j2 + src: files/pgsodium_getkey_readonly.sh.j2 dest: "/usr/lib/postgresql/bin/pgsodium_getkey.sh" owner: postgres group: postgres @@ -231,4 +231,4 @@ ansible.builtin.lineinfile: path: /etc/environment.d/postgresql.env line: 'GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins' - become: yes \ No newline at end of file + become: yes diff --git a/ansible/vars.yml b/ansible/vars.yml index 28a499a7f..d26ed2299 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -50,7 +50,7 @@ postgres_exporter_release_checksum: arm64: sha256:29ba62d538b92d39952afe12ee2e1f4401250d678ff4b354ff2752f4321c87a0 amd64: sha256:cb89fc5bf4485fb554e0d640d9684fae143a4b2d5fa443009bd29c59f9129e84 -adminapi_release: 0.71.1 +adminapi_release: 0.71.1-2-g1333ba1 adminmgr_release: 0.24.0 # Postgres Extensions diff --git a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh index 3b0bb93c8..d8ffc9d24 100755 --- a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh +++ b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh @@ -29,185 +29,9 @@ function waitfor_boot_finished { } function install_packages { - # Setup Ansible on host VM - apt-get update && sudo apt-get install software-properties-common -y + apt-get update && sudo apt-get install software-properties-common e2fsprogs -y add-apt-repository --yes --update ppa:ansible/ansible && sudo apt-get install ansible -y ansible-galaxy collection install community.general - - # Update apt and install required packages - apt-get update - apt-get install -y \ - gdisk \ - e2fsprogs \ - debootstrap \ - nvme-cli -} - -# Partition the new root EBS volume -function create_partition_table { - - if [ "${ARCH}" = "arm64" ]; then - parted --script /dev/xvdf \ - mklabel gpt \ - mkpart UEFI 1MiB 100MiB \ - mkpart ROOT 100MiB 100% - set 1 esp on \ - set 1 boot on - parted --script /dev/xvdf print - else - sgdisk -Zg -n1:0:4095 -t1:EF02 -c1:GRUB -n2:0:0 -t2:8300 -c2:EXT4 /dev/xvdf - fi - - sleep 2 -} - -function device_partition_mappings { - # NVMe EBS launch device mappings (symlinks): /dev/nvme*n* to /dev/xvd* - declare -A blkdev_mappings - for blkdev in $(nvme list | awk '/^\/dev/ { print $1 }'); do # /dev/nvme*n* - # Mapping info from disk headers - header=$(nvme id-ctrl --raw-binary "${blkdev}" | cut -c3073-3104 | tr -s ' ' | sed 's/ $//g' | sed 's!/dev/!!') - mapping="/dev/${header%%[0-9]}" # normalize sda1 => sda - - # Create /dev/xvd* device symlink - if [[ ! -z "$mapping" ]] && [[ -b "${blkdev}" ]] && [[ ! -L "${mapping}" ]]; then - ln -s "$blkdev" "$mapping" - - blkdev_mappings["$blkdev"]="$mapping" - fi - done - - create_partition_table - - # NVMe EBS launch device partition mappings (symlinks): /dev/nvme*n*p* to /dev/xvd*[0-9]+ - declare -A partdev_mappings - for blkdev in "${!blkdev_mappings[@]}"; do # /dev/nvme*n* - mapping="${blkdev_mappings[$blkdev]}" - - # Create /dev/xvd*[0-9]+ partition device symlink - for partdev in "${blkdev}"p*; do - partnum=${partdev##*p} - if [[ ! -L "${mapping}${partnum}" ]]; then - ln -s "${blkdev}p${partnum}" "${mapping}${partnum}" - - partdev_mappings["${blkdev}p${partnum}"]="${mapping}${partnum}" - fi - done - done -} - - -#Download and install latest e2fsprogs for fast_commit feature,if required. -function format_and_mount_rootfs { - mkfs.ext4 -m0.1 /dev/xvdf2 - - mount -o noatime,nodiratime /dev/xvdf2 /mnt - if [ "${ARCH}" = "arm64" ]; then - mkfs.fat -F32 /dev/xvdf1 - mkdir -p /mnt/boot/efi - sleep 2 - mount /dev/xvdf1 /mnt/boot/efi - fi - - mkfs.ext4 /dev/xvdh - - # Explicitly reserving 100MiB worth of blocks for the data volume - RESERVED_DATA_VOLUME_BLOCK_COUNT=$((100 * 1024 * 1024 / 4096)) - tune2fs -r $RESERVED_DATA_VOLUME_BLOCK_COUNT /dev/xvdh - - mkdir -p /mnt/data - mount -o defaults,discard /dev/xvdh /mnt/data -} - -function create_swapfile { - fallocate -l 1G /mnt/swapfile - chmod 600 /mnt/swapfile - mkswap /mnt/swapfile -} - -function format_build_partition { - mkfs.ext4 -O ^has_journal /dev/xvdc -} -function pull_docker { - apt-get install -y docker.io - docker run -itd --name ccachedata "${DOCKER_IMAGE}:${DOCKER_IMAGE_TAG}" sh - docker exec -itd ccachedata mkdir -p /build/ccache -} - -# Create fstab -function create_fstab { - FMT="%-42s %-11s %-5s %-17s %-5s %s" -cat > "/mnt/etc/fstab" << EOF -$(printf "${FMT}" "# DEVICE UUID" "MOUNTPOINT" "TYPE" "OPTIONS" "DUMP" "FSCK") -$(findmnt -no SOURCE /mnt | xargs blkid -o export | awk -v FMT="${FMT}" '/^UUID=/ { printf(FMT, $0, "/", "ext4", "defaults,discard", "0", "1" ) }') -$(findmnt -no SOURCE /mnt/boot/efi | xargs blkid -o export | awk -v FMT="${FMT}" '/^UUID=/ { printf(FMT, $0, "/boot/efi", "vfat", "umask=0077", "0", "1" ) }') -$(findmnt -no SOURCE /mnt/data | xargs blkid -o export | awk -v FMT="${FMT}" '/^UUID=/ { printf(FMT, $0, "/data", "ext4", "defaults,discard", "0", "2" ) }') -$(printf "$FMT" "/swapfile" "none" "swap" "sw" "0" "0") -EOF - unset FMT -} - -function setup_chroot_environment { - UBUNTU_VERSION=$(lsb_release -cs) # 'focal' for Ubuntu 20.04 - - # Bootstrap Ubuntu into /mnt - debootstrap --arch ${ARCH} --variant=minbase "$UBUNTU_VERSION" /mnt - - # Update ec2-region - REGION=$(curl --silent --fail http://169.254.169.254/latest/meta-data/placement/availability-zone | sed -E 's|[a-z]+$||g') - sed -i "s/REGION/${REGION}/g" /tmp/sources.list - cp /tmp/sources.list /mnt/etc/apt/sources.list - - if [ "${ARCH}" = "arm64" ]; then - create_fstab - fi - - # Create mount points and mount the filesystem - mkdir -p /mnt/{dev,proc,sys} - mount --rbind /dev /mnt/dev - mount --rbind /proc /mnt/proc - mount --rbind /sys /mnt/sys - - # Create build mount point and mount - mkdir -p /mnt/tmp - mount /dev/xvdc /mnt/tmp - chmod 777 /mnt/tmp - - # Copy apparmor profiles - chmod 644 /tmp/apparmor_profiles/* - cp -r /tmp/apparmor_profiles /mnt/tmp/ - - # Copy migrations - cp -r /tmp/migrations /mnt/tmp/ - - # Copy unit tests - cp -r /tmp/unit-tests /mnt/tmp/ - - # Copy the bootstrap script into place and execute inside chroot - cp /tmp/chroot-bootstrap-nix.sh /mnt/tmp/chroot-bootstrap-nix.sh - chroot /mnt /tmp/chroot-bootstrap-nix.sh - rm -f /mnt/tmp/chroot-bootstrap-nix.sh - echo "${POSTGRES_SUPABASE_VERSION}" > /mnt/root/supabase-release - - # Copy the nvme identification script into /sbin inside the chroot - mkdir -p /mnt/sbin - cp /tmp/ebsnvme-id /mnt/sbin/ebsnvme-id - chmod +x /mnt/sbin/ebsnvme-id - - # Copy the udev rules for identifying nvme devices into the chroot - mkdir -p /mnt/etc/udev/rules.d - cp /tmp/70-ec2-nvme-devices.rules \ - /mnt/etc/udev/rules.d/70-ec2-nvme-devices.rules - - #Copy custom cloud-init - rm -f /mnt/etc/cloud/cloud.cfg - cp /tmp/cloud.cfg /mnt/etc/cloud/cloud.cfg - - sleep 2 -} - -function download_ccache { - docker cp ccachedata:/build/ccache/. /mnt/tmp/ccache } function execute_playbook { @@ -219,110 +43,100 @@ EOF # Run Ansible playbook #export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_DEBUG=True && export ANSIBLE_REMOTE_TEMP=/mnt/tmp export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_REMOTE_TEMP=/mnt/tmp - ansible-playbook -c chroot -i '/mnt,' /tmp/ansible-playbook/ansible/playbook.yml --extra-vars '{"nixpkg_mode": true, "debpkg_mode": false, "stage2_nix": false}' $ARGS + ansible-playbook ./ansible/playbook.yml --extra-vars '{"nixpkg_mode": true, "debpkg_mode": false, "stage2_nix": false}' # $ARGS - I think this is being not passed in correctly } -function update_systemd_services { - # Disable vector service and set timer unit. - cp -v /tmp/vector.timer /mnt/etc/systemd/system/vector.timer - rm -f /mnt/etc/systemd/system/multi-user.target.wants/vector.service - ln -s /etc/systemd/system/vector.timer /mnt/etc/systemd/system/multi-user.target.wants/vector.timer - - # Disable services during first boot. - rm -f /mnt/etc/systemd/system/sysinit.target.wants/apparmor.service - rm -f /mnt/etc/systemd/system/multi-user.target.wants/postgresql.service - rm -f /mnt/etc/systemd/system/multi-user.target.wants/salt-minion.service +function setup_postgesql_env { + # Create the directory if it doesn't exist + sudo mkdir -p /etc/environment.d - # Disable auditd - rm -f /mnt/etc/systemd/system/multi-user.target.wants/auditd.service + # Define the contents of the PostgreSQL environment file + cat </dev/null +LOCALE_ARCHIVE=/usr/lib/locale/locale-archive +LANG="en_US.UTF-8" +LANGUAGE="en_US.UTF-8" +LC_ALL="en_US.UTF-8" +LC_CTYPE="en_US.UTF-8" +EOF } +function setup_locale { +cat << EOF >> /etc/locale.gen +en_US.UTF-8 UTF-8 +EOF -function clean_system { - # Copy cleanup scripts - cp -v /tmp/ansible-playbook/scripts/90-cleanup.sh /mnt/tmp - chmod +x /mnt/tmp/90-cleanup.sh - chroot /mnt /tmp/90-cleanup.sh +cat << EOF > /etc/default/locale +LANG="C.UTF-8" +LC_CTYPE="C.UTF-8" +EOF + locale-gen en_US.UTF-8 +} - # Cleanup logs - rm -rf /mnt/var/log/* - # https://github.com/fail2ban/fail2ban/issues/1593 - touch /mnt/var/log/auth.log +waitfor_boot_finished +install_packages +setup_postgesql_env +setup_locale +execute_playbook - touch /mnt/var/log/pgbouncer.log - if [ -f /usr/bin/chown ]; then - chroot /mnt /usr/bin/chown pgbouncer:postgres /var/log/pgbouncer.log - fi +# stage 2 things +function install_nix() { + sudo su -c "curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install --no-confirm \ + --extra-conf \"substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com\" \ + --extra-conf \"trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=\" " -s /bin/bash root + . /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh - # Setup postgresql logs - mkdir -p /mnt/var/log/postgresql - if [ -f /usr/bin/chown ]; then - chroot /mnt /usr/bin/chown postgres:postgres /var/log/postgresql - fi +} - # Setup wal-g logs - mkdir /mnt/var/log/wal-g - touch /mnt/var/log/wal-g/{backup-push.log,backup-fetch.log,wal-push.log,wal-fetch.log,pitr.log} +function execute_stage2_playbook { + sudo tee /etc/ansible/ansible.cfg < /var/log/lastlog; cat /dev/null > /var/log/wtmp +# dd if=/dev/zero of=/zerofile & +# PID=$! +# while [ -d /proc/$PID ] +# do +# printf "." +# sleep 5 +# done +# sync; rm /zerofile; sync + +cat /dev/null > /var/log/lastlog +cat /dev/null > /var/log/wtmp diff --git a/scripts/nix-provision.sh b/scripts/nix-provision.sh index 5ed678d7a..998acdaa2 100644 --- a/scripts/nix-provision.sh +++ b/scripts/nix-provision.sh @@ -29,13 +29,13 @@ function execute_stage2_playbook { [defaults] callbacks_enabled = timer, profile_tasks, profile_roles EOF - sed -i 's/- hosts: all/- hosts: localhost/' /tmp/ansible-playbook/ansible/playbook.yml + # sed -i 's/- hosts: all/- hosts: localhost/' ./ansible/playbook.yml # Run Ansible playbook export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_REMOTE_TEMP=/tmp - ansible-playbook /tmp/ansible-playbook/ansible/playbook.yml \ + GIT_SHA=$(git rev-parse HEAD) + ansible-playbook ./ansible/playbook.yml \ --extra-vars '{"nixpkg_mode": false, "stage2_nix": true, "debpkg_mode": false}' \ - --extra-vars "git_commit_sha=${GIT_SHA}" \ - $ARGS + --extra-vars "git_commit_sha=${GIT_SHA}" } function cleanup_packages { diff --git a/stage2-nix-psql.pkr.hcl b/stage2-nix-psql.pkr.hcl index 57ddfd9e7..5a289b7e4 100644 --- a/stage2-nix-psql.pkr.hcl +++ b/stage2-nix-psql.pkr.hcl @@ -55,55 +55,54 @@ packer { } } -source "amazon-ebs" "ubuntu" { - ami_name = "${var.ami_name}-${var.postgres-version}" - instance_type = "c6g.4xlarge" - region = "${var.region}" - source_ami_filter { - filters = { - name = "${var.ami_name}-${var.postgres-version}-stage-1" - root-device-type = "ebs" - virtualization-type = "hvm" - } - most_recent = true - owners = ["amazon", "self"] - } - - communicator = "ssh" - ssh_pty = true - ssh_username = "ubuntu" - ssh_timeout = "5m" - - associate_public_ip_address = true - - - ena_support = true - - run_tags = { - creator = "packer" - appType = "postgres" - packerExecutionId = "${var.packer-execution-id}" - } - run_volume_tags = { - creator = "packer" - appType = "postgres" - } - snapshot_tags = { - creator = "packer" - appType = "postgres" - } - tags = { - creator = "packer" - appType = "postgres" - postgresVersion = "${var.postgres-version}" - sourceSha = "${var.git-head-version}" - } +source "qemu" "supabase_postgres" { + vm_name = "ubuntu-2004-amd64-iso.qcow2" + iso_url = "https://www.releases.ubuntu.com/20.04/ubuntu-20.04.6-live-server-amd64.iso" + iso_checksum = "sha256:b8f31413336b9393ad5d8ef0282717b2ab19f007df2e9ed5196c13d8f9153c8b" + # vm_name = "ubuntu-2404-amd64.raw" + # iso_url = "https://www.releases.ubuntu.com/24.04/ubuntu-24.04-live-server-amd64.iso" + memory = 20000 + disk_image = false + output_directory = "output_images" + shutdown_command = "echo 'packer' | sudo -S shutdown -P now" + disk_size = "9000M" + format = "qcow2" + accelerator = "kvm" + net_device = "virtio-net" + disk_interface = "virtio" + boot_wait = "10s" + + boot_command = [ + # Make the language selector appear... + " ", + # ...then get rid of it + " ", + + # Go to the other installation options menu and leave it + "", + + # Remove the kernel command-line that already exists + "", + "", + "", + + # Add kernel command-line and start install + "/casper/vmlinuz ", + "initrd=/casper/initrd ", + "autoinstall ", + "ds=nocloud-net;s=http://{{.HTTPIP}}:{{.HTTPPort}}/ ", + "" + ] + http_directory = "http" + ssh_username = "packer" + ssh_password = "packer" + ssh_timeout = "60m" } build { name = "nix-packer-ubuntu" sources = [ - "source.amazon-ebs.ubuntu" + "source.qemu.supabase_postgres" ] # Copy ansible playbook diff --git a/user-data-cloudimg b/user-data-cloudimg new file mode 100644 index 000000000..9a74d237a --- /dev/null +++ b/user-data-cloudimg @@ -0,0 +1,16 @@ +#cloud-config +users: + - name: root + lock_passwd: false + ssh_redirect_user: true + hashed_passwd: "$6$canonical.$0zWaW71A9ke9ASsaOcFTdQ2tx1gSmLxMPrsH0rF0Yb.2AEKNPV1lrF94n6YuPJmnUy2K2/JSDtxuiBDey6Lpa/" + - name: ubuntu + lock_passwd: false + hashed_passwd: "$6$canonical.$0zWaW71A9ke9ASsaOcFTdQ2tx1gSmLxMPrsH0rF0Yb.2AEKNPV1lrF94n6YuPJmnUy2K2/JSDtxuiBDey6Lpa/" + ssh_redired_user: false + sudo: "ALL=(ALL) NOPASSWD:ALL" + shell: /usr/bin/bash + groups: [adm, audio, cdrom, dialout, dip, floppy, lxd, netdev, plugdev, sudo, video] +ssh_pwauth: True +disable_root: false +preserve_hostname: true From 4c2aa5bce9390f5c81fcb15fecf170bb5395c177 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Wed, 13 Nov 2024 15:35:36 +0800 Subject: [PATCH 34/94] chore: avoid using ansible globbing --- amazon-arm64-nix.pkr.hcl | 212 ++++++++++++++++-------- ansible/tasks/setup-postgres.yml | 6 - ansible/tasks/stage2-setup-postgres.yml | 61 ++----- 3 files changed, 157 insertions(+), 122 deletions(-) diff --git a/amazon-arm64-nix.pkr.hcl b/amazon-arm64-nix.pkr.hcl index 8afc87d23..118196473 100644 --- a/amazon-arm64-nix.pkr.hcl +++ b/amazon-arm64-nix.pkr.hcl @@ -38,14 +38,10 @@ variable "environment" { default = "prod" } -variable "git_sha" { +variable "region" { type = string } -# variable "region" { -# type = string -# } - variable "build-vol" { type = string default = "xvdc" @@ -102,83 +98,129 @@ packer { source = "github.com/hashicorp/amazon" version = "~> 1" } - qemu = { - version = "~> 1.0" - source = "github.com/hashicorp/qemu" - } } } -source "null" "dependencies" { - communicator = "none" -} +# source block +source "amazon-ebssurrogate" "source" { + profile = "${var.profile}" + #access_key = "${var.aws_access_key}" + #ami_name = "${var.ami_name}-arm64-${formatdate("YYYY-MM-DD-hhmm", timestamp())}" + ami_name = "${var.ami_name}-${var.postgres-version}-stage-1" + ami_virtualization_type = "hvm" + ami_architecture = "arm64" + ami_regions = "${var.ami_regions}" + instance_type = "c6g.4xlarge" + region = "${var.region}" + #secret_key = "${var.aws_secret_key}" + force_deregister = var.force-deregister -build { - name = "cloudimg.deps" - sources = ["source.null.dependencies"] + # Use latest official ubuntu focal ami owned by Canonical. + source_ami_filter { + filters = { + virtualization-type = "hvm" + name = "${var.ami}" + root-device-type = "ebs" + } + owners = [ "099720109477" ] + most_recent = true + } + ena_support = true + launch_block_device_mappings { + device_name = "/dev/xvdf" + delete_on_termination = true + volume_size = 10 + volume_type = "gp3" + } - provisioner "shell-local" { - inline = [ - "cp /usr/share/AAVMF/AAVMF_VARS.fd AAVMF_VARS.fd", - "cloud-localds seeds-cloudimg.iso user-data-cloudimg meta-data" - ] - inline_shebang = "/bin/bash -e" - } -} - -source "qemu" "cloudimg" { - boot_wait = "2s" - cpus = 12 - disk_image = true - disk_size = "15G" - format = "qcow2" - # TODO (darora): disable backing image for qcow2 - headless = true - http_directory = "http" - iso_checksum = "file:https://cloud-images.ubuntu.com/focal/current/SHA256SUMS" - iso_url = "https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-arm64.img" - memory = 20000 - qemu_binary = "qemu-system-aarch64" - qemu_img_args { - create = ["-F", "qcow2"] - } - qemuargs = [ - ["-machine", "virt"], - ["-cpu", "host"], - ["-device", "virtio-gpu-pci"], - ["-drive", "if=pflash,format=raw,id=ovmf_code,readonly=on,file=/usr/share/AAVMF/AAVMF_CODE.fd"], - ["-drive", "if=pflash,format=raw,id=ovmf_vars,file=AAVMF_VARS.fd"], - ["-drive", "file=output-cloudimg/packer-cloudimg,format=qcow2"], - ["-drive", "file=seeds-cloudimg.iso,format=raw"], - ["--enable-kvm"] - ] - shutdown_command = "sudo -S shutdown -P now" - ssh_handshake_attempts = 500 - ssh_password = "ubuntu" - ssh_timeout = "1h" - ssh_username = "ubuntu" - ssh_wait_timeout = "1h" - use_backing_file = true - accelerator = "kvm" + launch_block_device_mappings { + device_name = "/dev/xvdh" + delete_on_termination = true + volume_size = 8 + volume_type = "gp3" + } + + launch_block_device_mappings { + device_name = "/dev/${var.build-vol}" + delete_on_termination = true + volume_size = 16 + volume_type = "gp2" + omit_from_artifact = true + } + + run_tags = { + creator = "packer" + appType = "postgres" + packerExecutionId = "${var.packer-execution-id}" + } + run_volume_tags = { + creator = "packer" + appType = "postgres" + } + snapshot_tags = { + creator = "packer" + appType = "postgres" + } + tags = { + creator = "packer" + appType = "postgres" + postgresVersion = "${var.postgres-version}-stage1" + sourceSha = "${var.git-head-version}" + } + + communicator = "ssh" + ssh_pty = true + ssh_username = "ubuntu" + ssh_timeout = "5m" + + ami_root_device { + source_device_name = "/dev/xvdf" + device_name = "/dev/xvda" + delete_on_termination = true + volume_size = 10 + volume_type = "gp2" + } + + associate_public_ip_address = true } +# a build block invokes sources and runs provisioning steps on them. build { - name = "cloudimg.image" - sources = ["source.qemu.cloudimg"] + sources = ["source.amazon-ebssurrogate.source"] - # Copy ansible playbook - provisioner "shell" { - inline = ["mkdir /tmp/ansible-playbook"] + provisioner "file" { + source = "ebssurrogate/files/sources-arm64.cfg" + destination = "/tmp/sources.list" } provisioner "file" { - source = "ansible" - destination = "/tmp/ansible-playbook" + source = "ebssurrogate/files/ebsnvme-id" + destination = "/tmp/ebsnvme-id" } provisioner "file" { - source = "scripts" - destination = "/tmp/ansible-playbook" + source = "ebssurrogate/files/70-ec2-nvme-devices.rules" + destination = "/tmp/70-ec2-nvme-devices.rules" + } + + provisioner "file" { + source = "ebssurrogate/scripts/chroot-bootstrap-nix.sh" + destination = "/tmp/chroot-bootstrap-nix.sh" + } + + provisioner "file" { + source = "ebssurrogate/files/cloud.cfg" + destination = "/tmp/cloud.cfg" + } + + provisioner "file" { + source = "ebssurrogate/files/vector.timer" + destination = "/tmp/vector.timer" + } + + provisioner "file" { + source = "ebssurrogate/files/apparmor_profiles" + destination = "/tmp" } provisioner "file" { @@ -191,15 +233,45 @@ build { destination = "/tmp" } + # Copy ansible playbook + provisioner "shell" { + inline = ["mkdir /tmp/ansible-playbook"] + } + + provisioner "file" { + source = "ansible" + destination = "/tmp/ansible-playbook" + } + + provisioner "file" { + source = "scripts" + destination = "/tmp/ansible-playbook" + } + + provisioner "file" { + source = "ansible/vars.yml" + destination = "/tmp/ansible-playbook/vars.yml" + } + provisioner "shell" { environment_vars = [ - "POSTGRES_SUPABASE_VERSION=${var.postgres-version}", - "GIT_SHA=${var.git_sha}" + "ARGS=${var.ansible_arguments}", + "DOCKER_USER=${var.docker_user}", + "DOCKER_PASSWD=${var.docker_passwd}", + "DOCKER_IMAGE=${var.docker_image}", + "DOCKER_IMAGE_TAG=${var.docker_image_tag}", + "POSTGRES_SUPABASE_VERSION=${var.postgres-version}" ] use_env_var_file = true script = "ebssurrogate/scripts/surrogate-bootstrap-nix.sh" - execute_command = "sudo -S sh -c '. {{.EnvVarFile}} && cd /tmp/ansible-playbook && {{.Path}}'" + execute_command = "sudo -S sh -c '. {{.EnvVarFile}} && {{.Path}}'" start_retry_timeout = "5m" skip_clean = true } + + provisioner "file" { + source = "/tmp/ansible.log" + destination = "/tmp/ansible.log" + direction = "download" + } } diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index 75806995f..ea48340db 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -218,12 +218,6 @@ shell: source /var/lib/postgresql/.bashrc && /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data initdb -o "--allow-group-access" -o "--username=supabase_admin" args: executable: /bin/bash - environment: - LANG: en_US.UTF-8 - LANGUAGE: en_US.UTF-8 - LC_ALL: en_US.UTF-8 - LC_CTYPE: en_US.UTF-8 - LOCALE_ARCHIVE: /usr/lib/locale/locale-archive vars: ansible_command_timeout: 60 # Circumvents the following error: diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index 711bea0b5..147aa9d37 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -107,12 +107,11 @@ when: stage2_nix - name: Create symbolic links from /var/lib/postgresql/.nix-profile/bin to /usr/lib/postgresql/bin - file: - src: "{{ item }}" - dest: "/usr/lib/postgresql/bin/{{ item | basename }}" - state: link - with_fileglob: - - "/var/lib/postgresql/.nix-profile/bin/*" + shell: >- + find /var/lib/postgresql/.nix-profile/bin/ -maxdepth 1 -type f,l -exec sh -c 'ln -s "$0" "{{ item }}/$(basename $0)"' {} \; + loop: + - /usr/lib/postgresql/bin + - /usr/bin become: yes when: stage2_nix @@ -129,23 +128,9 @@ when: pg_config_stat.stat.exists and not pg_config_stat.stat.islnk and stage2_nix become: yes -- name: Create symbolic links from /var/lib/postgresql/.nix-profile/bin to /usr/bin - file: - src: "{{ item }}" - dest: "/usr/bin/{{ item | basename }}" - state: link - with_fileglob: - - "/var/lib/postgresql/.nix-profile/bin/*" - become: yes - when: stage2_nix - - name: Ensure postgres user has ownership of symlink - file: - path: "/usr/bin/{{ item | basename }}" - owner: postgres - group: postgres - with_fileglob: - - "/var/lib/postgresql/.nix-profile/bin/*" + shell: >- + find /var/lib/postgresql/.nix-profile/bin/ -maxdepth 1 -type f,l -exec chown postgres:postgres "/usr/bin/$(basename {})" \; become: yes when: stage2_nix @@ -160,22 +145,14 @@ # It was decided to leave pljava disabled at https://github.com/supabase/postgres/pull/690 therefore removing this task - name: Create symbolic links from /var/lib/postgresql/.nix-profile/share/postgresql to /usr/lib/postgresql/share/postgresql - file: - src: "{{ item }}" - dest: "/usr/lib/postgresql/share/postgresql/{{ item | basename }}" - state: link - with_fileglob: - - "/var/lib/postgresql/.nix-profile/share/postgresql/*" + shell: >- + find /var/lib/postgresql/.nix-profile/share/postgresql/ -maxdepth 1 -type f,l -exec sh -c 'ln -s "$0" "/usr/lib/postgresql/share/postgresql/$(basename $0)"' {} \; become: yes when: stage2_nix - name: Create symbolic links from /var/lib/postgresql/.nix-profile/share/postgresql/extension to /usr/lib/postgresql/share/postgresql/extension - file: - src: "{{ item }}" - dest: "/usr/lib/postgresql/share/postgresql/extension/{{ item | basename }}" - state: link - with_fileglob: - - "/var/lib/postgresql/.nix-profile/share/postgresql/extension/*" + shell: >- + find /var/lib/postgresql/.nix-profile/share/postgresql/extension/ -maxdepth 1 -type f,l -exec sh -c 'ln -s "$0" "/usr/lib/postgresql/share/postgresql/extension/$(basename $0)"' {} \; become: yes when: stage2_nix @@ -195,22 +172,14 @@ when: stage2_nix - name: Create symbolic links from /var/lib/postgresql/.nix-profile/share/postgresql/timezonesets to /usr/lib/postgresql/share/postgresql/timeszonesets - file: - src: "{{ item }}" - dest: "/usr/lib/postgresql/share/postgresql/timezonesets/{{ item | basename }}" - state: link - with_fileglob: - - "/var/lib/postgresql/.nix-profile/share/postgresql/timezonesets/*" + shell: >- + find /var/lib/postgresql/.nix-profile/share/postgresql/timezonesets/ -maxdepth 1 -type f,l -exec sh -c 'ln -s "$0" "/usr/lib/postgresql/share/postgresql/timezonesets/$(basename $0)"' {} \; become: yes when: stage2_nix - name: Create symbolic links from /var/lib/postgresql/.nix-profile/share/postgresql/tsearch_data to /usr/lib/postgresql/share/postgresql/tsearch_data - file: - src: "{{ item }}" - dest: "/usr/lib/postgresql/share/postgresql/tsearch_data/{{ item | basename }}" - state: link - with_fileglob: - - "/var/lib/postgresql/.nix-profile/share/postgresql/tsearch_data/*" + shell: >- + find /var/lib/postgresql/.nix-profile/share/postgresql/tsearch_data/ -maxdepth 1 -type f,l -exec sh -c 'ln -s "$0" "/usr/lib/postgresql/share/postgresql/tsearch_data/$(basename $0)"' {} \; become: yes when: stage2_nix From ea0abaefd4126789184c4928647b3fe530401d78 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Tue, 26 Nov 2024 05:41:40 +0800 Subject: [PATCH 35/94] revert stage2 changes --- stage2-nix-psql.pkr.hcl | 87 +++++++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 43 deletions(-) diff --git a/stage2-nix-psql.pkr.hcl b/stage2-nix-psql.pkr.hcl index 5a289b7e4..57ddfd9e7 100644 --- a/stage2-nix-psql.pkr.hcl +++ b/stage2-nix-psql.pkr.hcl @@ -55,54 +55,55 @@ packer { } } -source "qemu" "supabase_postgres" { - vm_name = "ubuntu-2004-amd64-iso.qcow2" - iso_url = "https://www.releases.ubuntu.com/20.04/ubuntu-20.04.6-live-server-amd64.iso" - iso_checksum = "sha256:b8f31413336b9393ad5d8ef0282717b2ab19f007df2e9ed5196c13d8f9153c8b" - # vm_name = "ubuntu-2404-amd64.raw" - # iso_url = "https://www.releases.ubuntu.com/24.04/ubuntu-24.04-live-server-amd64.iso" - memory = 20000 - disk_image = false - output_directory = "output_images" - shutdown_command = "echo 'packer' | sudo -S shutdown -P now" - disk_size = "9000M" - format = "qcow2" - accelerator = "kvm" - net_device = "virtio-net" - disk_interface = "virtio" - boot_wait = "10s" - - boot_command = [ - # Make the language selector appear... - " ", - # ...then get rid of it - " ", - - # Go to the other installation options menu and leave it - "", - - # Remove the kernel command-line that already exists - "", - "", - "", - - # Add kernel command-line and start install - "/casper/vmlinuz ", - "initrd=/casper/initrd ", - "autoinstall ", - "ds=nocloud-net;s=http://{{.HTTPIP}}:{{.HTTPPort}}/ ", - "" - ] - http_directory = "http" - ssh_username = "packer" - ssh_password = "packer" - ssh_timeout = "60m" +source "amazon-ebs" "ubuntu" { + ami_name = "${var.ami_name}-${var.postgres-version}" + instance_type = "c6g.4xlarge" + region = "${var.region}" + source_ami_filter { + filters = { + name = "${var.ami_name}-${var.postgres-version}-stage-1" + root-device-type = "ebs" + virtualization-type = "hvm" + } + most_recent = true + owners = ["amazon", "self"] + } + + communicator = "ssh" + ssh_pty = true + ssh_username = "ubuntu" + ssh_timeout = "5m" + + associate_public_ip_address = true + + + ena_support = true + + run_tags = { + creator = "packer" + appType = "postgres" + packerExecutionId = "${var.packer-execution-id}" + } + run_volume_tags = { + creator = "packer" + appType = "postgres" + } + snapshot_tags = { + creator = "packer" + appType = "postgres" + } + tags = { + creator = "packer" + appType = "postgres" + postgresVersion = "${var.postgres-version}" + sourceSha = "${var.git-head-version}" + } } build { name = "nix-packer-ubuntu" sources = [ - "source.qemu.supabase_postgres" + "source.amazon-ebs.ubuntu" ] # Copy ansible playbook From 41a4f14ded64e848f143b9b8be87b49dcfb5ee76 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Tue, 26 Nov 2024 06:21:36 +0800 Subject: [PATCH 36/94] preserve AMI build process --- Makefile | 8 +- ebssurrogate/scripts/qemu-bootstrap-nix.sh | 142 ++++++++++++++ qemu-arm64-nix.pkr.hcl | 205 +++++++++++++++++++++ scripts/90-cleanup-qemu.sh | 64 +++++++ scripts/90-cleanup.sh | 19 -- 5 files changed, 415 insertions(+), 23 deletions(-) create mode 100755 ebssurrogate/scripts/qemu-bootstrap-nix.sh create mode 100644 qemu-arm64-nix.pkr.hcl create mode 100644 scripts/90-cleanup-qemu.sh diff --git a/Makefile b/Makefile index 386388385..37be10d4f 100644 --- a/Makefile +++ b/Makefile @@ -2,11 +2,11 @@ UPSTREAM_NIX_GIT_SHA := $(shell git rev-parse origin/release/15.6) GIT_SHA := $(shell git describe --tags --always --dirty) -init: amazon-arm64-nix.pkr.hcl - packer init amazon-arm64-nix.pkr.hcl +init: qemu-arm64-nix.pkr.hcl + packer init qemu-arm64-nix.pkr.hcl -output-cloudimg/packer-cloudimg: ansible amazon-arm64-nix.pkr.hcl - packer build -var "git_sha=$(UPSTREAM_NIX_GIT_SHA)" amazon-arm64-nix.pkr.hcl +output-cloudimg/packer-cloudimg: ansible qemu-arm64-nix.pkr.hcl + packer build -var "git_sha=$(UPSTREAM_NIX_GIT_SHA)" qemu-arm64-nix.pkr.hcl disk/focal-raw.img: output-cloudimg/packer-cloudimg mkdir -p disk diff --git a/ebssurrogate/scripts/qemu-bootstrap-nix.sh b/ebssurrogate/scripts/qemu-bootstrap-nix.sh new file mode 100755 index 000000000..4e3f6d78f --- /dev/null +++ b/ebssurrogate/scripts/qemu-bootstrap-nix.sh @@ -0,0 +1,142 @@ +#!/usr/bin/env bash +# +# This script creates filesystem and setups up chrooted +# enviroment for further processing. It also runs +# ansible playbook and finally does system cleanup. +# +# Adapted from: https://github.com/jen20/packer-ubuntu-zfs + +set -o errexit +set -o pipefail +set -o xtrace + +if [ $(dpkg --print-architecture) = "amd64" ]; +then + ARCH="amd64"; +else + ARCH="arm64"; +fi + +function waitfor_boot_finished { + export DEBIAN_FRONTEND=noninteractive + + echo "args: ${ARGS}" + # Wait for cloudinit on the surrogate to complete before making progress + while [[ ! -f /var/lib/cloud/instance/boot-finished ]]; do + echo 'Waiting for cloud-init...' + sleep 1 + done +} + +function install_packages { + apt-get update && sudo apt-get install software-properties-common e2fsprogs -y + add-apt-repository --yes --update ppa:ansible/ansible && sudo apt-get install ansible -y + ansible-galaxy collection install community.general +} + +function execute_playbook { + +tee /etc/ansible/ansible.cfg </dev/null +LOCALE_ARCHIVE=/usr/lib/locale/locale-archive +LANG="en_US.UTF-8" +LANGUAGE="en_US.UTF-8" +LC_ALL="en_US.UTF-8" +LC_CTYPE="en_US.UTF-8" +EOF +} + +function setup_locale { +cat << EOF >> /etc/locale.gen +en_US.UTF-8 UTF-8 +EOF + +cat << EOF > /etc/default/locale +LANG="C.UTF-8" +LC_CTYPE="C.UTF-8" +EOF + locale-gen en_US.UTF-8 +} + +waitfor_boot_finished +install_packages +setup_postgesql_env +setup_locale +execute_playbook + +# stage 2 things +function install_nix() { + sudo su -c "curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install --no-confirm \ + --extra-conf \"substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com\" \ + --extra-conf \"trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=\" " -s /bin/bash root + . /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh + +} + +function execute_stage2_playbook { + sudo tee /etc/ansible/ansible.cfg < /root/.bash_history +unset HISTFILE +find /var/log -mtime -1 -type f -exec truncate -s 0 {} \; +rm -rf /var/log/*.gz /var/log/*.[0-9] /var/log/*-???????? +rm -rf /var/lib/cloud/instances/* +rm -f /root/.ssh/authorized_keys /etc/ssh/*key* +touch /etc/ssh/revoked_keys +chmod 600 /etc/ssh/revoked_keys + +cat /dev/null > /var/log/lastlog +cat /dev/null > /var/log/wtmp diff --git a/scripts/90-cleanup.sh b/scripts/90-cleanup.sh index 9313d3df3..44707d812 100644 --- a/scripts/90-cleanup.sh +++ b/scripts/90-cleanup.sh @@ -60,24 +60,5 @@ rm -f /root/.ssh/authorized_keys /etc/ssh/*key* touch /etc/ssh/revoked_keys chmod 600 /etc/ssh/revoked_keys -# Securely erase the unused portion of the filesystem -# GREEN='\033[0;32m' -# NC='\033[0m' -# printf "\n${GREEN}Writing zeros to the remaining disk space to securely -# erase the unused portion of the file system. -# Depending on your disk size this may take several minutes. -# The secure erase will complete successfully when you see:${NC} -# dd: writing to '/zerofile': No space left on device\n -# Beginning secure erase now\n" - -# dd if=/dev/zero of=/zerofile & -# PID=$! -# while [ -d /proc/$PID ] -# do -# printf "." -# sleep 5 -# done -# sync; rm /zerofile; sync - cat /dev/null > /var/log/lastlog cat /dev/null > /var/log/wtmp From c7696d3914289bf8b26a1a21fa7b8274ac82cae1 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Tue, 26 Nov 2024 07:28:44 +0800 Subject: [PATCH 37/94] chore: check ec2ic perms if not in qemu mode --- ansible/files/permission_check.py | 44 +++++++++++++++++----- ansible/playbook.yml | 2 +- ebssurrogate/scripts/qemu-bootstrap-nix.sh | 2 +- 3 files changed, 36 insertions(+), 12 deletions(-) diff --git a/ansible/files/permission_check.py b/ansible/files/permission_check.py index e980e6e66..3f0218904 100644 --- a/ansible/files/permission_check.py +++ b/ansible/files/permission_check.py @@ -1,6 +1,8 @@ import subprocess import json import sys +import argparse + # Expected groups for each user expected_results = { @@ -88,6 +90,9 @@ "messagebus": [ {"groupname":"messagebus","username":"messagebus"} ], + "ec2-instance-connect": [ + {"groupname": "nogroup", "username": "ec2-instance-connect"} + ], "sshd": [ {"groupname":"nogroup","username":"sshd"} ], @@ -142,6 +147,7 @@ ] } + # This program depends on osquery being installed on the system # Function to run osquery def run_osquery(query): @@ -149,6 +155,7 @@ def run_osquery(query): output, error = process.communicate() return output.decode('utf-8') + def parse_json(json_str): try: return json.loads(json_str) @@ -156,6 +163,7 @@ def parse_json(json_str): print("Error decoding JSON:", e) sys.exit(1) + def compare_results(username, query_result): expected_result = expected_results.get(username) if expected_result is None: @@ -170,6 +178,7 @@ def compare_results(username, query_result): print("Got:", query_result) sys.exit(1) + def check_nixbld_users(): query = """ SELECT u.username, g.groupname @@ -188,15 +197,30 @@ def check_nixbld_users(): print("All nixbld users are in the 'nixbld' group.") -# Define usernames for which you want to compare results -usernames = ["postgres", "ubuntu", "root", "daemon", "bin", "sys", "sync", "games","man","lp","mail","news","uucp","proxy","www-data","backup","list","irc","gnats","nobody","systemd-network","systemd-resolve","systemd-timesync","messagebus","sshd","wal-g","pgbouncer","gotrue","envoy","kong","nginx","vector","adminapi","postgrest","tcpdump","systemd-coredump"] -# Iterate over usernames, run the query, and compare results -for username in usernames: - query = f"SELECT u.username, g.groupname FROM users u JOIN user_groups ug ON u.uid = ug.uid JOIN groups g ON ug.gid = g.gid WHERE u.username = '{username}' ORDER BY g.groupname;" - query_result = run_osquery(query) - parsed_result = parse_json(query_result) - compare_results(username, parsed_result) +def main(): + parser = argparse.ArgumentParser( + prog='Supabase Postgres Artifact Permissions Checker', + description='Checks the Postgres Artifact for the appropriate users and group memberships') + parser.add_argument('-q', '--qemu', action='store_true', help='Whether we are checking a QEMU artifact') + args = parser.parse_args() + qemu_artifact = args.qemu or False + + # Define usernames for which you want to compare results + usernames = ["postgres", "ubuntu", "root", "daemon", "bin", "sys", "sync", "games","man","lp","mail","news","uucp","proxy","www-data","backup","list","irc","gnats","nobody","systemd-network","systemd-resolve","systemd-timesync","messagebus","sshd","wal-g","pgbouncer","gotrue","envoy","kong","nginx","vector","adminapi","postgrest","tcpdump","systemd-coredump"] + if not qemu_artifact: + usernames.append("ec2-instance-connect") + + # Iterate over usernames, run the query, and compare results + for username in usernames: + query = f"SELECT u.username, g.groupname FROM users u JOIN user_groups ug ON u.uid = ug.uid JOIN groups g ON ug.gid = g.gid WHERE u.username = '{username}' ORDER BY g.groupname;" + query_result = run_osquery(query) + parsed_result = parse_json(query_result) + compare_results(username, parsed_result) + + # Check if all nixbld users are in the nixbld group + check_nixbld_users() + -# Check if all nixbld users are in the nixbld group -check_nixbld_users() +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 06d30a93d..24e58963a 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -201,7 +201,7 @@ - name: Run osquery permission checks become: yes shell: | - sudo -u ubuntu bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && /usr/bin/python3 /tmp/ansible-playbook/ansible/files/permission_check.py" + sudo -u ubuntu bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && /usr/bin/python3 /tmp/ansible-playbook/ansible/files/permission_check.py {{ '--qemu' if qemu_mode is defined else '' }}" when: stage2_nix - name: Remove osquery diff --git a/ebssurrogate/scripts/qemu-bootstrap-nix.sh b/ebssurrogate/scripts/qemu-bootstrap-nix.sh index 4e3f6d78f..b4691ba3a 100755 --- a/ebssurrogate/scripts/qemu-bootstrap-nix.sh +++ b/ebssurrogate/scripts/qemu-bootstrap-nix.sh @@ -96,7 +96,7 @@ EOF # Run Ansible playbook export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_REMOTE_TEMP=/tmp ansible-playbook ./ansible/playbook.yml \ - --extra-vars '{"nixpkg_mode": false, "stage2_nix": true, "debpkg_mode": false}' \ + --extra-vars '{"nixpkg_mode": false, "stage2_nix": true, "debpkg_mode": false, "qemu_mode": true}' \ --extra-vars "git_commit_sha=${GIT_SHA}" } From 98a87b5bba7fc43fdf4c7b84be6ac108cb547b59 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Tue, 26 Nov 2024 08:22:34 +0800 Subject: [PATCH 38/94] chore: further ami build things --- Dockerfile-kubevirt | 2 +- Makefile | 2 +- amazon-arm64-nix.pkr.hcl | 2 +- ansible/files/permission_check.py | 2 +- ansible/playbook.yml | 14 +- ansible/tasks/setup-postgres.yml | 2 +- ebssurrogate/scripts/qemu-bootstrap-nix.sh | 3 +- .../scripts/surrogate-bootstrap-nix.sh | 342 ++++++++++++++---- scripts/90-cleanup-qemu.sh | 3 +- scripts/90-cleanup.sh | 21 +- scripts/nix-provision.sh | 8 +- 11 files changed, 309 insertions(+), 92 deletions(-) diff --git a/Dockerfile-kubevirt b/Dockerfile-kubevirt index c58e3dbb3..2591ceb6b 100644 --- a/Dockerfile-kubevirt +++ b/Dockerfile-kubevirt @@ -1,2 +1,2 @@ FROM scratch -ADD --chown=107:107 focal.img /disk/focal.qcow2 +ADD --chown=107:107 focal-raw.img /disk/focal.qcow2 diff --git a/Makefile b/Makefile index 37be10d4f..241271e0e 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ disk/focal-raw.img: output-cloudimg/packer-cloudimg sudo qemu-img convert -O raw output-cloudimg/packer-cloudimg disk/focal-raw.img container-disk-image: disk/focal-raw.img - sudo nerdctl build . -t supabase-postgres-test:$(GIT_SHA) --namespace k8s.io -f ./Dockerfile-kubevirt + docker build . -t supabase-postgres-test:$(GIT_SHA) -f ./Dockerfile-kubevirt host-disk: disk/focal-raw.img sudo chown 107 -R disk diff --git a/amazon-arm64-nix.pkr.hcl b/amazon-arm64-nix.pkr.hcl index 118196473..ec427ff9f 100644 --- a/amazon-arm64-nix.pkr.hcl +++ b/amazon-arm64-nix.pkr.hcl @@ -264,7 +264,7 @@ build { ] use_env_var_file = true script = "ebssurrogate/scripts/surrogate-bootstrap-nix.sh" - execute_command = "sudo -S sh -c '. {{.EnvVarFile}} && {{.Path}}'" + execute_command = "sudo -S sh -c '. {{.EnvVarFile}} && cd /tmp/ansible-playbook && {{.Path}}'" start_retry_timeout = "5m" skip_clean = true } diff --git a/ansible/files/permission_check.py b/ansible/files/permission_check.py index 3f0218904..ef22ed575 100644 --- a/ansible/files/permission_check.py +++ b/ansible/files/permission_check.py @@ -223,4 +223,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 24e58963a..5c5f8da3a 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -1,4 +1,4 @@ -- hosts: localhost +- hosts: all become: yes pre_tasks: @@ -142,6 +142,18 @@ import_tasks: tasks/setup-fail2ban.yml when: debpkg_mode or nixpkg_mode + + # Install EC2 instance connect + # Only for AWS images + - name: install EC2 instance connect + become: yes + apt: + pkg: + - ec2-instance-connect + tags: + - aws-only + when: qemu_mode is undefined + # Install this at the end to prevent it from kicking in during the apt process, causing conflicts - name: Install security tools become: yes diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index ea48340db..eeb3654fa 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -241,7 +241,7 @@ shell: | mkdir -p /run/postgresql chown -R postgres:postgres /run/postgresql - when: stage2_nix + when: stage2_nix and qemu_mode - name: Restart Postgres Database without Systemd become: yes diff --git a/ebssurrogate/scripts/qemu-bootstrap-nix.sh b/ebssurrogate/scripts/qemu-bootstrap-nix.sh index b4691ba3a..0fdc3ae2f 100755 --- a/ebssurrogate/scripts/qemu-bootstrap-nix.sh +++ b/ebssurrogate/scripts/qemu-bootstrap-nix.sh @@ -72,6 +72,8 @@ EOF locale-gen en_US.UTF-8 } +sed -i 's/- hosts: all/- hosts: localhost/' ansible/playbook.yml + waitfor_boot_finished install_packages setup_postgesql_env @@ -92,7 +94,6 @@ function execute_stage2_playbook { [defaults] callbacks_enabled = timer, profile_tasks, profile_roles EOF - # sed -i 's/- hosts: all/- hosts: localhost/' /tmp/ansible-playbook/ansible/playbook.yml # Run Ansible playbook export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_REMOTE_TEMP=/tmp ansible-playbook ./ansible/playbook.yml \ diff --git a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh index d8ffc9d24..3b0bb93c8 100755 --- a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh +++ b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh @@ -29,114 +29,300 @@ function waitfor_boot_finished { } function install_packages { - apt-get update && sudo apt-get install software-properties-common e2fsprogs -y + # Setup Ansible on host VM + apt-get update && sudo apt-get install software-properties-common -y add-apt-repository --yes --update ppa:ansible/ansible && sudo apt-get install ansible -y ansible-galaxy collection install community.general + + # Update apt and install required packages + apt-get update + apt-get install -y \ + gdisk \ + e2fsprogs \ + debootstrap \ + nvme-cli } -function execute_playbook { +# Partition the new root EBS volume +function create_partition_table { -tee /etc/ansible/ansible.cfg < sda - # Define the contents of the PostgreSQL environment file - cat </dev/null -LOCALE_ARCHIVE=/usr/lib/locale/locale-archive -LANG="en_US.UTF-8" -LANGUAGE="en_US.UTF-8" -LC_ALL="en_US.UTF-8" -LC_CTYPE="en_US.UTF-8" -EOF + # Create /dev/xvd* device symlink + if [[ ! -z "$mapping" ]] && [[ -b "${blkdev}" ]] && [[ ! -L "${mapping}" ]]; then + ln -s "$blkdev" "$mapping" + + blkdev_mappings["$blkdev"]="$mapping" + fi + done + + create_partition_table + + # NVMe EBS launch device partition mappings (symlinks): /dev/nvme*n*p* to /dev/xvd*[0-9]+ + declare -A partdev_mappings + for blkdev in "${!blkdev_mappings[@]}"; do # /dev/nvme*n* + mapping="${blkdev_mappings[$blkdev]}" + + # Create /dev/xvd*[0-9]+ partition device symlink + for partdev in "${blkdev}"p*; do + partnum=${partdev##*p} + if [[ ! -L "${mapping}${partnum}" ]]; then + ln -s "${blkdev}p${partnum}" "${mapping}${partnum}" + + partdev_mappings["${blkdev}p${partnum}"]="${mapping}${partnum}" + fi + done + done } -function setup_locale { -cat << EOF >> /etc/locale.gen -en_US.UTF-8 UTF-8 -EOF -cat << EOF > /etc/default/locale -LANG="C.UTF-8" -LC_CTYPE="C.UTF-8" +#Download and install latest e2fsprogs for fast_commit feature,if required. +function format_and_mount_rootfs { + mkfs.ext4 -m0.1 /dev/xvdf2 + + mount -o noatime,nodiratime /dev/xvdf2 /mnt + if [ "${ARCH}" = "arm64" ]; then + mkfs.fat -F32 /dev/xvdf1 + mkdir -p /mnt/boot/efi + sleep 2 + mount /dev/xvdf1 /mnt/boot/efi + fi + + mkfs.ext4 /dev/xvdh + + # Explicitly reserving 100MiB worth of blocks for the data volume + RESERVED_DATA_VOLUME_BLOCK_COUNT=$((100 * 1024 * 1024 / 4096)) + tune2fs -r $RESERVED_DATA_VOLUME_BLOCK_COUNT /dev/xvdh + + mkdir -p /mnt/data + mount -o defaults,discard /dev/xvdh /mnt/data +} + +function create_swapfile { + fallocate -l 1G /mnt/swapfile + chmod 600 /mnt/swapfile + mkswap /mnt/swapfile +} + +function format_build_partition { + mkfs.ext4 -O ^has_journal /dev/xvdc +} +function pull_docker { + apt-get install -y docker.io + docker run -itd --name ccachedata "${DOCKER_IMAGE}:${DOCKER_IMAGE_TAG}" sh + docker exec -itd ccachedata mkdir -p /build/ccache +} + +# Create fstab +function create_fstab { + FMT="%-42s %-11s %-5s %-17s %-5s %s" +cat > "/mnt/etc/fstab" << EOF +$(printf "${FMT}" "# DEVICE UUID" "MOUNTPOINT" "TYPE" "OPTIONS" "DUMP" "FSCK") +$(findmnt -no SOURCE /mnt | xargs blkid -o export | awk -v FMT="${FMT}" '/^UUID=/ { printf(FMT, $0, "/", "ext4", "defaults,discard", "0", "1" ) }') +$(findmnt -no SOURCE /mnt/boot/efi | xargs blkid -o export | awk -v FMT="${FMT}" '/^UUID=/ { printf(FMT, $0, "/boot/efi", "vfat", "umask=0077", "0", "1" ) }') +$(findmnt -no SOURCE /mnt/data | xargs blkid -o export | awk -v FMT="${FMT}" '/^UUID=/ { printf(FMT, $0, "/data", "ext4", "defaults,discard", "0", "2" ) }') +$(printf "$FMT" "/swapfile" "none" "swap" "sw" "0" "0") EOF - locale-gen en_US.UTF-8 + unset FMT } -waitfor_boot_finished -install_packages -setup_postgesql_env -setup_locale -execute_playbook +function setup_chroot_environment { + UBUNTU_VERSION=$(lsb_release -cs) # 'focal' for Ubuntu 20.04 + + # Bootstrap Ubuntu into /mnt + debootstrap --arch ${ARCH} --variant=minbase "$UBUNTU_VERSION" /mnt + + # Update ec2-region + REGION=$(curl --silent --fail http://169.254.169.254/latest/meta-data/placement/availability-zone | sed -E 's|[a-z]+$||g') + sed -i "s/REGION/${REGION}/g" /tmp/sources.list + cp /tmp/sources.list /mnt/etc/apt/sources.list -# stage 2 things -function install_nix() { - sudo su -c "curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install --no-confirm \ - --extra-conf \"substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com\" \ - --extra-conf \"trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=\" " -s /bin/bash root - . /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh + if [ "${ARCH}" = "arm64" ]; then + create_fstab + fi + # Create mount points and mount the filesystem + mkdir -p /mnt/{dev,proc,sys} + mount --rbind /dev /mnt/dev + mount --rbind /proc /mnt/proc + mount --rbind /sys /mnt/sys + + # Create build mount point and mount + mkdir -p /mnt/tmp + mount /dev/xvdc /mnt/tmp + chmod 777 /mnt/tmp + + # Copy apparmor profiles + chmod 644 /tmp/apparmor_profiles/* + cp -r /tmp/apparmor_profiles /mnt/tmp/ + + # Copy migrations + cp -r /tmp/migrations /mnt/tmp/ + + # Copy unit tests + cp -r /tmp/unit-tests /mnt/tmp/ + + # Copy the bootstrap script into place and execute inside chroot + cp /tmp/chroot-bootstrap-nix.sh /mnt/tmp/chroot-bootstrap-nix.sh + chroot /mnt /tmp/chroot-bootstrap-nix.sh + rm -f /mnt/tmp/chroot-bootstrap-nix.sh + echo "${POSTGRES_SUPABASE_VERSION}" > /mnt/root/supabase-release + + # Copy the nvme identification script into /sbin inside the chroot + mkdir -p /mnt/sbin + cp /tmp/ebsnvme-id /mnt/sbin/ebsnvme-id + chmod +x /mnt/sbin/ebsnvme-id + + # Copy the udev rules for identifying nvme devices into the chroot + mkdir -p /mnt/etc/udev/rules.d + cp /tmp/70-ec2-nvme-devices.rules \ + /mnt/etc/udev/rules.d/70-ec2-nvme-devices.rules + + #Copy custom cloud-init + rm -f /mnt/etc/cloud/cloud.cfg + cp /tmp/cloud.cfg /mnt/etc/cloud/cloud.cfg + + sleep 2 } -function execute_stage2_playbook { - sudo tee /etc/ansible/ansible.cfg < /var/log/lastlog -cat /dev/null > /var/log/wtmp +# Securely erase the unused portion of the filesystem +GREEN='\033[0;32m' +NC='\033[0m' +printf "\n${GREEN}Writing zeros to the remaining disk space to securely +erase the unused portion of the file system. +Depending on your disk size this may take several minutes. +The secure erase will complete successfully when you see:${NC} + dd: writing to '/zerofile': No space left on device\n +Beginning secure erase now\n" + +dd if=/dev/zero of=/zerofile & + PID=$! + while [ -d /proc/$PID ] + do + printf "." + sleep 5 + done +sync; rm /zerofile; sync +cat /dev/null > /var/log/lastlog; cat /dev/null > /var/log/wtmp diff --git a/scripts/nix-provision.sh b/scripts/nix-provision.sh index 998acdaa2..5ed678d7a 100644 --- a/scripts/nix-provision.sh +++ b/scripts/nix-provision.sh @@ -29,13 +29,13 @@ function execute_stage2_playbook { [defaults] callbacks_enabled = timer, profile_tasks, profile_roles EOF - # sed -i 's/- hosts: all/- hosts: localhost/' ./ansible/playbook.yml + sed -i 's/- hosts: all/- hosts: localhost/' /tmp/ansible-playbook/ansible/playbook.yml # Run Ansible playbook export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_REMOTE_TEMP=/tmp - GIT_SHA=$(git rev-parse HEAD) - ansible-playbook ./ansible/playbook.yml \ + ansible-playbook /tmp/ansible-playbook/ansible/playbook.yml \ --extra-vars '{"nixpkg_mode": false, "stage2_nix": true, "debpkg_mode": false}' \ - --extra-vars "git_commit_sha=${GIT_SHA}" + --extra-vars "git_commit_sha=${GIT_SHA}" \ + $ARGS } function cleanup_packages { From 841505a42442b179cb531a66cbae0e7738fb4593 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Tue, 26 Nov 2024 15:22:01 +0800 Subject: [PATCH 39/94] chore: ship qcow2 image instead of raw --- Dockerfile-kubevirt | 2 +- Makefile | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Dockerfile-kubevirt b/Dockerfile-kubevirt index 2591ceb6b..c6b00b667 100644 --- a/Dockerfile-kubevirt +++ b/Dockerfile-kubevirt @@ -1,2 +1,2 @@ FROM scratch -ADD --chown=107:107 focal-raw.img /disk/focal.qcow2 +ADD --chown=107:107 disk/focal.img /disk/focal.qcow2 diff --git a/Makefile b/Makefile index 241271e0e..2f2db8785 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,10 @@ disk/focal-raw.img: output-cloudimg/packer-cloudimg mkdir -p disk sudo qemu-img convert -O raw output-cloudimg/packer-cloudimg disk/focal-raw.img -container-disk-image: disk/focal-raw.img +disk/focal.img: output-cloudimg/packer-cloudimg + sudo qemu-img convert -O qcow2 output-cloudimg/packer-cloudimg disk/focal.img + +container-disk-image: disk/focal.img docker build . -t supabase-postgres-test:$(GIT_SHA) -f ./Dockerfile-kubevirt host-disk: disk/focal-raw.img From 36510ea81e8a18d63b79e6cc25dfcabc6ca9a5a3 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Wed, 27 Nov 2024 08:37:44 +0800 Subject: [PATCH 40/94] chore: cleanup + start building nix artifacts on qemu branches --- .github/workflows/nix-build.yml | 1 + Dockerfile-kubevirt | 2 +- Makefile | 8 +--- qemu-arm64-nix.pkr.hcl | 73 +-------------------------------- 4 files changed, 6 insertions(+), 78 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 08c316bce..6674bad28 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -5,6 +5,7 @@ on: branches: - develop - release/* + - da/qemu* pull_request: workflow_dispatch: diff --git a/Dockerfile-kubevirt b/Dockerfile-kubevirt index c6b00b667..7cb749aed 100644 --- a/Dockerfile-kubevirt +++ b/Dockerfile-kubevirt @@ -1,2 +1,2 @@ FROM scratch -ADD --chown=107:107 disk/focal.img /disk/focal.qcow2 +ADD --chown=107:107 output-cloudimg/packer-cloudimg /disk/focal.qcow2 diff --git a/Makefile b/Makefile index 2f2db8785..411e94451 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,4 @@ -# TODO (darora): we can get rid of this once we're actually building nix things on this -UPSTREAM_NIX_GIT_SHA := $(shell git rev-parse origin/release/15.6) +UPSTREAM_NIX_GIT_SHA := $(shell git rev-parse HEAD) GIT_SHA := $(shell git describe --tags --always --dirty) init: qemu-arm64-nix.pkr.hcl @@ -12,10 +11,7 @@ disk/focal-raw.img: output-cloudimg/packer-cloudimg mkdir -p disk sudo qemu-img convert -O raw output-cloudimg/packer-cloudimg disk/focal-raw.img -disk/focal.img: output-cloudimg/packer-cloudimg - sudo qemu-img convert -O qcow2 output-cloudimg/packer-cloudimg disk/focal.img - -container-disk-image: disk/focal.img +container-disk-image: output-cloudimg/packer-cloudimg docker build . -t supabase-postgres-test:$(GIT_SHA) -f ./Dockerfile-kubevirt host-disk: disk/focal-raw.img diff --git a/qemu-arm64-nix.pkr.hcl b/qemu-arm64-nix.pkr.hcl index 6ba1120e4..01a0d164c 100644 --- a/qemu-arm64-nix.pkr.hcl +++ b/qemu-arm64-nix.pkr.hcl @@ -1,38 +1,8 @@ -variable "ami" { - type = string - default = "ubuntu/images/hvm-ssd/ubuntu-focal-20.04-arm64-server-*" -} - -variable "profile" { - type = string - default = "${env("AWS_PROFILE")}" -} - -variable "ami_name" { - type = string - default = "supabase-postgres" -} - -variable "ami_regions" { - type = list(string) - default = ["ap-southeast-2"] -} - variable "ansible_arguments" { type = string default = "--skip-tags install-postgrest,install-pgbouncer,install-supabase-internal" } -variable "aws_access_key" { - type = string - default = "" -} - -variable "aws_secret_key" { - type = string - default = "" -} - variable "environment" { type = string default = "prod" @@ -42,36 +12,6 @@ variable "git_sha" { type = string } -# variable "region" { -# type = string -# } - -variable "build-vol" { - type = string - default = "xvdc" -} - -# ccache docker image details -variable "docker_user" { - type = string - default = "" -} - -variable "docker_passwd" { - type = string - default = "" -} - -variable "docker_image" { - type = string - default = "" -} - -variable "docker_image_tag" { - type = string - default = "latest" -} - locals { creator = "packer" } @@ -91,11 +31,6 @@ variable "packer-execution-id" { default = "unknown" } -variable "force-deregister" { - type = bool - default = false -} - packer { required_plugins { amazon = { @@ -132,16 +67,12 @@ source "qemu" "cloudimg" { disk_image = true disk_size = "15G" format = "qcow2" - # TODO (darora): disable backing image for qcow2 headless = true http_directory = "http" iso_checksum = "file:https://cloud-images.ubuntu.com/focal/current/SHA256SUMS" iso_url = "https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-arm64.img" memory = 20000 qemu_binary = "qemu-system-aarch64" - qemu_img_args { - create = ["-F", "qcow2"] - } qemuargs = [ ["-machine", "virt"], ["-cpu", "host"], @@ -158,8 +89,8 @@ source "qemu" "cloudimg" { ssh_timeout = "1h" ssh_username = "ubuntu" ssh_wait_timeout = "1h" - use_backing_file = true - accelerator = "kvm" + use_backing_file = false + accelerator = "kvm" } build { From 03eb5cfcccf385d532d19ffdbe15b0169df3d340 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Wed, 27 Nov 2024 09:42:05 +0800 Subject: [PATCH 41/94] chore: add rudimentary docs on the QEMU artifact --- Makefile | 3 + ebssurrogate/scripts/qemu-bootstrap-nix.sh | 12 +-- qemu_artifact.md | 95 ++++++++++++++++++++++ 3 files changed, 102 insertions(+), 8 deletions(-) create mode 100644 qemu_artifact.md diff --git a/Makefile b/Makefile index 411e94451..e656f3607 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,9 @@ disk/focal-raw.img: output-cloudimg/packer-cloudimg container-disk-image: output-cloudimg/packer-cloudimg docker build . -t supabase-postgres-test:$(GIT_SHA) -f ./Dockerfile-kubevirt +eks-node-container-disk-image: output-cloudimg/packer-cloudimg + sudo nerdctl build . -t supabase-postgres-test:$(GIT_SHA) --namespace k8s.io -f ./Dockerfile-kubevirt + host-disk: disk/focal-raw.img sudo chown 107 -R disk diff --git a/ebssurrogate/scripts/qemu-bootstrap-nix.sh b/ebssurrogate/scripts/qemu-bootstrap-nix.sh index 0fdc3ae2f..6674552c0 100755 --- a/ebssurrogate/scripts/qemu-bootstrap-nix.sh +++ b/ebssurrogate/scripts/qemu-bootstrap-nix.sh @@ -1,10 +1,4 @@ #!/usr/bin/env bash -# -# This script creates filesystem and setups up chrooted -# enviroment for further processing. It also runs -# ansible playbook and finally does system cleanup. -# -# Adapted from: https://github.com/jen20/packer-ubuntu-zfs set -o errexit set -o pipefail @@ -41,9 +35,8 @@ tee /etc/ansible/ansible.cfg < Date: Wed, 27 Nov 2024 10:05:36 +0800 Subject: [PATCH 42/94] chore: upgrade to latest adminapi release --- ansible/vars.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index d26ed2299..821427a17 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -50,7 +50,7 @@ postgres_exporter_release_checksum: arm64: sha256:29ba62d538b92d39952afe12ee2e1f4401250d678ff4b354ff2752f4321c87a0 amd64: sha256:cb89fc5bf4485fb554e0d640d9684fae143a4b2d5fa443009bd29c59f9129e84 -adminapi_release: 0.71.1-2-g1333ba1 +adminapi_release: 0.72.0 adminmgr_release: 0.24.0 # Postgres Extensions From 6a233caa56b6f5475559052df42f691460b388fe Mon Sep 17 00:00:00 2001 From: Angelico Date: Thu, 28 Nov 2024 16:45:14 +0800 Subject: [PATCH 43/94] chore: first pass at adding details to the docs --- qemu_artifact.md | 57 +++++++++++++++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 20 deletions(-) diff --git a/qemu_artifact.md b/qemu_artifact.md index ef5d8a68b..d841a2a3f 100644 --- a/qemu_artifact.md +++ b/qemu_artifact.md @@ -10,38 +10,56 @@ Given the size of the image, the first VM using it on a node might take a while ## Creating a bare-metal instance -We launch an Ubuntu 22 bare-metal instance; we're using the `c6g.metal` instance type in this case, but any ARM instance type is sufficient for our purposes. +We launch an Ubuntu 22 bare-metal instance; we're using the `c6g.metal` instance type in this case, but any ARM instance type is sufficient for our purposes. In the example below the region used is: `ap-south-1`. - aws ec2 create-security-group --group-name "launch-wizard-1" --description "launch-wizard-1 created 2024-11-26T00:32:56.039Z" --vpc-id "vpc-0fbfcc428751ce76b" - aws ec2 authorize-security-group-ingress --group-id "sg-preview-1" --ip-permissions '{"IpProtocol":"tcp","FromPort":22,"ToPort":22,"IpRanges":[{"CidrIp":"0.0.0.0/0"}]}' - aws ec2 run-instances --image-id "ami-0a87daabd88e93b1f" --instance-type "c6g.metal" --key-name "darora-aps1" --block-device-mappings '{"DeviceName":"/dev/sda1","Ebs":{"Encrypted":false,"DeleteOnTermination":true,"Iops":3000,"SnapshotId":"snap-0fe84a34403e3da8b","VolumeSize":200,"VolumeType":"gp3","Throughput":125}}' --network-interfaces '{"AssociatePublicIpAddress":true,"DeviceIndex":0,"Groups":["sg-preview-1"]}' --tag-specifications '{"ResourceType":"instance","Tags":[{"Key":"Name","Value":"darora-pg-image"}]}' --metadata-options '{"HttpEndpoint":"enabled","HttpPutResponseHopLimit":2,"HttpTokens":"required"}' --private-dns-name-options '{"HostnameType":"ip-name","EnableResourceNameDnsARecord":true,"EnableResourceNameDnsAAAARecord":false}' --count "1" +```bash +aws ec2 create-security-group --group-name "launch-wizard-1" --description "launch-wizard-1 created 2024-11-26T00:32:56.039Z" --vpc-id "insert-vpc-id" + +aws ec2 authorize-security-group-ingress --group-id "insert-sg-group" --ip-permissions '{"IpProtocol":"tcp","FromPort":22,"ToPort":22,"IpRanges":[{"CidrIp":"0.0.0.0/0"}]}' + +aws ec2 run-instances \ +--image-id "ami-0a87daabd88e93b1f" \ +--instance-type "c6g.metal" \ +--key-name "insert-key-pair" \ +--block-device-mappings '{"DeviceName":"/dev/sda1","Ebs":{"Encrypted":false,"DeleteOnTermination":true,"Iops":3000,"SnapshotId":"snap-0fe84a34403e3da8b","VolumeSize":200,"VolumeType":"gp3","Throughput":125}}' \ +--network-interfaces '{"AssociatePublicIpAddress":true,"DeviceIndex":0,"Groups":["insert-sg-group"]}' \ +--tag-specifications '{"ResourceType":"instance","Tags":[{"Key":"Name","Value":"qemu-pg-image"}]}' \ +--metadata-options '{"HttpEndpoint":"enabled","HttpPutResponseHopLimit":2,"HttpTokens":"required"}' \ +--private-dns-name-options '{"HostnameType":"ip-name","EnableResourceNameDnsARecord":true,"EnableResourceNameDnsAAAARecord":false}' \ +--count "1" + +``` ## Install deps On the instance, install the dependencies we require for producing QEMU artifacts: - sudo apt-get update - sudo apt-get install -y qemu-system qemu-system-arm qemu-utils qemu-efi-aarch64 libvirt-clients libvirt-daemon libqcow-utils software-properties-common git make libnbd-bin nbdkit fuse2fs cloud-image-utils awscli - sudo usermod -aG kvm ubuntu - curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add - - sudo apt-add-repository "deb [arch=arm64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" - sudo apt-get update && sudo apt-get install packer=1.11.2-1 - sudo apt-get install -y docker.io - +```bash +sudo apt-get update +sudo apt-get install -y qemu-system qemu-system-arm qemu-utils qemu-efi-aarch64 libvirt-clients libvirt-daemon libqcow-utils software-properties-common git make libnbd-bin nbdkit fuse2fs cloud-image-utils awscli + sudo usermod -aG kvm ubuntu +curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add - +sudo apt-add-repository "deb [arch=arm64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" +sudo apt-get update && sudo apt-get install packer=1.11.2-1 +sudo apt-get install -y docker.io +``` Some dev deps that might be useful: - sudo apt-get install -y emacs ripgrep vim-tiny byobu - +```bash +sudo apt-get install -y emacs ripgrep vim-tiny byobu +``` ## Clone repo and build Logout/login first to pick up new group memberships! - git clone https://github.com/supabase/postgres.git - cd postgres - git checkout da/qemu-rebasing # choose appropriate branch here - make init container-disk-image +``` bash +git clone https://github.com/supabase/postgres.git +cd postgres +git checkout da/qemu-rebasing # choose appropriate branch here +make init container-disk-image +``` ### Build process @@ -67,8 +85,7 @@ b. packer build (`qemu-arm64-nix.pkr.hcl`) ## Publish image for later use -Publish the built image to a registry of your choosing, and use the published image with KubeVirt. - +Following `make init container-disk-image`, the generated image should be found in: `/path/to/postgres/output-cloudimg`. For portability the image is also bundled up as a docker image with the name: `supabase-postgres-test` . Publish the built docker image to a registry of your choosing, and use the published image with KubeVirt. # Iterating on the QEMU artifact From 9fc808bfe72f321877f58914fb1e6917772ee02b Mon Sep 17 00:00:00 2001 From: Angelico Date: Fri, 29 Nov 2024 12:00:49 +0800 Subject: [PATCH 44/94] chore: update doc on steps when running within the cluster --- qemu_artifact.md | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/qemu_artifact.md b/qemu_artifact.md index d841a2a3f..092557236 100644 --- a/qemu_artifact.md +++ b/qemu_artifact.md @@ -13,11 +13,13 @@ Given the size of the image, the first VM using it on a node might take a while We launch an Ubuntu 22 bare-metal instance; we're using the `c6g.metal` instance type in this case, but any ARM instance type is sufficient for our purposes. In the example below the region used is: `ap-south-1`. ```bash - +# create a security group for your instance aws ec2 create-security-group --group-name "launch-wizard-1" --description "launch-wizard-1 created 2024-11-26T00:32:56.039Z" --vpc-id "insert-vpc-id" +# using the generated security group ID (insert-sg-group), ensure that it allows for SSH access aws ec2 authorize-security-group-ingress --group-id "insert-sg-group" --ip-permissions '{"IpProtocol":"tcp","FromPort":22,"ToPort":22,"IpRanges":[{"CidrIp":"0.0.0.0/0"}]}' +# spin up your instance with the generated security group ID (insert-sg-group) aws ec2 run-instances \ --image-id "ami-0a87daabd88e93b1f" \ --instance-type "c6g.metal" \ @@ -32,22 +34,22 @@ aws ec2 run-instances \ ``` ## Install deps -On the instance, install the dependencies we require for producing QEMU artifacts: +On the instance, install the dependencies we require for producing QEMU artifacts. Assuming you are the root user: ```bash -sudo apt-get update -sudo apt-get install -y qemu-system qemu-system-arm qemu-utils qemu-efi-aarch64 libvirt-clients libvirt-daemon libqcow-utils software-properties-common git make libnbd-bin nbdkit fuse2fs cloud-image-utils awscli - sudo usermod -aG kvm ubuntu -curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add - -sudo apt-add-repository "deb [arch=arm64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" -sudo apt-get update && sudo apt-get install packer=1.11.2-1 -sudo apt-get install -y docker.io +apt-get update +apt-get install -y qemu-system qemu-system-arm qemu-utils qemu-efi-aarch64 libvirt-clients libvirt-daemon libqcow-utils software-properties-common git make libnbd-bin nbdkit fuse2fs cloud-image-utils awscli +usermod -aG kvm ubuntu +curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - +apt-add-repository "deb [arch=arm64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" +apt-get update && apt-get install packer=1.11.2-1 +apt-get install -y docker.io ``` Some dev deps that might be useful: ```bash -sudo apt-get install -y emacs ripgrep vim-tiny byobu +apt-get install -y emacs ripgrep vim-tiny byobu ``` ## Clone repo and build @@ -91,8 +93,9 @@ Following `make init container-disk-image`, the generated image should be found For a tighter iteration loop on the Postgres artifact, the recommended workflow is to do so on an Ubuntu bare-metal node that's part of the EKS cluster that you're deploying to. -- Use the `host-disk` make target to build the raw image file on disk. (`/path/to/postgres/disk/focal-raw.img`) -- Update the VM spec to use `hostDisk` instead of `containerDisk`. Note that only one VM can use an image at a time, so you can't create multiple VMs backed by the same host disk. +- Instead of running `make init container-disk-image`, use `make init host-disk` instead to build the raw image file on disk. (`/path/to/postgres/disk/focal-raw.img`) +- Update the VM spec to use `hostDisk` instead of `containerDisk` + - Note that only one VM can use an image at a time, so you can't create multiple VMs backed by the same host disk. - Enable the `HostDisk` feature flag for KubeVirt - Deploy the VM to the node @@ -102,11 +105,13 @@ Additionally, to iterate on the container image part of things, you can build th Installing `docker.io` on an EKS node might interfere with the k8s setup of the node. You can instead install `nerdctl` and `buildkit`: - curl -L -O https://github.com/containerd/nerdctl/releases/download/v2.0.0/nerdctl-2.0.0-linux-arm64.tar.gz - tar -xzf nerdctl-2.0.0-linux-arm64.tar.gz - sudo mv ./nerdctl /usr/local/bin/ - curl -O -L https://github.com/moby/buildkit/releases/download/v0.17.1/buildkit-v0.17.1.linux-arm64.tar.gz - tar -xzf buildkit-v0.17.1.linux-arm64.tar.gz - sudo mv bin/* /usr/local/bin/ +```bash +curl -L -O https://github.com/containerd/nerdctl/releases/download/v2.0.0/nerdctl-2.0.0-linux-arm64.tar.gz +tar -xzf nerdctl-2.0.0-linux-arm64.tar.gz +mv ./nerdctl /usr/local/bin/ +curl -O -L https://github.com/moby/buildkit/releases/download/v0.17.1/buildkit-v0.17.1.linux-arm64.tar.gz +tar -xzf buildkit-v0.17.1.linux-arm64.tar.gz +mv bin/* /usr/local/bin/ +``` -You'll need to run buildkit: `sudo buildkitd` +You'll need to run buildkit: `buildkitd` From 78e7b0dae684d46dce92ced19b5ef45a848d6b9d Mon Sep 17 00:00:00 2001 From: Div Arora Date: Fri, 29 Nov 2024 12:16:29 +0800 Subject: [PATCH 45/94] Update qemu_artifact.md --- qemu_artifact.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qemu_artifact.md b/qemu_artifact.md index 092557236..9fa05e909 100644 --- a/qemu_artifact.md +++ b/qemu_artifact.md @@ -23,7 +23,7 @@ aws ec2 authorize-security-group-ingress --group-id "insert-sg-group" --ip-permi aws ec2 run-instances \ --image-id "ami-0a87daabd88e93b1f" \ --instance-type "c6g.metal" \ ---key-name "insert-key-pair" \ +--key-name "INSERT_KEY_PAIR_NAME" \ # create a key pair, or use other mechanism of getting on to the box --block-device-mappings '{"DeviceName":"/dev/sda1","Ebs":{"Encrypted":false,"DeleteOnTermination":true,"Iops":3000,"SnapshotId":"snap-0fe84a34403e3da8b","VolumeSize":200,"VolumeType":"gp3","Throughput":125}}' \ --network-interfaces '{"AssociatePublicIpAddress":true,"DeviceIndex":0,"Groups":["insert-sg-group"]}' \ --tag-specifications '{"ResourceType":"instance","Tags":[{"Key":"Name","Value":"qemu-pg-image"}]}' \ From 46c5c514bd06d90e13e401edaea767b8a81559e3 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Fri, 31 Jan 2025 12:55:37 +0800 Subject: [PATCH 46/94] chore: add image built on alpine with some qemu deps --- Dockerfile-kubernetes | 9 +++++++++ Makefile | 3 +++ 2 files changed, 12 insertions(+) create mode 100644 Dockerfile-kubernetes diff --git a/Dockerfile-kubernetes b/Dockerfile-kubernetes new file mode 100644 index 000000000..74b862e4e --- /dev/null +++ b/Dockerfile-kubernetes @@ -0,0 +1,9 @@ +FROM alpine:3.19 + +ADD output-cloudimg/packer-cloudimg /disk/focal.qcow2 + +RUN apk add --no-cache qemu-system-aarch64 qemu-img openssh nftables cloud-utils-localds aavmf +# dev stuff +# RUN apk add --no-cache iproute2 + +CMD exec /bin/sh -c "trap : TERM INT; sleep 9999999999d & wait" diff --git a/Makefile b/Makefile index e656f3607..1b06d7f0c 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,9 @@ container-disk-image: output-cloudimg/packer-cloudimg eks-node-container-disk-image: output-cloudimg/packer-cloudimg sudo nerdctl build . -t supabase-postgres-test:$(GIT_SHA) --namespace k8s.io -f ./Dockerfile-kubevirt +alpine-image: output-cloudimg/packer-cloudimg + sudo nerdctl build . -t supabase-postgres-test:$(GIT_SHA) -f ./Dockerfile-kubernetes + host-disk: disk/focal-raw.img sudo chown 107 -R disk From c9c3be4168958ccb33b2047d84d10ff586a11f5a Mon Sep 17 00:00:00 2001 From: Div Arora Date: Fri, 31 Jan 2025 13:12:31 +0800 Subject: [PATCH 47/94] chore: upgrade to latest alpine --- Dockerfile-kubernetes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile-kubernetes b/Dockerfile-kubernetes index 74b862e4e..a8231b5ee 100644 --- a/Dockerfile-kubernetes +++ b/Dockerfile-kubernetes @@ -1,4 +1,4 @@ -FROM alpine:3.19 +FROM alpine:3.21 ADD output-cloudimg/packer-cloudimg /disk/focal.qcow2 From a582a0c65f6137ed6049e131da55c072e225ad93 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Sat, 1 Feb 2025 11:02:02 +0800 Subject: [PATCH 48/94] chore: remove temporary changes that are no longer necessary --- Dockerfile-kubevirt | 2 -- Makefile | 11 +---------- ansible/tasks/internal/admin-api.yml | 2 +- 3 files changed, 2 insertions(+), 13 deletions(-) delete mode 100644 Dockerfile-kubevirt diff --git a/Dockerfile-kubevirt b/Dockerfile-kubevirt deleted file mode 100644 index 7cb749aed..000000000 --- a/Dockerfile-kubevirt +++ /dev/null @@ -1,2 +0,0 @@ -FROM scratch -ADD --chown=107:107 output-cloudimg/packer-cloudimg /disk/focal.qcow2 diff --git a/Makefile b/Makefile index 1b06d7f0c..5bef8a430 100644 --- a/Makefile +++ b/Makefile @@ -11,19 +11,10 @@ disk/focal-raw.img: output-cloudimg/packer-cloudimg mkdir -p disk sudo qemu-img convert -O raw output-cloudimg/packer-cloudimg disk/focal-raw.img -container-disk-image: output-cloudimg/packer-cloudimg - docker build . -t supabase-postgres-test:$(GIT_SHA) -f ./Dockerfile-kubevirt - -eks-node-container-disk-image: output-cloudimg/packer-cloudimg - sudo nerdctl build . -t supabase-postgres-test:$(GIT_SHA) --namespace k8s.io -f ./Dockerfile-kubevirt - alpine-image: output-cloudimg/packer-cloudimg sudo nerdctl build . -t supabase-postgres-test:$(GIT_SHA) -f ./Dockerfile-kubernetes -host-disk: disk/focal-raw.img - sudo chown 107 -R disk - clean: rm -rf output-cloudimg -.PHONY: container-disk-image host-disk init clean +.PHONY: alpine-image init clean diff --git a/ansible/tasks/internal/admin-api.yml b/ansible/tasks/internal/admin-api.yml index f23849a58..da93fef62 100644 --- a/ansible/tasks/internal/admin-api.yml +++ b/ansible/tasks/internal/admin-api.yml @@ -37,7 +37,7 @@ - name: Download adminapi archive get_url: - url: "https://supabase-public-artifacts-bucket.s3.us-east-1.amazonaws.com/supabase-admin-api/v3-temp/supabase-admin-api-v{{ adminapi_release }}.tar.gz" + url: "https://supabase-public-artifacts-bucket.s3.amazonaws.com/supabase-admin-api/v{{ adminapi_release }}/supabase-admin-api_{{ adminapi_release }}_linux_{{ arch }}.tar.gz" dest: "/tmp/adminapi.tar.gz" timeout: 90 From 5658e7890c47665711103d75ef21c15cefbc527e Mon Sep 17 00:00:00 2001 From: Div Arora Date: Sat, 1 Feb 2025 17:52:36 +0800 Subject: [PATCH 49/94] chore: upgrade to latest awscli release --- ansible/vars.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 4e3feb751..6b1a1c1bc 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -25,7 +25,7 @@ postgrest_x86_release_checksum: sha1:61c513f91a8931be4062587b9d4a18b42acf5c05 gotrue_release: 2.169.0 gotrue_release_checksum: sha1:1419b94683aac7ddc30355408b8e8b79e61146c4 -aws_cli_release: "2.2.7" +aws_cli_release: "2.23.11" salt_minion_version: 3007 From 67bc2c94a2f2827aa28fd8e98043987365432578 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Sat, 1 Feb 2025 18:10:33 +0800 Subject: [PATCH 50/94] fix: define postgres major version --- ebssurrogate/scripts/qemu-bootstrap-nix.sh | 6 ++++-- qemu-arm64-nix.pkr.hcl | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ebssurrogate/scripts/qemu-bootstrap-nix.sh b/ebssurrogate/scripts/qemu-bootstrap-nix.sh index 6674552c0..d05c46bd1 100755 --- a/ebssurrogate/scripts/qemu-bootstrap-nix.sh +++ b/ebssurrogate/scripts/qemu-bootstrap-nix.sh @@ -36,7 +36,8 @@ callbacks_enabled = timer, profile_tasks, profile_roles EOF # Run Ansible playbook export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_REMOTE_TEMP=/mnt/tmp - ansible-playbook ./ansible/playbook.yml --extra-vars '{"nixpkg_mode": true, "debpkg_mode": false, "stage2_nix": false}' + ansible-playbook ./ansible/playbook.yml --extra-vars '{"nixpkg_mode": true, "debpkg_mode": false, "stage2_nix": false}' \ + --extra-vars "psql_version=psql_${POSTGRES_MAJOR_VERSION}" } function setup_postgesql_env { @@ -94,7 +95,8 @@ EOF export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_REMOTE_TEMP=/tmp ansible-playbook ./ansible/playbook.yml \ --extra-vars '{"nixpkg_mode": false, "stage2_nix": true, "debpkg_mode": false, "qemu_mode": true}' \ - --extra-vars "git_commit_sha=${GIT_SHA}" + --extra-vars "git_commit_sha=${GIT_SHA}" \ + --extra-vars "psql_version=psql_${POSTGRES_MAJOR_VERSION}" } function clean_system { diff --git a/qemu-arm64-nix.pkr.hcl b/qemu-arm64-nix.pkr.hcl index 01a0d164c..d8f5ba38b 100644 --- a/qemu-arm64-nix.pkr.hcl +++ b/qemu-arm64-nix.pkr.hcl @@ -125,7 +125,8 @@ build { provisioner "shell" { environment_vars = [ "POSTGRES_SUPABASE_VERSION=${var.postgres-version}", - "GIT_SHA=${var.git_sha}" + "GIT_SHA=${var.git_sha}", + "POSTGRES_MAJOR_VERSION=15" # TODO (darora): pass this in from vars ] use_env_var_file = true script = "ebssurrogate/scripts/qemu-bootstrap-nix.sh" From 035dc13ab6301e244607795f7a0cfa0d90504caf Mon Sep 17 00:00:00 2001 From: Div Arora Date: Sat, 1 Feb 2025 18:22:06 +0800 Subject: [PATCH 51/94] fix: more new vars --- ebssurrogate/scripts/qemu-bootstrap-nix.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ebssurrogate/scripts/qemu-bootstrap-nix.sh b/ebssurrogate/scripts/qemu-bootstrap-nix.sh index d05c46bd1..59381c780 100755 --- a/ebssurrogate/scripts/qemu-bootstrap-nix.sh +++ b/ebssurrogate/scripts/qemu-bootstrap-nix.sh @@ -37,6 +37,8 @@ EOF # Run Ansible playbook export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_REMOTE_TEMP=/mnt/tmp ansible-playbook ./ansible/playbook.yml --extra-vars '{"nixpkg_mode": true, "debpkg_mode": false, "stage2_nix": false}' \ + --extra-vars "postgresql_version=postgresql_${POSTGRES_MAJOR_VERSION}" \ + --extra-vars "postgresql_major_version=${POSTGRES_MAJOR_VERSION}" \ --extra-vars "psql_version=psql_${POSTGRES_MAJOR_VERSION}" } @@ -96,6 +98,8 @@ EOF ansible-playbook ./ansible/playbook.yml \ --extra-vars '{"nixpkg_mode": false, "stage2_nix": true, "debpkg_mode": false, "qemu_mode": true}' \ --extra-vars "git_commit_sha=${GIT_SHA}" \ + --extra-vars "postgresql_version=postgresql_${POSTGRES_MAJOR_VERSION}" \ + --extra-vars "postgresql_major_version=${POSTGRES_MAJOR_VERSION}" \ --extra-vars "psql_version=psql_${POSTGRES_MAJOR_VERSION}" } From 85c7e82dcd0fdfdae8a994f6d19a723b31b21e28 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Sat, 1 Feb 2025 18:30:28 +0800 Subject: [PATCH 52/94] fix: yet another pg var --- .github/workflows/ami-release-nix.yml | 1 + ebssurrogate/scripts/qemu-bootstrap-nix.sh | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 3242a4e1d..10c419fb1 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -78,6 +78,7 @@ jobs: run: | packer init amazon-arm64-nix.pkr.hcl GIT_SHA=${{github.sha}} + # why is postgresql_major defined here instead of where the _three_ other postgresql_* variables are defined? packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl - name: Build AMI stage 2 diff --git a/ebssurrogate/scripts/qemu-bootstrap-nix.sh b/ebssurrogate/scripts/qemu-bootstrap-nix.sh index 59381c780..ca52c5cd5 100755 --- a/ebssurrogate/scripts/qemu-bootstrap-nix.sh +++ b/ebssurrogate/scripts/qemu-bootstrap-nix.sh @@ -39,6 +39,7 @@ EOF ansible-playbook ./ansible/playbook.yml --extra-vars '{"nixpkg_mode": true, "debpkg_mode": false, "stage2_nix": false}' \ --extra-vars "postgresql_version=postgresql_${POSTGRES_MAJOR_VERSION}" \ --extra-vars "postgresql_major_version=${POSTGRES_MAJOR_VERSION}" \ + --extra-vars "postgresql_major=${POSTGRES_MAJOR_VERSION}" \ --extra-vars "psql_version=psql_${POSTGRES_MAJOR_VERSION}" } @@ -100,6 +101,7 @@ EOF --extra-vars "git_commit_sha=${GIT_SHA}" \ --extra-vars "postgresql_version=postgresql_${POSTGRES_MAJOR_VERSION}" \ --extra-vars "postgresql_major_version=${POSTGRES_MAJOR_VERSION}" \ + --extra-vars "postgresql_major=${POSTGRES_MAJOR_VERSION}" \ --extra-vars "psql_version=psql_${POSTGRES_MAJOR_VERSION}" } From bcf0e0e9cae9ce600cefad9217397fd0087a58ad Mon Sep 17 00:00:00 2001 From: Div Arora Date: Sun, 2 Feb 2025 09:48:28 +0800 Subject: [PATCH 53/94] chore: initial version of GHA to build qemu artifact --- .github/workflows/qemu-image-build.yml | 173 +++++++++++++++++++++++++ qemu-arm64-nix.pkr.hcl | 2 +- 2 files changed, 174 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/qemu-image-build.yml diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml new file mode 100644 index 000000000..f57513ffa --- /dev/null +++ b/.github/workflows/qemu-image-build.yml @@ -0,0 +1,173 @@ +name: Build QEMU image + +on: + push: + branches: + - develop + - release/* + - da/merging-qemu + paths: + - '.github/workflows/qemu-image-build.yml' + - 'qemu-arm64-nix.pkr.hcl' + - 'common-nix.vars.pkr.hcl' + - 'ansible/vars.yml' + workflow_dispatch: + +jobs: + prepare: + runs-on: ubuntu-latest + outputs: + postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }} + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + + - uses: DeterminateSystems/nix-installer-action@main + + - name: Set PostgreSQL versions - only builds pg15 atm + id: set-versions + run: | + VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[0]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]') + echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT + + build: + needs: prepare + strategy: + matrix: + postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} + # include: + # # TODO (darora): change to use non-self-hosted runners, since they now allow nested virtualization + # - runner: arm-runner + # arch: arm64 + # ubuntu_release: focal + # ubuntu_version: 20.04 + # mcpu: neoverse-n1 + runs-on: + group: pg-arm-large + timeout-minutes: 150 + permissions: + contents: write + packages: write + id-token: write + + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + + - uses: DeterminateSystems/nix-installer-action@main + + # TODO (darora): might need to temporarily disable this + - name: Run checks if triggered manually + if: ${{ github.event_name == 'workflow_dispatch' }} + run: | + SUFFIX=$(sudo nix run nixpkgs#yq -- ".postgres_release[\"postgres${{ matrix.postgres_version }}\"]" ansible/vars.yml | sed -E 's/[0-9\.]+(.*)$/\1/') + if [[ -z $SUFFIX ]] ; then + echo "Version must include non-numeric characters if built manually." + exit 1 + fi + + - name: Set PostgreSQL version environment variable + run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV + + - name: Generate common-nix.vars.pkr.hcl + run: | + PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes + echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl + echo 'postgres_major_version = "'$POSTGRES_MAJOR_VERSION'"' >> common-nix.vars.pkr.hcl + echo 'postgresql_major = "'$POSTGRES_MAJOR_VERSION'"' >> common-nix.vars.pkr.hcl + # Ensure there's a newline at the end of the file + echo "" >> common-nix.vars.pkr.hcl + + - name: Get binaries + run: | + hash + + - name: Build QEMU artifact + run: | + make init + GIT_SHA=${{github.sha}} + # TODO (darora): possibly need to install qemu deps here + packer build -var "git_sha=$(UPSTREAM_NIX_GIT_SHA)" -var-file="common-nix.vars.pkr.hcl" qemu-arm64-nix.pkr.hcl + # packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl + + # TODO (darora): publishing things is temporarily disabled + # - name: Grab release version + # id: process_release_version + # run: | + # VERSION=$(cat common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g') + # echo "version=$VERSION" >> $GITHUB_OUTPUT + + # - name: Create nix flake revision tarball + # run: | + # GIT_SHA=${{github.sha}} + # MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }} + + # mkdir -p "/tmp/pg_upgrade_bin/${MAJOR_VERSION}" + # echo "$GIT_SHA" >> "/tmp/pg_upgrade_bin/${MAJOR_VERSION}/nix_flake_version" + # tar -czf "/tmp/pg_binaries.tar.gz" -C "/tmp/pg_upgrade_bin" . + + # - name: configure aws credentials - staging + # uses: aws-actions/configure-aws-credentials@v4 + # with: + # role-to-assume: ${{ secrets.DEV_AWS_ROLE }} + # aws-region: "us-east-1" + + # - name: Upload software manifest to s3 staging + # run: | + # cd ansible + # ansible-playbook -i localhost \ + # -e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \ + # -e "internal_artifacts_bucket=${{ secrets.ARTIFACTS_BUCKET }}" \ + # -e "postgres_major_version=${{ env.POSTGRES_MAJOR_VERSION }}" \ + # manifest-playbook.yml + + # - name: Upload nix flake revision to s3 staging + # run: | + # aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz + + # - name: configure aws credentials - prod + # uses: aws-actions/configure-aws-credentials@v4 + # with: + # role-to-assume: ${{ secrets.PROD_AWS_ROLE }} + # aws-region: "us-east-1" + + # - name: Upload software manifest to s3 prod + # run: | + # cd ansible + # ansible-playbook -i localhost \ + # -e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \ + # -e "internal_artifacts_bucket=${{ secrets.PROD_ARTIFACTS_BUCKET }}" \ + # -e "postgres_major_version=${{ env.POSTGRES_MAJOR_VERSION }}" \ + # manifest-playbook.yml + + # - name: Upload nix flake revision to s3 prod + # run: | + # aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz + + # - name: Create release + # uses: softprops/action-gh-release@v1 + # with: + # name: ${{ steps.process_release_version.outputs.version }} + # tag_name: ${{ steps.process_release_version.outputs.version }} + # target_commitish: ${{github.sha}} + + # - name: Slack Notification on Failure + # if: ${{ failure() }} + # uses: rtCamp/action-slack-notify@v2 + # env: + # SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} + # SLACK_USERNAME: 'gha-failures-notifier' + # SLACK_COLOR: 'danger' + # SLACK_MESSAGE: 'Building Postgres AMI failed' + # SLACK_FOOTER: '' + + - name: Cleanup resources after build + if: ${{ always() }} + run: | + aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids + + - name: Cleanup resources on build cancellation + if: ${{ cancelled() }} + run: | + aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids diff --git a/qemu-arm64-nix.pkr.hcl b/qemu-arm64-nix.pkr.hcl index d8f5ba38b..fded0b696 100644 --- a/qemu-arm64-nix.pkr.hcl +++ b/qemu-arm64-nix.pkr.hcl @@ -124,9 +124,9 @@ build { provisioner "shell" { environment_vars = [ + "POSTGRES_MAJOR_VERSION=${var.postgres_major_version}" "POSTGRES_SUPABASE_VERSION=${var.postgres-version}", "GIT_SHA=${var.git_sha}", - "POSTGRES_MAJOR_VERSION=15" # TODO (darora): pass this in from vars ] use_env_var_file = true script = "ebssurrogate/scripts/qemu-bootstrap-nix.sh" From 277f761c584aa775dc78034a85a38df75d072338 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Sun, 2 Feb 2025 10:03:44 +0800 Subject: [PATCH 54/94] . --- .github/workflows/qemu-image-build.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index f57513ffa..378cd1bcf 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -21,7 +21,7 @@ jobs: steps: - name: Checkout Repo uses: actions/checkout@v3 - + - uses: DeterminateSystems/nix-installer-action@main - name: Set PostgreSQL versions - only builds pg15 atm @@ -54,6 +54,11 @@ jobs: - name: Checkout Repo uses: actions/checkout@v3 + - name: Get binaries + run: | + hash + echo $PATH + - uses: DeterminateSystems/nix-installer-action@main # TODO (darora): might need to temporarily disable this @@ -66,11 +71,17 @@ jobs: exit 1 fi + - name: Get binaries + run: | + hash + echo $PATH + - name: Set PostgreSQL version environment variable run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV - name: Generate common-nix.vars.pkr.hcl run: | + . /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl @@ -79,10 +90,6 @@ jobs: # Ensure there's a newline at the end of the file echo "" >> common-nix.vars.pkr.hcl - - name: Get binaries - run: | - hash - - name: Build QEMU artifact run: | make init From aa07a3852ccf9775aa8d14819b9d54ecb8b611cc Mon Sep 17 00:00:00 2001 From: Div Arora Date: Sun, 2 Feb 2025 10:07:53 +0800 Subject: [PATCH 55/94] . --- .github/workflows/qemu-image-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index 378cd1bcf..6c4d3bba2 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -82,7 +82,7 @@ jobs: - name: Generate common-nix.vars.pkr.hcl run: | . /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh - PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl echo 'postgres_major_version = "'$POSTGRES_MAJOR_VERSION'"' >> common-nix.vars.pkr.hcl From a65a8c5dd3cf29aa3675419fc0effd22dac44d3d Mon Sep 17 00:00:00 2001 From: Div Arora Date: Sun, 2 Feb 2025 10:11:48 +0800 Subject: [PATCH 56/94] . --- qemu-arm64-nix.pkr.hcl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qemu-arm64-nix.pkr.hcl b/qemu-arm64-nix.pkr.hcl index fded0b696..277b3d886 100644 --- a/qemu-arm64-nix.pkr.hcl +++ b/qemu-arm64-nix.pkr.hcl @@ -124,9 +124,9 @@ build { provisioner "shell" { environment_vars = [ - "POSTGRES_MAJOR_VERSION=${var.postgres_major_version}" + "POSTGRES_MAJOR_VERSION=${var.postgres_major_version}", "POSTGRES_SUPABASE_VERSION=${var.postgres-version}", - "GIT_SHA=${var.git_sha}", + "GIT_SHA=${var.git_sha}" ] use_env_var_file = true script = "ebssurrogate/scripts/qemu-bootstrap-nix.sh" From 4823c8a310d2492c672fe30e41bf53bc7b06f269 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Sun, 2 Feb 2025 10:41:52 +0800 Subject: [PATCH 57/94] chore: install system dependencies --- .github/workflows/qemu-image-build.yml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index 6c4d3bba2..fc24838fd 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -35,13 +35,6 @@ jobs: strategy: matrix: postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} - # include: - # # TODO (darora): change to use non-self-hosted runners, since they now allow nested virtualization - # - runner: arm-runner - # arch: arm64 - # ubuntu_release: focal - # ubuntu_version: 20.04 - # mcpu: neoverse-n1 runs-on: group: pg-arm-large timeout-minutes: 150 @@ -71,10 +64,9 @@ jobs: exit 1 fi - - name: Get binaries + - name: Install dependencies run: | - hash - echo $PATH + sudo apt-get install -y qemu-efi-aarch64 cloud-image-utils qemu-system-arm - name: Set PostgreSQL version environment variable run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV From 6ad5807cdf385cc2f669faa7066d40997030de46 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Sun, 2 Feb 2025 11:19:16 +0800 Subject: [PATCH 58/94] chore: pass in pg version var correctly --- .github/workflows/qemu-image-build.yml | 4 ++-- qemu-arm64-nix.pkr.hcl | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index fc24838fd..bf17bddc8 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -77,8 +77,8 @@ jobs: PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl - echo 'postgres_major_version = "'$POSTGRES_MAJOR_VERSION'"' >> common-nix.vars.pkr.hcl - echo 'postgresql_major = "'$POSTGRES_MAJOR_VERSION'"' >> common-nix.vars.pkr.hcl + echo 'postgres-major-version = "'$POSTGRES_MAJOR_VERSION'"' >> common-nix.vars.pkr.hcl + # echo 'postgresql_major = "'$POSTGRES_MAJOR_VERSION'"' >> common-nix.vars.pkr.hcl # Ensure there's a newline at the end of the file echo "" >> common-nix.vars.pkr.hcl diff --git a/qemu-arm64-nix.pkr.hcl b/qemu-arm64-nix.pkr.hcl index 277b3d886..3b1b212be 100644 --- a/qemu-arm64-nix.pkr.hcl +++ b/qemu-arm64-nix.pkr.hcl @@ -21,6 +21,11 @@ variable "postgres-version" { default = "" } +variable "postgres-major-version" { + type = string + default = "" +} + variable "git-head-version" { type = string default = "unknown" @@ -124,7 +129,7 @@ build { provisioner "shell" { environment_vars = [ - "POSTGRES_MAJOR_VERSION=${var.postgres_major_version}", + "POSTGRES_MAJOR_VERSION=${var.postgres-major-version}", "POSTGRES_SUPABASE_VERSION=${var.postgres-version}", "GIT_SHA=${var.git_sha}" ] From 09fecce6e0cbe4f3baa8bcb226e495bc6bb400f0 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Sun, 2 Feb 2025 11:29:48 +0800 Subject: [PATCH 59/94] enable logs --- .github/workflows/qemu-image-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index bf17bddc8..a3576f36c 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -87,6 +87,7 @@ jobs: make init GIT_SHA=${{github.sha}} # TODO (darora): possibly need to install qemu deps here + export PACKER_LOG=1 packer build -var "git_sha=$(UPSTREAM_NIX_GIT_SHA)" -var-file="common-nix.vars.pkr.hcl" qemu-arm64-nix.pkr.hcl # packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl From b9fd20baf60119048f241d3f1e67634ed5a07f14 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Sun, 2 Feb 2025 11:41:07 +0800 Subject: [PATCH 60/94] . --- .github/workflows/qemu-image-build.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index a3576f36c..bf2a1a0b6 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -66,7 +66,14 @@ jobs: - name: Install dependencies run: | - sudo apt-get install -y qemu-efi-aarch64 cloud-image-utils qemu-system-arm + sudo apt-get install -y qemu-efi-aarch64 cloud-image-utils qemu-system-arm qemu-utils + sudo modprobe kvm + + - name: Enable KVM group perms + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm - name: Set PostgreSQL version environment variable run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV From bdbd500af0e8f1169fa456207a0462c4435dfca6 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Sun, 2 Feb 2025 11:50:38 +0800 Subject: [PATCH 61/94] . --- .github/workflows/qemu-image-build.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index bf2a1a0b6..65a305cfa 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -67,13 +67,6 @@ jobs: - name: Install dependencies run: | sudo apt-get install -y qemu-efi-aarch64 cloud-image-utils qemu-system-arm qemu-utils - sudo modprobe kvm - - - name: Enable KVM group perms - run: | - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm - name: Set PostgreSQL version environment variable run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV From 2f0f1bea8226a295bcf4fab4fd8b093e6b8d1aef Mon Sep 17 00:00:00 2001 From: Div Arora Date: Sun, 2 Feb 2025 13:35:04 +0800 Subject: [PATCH 62/94] chore: temporarily disable KVM to test sw emulation performance --- qemu-arm64-nix.pkr.hcl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qemu-arm64-nix.pkr.hcl b/qemu-arm64-nix.pkr.hcl index 3b1b212be..f0fad533a 100644 --- a/qemu-arm64-nix.pkr.hcl +++ b/qemu-arm64-nix.pkr.hcl @@ -85,8 +85,8 @@ source "qemu" "cloudimg" { ["-drive", "if=pflash,format=raw,id=ovmf_code,readonly=on,file=/usr/share/AAVMF/AAVMF_CODE.fd"], ["-drive", "if=pflash,format=raw,id=ovmf_vars,file=AAVMF_VARS.fd"], ["-drive", "file=output-cloudimg/packer-cloudimg,format=qcow2"], - ["-drive", "file=seeds-cloudimg.iso,format=raw"], - ["--enable-kvm"] + ["-drive", "file=seeds-cloudimg.iso,format=raw"] + # ["--enable-kvm"] ] shutdown_command = "sudo -S shutdown -P now" ssh_handshake_attempts = 500 @@ -95,7 +95,7 @@ source "qemu" "cloudimg" { ssh_username = "ubuntu" ssh_wait_timeout = "1h" use_backing_file = false - accelerator = "kvm" + # accelerator = "kvm" } build { From cb9ffbd249e6981a977768896b5e227581346e86 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Sun, 2 Feb 2025 13:44:16 +0800 Subject: [PATCH 63/94] chore: specify number of CPU cores --- qemu-arm64-nix.pkr.hcl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qemu-arm64-nix.pkr.hcl b/qemu-arm64-nix.pkr.hcl index f0fad533a..87c5f544f 100644 --- a/qemu-arm64-nix.pkr.hcl +++ b/qemu-arm64-nix.pkr.hcl @@ -68,7 +68,7 @@ build { source "qemu" "cloudimg" { boot_wait = "2s" - cpus = 12 + cpus = 8 disk_image = true disk_size = "15G" format = "qcow2" @@ -76,11 +76,11 @@ source "qemu" "cloudimg" { http_directory = "http" iso_checksum = "file:https://cloud-images.ubuntu.com/focal/current/SHA256SUMS" iso_url = "https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-arm64.img" - memory = 20000 + memory = 40000 qemu_binary = "qemu-system-aarch64" qemuargs = [ ["-machine", "virt"], - ["-cpu", "host"], + # ["-cpu", "host"], ["-device", "virtio-gpu-pci"], ["-drive", "if=pflash,format=raw,id=ovmf_code,readonly=on,file=/usr/share/AAVMF/AAVMF_CODE.fd"], ["-drive", "if=pflash,format=raw,id=ovmf_vars,file=AAVMF_VARS.fd"], From 2965425fb0b7ae334d8d9c93687bf67547a6b767 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Tue, 4 Feb 2025 10:45:42 +0800 Subject: [PATCH 64/94] trigger build From e1b27e4a344deb6e43706ff0d71579d16fcedfc1 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Tue, 4 Feb 2025 10:49:10 +0800 Subject: [PATCH 65/94] Revert "chore: temporarily disable KVM to test sw emulation performance" This reverts commit 2f0f1bea8226a295bcf4fab4fd8b093e6b8d1aef. --- qemu-arm64-nix.pkr.hcl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qemu-arm64-nix.pkr.hcl b/qemu-arm64-nix.pkr.hcl index 87c5f544f..2dd549bb2 100644 --- a/qemu-arm64-nix.pkr.hcl +++ b/qemu-arm64-nix.pkr.hcl @@ -85,8 +85,8 @@ source "qemu" "cloudimg" { ["-drive", "if=pflash,format=raw,id=ovmf_code,readonly=on,file=/usr/share/AAVMF/AAVMF_CODE.fd"], ["-drive", "if=pflash,format=raw,id=ovmf_vars,file=AAVMF_VARS.fd"], ["-drive", "file=output-cloudimg/packer-cloudimg,format=qcow2"], - ["-drive", "file=seeds-cloudimg.iso,format=raw"] - # ["--enable-kvm"] + ["-drive", "file=seeds-cloudimg.iso,format=raw"], + ["--enable-kvm"] ] shutdown_command = "sudo -S shutdown -P now" ssh_handshake_attempts = 500 @@ -95,7 +95,7 @@ source "qemu" "cloudimg" { ssh_username = "ubuntu" ssh_wait_timeout = "1h" use_backing_file = false - # accelerator = "kvm" + accelerator = "kvm" } build { From 0e96fe8e991bdda8e1ec2cd913bfbd1262d850bd Mon Sep 17 00:00:00 2001 From: Div Arora Date: Tue, 4 Feb 2025 10:50:57 +0800 Subject: [PATCH 66/94] chore: use native runners --- .github/workflows/qemu-image-build.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index 65a305cfa..a214db507 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -35,8 +35,7 @@ jobs: strategy: matrix: postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} - runs-on: - group: pg-arm-large + runs-on: arm-native-runner timeout-minutes: 150 permissions: contents: write From 752832507cd29cad0b5e82e205477ee99c189db4 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Tue, 4 Feb 2025 10:54:43 +0800 Subject: [PATCH 67/94] update apt sources --- .github/workflows/qemu-image-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index a214db507..75683cda4 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -65,6 +65,7 @@ jobs: - name: Install dependencies run: | + sudo apt-get update sudo apt-get install -y qemu-efi-aarch64 cloud-image-utils qemu-system-arm qemu-utils - name: Set PostgreSQL version environment variable From 30347043765cbf04e112b8b6f68212ceef762f39 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Tue, 4 Feb 2025 10:57:15 +0800 Subject: [PATCH 68/94] . --- .github/workflows/qemu-image-build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index 75683cda4..4a4d61f50 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -73,7 +73,6 @@ jobs: - name: Generate common-nix.vars.pkr.hcl run: | - . /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl From 66c7850f1384501322918c90c8c8fa7b61c3829b Mon Sep 17 00:00:00 2001 From: Div Arora Date: Tue, 4 Feb 2025 13:36:14 +0800 Subject: [PATCH 69/94] attempt to enable kvm --- .github/workflows/qemu-image-build.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index 4a4d61f50..a2d01c3e2 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -63,6 +63,13 @@ jobs: exit 1 fi + - name: enable KVM support + run: | + sudo mkdir -p /etc/udev/rules.d/ + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-math=kvm + - name: Install dependencies run: | sudo apt-get update From 3a37d03a796cb163779beb8d5516804888e722c9 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Tue, 4 Feb 2025 14:24:23 +0800 Subject: [PATCH 70/94] . --- .github/workflows/qemu-image-build.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index a2d01c3e2..6ae568a69 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -65,10 +65,8 @@ jobs: - name: enable KVM support run: | - sudo mkdir -p /etc/udev/rules.d/ - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --name-math=kvm + sudo ls /dev/kvm + sudo chmod 666 /dev/kvm - name: Install dependencies run: | From f9643937bbe0cea54a569a2eb1ed1ac741fee875 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Tue, 4 Feb 2025 14:26:44 +0800 Subject: [PATCH 71/94] chore; REVERT this later --- .github/workflows/qemu-image-build.yml | 34 -------------------------- 1 file changed, 34 deletions(-) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index 6ae568a69..564542cfa 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -14,27 +14,8 @@ on: workflow_dispatch: jobs: - prepare: - runs-on: ubuntu-latest - outputs: - postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }} - steps: - - name: Checkout Repo - uses: actions/checkout@v3 - - - uses: DeterminateSystems/nix-installer-action@main - - - name: Set PostgreSQL versions - only builds pg15 atm - id: set-versions - run: | - VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[0]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]') - echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT build: - needs: prepare - strategy: - matrix: - postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} runs-on: arm-native-runner timeout-minutes: 150 permissions: @@ -46,23 +27,8 @@ jobs: - name: Checkout Repo uses: actions/checkout@v3 - - name: Get binaries - run: | - hash - echo $PATH - - uses: DeterminateSystems/nix-installer-action@main - # TODO (darora): might need to temporarily disable this - - name: Run checks if triggered manually - if: ${{ github.event_name == 'workflow_dispatch' }} - run: | - SUFFIX=$(sudo nix run nixpkgs#yq -- ".postgres_release[\"postgres${{ matrix.postgres_version }}\"]" ansible/vars.yml | sed -E 's/[0-9\.]+(.*)$/\1/') - if [[ -z $SUFFIX ]] ; then - echo "Version must include non-numeric characters if built manually." - exit 1 - fi - - name: enable KVM support run: | sudo ls /dev/kvm From 23fbb98030f18addd3e3e255b2b98723e6836b40 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Tue, 4 Feb 2025 14:29:14 +0800 Subject: [PATCH 72/94] . --- .github/workflows/qemu-image-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index 564542cfa..c74187bf6 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -44,7 +44,7 @@ jobs: - name: Generate common-nix.vars.pkr.hcl run: | - PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl echo 'postgres-major-version = "'$POSTGRES_MAJOR_VERSION'"' >> common-nix.vars.pkr.hcl From 2f36aab7fe253cafc4e33dd07e290b792db74b79 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Tue, 4 Feb 2025 14:29:56 +0800 Subject: [PATCH 73/94] Revert "chore; REVERT this later" This reverts commit f9643937bbe0cea54a569a2eb1ed1ac741fee875. --- .github/workflows/qemu-image-build.yml | 34 ++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index c74187bf6..4f0678070 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -14,8 +14,27 @@ on: workflow_dispatch: jobs: + prepare: + runs-on: ubuntu-latest + outputs: + postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }} + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + + - uses: DeterminateSystems/nix-installer-action@main + + - name: Set PostgreSQL versions - only builds pg15 atm + id: set-versions + run: | + VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[0]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]') + echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT build: + needs: prepare + strategy: + matrix: + postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} runs-on: arm-native-runner timeout-minutes: 150 permissions: @@ -27,8 +46,23 @@ jobs: - name: Checkout Repo uses: actions/checkout@v3 + - name: Get binaries + run: | + hash + echo $PATH + - uses: DeterminateSystems/nix-installer-action@main + # TODO (darora): might need to temporarily disable this + - name: Run checks if triggered manually + if: ${{ github.event_name == 'workflow_dispatch' }} + run: | + SUFFIX=$(sudo nix run nixpkgs#yq -- ".postgres_release[\"postgres${{ matrix.postgres_version }}\"]" ansible/vars.yml | sed -E 's/[0-9\.]+(.*)$/\1/') + if [[ -z $SUFFIX ]] ; then + echo "Version must include non-numeric characters if built manually." + exit 1 + fi + - name: enable KVM support run: | sudo ls /dev/kvm From 8d0ef7a57d6dfd9f5c674c062340260bef4fc80f Mon Sep 17 00:00:00 2001 From: Div Arora Date: Tue, 4 Feb 2025 14:30:17 +0800 Subject: [PATCH 74/94] chore: remove path output --- .github/workflows/qemu-image-build.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index 4f0678070..a9d6294ca 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -46,11 +46,6 @@ jobs: - name: Checkout Repo uses: actions/checkout@v3 - - name: Get binaries - run: | - hash - echo $PATH - - uses: DeterminateSystems/nix-installer-action@main # TODO (darora): might need to temporarily disable this From a24e13f6e81d71f32b3a4e1c5dd3e62bfdca64ef Mon Sep 17 00:00:00 2001 From: Div Arora Date: Tue, 4 Feb 2025 14:33:04 +0800 Subject: [PATCH 75/94] . --- .github/workflows/qemu-image-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index a9d6294ca..790d790ee 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -60,7 +60,7 @@ jobs: - name: enable KVM support run: | - sudo ls /dev/kvm + sudo chown runner /dev/kvm sudo chmod 666 /dev/kvm - name: Install dependencies From cf3b0ecafefe89473b9e7ae0c4482dfa8aa60c97 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Tue, 4 Feb 2025 15:02:53 +0800 Subject: [PATCH 76/94] chore: install packer since we're experimenting with a base image --- .github/workflows/qemu-image-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index 790d790ee..3db3c7013 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -67,6 +67,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y qemu-efi-aarch64 cloud-image-utils qemu-system-arm qemu-utils + cd && curl -L "https://releases.hashicorp.com/packer/1.7.8/packer_1.7.8_linux_$(dpkg --print-architecture).zip" -o packer.zip && unzip packer.zip && rm -f packer.zip && sudo mv packer /usr/local/bin/ - name: Set PostgreSQL version environment variable run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV @@ -85,7 +86,6 @@ jobs: run: | make init GIT_SHA=${{github.sha}} - # TODO (darora): possibly need to install qemu deps here export PACKER_LOG=1 packer build -var "git_sha=$(UPSTREAM_NIX_GIT_SHA)" -var-file="common-nix.vars.pkr.hcl" qemu-arm64-nix.pkr.hcl # packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl From 9ad843211bc27c27b443bece08a3527f39aa8f16 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Tue, 4 Feb 2025 15:09:10 +0800 Subject: [PATCH 77/94] reenable host cpu --- qemu-arm64-nix.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qemu-arm64-nix.pkr.hcl b/qemu-arm64-nix.pkr.hcl index 2dd549bb2..4bb56868a 100644 --- a/qemu-arm64-nix.pkr.hcl +++ b/qemu-arm64-nix.pkr.hcl @@ -80,7 +80,7 @@ source "qemu" "cloudimg" { qemu_binary = "qemu-system-aarch64" qemuargs = [ ["-machine", "virt"], - # ["-cpu", "host"], + ["-cpu", "host"], ["-device", "virtio-gpu-pci"], ["-drive", "if=pflash,format=raw,id=ovmf_code,readonly=on,file=/usr/share/AAVMF/AAVMF_CODE.fd"], ["-drive", "if=pflash,format=raw,id=ovmf_vars,file=AAVMF_VARS.fd"], From dc080fca525cf4ea9758f5860a9005e3df59c27b Mon Sep 17 00:00:00 2001 From: Div Arora Date: Tue, 4 Feb 2025 15:14:33 +0800 Subject: [PATCH 78/94] . --- qemu-arm64-nix.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qemu-arm64-nix.pkr.hcl b/qemu-arm64-nix.pkr.hcl index 4bb56868a..a9843d1ad 100644 --- a/qemu-arm64-nix.pkr.hcl +++ b/qemu-arm64-nix.pkr.hcl @@ -79,7 +79,7 @@ source "qemu" "cloudimg" { memory = 40000 qemu_binary = "qemu-system-aarch64" qemuargs = [ - ["-machine", "virt"], + ["-machine", "virt,gic-version=3"], ["-cpu", "host"], ["-device", "virtio-gpu-pci"], ["-drive", "if=pflash,format=raw,id=ovmf_code,readonly=on,file=/usr/share/AAVMF/AAVMF_CODE.fd"], From add5d2b058703f8e6da2f73f1e62c849d3c96be0 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Tue, 4 Feb 2025 16:15:08 +0800 Subject: [PATCH 79/94] try publishing --- .github/workflows/qemu-image-build.yml | 45 +++++++++++++++++++------- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index 3db3c7013..dd96a0b51 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -48,7 +48,6 @@ jobs: - uses: DeterminateSystems/nix-installer-action@main - # TODO (darora): might need to temporarily disable this - name: Run checks if triggered manually if: ${{ github.event_name == 'workflow_dispatch' }} run: | @@ -90,12 +89,11 @@ jobs: packer build -var "git_sha=$(UPSTREAM_NIX_GIT_SHA)" -var-file="common-nix.vars.pkr.hcl" qemu-arm64-nix.pkr.hcl # packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl - # TODO (darora): publishing things is temporarily disabled - # - name: Grab release version - # id: process_release_version - # run: | - # VERSION=$(cat common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g') - # echo "version=$VERSION" >> $GITHUB_OUTPUT + - name: Grab release version + id: process_release_version + run: | + VERSION=$(cat common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g') + echo "version=$VERSION" >> $GITHUB_OUTPUT # - name: Create nix flake revision tarball # run: | @@ -106,11 +104,34 @@ jobs: # echo "$GIT_SHA" >> "/tmp/pg_upgrade_bin/${MAJOR_VERSION}/nix_flake_version" # tar -czf "/tmp/pg_binaries.tar.gz" -C "/tmp/pg_upgrade_bin" . - # - name: configure aws credentials - staging - # uses: aws-actions/configure-aws-credentials@v4 - # with: - # role-to-assume: ${{ secrets.DEV_AWS_ROLE }} - # aws-region: "us-east-1" + - name: configure aws credentials - staging + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.DEV_AWS_ROLE }} + aws-region: "us-east-1" + + - run: docker context create builders + + - uses: docker/setup-buildx-action@v3 + with: + endpoint: builders + + - name: Login to ECR + uses: docker/login-action@v2 + with: + registry: public.ecr.aws + + - id: build + uses: docker/build-push-action@v5 + with: + file: Dockerfile-kubernetes + push: true + target: production + tags: ${{ steps.process_release_version.outputs.version }} + platforms: linux/arm64 + cache-from: type=gha,scope=${{ github.ref_name }}-qemu-${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-qemu-${{ matrix.arch }} + # - name: Upload software manifest to s3 staging # run: | From 48779031a70fbf9f15d1e072050832eecb8971e9 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Tue, 4 Feb 2025 16:28:53 +0800 Subject: [PATCH 80/94] cleanup --- .github/workflows/qemu-image-build.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index dd96a0b51..038e534ab 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -62,18 +62,19 @@ jobs: sudo chown runner /dev/kvm sudo chmod 666 /dev/kvm - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y qemu-efi-aarch64 cloud-image-utils qemu-system-arm qemu-utils - cd && curl -L "https://releases.hashicorp.com/packer/1.7.8/packer_1.7.8_linux_$(dpkg --print-architecture).zip" -o packer.zip && unzip packer.zip && rm -f packer.zip && sudo mv packer /usr/local/bin/ + # - name: Install dependencies + # run: | + # sudo apt-get update + # sudo apt-get install -y qemu-efi-aarch64 cloud-image-utils qemu-system-arm qemu-utils + # cd && curl -L "https://releases.hashicorp.com/packer/1.7.8/packer_1.7.8_linux_$(dpkg --print-architecture).zip" -o packer.zip && unzip packer.zip && rm -f packer.zip && sudo mv packer /usr/local/bin/ - name: Set PostgreSQL version environment variable run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV - name: Generate common-nix.vars.pkr.hcl run: | - PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + curl -L https://github.com/mikefarah/yq/releases/download/v4.45.1/yq_linux_arm64 -o yq && sudo mv yq /usr/local/bin + PG_VERSION=$(yq '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl echo 'postgres-major-version = "'$POSTGRES_MAJOR_VERSION'"' >> common-nix.vars.pkr.hcl @@ -87,7 +88,6 @@ jobs: GIT_SHA=${{github.sha}} export PACKER_LOG=1 packer build -var "git_sha=$(UPSTREAM_NIX_GIT_SHA)" -var-file="common-nix.vars.pkr.hcl" qemu-arm64-nix.pkr.hcl - # packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl - name: Grab release version id: process_release_version @@ -132,7 +132,6 @@ jobs: cache-from: type=gha,scope=${{ github.ref_name }}-qemu-${{ matrix.arch }} cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-qemu-${{ matrix.arch }} - # - name: Upload software manifest to s3 staging # run: | # cd ansible From e423ffc59b47ccf61d70272c881924fcaa4b20af Mon Sep 17 00:00:00 2001 From: Div Arora Date: Tue, 4 Feb 2025 16:31:14 +0800 Subject: [PATCH 81/94] . --- .github/workflows/qemu-image-build.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index 038e534ab..f509bb833 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -82,6 +82,13 @@ jobs: # Ensure there's a newline at the end of the file echo "" >> common-nix.vars.pkr.hcl + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get remove -y qemu-efi-aarch64 cloud-image-utils qemu-system-arm qemu-utils + sudo apt-get install -y qemu-efi-aarch64 cloud-image-utils qemu-system-arm qemu-utils + - name: Build QEMU artifact run: | make init From 08e6311940980d21144b94f510b4a7212483e025 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Tue, 4 Feb 2025 16:33:55 +0800 Subject: [PATCH 82/94] fix --- .github/workflows/qemu-image-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index f509bb833..eb3c1b262 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -73,7 +73,7 @@ jobs: - name: Generate common-nix.vars.pkr.hcl run: | - curl -L https://github.com/mikefarah/yq/releases/download/v4.45.1/yq_linux_arm64 -o yq && sudo mv yq /usr/local/bin + curl -L https://github.com/mikefarah/yq/releases/download/v4.45.1/yq_linux_arm64 -o yq && mv yq /home/runner/.local/bin/ PG_VERSION=$(yq '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl From 44f85633181ac3ee673bdd2a4a723e3b9ad09e54 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Tue, 4 Feb 2025 16:42:45 +0800 Subject: [PATCH 83/94] . --- .github/workflows/qemu-image-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index eb3c1b262..a690bb7d3 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -73,8 +73,8 @@ jobs: - name: Generate common-nix.vars.pkr.hcl run: | - curl -L https://github.com/mikefarah/yq/releases/download/v4.45.1/yq_linux_arm64 -o yq && mv yq /home/runner/.local/bin/ - PG_VERSION=$(yq '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + curl -L https://github.com/mikefarah/yq/releases/download/v4.45.1/yq_linux_arm64 -o yq + PG_VERSION=$(./yq '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl echo 'postgres-major-version = "'$POSTGRES_MAJOR_VERSION'"' >> common-nix.vars.pkr.hcl From b245fc2a62145255de3ec4209fbeb51c8c0ff4c2 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Tue, 4 Feb 2025 16:44:24 +0800 Subject: [PATCH 84/94] . --- .github/workflows/qemu-image-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index a690bb7d3..d8073ced9 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -74,6 +74,7 @@ jobs: - name: Generate common-nix.vars.pkr.hcl run: | curl -L https://github.com/mikefarah/yq/releases/download/v4.45.1/yq_linux_arm64 -o yq + chmod +x yq PG_VERSION=$(./yq '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl From 8c478bd70a0d8011e7d28d37b56d5331f52c4538 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Tue, 4 Feb 2025 16:59:53 +0800 Subject: [PATCH 85/94] . --- .github/workflows/qemu-image-build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index d8073ced9..712b607f7 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -134,7 +134,6 @@ jobs: with: file: Dockerfile-kubernetes push: true - target: production tags: ${{ steps.process_release_version.outputs.version }} platforms: linux/arm64 cache-from: type=gha,scope=${{ github.ref_name }}-qemu-${{ matrix.arch }} From 8d5d14a800785a81714e7e2447901158de3eceb8 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Tue, 4 Feb 2025 17:00:08 +0800 Subject: [PATCH 86/94] test using builtin deps --- .github/workflows/qemu-image-build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index 712b607f7..8151ab4b6 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -84,11 +84,11 @@ jobs: echo "" >> common-nix.vars.pkr.hcl - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get remove -y qemu-efi-aarch64 cloud-image-utils qemu-system-arm qemu-utils - sudo apt-get install -y qemu-efi-aarch64 cloud-image-utils qemu-system-arm qemu-utils + # - name: Install dependencies + # run: | + # sudo apt-get update + # sudo apt-get remove -y qemu-efi-aarch64 cloud-image-utils qemu-system-arm qemu-utils + # sudo apt-get install -y qemu-efi-aarch64 cloud-image-utils qemu-system-arm qemu-utils - name: Build QEMU artifact run: | From 75b76b7bcb4d19ac9889d08532a4a776996b88e7 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Tue, 4 Feb 2025 17:08:05 +0800 Subject: [PATCH 87/94] Revert "test using builtin deps" This reverts commit 8d5d14a800785a81714e7e2447901158de3eceb8. --- .github/workflows/qemu-image-build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index 8151ab4b6..712b607f7 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -84,11 +84,11 @@ jobs: echo "" >> common-nix.vars.pkr.hcl - # - name: Install dependencies - # run: | - # sudo apt-get update - # sudo apt-get remove -y qemu-efi-aarch64 cloud-image-utils qemu-system-arm qemu-utils - # sudo apt-get install -y qemu-efi-aarch64 cloud-image-utils qemu-system-arm qemu-utils + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get remove -y qemu-efi-aarch64 cloud-image-utils qemu-system-arm qemu-utils + sudo apt-get install -y qemu-efi-aarch64 cloud-image-utils qemu-system-arm qemu-utils - name: Build QEMU artifact run: | From e8d5da40670682d1411b04e6c6fabccc14e52d82 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Tue, 4 Feb 2025 17:18:00 +0800 Subject: [PATCH 88/94] . --- .github/workflows/qemu-image-build.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index 712b607f7..7f4823485 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -97,6 +97,10 @@ jobs: export PACKER_LOG=1 packer build -var "git_sha=$(UPSTREAM_NIX_GIT_SHA)" -var-file="common-nix.vars.pkr.hcl" qemu-arm64-nix.pkr.hcl + - name: Install dependencies + run: | + ls -lh output-cloudimg/ + - name: Grab release version id: process_release_version run: | @@ -136,8 +140,8 @@ jobs: push: true tags: ${{ steps.process_release_version.outputs.version }} platforms: linux/arm64 - cache-from: type=gha,scope=${{ github.ref_name }}-qemu-${{ matrix.arch }} - cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-qemu-${{ matrix.arch }} + cache-from: type=gha,scope=${{ github.ref_name }}-qemu + cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-qemu # - name: Upload software manifest to s3 staging # run: | From ec51f422e943adf50b15ab5925f8cc544bce460c Mon Sep 17 00:00:00 2001 From: Div Arora Date: Tue, 4 Feb 2025 17:29:49 +0800 Subject: [PATCH 89/94] simplify --- .github/workflows/qemu-image-build.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index 7f4823485..db40c000b 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -122,11 +122,7 @@ jobs: role-to-assume: ${{ secrets.DEV_AWS_ROLE }} aws-region: "us-east-1" - - run: docker context create builders - - uses: docker/setup-buildx-action@v3 - with: - endpoint: builders - name: Login to ECR uses: docker/login-action@v2 @@ -140,8 +136,6 @@ jobs: push: true tags: ${{ steps.process_release_version.outputs.version }} platforms: linux/arm64 - cache-from: type=gha,scope=${{ github.ref_name }}-qemu - cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-qemu # - name: Upload software manifest to s3 staging # run: | From a7496be03858eee1de1e2641737c94305c8802c5 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Tue, 4 Feb 2025 17:53:03 +0800 Subject: [PATCH 90/94] . --- Dockerfile-kubernetes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile-kubernetes b/Dockerfile-kubernetes index a8231b5ee..135b42c7c 100644 --- a/Dockerfile-kubernetes +++ b/Dockerfile-kubernetes @@ -1,6 +1,6 @@ FROM alpine:3.21 -ADD output-cloudimg/packer-cloudimg /disk/focal.qcow2 +ADD ./output-cloudimg/packer-cloudimg /disk/focal.qcow2 RUN apk add --no-cache qemu-system-aarch64 qemu-img openssh nftables cloud-utils-localds aavmf # dev stuff From 1b23658b9d737dffc538da60a7031606fe23bd1f Mon Sep 17 00:00:00 2001 From: Div Arora Date: Tue, 4 Feb 2025 17:56:28 +0800 Subject: [PATCH 91/94] context --- .github/workflows/qemu-image-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index db40c000b..2061c40d9 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -136,6 +136,7 @@ jobs: push: true tags: ${{ steps.process_release_version.outputs.version }} platforms: linux/arm64 + context: . # - name: Upload software manifest to s3 staging # run: | From 45a2f3eb6023e195092f6cb66b2fa2d9dfaef70b Mon Sep 17 00:00:00 2001 From: Div Arora Date: Wed, 5 Feb 2025 07:58:48 +0800 Subject: [PATCH 92/94] fix: correct tag --- .github/workflows/qemu-image-build.yml | 2 +- Dockerfile-kubernetes | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index 2061c40d9..255f39e90 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -134,7 +134,7 @@ jobs: with: file: Dockerfile-kubernetes push: true - tags: ${{ steps.process_release_version.outputs.version }} + tags: public.ecr.aws/w9p6e7k7/supabase/postgres-v3:${{ steps.process_release_version.outputs.version }} platforms: linux/arm64 context: . diff --git a/Dockerfile-kubernetes b/Dockerfile-kubernetes index 135b42c7c..716e72b42 100644 --- a/Dockerfile-kubernetes +++ b/Dockerfile-kubernetes @@ -2,7 +2,7 @@ FROM alpine:3.21 ADD ./output-cloudimg/packer-cloudimg /disk/focal.qcow2 -RUN apk add --no-cache qemu-system-aarch64 qemu-img openssh nftables cloud-utils-localds aavmf +RUN apk add --no-cache qemu-system-aarch64 qemu-img openssh-client nftables cloud-utils-localds aavmf # dev stuff # RUN apk add --no-cache iproute2 From e53bd629a418c7a85a8262b89d567af13cc48893 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Wed, 5 Feb 2025 08:18:48 +0800 Subject: [PATCH 93/94] chore: cleanup instructions for building image --- .github/workflows/nix-build.yml | 1 - .github/workflows/qemu-image-build.yml | 47 ++++++---------- ansible/tasks/setup-postgres.yml | 2 +- qemu_artifact.md | 77 ++------------------------ 4 files changed, 23 insertions(+), 104 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 49dc676d3..c1811fd00 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -5,7 +5,6 @@ on: branches: - develop - release/* - - da/qemu* pull_request: workflow_dispatch: diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index 255f39e90..464ee01fe 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -5,7 +5,6 @@ on: branches: - develop - release/* - - da/merging-qemu paths: - '.github/workflows/qemu-image-build.yml' - 'qemu-arm64-nix.pkr.hcl' @@ -62,28 +61,20 @@ jobs: sudo chown runner /dev/kvm sudo chmod 666 /dev/kvm - # - name: Install dependencies - # run: | - # sudo apt-get update - # sudo apt-get install -y qemu-efi-aarch64 cloud-image-utils qemu-system-arm qemu-utils - # cd && curl -L "https://releases.hashicorp.com/packer/1.7.8/packer_1.7.8_linux_$(dpkg --print-architecture).zip" -o packer.zip && unzip packer.zip && rm -f packer.zip && sudo mv packer /usr/local/bin/ - - name: Set PostgreSQL version environment variable run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV - name: Generate common-nix.vars.pkr.hcl run: | - curl -L https://github.com/mikefarah/yq/releases/download/v4.45.1/yq_linux_arm64 -o yq - chmod +x yq + curl -L https://github.com/mikefarah/yq/releases/download/v4.45.1/yq_linux_arm64 -o yq && chmod +x yq PG_VERSION=$(./yq '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl echo 'postgres-major-version = "'$POSTGRES_MAJOR_VERSION'"' >> common-nix.vars.pkr.hcl - # echo 'postgresql_major = "'$POSTGRES_MAJOR_VERSION'"' >> common-nix.vars.pkr.hcl # Ensure there's a newline at the end of the file echo "" >> common-nix.vars.pkr.hcl - + # TODO (darora): not quite sure why I'm having to uninstall and re-install these deps, but the build fails w/o this - name: Install dependencies run: | sudo apt-get update @@ -95,11 +86,7 @@ jobs: make init GIT_SHA=${{github.sha}} export PACKER_LOG=1 - packer build -var "git_sha=$(UPSTREAM_NIX_GIT_SHA)" -var-file="common-nix.vars.pkr.hcl" qemu-arm64-nix.pkr.hcl - - - name: Install dependencies - run: | - ls -lh output-cloudimg/ + packer build -var "git_sha=${GIT_SHA}" -var-file="common-nix.vars.pkr.hcl" qemu-arm64-nix.pkr.hcl - name: Grab release version id: process_release_version @@ -122,21 +109,21 @@ jobs: role-to-assume: ${{ secrets.DEV_AWS_ROLE }} aws-region: "us-east-1" - - uses: docker/setup-buildx-action@v3 - - - name: Login to ECR - uses: docker/login-action@v2 + - name: Login to Amazon ECR Public + id: login-ecr-public + uses: aws-actions/amazon-ecr-login@v2 with: - registry: public.ecr.aws - - - id: build - uses: docker/build-push-action@v5 - with: - file: Dockerfile-kubernetes - push: true - tags: public.ecr.aws/w9p6e7k7/supabase/postgres-v3:${{ steps.process_release_version.outputs.version }} - platforms: linux/arm64 - context: . + registry-type: public + + - name: Build, tag, and push docker image to Amazon ECR Public + env: + REGISTRY: public.ecr.aws/w9p6e7k7 + REGISTRY_ALIAS: supabase + REPOSITORY: postgres-vm-image + IMAGE_TAG: ${{ steps.process_release_version.outputs.version }} + run: | + docker build -f Dockerfile-kubernetes -t $REGISTRY/$REGISTRY_ALIAS/$REPOSITORY:$IMAGE_TAG . + docker push $REGISTRY/$REGISTRY_ALIAS/$REPOSITORY:$IMAGE_TAG # - name: Upload software manifest to s3 staging # run: | diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index 08666d23c..a45b7a502 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -273,7 +273,7 @@ shell: | mkdir -p /run/postgresql chown -R postgres:postgres /run/postgresql - when: stage2_nix and qemu_mode + when: stage2_nix and qemu_mode is defined - name: Restart Postgres Database without Systemd become: yes diff --git a/qemu_artifact.md b/qemu_artifact.md index 9fa05e909..1ac61beb3 100644 --- a/qemu_artifact.md +++ b/qemu_artifact.md @@ -1,68 +1,9 @@ # QEMU artifact -We build a container image that contains a QEMU qcow2 disk image. This container image can be use with KubeVirt's [containerDisk](https://kubevirt.io/user-guide/storage/disks_and_volumes/#containerdisk) functionality to boot up VMs off the qcow2 image. - -Container images are a convenient mechanism to ship the disk image to the nodes where they're needed. +We build a container image that contains a QEMU qcow2 disk image. Container images are a convenient mechanism to ship the disk image to the nodes where they're needed. Given the size of the image, the first VM using it on a node might take a while to come up, while the image is being pulled down. The image can be pre-fetched to avoid this; we might also switch to other deployment mechanisms in the future. -# Building QEMU artifact - -## Creating a bare-metal instance - -We launch an Ubuntu 22 bare-metal instance; we're using the `c6g.metal` instance type in this case, but any ARM instance type is sufficient for our purposes. In the example below the region used is: `ap-south-1`. - -```bash -# create a security group for your instance -aws ec2 create-security-group --group-name "launch-wizard-1" --description "launch-wizard-1 created 2024-11-26T00:32:56.039Z" --vpc-id "insert-vpc-id" - -# using the generated security group ID (insert-sg-group), ensure that it allows for SSH access -aws ec2 authorize-security-group-ingress --group-id "insert-sg-group" --ip-permissions '{"IpProtocol":"tcp","FromPort":22,"ToPort":22,"IpRanges":[{"CidrIp":"0.0.0.0/0"}]}' - -# spin up your instance with the generated security group ID (insert-sg-group) -aws ec2 run-instances \ ---image-id "ami-0a87daabd88e93b1f" \ ---instance-type "c6g.metal" \ ---key-name "INSERT_KEY_PAIR_NAME" \ # create a key pair, or use other mechanism of getting on to the box ---block-device-mappings '{"DeviceName":"/dev/sda1","Ebs":{"Encrypted":false,"DeleteOnTermination":true,"Iops":3000,"SnapshotId":"snap-0fe84a34403e3da8b","VolumeSize":200,"VolumeType":"gp3","Throughput":125}}' \ ---network-interfaces '{"AssociatePublicIpAddress":true,"DeviceIndex":0,"Groups":["insert-sg-group"]}' \ ---tag-specifications '{"ResourceType":"instance","Tags":[{"Key":"Name","Value":"qemu-pg-image"}]}' \ ---metadata-options '{"HttpEndpoint":"enabled","HttpPutResponseHopLimit":2,"HttpTokens":"required"}' \ ---private-dns-name-options '{"HostnameType":"ip-name","EnableResourceNameDnsARecord":true,"EnableResourceNameDnsAAAARecord":false}' \ ---count "1" - -``` -## Install deps - -On the instance, install the dependencies we require for producing QEMU artifacts. Assuming you are the root user: - -```bash -apt-get update -apt-get install -y qemu-system qemu-system-arm qemu-utils qemu-efi-aarch64 libvirt-clients libvirt-daemon libqcow-utils software-properties-common git make libnbd-bin nbdkit fuse2fs cloud-image-utils awscli -usermod -aG kvm ubuntu -curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - -apt-add-repository "deb [arch=arm64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" -apt-get update && apt-get install packer=1.11.2-1 -apt-get install -y docker.io -``` - -Some dev deps that might be useful: - -```bash -apt-get install -y emacs ripgrep vim-tiny byobu -``` - -## Clone repo and build - -Logout/login first to pick up new group memberships! - -``` bash -git clone https://github.com/supabase/postgres.git -cd postgres -git checkout da/qemu-rebasing # choose appropriate branch here -make init container-disk-image -``` - ### Build process The current AMI process involves a few steps: @@ -87,21 +28,13 @@ b. packer build (`qemu-arm64-nix.pkr.hcl`) ## Publish image for later use -Following `make init container-disk-image`, the generated image should be found in: `/path/to/postgres/output-cloudimg`. For portability the image is also bundled up as a docker image with the name: `supabase-postgres-test` . Publish the built docker image to a registry of your choosing, and use the published image with KubeVirt. - -# Iterating on the QEMU artifact - -For a tighter iteration loop on the Postgres artifact, the recommended workflow is to do so on an Ubuntu bare-metal node that's part of the EKS cluster that you're deploying to. +Following `make init alpine-image`, the generated VM image should be bundled as a container image with the name: `supabase-postgres-test` . Publish the built docker image to a registry of your choosing, and use the published image with e.g. KubeVirt. -- Instead of running `make init container-disk-image`, use `make init host-disk` instead to build the raw image file on disk. (`/path/to/postgres/disk/focal-raw.img`) -- Update the VM spec to use `hostDisk` instead of `containerDisk` - - Note that only one VM can use an image at a time, so you can't create multiple VMs backed by the same host disk. -- Enable the `HostDisk` feature flag for KubeVirt -- Deploy the VM to the node +## Iterating on image -Additionally, to iterate on the container image part of things, you can build the image on the bare-metal node (`eks-node-container-disk-image` target), rather than needing to publish it to ECR or similar registry. However, this part can take a while, so iterating using host disks remains the fastest dev loop. +For faster iteration, it's more convenient to build the image on an ubuntu bare-metal node that's part of the EKS cluster you're using. Build the image in the `k8s.io` namespace in order for it to be available for immediate use on that node. -## Dependencies note +### Dependencies note Installing `docker.io` on an EKS node might interfere with the k8s setup of the node. You can instead install `nerdctl` and `buildkit`: From bbd589fe15cfc8666a1e89d9c22b724168f6efa7 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Fri, 7 Feb 2025 11:01:52 +0800 Subject: [PATCH 94/94] chore: run scripts through an auto-formatter Used black for python script, shfmt for shell. Default configurations were used for both. --- ansible/files/permission_check.py | 253 ++++++++++----------- ebssurrogate/scripts/qemu-bootstrap-nix.sh | 125 +++++----- qemu_artifact.md | 2 +- 3 files changed, 188 insertions(+), 192 deletions(-) diff --git a/ansible/files/permission_check.py b/ansible/files/permission_check.py index ef22ed575..5bf421018 100644 --- a/ansible/files/permission_check.py +++ b/ansible/files/permission_check.py @@ -8,152 +8,102 @@ expected_results = { "postgres": [ {"groupname": "postgres", "username": "postgres"}, - {"groupname": "ssl-cert", "username": "postgres"} + {"groupname": "ssl-cert", "username": "postgres"}, ], "ubuntu": [ - {'groupname': 'adm', 'username': 'ubuntu'}, - {'groupname': 'audio', 'username': 'ubuntu'}, - {'groupname': 'cdrom', 'username': 'ubuntu'}, - {'groupname': 'dialout', 'username': 'ubuntu'}, - {'groupname': 'dip', 'username': 'ubuntu'}, - {'groupname': 'floppy', 'username': 'ubuntu'}, - {'groupname': 'lxd', 'username': 'ubuntu'}, - {'groupname': 'netdev', 'username': 'ubuntu'}, - {'groupname': 'plugdev', 'username': 'ubuntu'}, - {'groupname': 'sudo', 'username': 'ubuntu'}, - {'groupname': 'ubuntu', 'username': 'ubuntu'}, - {'groupname': 'video', 'username': 'ubuntu'} - ], - "root": [ - {"groupname":"root","username":"root"} - ], - "daemon": [ - {"groupname":"daemon","username":"daemon"} - ], - "bin": [ - {"groupname":"bin","username":"bin"} - ], - "sys": [ - {"groupname":"sys","username":"sys"} - ], - "sync": [ - {"groupname":"nogroup","username":"sync"} - ], - "games": [ - {"groupname":"games","username":"games"} - ], - "man": [ - {"groupname":"man","username":"man"} - ], - "lp": [ - {"groupname":"lp","username":"lp"} - ], - "mail": [ - {"groupname":"mail","username":"mail"} - ], - "news": [ - {"groupname":"news","username":"news"} - ], - "uucp": [ - {"groupname":"uucp","username":"uucp"} - ], - "proxy": [ - {"groupname":"proxy","username":"proxy"} - ], - "www-data": [ - {"groupname":"www-data","username":"www-data"} - ], - "backup": [ - {"groupname":"backup","username":"backup"} - ], - "list": [ - {"groupname":"list","username":"list"} - ], - "irc": [ - {"groupname":"irc","username":"irc"} - ], - "gnats": [ - {"groupname":"gnats","username":"gnats"} - ], - "nobody": [ - {"groupname":"nogroup","username":"nobody"} - ], + {"groupname": "adm", "username": "ubuntu"}, + {"groupname": "audio", "username": "ubuntu"}, + {"groupname": "cdrom", "username": "ubuntu"}, + {"groupname": "dialout", "username": "ubuntu"}, + {"groupname": "dip", "username": "ubuntu"}, + {"groupname": "floppy", "username": "ubuntu"}, + {"groupname": "lxd", "username": "ubuntu"}, + {"groupname": "netdev", "username": "ubuntu"}, + {"groupname": "plugdev", "username": "ubuntu"}, + {"groupname": "sudo", "username": "ubuntu"}, + {"groupname": "ubuntu", "username": "ubuntu"}, + {"groupname": "video", "username": "ubuntu"}, + ], + "root": [{"groupname": "root", "username": "root"}], + "daemon": [{"groupname": "daemon", "username": "daemon"}], + "bin": [{"groupname": "bin", "username": "bin"}], + "sys": [{"groupname": "sys", "username": "sys"}], + "sync": [{"groupname": "nogroup", "username": "sync"}], + "games": [{"groupname": "games", "username": "games"}], + "man": [{"groupname": "man", "username": "man"}], + "lp": [{"groupname": "lp", "username": "lp"}], + "mail": [{"groupname": "mail", "username": "mail"}], + "news": [{"groupname": "news", "username": "news"}], + "uucp": [{"groupname": "uucp", "username": "uucp"}], + "proxy": [{"groupname": "proxy", "username": "proxy"}], + "www-data": [{"groupname": "www-data", "username": "www-data"}], + "backup": [{"groupname": "backup", "username": "backup"}], + "list": [{"groupname": "list", "username": "list"}], + "irc": [{"groupname": "irc", "username": "irc"}], + "gnats": [{"groupname": "gnats", "username": "gnats"}], + "nobody": [{"groupname": "nogroup", "username": "nobody"}], "systemd-network": [ - {"groupname":"systemd-network","username":"systemd-network"} + {"groupname": "systemd-network", "username": "systemd-network"} ], "systemd-resolve": [ - {"groupname":"systemd-resolve","username":"systemd-resolve"} + {"groupname": "systemd-resolve", "username": "systemd-resolve"} ], "systemd-timesync": [ - {"groupname":"systemd-timesync","username":"systemd-timesync"} - ], - "messagebus": [ - {"groupname":"messagebus","username":"messagebus"} + {"groupname": "systemd-timesync", "username": "systemd-timesync"} ], + "messagebus": [{"groupname": "messagebus", "username": "messagebus"}], "ec2-instance-connect": [ {"groupname": "nogroup", "username": "ec2-instance-connect"} ], - "sshd": [ - {"groupname":"nogroup","username":"sshd"} - ], + "sshd": [{"groupname": "nogroup", "username": "sshd"}], "wal-g": [ - {"groupname":"postgres","username":"wal-g"}, - {"groupname":"wal-g","username":"wal-g"} + {"groupname": "postgres", "username": "wal-g"}, + {"groupname": "wal-g", "username": "wal-g"}, ], "pgbouncer": [ - {"groupname":"pgbouncer","username":"pgbouncer"}, - {"groupname":"postgres","username":"pgbouncer"}, - {"groupname":"ssl-cert","username":"pgbouncer"} - ], - "gotrue": [ - {"groupname":"gotrue","username":"gotrue"} - ], - "envoy": [ - {"groupname":"envoy","username":"envoy"} - ], - "kong": [ - {"groupname":"kong","username":"kong"} - ], - "nginx": [ - {"groupname":"nginx","username":"nginx"} - ], + {"groupname": "pgbouncer", "username": "pgbouncer"}, + {"groupname": "postgres", "username": "pgbouncer"}, + {"groupname": "ssl-cert", "username": "pgbouncer"}, + ], + "gotrue": [{"groupname": "gotrue", "username": "gotrue"}], + "envoy": [{"groupname": "envoy", "username": "envoy"}], + "kong": [{"groupname": "kong", "username": "kong"}], + "nginx": [{"groupname": "nginx", "username": "nginx"}], "vector": [ - {"groupname":"adm","username":"vector"}, - {"groupname":"postgres","username":"vector"}, - {"groupname":"systemd-journal","username":"vector"}, - {"groupname":"vector","username":"vector"} + {"groupname": "adm", "username": "vector"}, + {"groupname": "postgres", "username": "vector"}, + {"groupname": "systemd-journal", "username": "vector"}, + {"groupname": "vector", "username": "vector"}, ], "adminapi": [ - {"groupname":"admin","username":"adminapi"}, - {"groupname":"adminapi","username":"adminapi"}, - {"groupname":"envoy","username":"adminapi"}, - {"groupname":"kong","username":"adminapi"}, - {"groupname":"pgbouncer","username":"adminapi"}, - {"groupname":"postgres","username":"adminapi"}, - {"groupname":"postgrest","username":"adminapi"}, - {"groupname":"root","username":"adminapi"}, - {"groupname":"systemd-journal","username":"adminapi"}, - {"groupname":"vector","username":"adminapi"}, - {"groupname":"wal-g","username":"adminapi"} - ], - "postgrest": [ - {"groupname":"postgrest","username":"postgrest"} - ], - "tcpdump": [ - {"groupname":"tcpdump","username":"tcpdump"} - ], + {"groupname": "admin", "username": "adminapi"}, + {"groupname": "adminapi", "username": "adminapi"}, + {"groupname": "envoy", "username": "adminapi"}, + {"groupname": "kong", "username": "adminapi"}, + {"groupname": "pgbouncer", "username": "adminapi"}, + {"groupname": "postgres", "username": "adminapi"}, + {"groupname": "postgrest", "username": "adminapi"}, + {"groupname": "root", "username": "adminapi"}, + {"groupname": "systemd-journal", "username": "adminapi"}, + {"groupname": "vector", "username": "adminapi"}, + {"groupname": "wal-g", "username": "adminapi"}, + ], + "postgrest": [{"groupname": "postgrest", "username": "postgrest"}], + "tcpdump": [{"groupname": "tcpdump", "username": "tcpdump"}], "systemd-coredump": [ - {"groupname":"systemd-coredump","username":"systemd-coredump"} - ] + {"groupname": "systemd-coredump", "username": "systemd-coredump"} + ], } # This program depends on osquery being installed on the system # Function to run osquery def run_osquery(query): - process = subprocess.Popen(['osqueryi', '--json', query], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + process = subprocess.Popen( + ["osqueryi", "--json", query], stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) output, error = process.communicate() - return output.decode('utf-8') + return output.decode("utf-8") def parse_json(json_str): @@ -173,7 +123,9 @@ def compare_results(username, query_result): if query_result == expected_result: print(f"The query result for user '{username}' matches the expected result.") else: - print(f"The query result for user '{username}' does not match the expected result.") + print( + f"The query result for user '{username}' does not match the expected result." + ) print("Expected:", expected_result) print("Got:", query_result) sys.exit(1) @@ -191,8 +143,10 @@ def check_nixbld_users(): parsed_result = parse_json(query_result) for user in parsed_result: - if user['groupname'] != 'nixbld': - print(f"User '{user['username']}' is in group '{user['groupname']}' instead of 'nixbld'.") + if user["groupname"] != "nixbld": + print( + f"User '{user['username']}' is in group '{user['groupname']}' instead of 'nixbld'." + ) sys.exit(1) print("All nixbld users are in the 'nixbld' group.") @@ -200,14 +154,57 @@ def check_nixbld_users(): def main(): parser = argparse.ArgumentParser( - prog='Supabase Postgres Artifact Permissions Checker', - description='Checks the Postgres Artifact for the appropriate users and group memberships') - parser.add_argument('-q', '--qemu', action='store_true', help='Whether we are checking a QEMU artifact') + prog="Supabase Postgres Artifact Permissions Checker", + description="Checks the Postgres Artifact for the appropriate users and group memberships", + ) + parser.add_argument( + "-q", + "--qemu", + action="store_true", + help="Whether we are checking a QEMU artifact", + ) args = parser.parse_args() qemu_artifact = args.qemu or False # Define usernames for which you want to compare results - usernames = ["postgres", "ubuntu", "root", "daemon", "bin", "sys", "sync", "games","man","lp","mail","news","uucp","proxy","www-data","backup","list","irc","gnats","nobody","systemd-network","systemd-resolve","systemd-timesync","messagebus","sshd","wal-g","pgbouncer","gotrue","envoy","kong","nginx","vector","adminapi","postgrest","tcpdump","systemd-coredump"] + usernames = [ + "postgres", + "ubuntu", + "root", + "daemon", + "bin", + "sys", + "sync", + "games", + "man", + "lp", + "mail", + "news", + "uucp", + "proxy", + "www-data", + "backup", + "list", + "irc", + "gnats", + "nobody", + "systemd-network", + "systemd-resolve", + "systemd-timesync", + "messagebus", + "sshd", + "wal-g", + "pgbouncer", + "gotrue", + "envoy", + "kong", + "nginx", + "vector", + "adminapi", + "postgrest", + "tcpdump", + "systemd-coredump", + ] if not qemu_artifact: usernames.append("ec2-instance-connect") diff --git a/ebssurrogate/scripts/qemu-bootstrap-nix.sh b/ebssurrogate/scripts/qemu-bootstrap-nix.sh index ca52c5cd5..61606c81e 100755 --- a/ebssurrogate/scripts/qemu-bootstrap-nix.sh +++ b/ebssurrogate/scripts/qemu-bootstrap-nix.sh @@ -4,11 +4,10 @@ set -o errexit set -o pipefail set -o xtrace -if [ $(dpkg --print-architecture) = "amd64" ]; -then - ARCH="amd64"; +if [ $(dpkg --print-architecture) = "amd64" ]; then + ARCH="amd64" else - ARCH="arm64"; + ARCH="arm64" fi function waitfor_boot_finished { @@ -17,8 +16,8 @@ function waitfor_boot_finished { echo "args: ${ARGS}" # Wait for cloudinit on the surrogate to complete before making progress while [[ ! -f /var/lib/cloud/instance/boot-finished ]]; do - echo 'Waiting for cloud-init...' - sleep 1 + echo 'Waiting for cloud-init...' + sleep 1 done } @@ -30,25 +29,25 @@ function install_packages { function execute_playbook { -tee /etc/ansible/ansible.cfg </dev/null + # Define the contents of the PostgreSQL environment file + cat </dev/null LOCALE_ARCHIVE=/usr/lib/locale/locale-archive LANG="en_US.UTF-8" LANGUAGE="en_US.UTF-8" @@ -58,11 +57,11 @@ EOF } function setup_locale { -cat << EOF >> /etc/locale.gen + cat <>/etc/locale.gen en_US.UTF-8 UTF-8 EOF -cat << EOF > /etc/default/locale + cat </etc/default/locale LANG="C.UTF-8" LC_CTYPE="C.UTF-8" EOF @@ -82,64 +81,64 @@ execute_playbook #################### function install_nix() { - sudo su -c "curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install --no-confirm \ + sudo su -c "curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install --no-confirm \ --extra-conf \"substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com\" \ --extra-conf \"trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=\" " -s /bin/bash root - . /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh + . /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh } function execute_stage2_playbook { - sudo tee /etc/ansible/ansible.cfg <