rework migrations between differerent databases (#601)

* extract `migrate_to` fn's to make them reusable * fix errors + clippy * `impl<'r> FromRow<'r, SqliteRow> for Event` * leftover errors + make clippy happy * move all new insert fns into separate module * move `init_prod` migrations into separate module * cleanup old migration impls * add a catch block in `main.rs` to catch misconfiguration early * fix some query errors and remove migration from `AppState` and use dedicated `migrate()` in `database.rs` instead * make migrations `postgres` -> `hiqlite` + `hiqlite` -> `postgres` work * remove feature separation for `postgres` and `sqlite` everywhere * cleanup `justfile`
sebadob · Nov 6, 2024 · dac3892 · dac3892
1 parent af54cbc
commit dac3892
Show file tree

Hide file tree

Showing 62 changed files with 2,102 additions and 3,526 deletions.
diff --git a/justfile b/justfile
@@ -17,14 +17,7 @@ container_mailcrab := "rauthy-mailcrab"
 container_postgres := "rauthy-db-postgres"
 container_cargo_registry := "/usr/local/cargo/registry"
 file_test_pid := ".test_pid"
-
-# The sqlite db url will only be present until the migration has been completed
-# to avoid errors from sqlx, because the current pool setup has not been adjusted yet.
-# This will be the very last step of the migration.
-
-db_url_hiqlite := "DATABASE_URL=sqlite:data/rauthy.db HIQLITE=true"
-db_url_sqlite := "DATABASE_URL=sqlite:data/rauthy.db"
-db_url_postgres := "DATABASE_URL=postgresql://rauthy:123SuperSafe@$DEV_HOST:5432/rauthy"
+hiqlite := "HIQLITE=true"
 
 [private]
 default:
@@ -72,7 +65,7 @@ setup:
     just backend-start
     just migrate
 
-    {{ db_url_sqlite }} cargo build
+    cargo build
 
 # start the backend containers for local dev
 @backend-start:
@@ -182,19 +175,11 @@ postgres-stop:
 fmt:
     cargo fmt --all
 
-# clippy with sqlite features
 clippy:
     #!/usr/bin/env bash
     set -euxo pipefail
     clear
-    {{ db_url_sqlite }} cargo clippy
-
-# clippy with postgres features
-clippy-postgres:
-    #!/usr/bin/env bash
-    set -euxo pipefail
-    clear
-    {{ db_url_postgres }} cargo clippy --features postgres
+    cargo clippy
 
 # delete the local hiqlite database
 delete-hiqlite:
@@ -205,35 +190,23 @@ delete-hiqlite:
     rm -rf data/logs
     rm -rf data/state_machine
 
-# re-create and migrate the sqlite database with sqlx
-migrate:
-    #!/usr/bin/env bash
-    set -euxo pipefail
-
-    mkdir -p data/
-    rm -f data/rauthy.db*
-    {{ db_url_sqlite }} sqlx database create
-    {{ db_url_sqlite }} sqlx migrate run --source migrations/sqlite
-
 # migrate the postgres database with sqlx
 migrate-postgres:
-    {{ db_url_postgres }} sqlx migrate run --source migrations/postgres
+    sqlx migrate run --source migrations/postgres
 
-# runs any of: none (sqlite), postgres, ui
+# runs any of: none (hiqlite), postgres, ui
 run ty="hiqlite":
     #!/usr/bin/env bash
     set -euxo pipefail
     clear
 
     if [[ {{ ty }} == "postgres" ]]; then
-      {{ db_url_postgres }} cargo run --features postgres
+      cargo run
     elif [[ {{ ty }} == "ui" ]]; then
       cd frontend
       {{ npm }} run dev -- --host=0.0.0.0
     elif [[ {{ ty }} == "hiqlite" ]]; then
-      {{ db_url_hiqlite }} cargo run
-    elif [[ {{ ty }} == "sqlite" ]]; then
-      {{ db_url_sqlite }} cargo run
+      {{ hiqlite }} cargo run
     fi
 
 # prints out the currently set version
@@ -242,11 +215,11 @@ version:
     echo "v$TAG"
 
 # only starts the backend in test mode with hiqlite database for easier test debugging
-test-backend: test-backend-stop migrate delete-hiqlite
+test-backend: test-backend-stop delete-hiqlite
     #!/usr/bin/env bash
     set -euxo pipefail
     clear
-    {{ db_url_hiqlite }} cargo run test
+    {{ hiqlite }} cargo run test
 
 # stops a possibly running test backend that may have spawned in the background for integration tests
 test-backend-stop:
@@ -261,39 +234,21 @@ test *test:
     #!/usr/bin/env bash
     set -euxo pipefail
     clear
-    {{ db_url_hiqlite }} cargo test {{ test }}
+    {{ hiqlite }} cargo test {{ test }}
 
 # runs the full set of tests with sqlite
-test-hiqlite *test: test-backend-stop migrate delete-hiqlite
+test-hiqlite *test: test-backend-stop delete-hiqlite
     #!/usr/bin/env bash
     clear
 
-    {{ db_url_hiqlite }} cargo build
-    {{ db_url_hiqlite }} ./target/debug/rauthy test &
+    {{ hiqlite }} cargo build
+    {{ hiqlite }} ./target/debug/rauthy test &
     echo $! > {{ file_test_pid }}
 
     # a fresh Hiqlite instance needs ~1 - 1.5 seconds for the raft initialization
     sleep 3
 
-    if {{ db_url_hiqlite }} cargo test {{ test }}; then
-      echo "All SQLite tests successful"
-      just test-backend-stop
-    else
-      echo "Failed Tests"
-      just test-backend-stop
-      exit 1
-    fi
-
-# runs the full set of tests with sqlite
-test-sqlite *test: test-backend-stop migrate
-    #!/usr/bin/env bash
-    clear
-
-    {{ db_url_sqlite }} cargo build
-    {{ db_url_sqlite }} ./target/debug/rauthy test &
-    echo $! > {{ file_test_pid }}
-
-    if {{ db_url_sqlite }} cargo test {{ test }}; then
+    if {{ hiqlite }} cargo test {{ test }}; then
       echo "All SQLite tests successful"
       just test-backend-stop
     else
@@ -307,11 +262,11 @@ test-postgres test="": test-backend-stop postgres-stop postgres-start
     #!/usr/bin/env bash
     clear
 
-    {{ db_url_postgres }} cargo build --features postgres
-    {{ db_url_postgres }} ./target/debug/rauthy test &
+    cargo build
+    ./target/debug/rauthy test &
     echo $! > {{ file_test_pid }}
 
-    if {{ db_url_postgres }} cargo test --features postgres {{ test }}; then
+    if cargo test {{ test }}; then
       echo "All SQLite tests successful"
       just test-backend-stop
     else
@@ -385,68 +340,17 @@ build-docs:
     cd book
     mdbook build -d ../docs
 
-# Build the final container image. Skip tests with `no-test`.
-build no-test="test" image="ghcr.io/sebadob/rauthy": build-ui
+# sqlx prepare for the workspace
+prepare:
+    cargo sqlx prepare --workspace
+
+# Build the final container image.
+build image="ghcr.io/sebadob/rauthy": build-ui prepare
     #!/usr/bin/env bash
     set -euxo pipefail
 
-    # sqlite
-    if [ {{ no-test }} != "no-test" ]; then
-        echo "make sure clippy is fine with sqlite"
-        {{ db_url_sqlite }} cargo clippy -- -D warnings
-        echo "run tests against sqlite"
-        just test-sqlite
-    fi
-
-    # make sure any big testing sqlite backups are cleaned up to speed up docker build
-    rm -rf data/backup out
     mkdir -p out/empty
 
-    echo "build sqlite release"
-    ## IMPORTANT: We can't use `cross` for the x86 build because it uses a way too old
-    ## `gcc`which has a known `memcmp` issue, which they decided to ignore:
-    ## https://github.com/cross-rs/cross/security/advisories/GHSA-2r9g-5qvw-fgmf
-    ## https://gcc.gnu.org/bugzilla/show_bug.cgi?id=95189
-    {{ docker }} run \
-      -v {{ cargo_home }}/registry:{{ container_cargo_registry }} \
-      -v {{ invocation_directory() }}/:/work/ \
-      -w /work \
-      {{ map_docker_user }} \
-      -e {{ db_url_sqlite }} \
-      --net host \
-      {{ builder_image }}:{{ builder_tag_date }} \
-      cargo build --release --target x86_64-unknown-linux-gnu
-    cp target/x86_64-unknown-linux-gnu/release/rauthy out/rauthy_amd64
-
-    {{ docker }} run \
-      -v {{ cargo_home }}/registry:{{ container_cargo_registry }} \
-      -v {{ invocation_directory() }}/:/work/ \
-      -w /work \
-      {{ map_docker_user }} \
-      -e {{ db_url_sqlite }} \
-      --net host \
-      {{ builder_image }}:{{ builder_tag_date }} \
-      cargo build --release --target aarch64-unknown-linux-gnu
-    cp target/aarch64-unknown-linux-gnu/release/rauthy out/rauthy_arm64
-
-    {{ docker }} buildx build \
-        -t {{ image }}:$TAG-lite \
-        --platform linux/amd64,linux/arm64 \
-        --push \
-        .
-
-    rm -rf out/rauthy_*
-
-    # postgres
-    if [ {{ no-test }} != "no-test" ]; then
-        echo "make sure clippy is fine with postgres"
-        {{ db_url_postgres }} cargo clippy --features postgres -- -D warnings
-        echo "run tests against postgres"
-        just test-postgres
-    fi
-
-    echo "build postgres release"
-    {{ db_url_postgres }} cargo sqlx prepare --workspace -- --features postgres
     # IMPORTANT: We can't use `cross` for the x86 build because it uses a way too old
     # `gcc`which has a known `memcmp` issue, which they decided to ignore:
     # https://github.com/cross-rs/cross/security/advisories/GHSA-2r9g-5qvw-fgmf
@@ -458,7 +362,7 @@ build no-test="test" image="ghcr.io/sebadob/rauthy": build-ui
       {{ map_docker_user }} \
       --net {{ container_network }} \
       {{ builder_image }}:{{ builder_tag_date }} \
-      cargo build --release --features postgres --target x86_64-unknown-linux-gnu
+      cargo build --release --target x86_64-unknown-linux-gnu
     cp target/x86_64-unknown-linux-gnu/release/rauthy out/rauthy_amd64
 
     {{ docker }} run \
@@ -468,7 +372,7 @@ build no-test="test" image="ghcr.io/sebadob/rauthy": build-ui
       {{ map_docker_user }} \
       --net {{ container_network }} \
       {{ builder_image }}:{{ builder_tag_date }} \
-      cargo build --release --features postgres --target aarch64-unknown-linux-gnu
+      cargo build --release --target aarch64-unknown-linux-gnu
     cp target/aarch64-unknown-linux-gnu/release/rauthy out/rauthy_arm64
 
     {{ docker }} buildx build \
@@ -484,41 +388,20 @@ build-builder image="ghcr.io/sebadob/rauthy-builder" push="push":
     #!/usr/bin/env bash
     set -euxo pipefail
 
-    # using bookworm instead of bullseye because of the newer, bug free gcc
     {{ docker }} build \
           -t {{ image }}:$TODAY \
           -f Dockerfile_builder \
           --build-arg="IMAGE=rust:1.81-bookworm" \
           .
     {{ docker }} push {{ image }}:$TODAY
 
-#    # using bookworm instead of bullseye because of the newer, bug free gcc
-#    {{docker}} build \
-#          -t {{image}}:amd64-$TODAY \
-#          -f Dockerfile_builder \
-#          --build-arg="IMAGE=rust:1.81-bookworm" \
-#          --no-cache \
-#          .
-#
-#    {{docker}} pull ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main
-#    {{docker}} build \
-#          -t {{image}}:arm64-$TODAY \
-#          -f Dockerfile_builder \
-#          --build-arg="IMAGE=ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main" \
-#          --no-cache \
-#          .
-#
-#    {{docker}} push {{image}}:amd64-$TODAY
-#    {{docker}} push {{image}}:arm64-$TODAY
-
 # makes sure everything is fine
 is-clean:
     #!/usr/bin/env bash
     set -euxo pipefail
 
     # exit early if clippy emits warnings
-    {{ db_url_sqlite }} cargo clippy -- -D warnings
-    {{ db_url_postgres }} cargo clippy --features postgres -- -D warnings
+    cargo clippy postgres -- -D warnings
 
     # make sure everything has been committed
     git diff --exit-code
@@ -537,7 +420,7 @@ release:
     git push origin "v$TAG"
 
 # publishes the application images - full pipeline incl clippy and testing  you can provide a custom image name as variable
-publish: build-docs fmt build
+publish: build-docs fmt test-hiqlite test-postgres build
     #!/usr/bin/env bash
     set -euxo pipefail
 
@@ -552,13 +435,13 @@ publish-latest:
     {{ docker }} push ghcr.io/sebadob/rauthy:latest
 
 # should be run before submitting a PR to make sure everything is fine
-pre-pr-checks: build-ui fmt test-sqlite test-postgres clippy clippy-postgres
+pre-pr-checks: build-ui fmt test-hiqlite test-postgres clippy clippy
     #!/usr/bin/env bash
     set -euxo pipefail
 
 # does a `cargo update` + `npm update` for the UI
 update-deps:
     #!/usr/bin/env bash
-    {{ db_url_sqlite }} cargo update
+    cargo update
     cd frontend
     {{ npm }} update
diff --git a/migrations/hiqlite/1_init_schema.sql b/migrations/hiqlite/1_init_schema.sql
@@ -136,7 +136,7 @@ CREATE TABLE events
     level     INTEGER NOT NULL,
     ip        TEXT,
     data      INTEGER,
-    TEXT      TEXT,
+    text      TEXT,
     CONSTRAINT events_pk
         PRIMARY KEY (id, timestamp)
 ) STRICT;

diff --git a/rauthy.cfg b/rauthy.cfg
@@ -406,11 +406,12 @@ HQL_NODES="
 #
 #DATABASE_URL=sqlite::memory:
 #DATABASE_URL=sqlite:data/rauthy.db
+DATABASE_URL=postgresql://rauthy:123SuperSafe@localhost:5432/rauthy
 
 # Max DB connections - irrelevant for SQLite (default: 5)
 #DATABASE_MAX_CONN=5
 
-# If specified, the current Database, set with DATABASE_URL, will be DELETED and OVERWRITTEN with a migration from the
+# If specified, the currently configured Database will be DELETED and OVERWRITTEN with a migration from the
 # given database with this variable. Can be used to migrate between different databases.
 # !!! USE WITH CARE !!!
 #MIGRATE_DB_FROM=sqlite:data/rauthy.db
@@ -421,7 +422,7 @@ HQL_NODES="
 #
 # To use the EXPERIMENTAL Hiqlite database, set to `true`
 # default: false
-#HIQLITE=true
+HIQLITE=true
 
 # The data dir hiqlite will store raft logs and state machine data in.
 # default: data

diff --git a/src/api/src/api_keys.rs b/src/api/src/api_keys.rs
@@ -55,15 +55,14 @@ pub async fn get_api_keys(
 )]
 #[post("/api_keys")]
 pub async fn post_api_key(
-    data: web::Data<AppState>,
     principal: ReqPrincipal,
     payload: Json<ApiKeyRequest>,
 ) -> Result<HttpResponse, ErrorResponse> {
     principal.validate_admin_session()?;
 
     let req = payload.into_inner();
     let access = req.access.into_iter().map(|a| a.into()).collect();
-    let secret = ApiKeyEntity::create(&data.db, req.name, req.exp, access).await?;
+    let secret = ApiKeyEntity::create(req.name, req.exp, access).await?;
 
     Ok(HttpResponse::Ok()
         .content_type(TEXT_PLAIN_UTF_8)

diff --git a/src/api/src/generic.rs b/src/api/src/generic.rs
@@ -16,6 +16,7 @@ use rauthy_common::constants::{
 use rauthy_common::utils::real_ip_from_req;
 use rauthy_error::ErrorResponse;
 use rauthy_models::app_state::AppState;
+use rauthy_models::database::{Cache, DB};
 use rauthy_models::entity::api_keys::{AccessGroup, AccessRights};
 use rauthy_models::entity::app_version::LatestAppVersion;
 use rauthy_models::entity::auth_providers::AuthProviderTemplate;
@@ -27,7 +28,6 @@ use rauthy_models::entity::sessions::Session;
 use rauthy_models::entity::users::User;
 use rauthy_models::events::event::Event;
 use rauthy_models::events::ip_blacklist_handler::{IpBlacklist, IpBlacklistReq};
-use rauthy_models::hiqlite::{Cache, DB};
 use rauthy_models::i18n::account::I18nAccount;
 use rauthy_models::i18n::authorize::I18nAuthorize;
 use rauthy_models::i18n::device::I18nDevice;

diff --git a/src/bin/Cargo.toml b/src/bin/Cargo.toml
@@ -5,10 +5,6 @@ edition.workspace = true
 authors.workspace = true
 license.workspace = true
 
-[features]
-default = []
-postgres = []
-
 [dependencies]
 rauthy-common = { path = "../common" }
 rauthy-error = { path = "../error" }