From 62e62359198fc718e583516095214a527dd10554 Mon Sep 17 00:00:00 2001 From: Mats Kindahl Date: Thu, 28 Sep 2023 17:22:03 +0200 Subject: [PATCH 01/32] Restart scheduler on error If the scheduler receives an error, it will never restart again since `bgw_restart_time` is set to `BGW_NEVER_RESTART`, which will prevent all jobs from executing. This commit adds the GUC `timescaledb.bgw_scheduler_restart_time` that can be set to the restart time for the scheduler. It defaults to 60 seconds, which is the default restart interval for background workers defined by PostgreSQL. It also adds `timescaledb.debug_bgw_scheduler_exit_status` to be able to shutdown the scheduler with a non-zero exit status, which allows the restart functionality to be tested. It also ensures that `backend_type` is explicitly set up rather than copied from `application_name` and add some more information to `application_name`. It also updates the tests to use `backend_type` where applicable. To avoid exhausting slots when the launcher restarts, it will kill all existing schedulers and start new ones. Since background worker slots are easily exhausted in the `bgw_launcher` test, we do not run it repeatedly in the flakes workflow. --- .github/gh_matrix_builder.py | 17 +- .unreleased/pr_7527 | 1 + src/bgw/scheduler.c | 14 +- src/compat/compat.h | 6 + src/guc.c | 27 +++ src/guc.h | 6 + src/loader/bgw_launcher.c | 121 ++++++++++++- src/loader/bgw_launcher.h | 7 +- src/loader/loader.c | 2 +- test/expected/bgw_launcher.out | 42 +++-- test/pgtest/CMakeLists.txt | 3 + test/sql/bgw_launcher.sql | 20 +- test/sql/include/bgw_launcher_utils.sql | 11 +- tsl/test/expected/bgw_scheduler_control.out | 2 +- tsl/test/expected/bgw_scheduler_restart.out | 191 ++++++++++++++++++++ tsl/test/sql/CMakeLists.txt | 2 + tsl/test/sql/bgw_scheduler_restart.sql | 135 ++++++++++++++ 17 files changed, 552 insertions(+), 55 deletions(-) create mode 100644 .unreleased/pr_7527 create mode 100644 tsl/test/expected/bgw_scheduler_restart.out create mode 100644 tsl/test/sql/bgw_scheduler_restart.sql diff --git a/.github/gh_matrix_builder.py b/.github/gh_matrix_builder.py index 441536753f8..f50c1176eae 100755 --- a/.github/gh_matrix_builder.py +++ b/.github/gh_matrix_builder.py @@ -54,6 +54,16 @@ "memoize", } +# Tests that we do not run as part of a Flake tests +flaky_exclude_tests = { + # Not executed as a flake test since it easily exhausts available + # background worker slots. + "bgw_launcher", + # Not executed as a flake test since it takes a very long time and + # easily interferes with other tests. + "bgw_scheduler_restart", +} + # helper functions to generate matrix entries # the release and apache config inherit from the @@ -306,13 +316,14 @@ def macos_config(overrides): sys.exit(1) if tests: - to_run = list(tests) * 20 + to_run = [t for t in list(tests) if t not in flaky_exclude_tests] * 20 random.shuffle(to_run) + installcheck_args = f'TESTS="{" ".join(to_run)}"' m["include"].append( build_debug_config( { "coverage": False, - "installcheck_args": f'TESTS="{" ".join(to_run)}"', + "installcheck_args": installcheck_args, "name": "Flaky Check Debug", "pg": PG16_LATEST, "pginstallcheck": False, @@ -323,7 +334,7 @@ def macos_config(overrides): build_debug_config( { "coverage": False, - "installcheck_args": f'TESTS="{" ".join(to_run)}"', + "installcheck_args": installcheck_args, "name": "Flaky Check Debug", "pg": PG17_LATEST, "pginstallcheck": False, diff --git a/.unreleased/pr_7527 b/.unreleased/pr_7527 new file mode 100644 index 00000000000..534b8bec629 --- /dev/null +++ b/.unreleased/pr_7527 @@ -0,0 +1 @@ +Fixes: #7527 Restart scheduler on error diff --git a/src/bgw/scheduler.c b/src/bgw/scheduler.c index d44fc59d433..fefe4cf2602 100644 --- a/src/bgw/scheduler.c +++ b/src/bgw/scheduler.c @@ -795,9 +795,11 @@ ts_bgw_scheduler_process(int32 run_for_interval_ms, * exit. */ if (ts_guc_restoring || IsBinaryUpgrade) { - elog(LOG, - "scheduler for database %u exiting since the database is restoring or upgrading", - MyDatabaseId); + ereport(LOG, + errmsg("scheduler for database %u exiting with exit status %d", + MyDatabaseId, + ts_debug_bgw_scheduler_exit_status), + errdetail("the database is restoring or upgrading")); terminate_all_jobs_and_release_workers(); goto scheduler_exit; } @@ -866,7 +868,10 @@ ts_bgw_scheduler_process(int32 run_for_interval_ms, MemoryContextReset(scratch_mctx); } - elog(DEBUG1, "database scheduler for database %u exiting", MyDatabaseId); + elog(DEBUG1, + "scheduler for database %u exiting with exit status %d", + MyDatabaseId, + ts_debug_bgw_scheduler_exit_status); #ifdef TS_DEBUG if (ts_shutdown_bgw) @@ -879,6 +884,7 @@ ts_bgw_scheduler_process(int32 run_for_interval_ms, wait_for_all_jobs_to_shutdown(); check_for_stopped_and_timed_out_jobs(); scheduled_jobs = NIL; + proc_exit(ts_debug_bgw_scheduler_exit_status); } static void diff --git a/src/compat/compat.h b/src/compat/compat.h index 298f74fb716..58654de9f9e 100644 --- a/src/compat/compat.h +++ b/src/compat/compat.h @@ -655,6 +655,12 @@ RelationGetSmgr(Relation rel) GenerationContextCreate(parent, name, blockSize) #endif +#if PG16_GE +#define pgstat_get_local_beentry_by_index_compat(idx) pgstat_get_local_beentry_by_index(idx) +#else +#define pgstat_get_local_beentry_by_index_compat(idx) pgstat_fetch_stat_local_beentry(idx) +#endif + /* * PG16 adds a new parameter to DefineIndex, total_parts, that takes * in the total number of direct and indirect partitions of the relation. diff --git a/src/guc.c b/src/guc.c index c71035a2979..2f3c488421d 100644 --- a/src/guc.c +++ b/src/guc.c @@ -183,6 +183,20 @@ bool ts_guc_debug_require_batch_sorted_merge = false; bool ts_guc_debug_allow_cagg_with_deprecated_funcs = false; +/* + * Exit code for the scheduler. + * + * Normally it exits with a zero which means that it will not restart. If an + * error is raised, it exits with error code 1, which will trigger a + * restart. + * + * This variable exists to be able to trigger a restart for a normal exit, + * which is useful when debugging. + * + * See backend/postmaster/bgworker.c + */ +int ts_debug_bgw_scheduler_exit_status = 0; + #ifdef TS_DEBUG bool ts_shutdown_bgw = false; char *ts_current_timestamp_mock = NULL; @@ -1055,6 +1069,19 @@ _guc_init(void) NULL, NULL); + DefineCustomIntVariable(/* name= */ MAKE_EXTOPTION("debug_bgw_scheduler_exit_status"), + /* short_desc= */ "exit status to use when shutting down the scheduler", + /* long_desc= */ "this is for debugging purposes", + /* valueAddr= */ &ts_debug_bgw_scheduler_exit_status, + /* bootValue= */ 0, + /* minValue= */ 0, + /* maxValue= */ 255, + /* context= */ PGC_SIGHUP, + /* flags= */ 0, + /* check_hook= */ NULL, + /* assign_hook= */ NULL, + /* show_hook= */ NULL); + #ifdef TS_DEBUG DefineCustomBoolVariable(/* name= */ MAKE_EXTOPTION("shutdown_bgw_scheduler"), /* short_desc= */ "immediately shutdown the bgw scheduler", diff --git a/src/guc.h b/src/guc.h index 34ebc0ef2d6..54a341b7a3b 100644 --- a/src/guc.h +++ b/src/guc.h @@ -92,6 +92,12 @@ extern TSDLLEXPORT bool ts_guc_auto_sparse_indexes; extern TSDLLEXPORT bool ts_guc_enable_columnarscan; extern TSDLLEXPORT int ts_guc_bgw_log_level; +/* + * Exit code to use when scheduler exits. + * + * Used for debugging. + */ +extern TSDLLEXPORT int ts_debug_bgw_scheduler_exit_status; #ifdef TS_DEBUG extern bool ts_shutdown_bgw; extern char *ts_current_timestamp_mock; diff --git a/src/loader/bgw_launcher.c b/src/loader/bgw_launcher.c index af2a2cc3e13..02daa180b64 100644 --- a/src/loader/bgw_launcher.c +++ b/src/loader/bgw_launcher.c @@ -84,6 +84,8 @@ typedef enum SchedulerState static volatile sig_atomic_t got_SIGHUP = false; +int ts_guc_bgw_scheduler_restart_time_sec = BGW_DEFAULT_RESTART_INTERVAL; + static void launcher_sighup(SIGNAL_ARGS) { @@ -238,13 +240,27 @@ terminate_background_worker(BackgroundWorkerHandle *handle) } extern void -ts_bgw_cluster_launcher_register(void) +ts_bgw_cluster_launcher_init(void) { BackgroundWorker worker; + DefineCustomIntVariable(/* name= */ MAKE_EXTOPTION("bgw_scheduler_restart_time"), + /* short_desc= */ "Restart time for scheduler in seconds", + /* long_desc= */ + "The number of seconds until the scheduler restart on failure.", + /* valueAddr= */ &ts_guc_bgw_scheduler_restart_time_sec, + /* bootValue= */ BGW_DEFAULT_RESTART_INTERVAL, + /* minValue= */ 1, + /* maxValue= */ 3600, + /* context= */ PGC_SIGHUP, + /* flags= */ GUC_UNIT_S, + /* check_hook= */ NULL, + /* assign_hook= */ NULL, + /* show_hook= */ NULL); + memset(&worker, 0, sizeof(worker)); /* set up worker settings for our main worker */ - snprintf(worker.bgw_name, BGW_MAXLEN, "TimescaleDB Background Worker Launcher"); + snprintf(worker.bgw_name, BGW_MAXLEN, TS_BGW_TYPE_LAUNCHER); worker.bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION; worker.bgw_restart_time = BGW_LAUNCHER_RESTART_TIME_S; @@ -274,9 +290,10 @@ register_entrypoint_for_db(Oid db_id, VirtualTransactionId vxid, BackgroundWorke BackgroundWorker worker; memset(&worker, 0, sizeof(worker)); - snprintf(worker.bgw_name, BGW_MAXLEN, "TimescaleDB Background Worker Scheduler"); + snprintf(worker.bgw_type, BGW_MAXLEN, TS_BGW_TYPE_SCHEDULER); + snprintf(worker.bgw_name, BGW_MAXLEN, "%s for database %d", TS_BGW_TYPE_SCHEDULER, db_id); worker.bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION; - worker.bgw_restart_time = BGW_NEVER_RESTART; + worker.bgw_restart_time = ts_guc_bgw_scheduler_restart_time_sec, worker.bgw_start_time = BgWorkerStart_RecoveryFinished; snprintf(worker.bgw_library_name, BGW_MAXLEN, EXTENSION_NAME); snprintf(worker.bgw_function_name, BGW_MAXLEN, BGW_ENTRYPOINT_FUNCNAME); @@ -332,15 +349,89 @@ db_hash_entry_create_if_not_exists(HTAB *db_htab, Oid db_oid) return db_he; } +/* + * Result from signalling a backend. + * + * Error codes are non-zero, and success is zero. + */ +enum SignalBackendResult +{ + SIGNAL_BACKEND_SUCCESS = 0, + SIGNAL_BACKEND_ERROR, + SIGNAL_BACKEND_NOPERMISSION, + SIGNAL_BACKEND_NOSUPERUSER, +}; + +/* + * Terminate a background worker. + * + * This is copied from pg_signal_backend() in + * src/backend/storage/ipc/signalfuncs.c but tweaked to not require a database + * connection since the launcher does not have one. + */ +static enum SignalBackendResult +ts_signal_backend(int pid, int sig) +{ + PGPROC *proc = BackendPidGetProc(pid); + + if (unlikely(proc == NULL)) + { + ereport(WARNING, (errmsg("PID %d is not a PostgreSQL backend process", pid))); + return SIGNAL_BACKEND_ERROR; + } + + if (unlikely(kill(pid, sig))) + { + /* Again, just a warning to allow loops */ + ereport(WARNING, (errmsg("could not send signal to process %d: %m", pid))); + return SIGNAL_BACKEND_ERROR; + } + + return SIGNAL_BACKEND_SUCCESS; +} + +/* + * Terminate backends by backend type. + * + * We iterate through all backends and mark those that match the given backend + * type as terminated. + * + * Note that there is potentially a delay between marking backends as + * terminated and their actual termination, so the backends have to be able to + * run even if there are multiple instances accessing the same data. + * + * Parts of this code is taken from pg_stat_get_activity() in + * src/backend/utils/adt/pgstatfuncs.c. + */ +static void +terminate_backends_by_backend_type(const char *backend_type) +{ + Assert(backend_type); + + const int num_backends = pgstat_fetch_stat_numbackends(); + for (int curr_backend = 1; curr_backend <= num_backends; ++curr_backend) + { + const LocalPgBackendStatus *local_beentry = + pgstat_get_local_beentry_by_index_compat(curr_backend); + const PgBackendStatus *beentry = &local_beentry->backendStatus; + const char *bgw_type = GetBackgroundWorkerTypeByPid(beentry->st_procpid); + if (bgw_type && strcmp(backend_type, bgw_type) == 0) + { + int error = ts_signal_backend(beentry->st_procpid, SIGTERM); + if (error) + elog(LOG, "failed to terminate backend with pid %d", beentry->st_procpid); + } + } +} + /* * Model this on autovacuum.c -> get_database_list. * - * Note that we are not doing - * all the things around memory context that they do, because the hashtable - * we're using to store db entries is automatically created in its own memory - * context (a child of TopMemoryContext) This can get called at two different - * times 1) when the cluster launcher starts and is looking for dbs and 2) if - * it restarts due to a postmaster signal. + * Note that we are not doing all the things around memory context that they + * do, because the hashtable we're using to store db entries is automatically + * created in its own memory context (a child of TopMemoryContext) This can + * get called at two different times 1) when the cluster launcher starts and + * is looking for dbs and 2) if it restarts due to a postmaster signal. */ static void populate_database_htab(HTAB *db_htab) @@ -757,6 +848,16 @@ ts_bgw_cluster_launcher_main(PG_FUNCTION_ARGS) db_htab = init_database_htab(); *htab_storage = db_htab; + /* + * If the launcher was restarted and discovers old schedulers, these has + * to be terminated to avoid exhausting the worker slots. + * + * We cannot easily pick up the old schedulers since we do not have access + * to the slots array in PostgreSQL, so instead we scan for something that + * looks like schedulers for databases, and kill them. New ones will then + * be spawned below. + */ + terminate_backends_by_backend_type(TS_BGW_TYPE_SCHEDULER); populate_database_htab(db_htab); while (true) diff --git a/src/loader/bgw_launcher.h b/src/loader/bgw_launcher.h index f90a65cb3a9..82c2ec1893b 100644 --- a/src/loader/bgw_launcher.h +++ b/src/loader/bgw_launcher.h @@ -8,7 +8,12 @@ #include #include -extern void ts_bgw_cluster_launcher_register(void); +#define TS_BGW_TYPE_LAUNCHER "TimescaleDB Background Worker Launcher" +#define TS_BGW_TYPE_SCHEDULER "TimescaleDB Background Worker Scheduler" + +extern int ts_guc_bgw_scheduler_restart_time_sec; + +extern void ts_bgw_cluster_launcher_init(void); /*called by postmaster at launcher bgw startup*/ TSDLLEXPORT extern Datum ts_bgw_cluster_launcher_main(PG_FUNCTION_ARGS); diff --git a/src/loader/loader.c b/src/loader/loader.c index 6537e49d8c5..fdff65236fa 100644 --- a/src/loader/loader.c +++ b/src/loader/loader.c @@ -591,7 +591,7 @@ _PG_init(void) timescaledb_shmem_request_hook(); #endif - ts_bgw_cluster_launcher_register(); + ts_bgw_cluster_launcher_init(); ts_bgw_counter_setup_gucs(); ts_bgw_interface_register_api_version(); diff --git a/test/expected/bgw_launcher.out b/test/expected/bgw_launcher.out index 84c2b30bf5a..ff8a61559b6 100644 --- a/test/expected/bgw_launcher.out +++ b/test/expected/bgw_launcher.out @@ -21,11 +21,12 @@ CREATE DATABASE :TEST_DBNAME_2; -- Further Note: PG 9.6 changed what appeared in pg_stat_activity, so the launcher doesn't actually show up. -- we can still test its interactions with its children, but can't test some of the things specific to the launcher. -- So we've added some bits about the version number as needed. -CREATE VIEW worker_counts as SELECT count(*) filter (WHERE application_name = 'TimescaleDB Background Worker Launcher') as launcher, -count(*) filter (WHERE application_name = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME') as single_scheduler, -count(*) filter (WHERE application_name = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME_2') as single_2_scheduler, -count(*) filter (WHERE application_name = 'TimescaleDB Background Worker Scheduler' AND datname = 'template1') as template1_scheduler -FROM pg_stat_activity; +CREATE VIEW worker_counts as +SELECT count(*) filter (WHERE backend_type = 'TimescaleDB Background Worker Launcher') as launcher, + count(*) filter (WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME') as single_scheduler, + count(*) filter (WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME_2') as single_2_scheduler, + count(*) filter (WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = 'template1') as template1_scheduler + FROM pg_stat_activity; CREATE FUNCTION wait_worker_counts(launcher_ct INTEGER, scheduler1_ct INTEGER, scheduler2_ct INTEGER, template1_ct INTEGER) RETURNS BOOLEAN LANGUAGE PLPGSQL AS $BODY$ DECLARE @@ -103,7 +104,7 @@ SELECT wait_worker_counts(1,0,1,0); -- Now let's restart the scheduler in test db 2 and make sure our backend_start changed SELECT backend_start as orig_backend_start FROM pg_stat_activity -WHERE application_name = 'TimescaleDB Background Worker Scheduler' +WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME_2' \gset -- We'll do this in a txn so that we can see that the worker locks on our txn before continuing BEGIN; @@ -122,7 +123,7 @@ SELECT wait_worker_counts(1,0,1,0); SELECT (backend_start > :'orig_backend_start'::timestamptz) backend_start_changed, (wait_event = 'virtualxid') wait_event_changed FROM pg_stat_activity -WHERE application_name = 'TimescaleDB Background Worker Scheduler' +WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME_2'; backend_start_changed | wait_event_changed -----------------------+-------------------- @@ -138,7 +139,7 @@ SELECT wait_worker_counts(1,0,1,0); SELECT (wait_event IS DISTINCT FROM 'virtualxid') wait_event_changed FROM pg_stat_activity -WHERE application_name = 'TimescaleDB Background Worker Scheduler' +WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME_2'; wait_event_changed -------------------- @@ -187,7 +188,7 @@ SELECT wait_worker_counts(1,0,1,0); -- make sure start is idempotent SELECT backend_start as orig_backend_start FROM pg_stat_activity -WHERE application_name = 'TimescaleDB Background Worker Scheduler' +WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME_2' \gset -- Since we're doing idempotency tests, we're also going to exercise our queue and start 20 times SELECT _timescaledb_functions.start_background_workers() as start_background_workers, * FROM generate_series(1,20); @@ -227,7 +228,7 @@ FOR i in 1..5 LOOP SELECT (backend_start = $1::timestamptz) backend_start_unchanged FROM pg_stat_activity -WHERE application_name = 'TimescaleDB Background Worker Scheduler' +WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = $2 into r; if(r) THEN PERFORM pg_sleep(0.1); @@ -274,7 +275,7 @@ SELECT wait_worker_counts(1,0,1,0); -- Now let's restart the scheduler and make sure our backend_start changed SELECT backend_start as orig_backend_start FROM pg_stat_activity -WHERE application_name = 'TimescaleDB Background Worker Scheduler' +WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME_2' \gset BEGIN; DROP EXTENSION timescaledb; @@ -294,7 +295,7 @@ FOR i in 1..10 LOOP SELECT (backend_start > $1::timestamptz) backend_start_changed FROM pg_stat_activity -WHERE application_name = 'TimescaleDB Background Worker Scheduler' +WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = $2 into r; if(NOT r) THEN PERFORM pg_sleep(0.1); @@ -315,9 +316,9 @@ SELECT wait_greater(:'orig_backend_start',:'TEST_DBNAME_2'); -- Make sure canceling the launcher backend causes a restart of schedulers SELECT backend_start as orig_backend_start FROM pg_stat_activity -WHERE application_name = 'TimescaleDB Background Worker Scheduler' +WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME_2' \gset -SELECT pg_cancel_backend(pid) FROM pg_stat_activity WHERE application_name = 'TimescaleDB Background Worker Launcher'; +SELECT pg_cancel_backend(pid) FROM pg_stat_activity WHERE backend_type = 'TimescaleDB Background Worker Launcher'; pg_cancel_backend ------------------- t @@ -445,11 +446,12 @@ ALTER ROLE :ROLE_DEFAULT_PERM_USER WITH NOSUPERUSER; -- Further Note: PG 9.6 changed what appeared in pg_stat_activity, so the launcher doesn't actually show up. -- we can still test its interactions with its children, but can't test some of the things specific to the launcher. -- So we've added some bits about the version number as needed. -CREATE VIEW worker_counts as SELECT count(*) filter (WHERE application_name = 'TimescaleDB Background Worker Launcher') as launcher, -count(*) filter (WHERE application_name = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME') as single_scheduler, -count(*) filter (WHERE application_name = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME_2') as single_2_scheduler, -count(*) filter (WHERE application_name = 'TimescaleDB Background Worker Scheduler' AND datname = 'template1') as template1_scheduler -FROM pg_stat_activity; +CREATE VIEW worker_counts as +SELECT count(*) filter (WHERE backend_type = 'TimescaleDB Background Worker Launcher') as launcher, + count(*) filter (WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME') as single_scheduler, + count(*) filter (WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME_2') as single_2_scheduler, + count(*) filter (WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = 'template1') as template1_scheduler + FROM pg_stat_activity; CREATE FUNCTION wait_worker_counts(launcher_ct INTEGER, scheduler1_ct INTEGER, scheduler2_ct INTEGER, template1_ct INTEGER) RETURNS BOOLEAN LANGUAGE PLPGSQL AS $BODY$ DECLARE @@ -602,7 +604,7 @@ SELECT _timescaledb_functions.stop_background_workers(); t (1 row) -SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE application_name = 'TimescaleDB Background Worker Launcher'; +SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE backend_type = 'TimescaleDB Background Worker Launcher'; pg_terminate_backend ---------------------- t diff --git a/test/pgtest/CMakeLists.txt b/test/pgtest/CMakeLists.txt index 2d65a493eb2..8d7dc3779c1 100644 --- a/test/pgtest/CMakeLists.txt +++ b/test/pgtest/CMakeLists.txt @@ -29,6 +29,9 @@ set(PG_IGNORE_TESTS sanity_check type_sanity create_am + # Ignoring because it spawns different number of workers in different + # versions. + select_parallel psql) # Modify the test schedule to ignore some tests diff --git a/test/sql/bgw_launcher.sql b/test/sql/bgw_launcher.sql index db3d240c6fb..19a01789667 100644 --- a/test/sql/bgw_launcher.sql +++ b/test/sql/bgw_launcher.sql @@ -33,7 +33,7 @@ SELECT wait_worker_counts(1,0,1,0); -- Now let's restart the scheduler in test db 2 and make sure our backend_start changed SELECT backend_start as orig_backend_start FROM pg_stat_activity -WHERE application_name = 'TimescaleDB Background Worker Scheduler' +WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME_2' \gset -- We'll do this in a txn so that we can see that the worker locks on our txn before continuing BEGIN; @@ -43,14 +43,14 @@ SELECT wait_worker_counts(1,0,1,0); SELECT (backend_start > :'orig_backend_start'::timestamptz) backend_start_changed, (wait_event = 'virtualxid') wait_event_changed FROM pg_stat_activity -WHERE application_name = 'TimescaleDB Background Worker Scheduler' +WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME_2'; COMMIT; SELECT wait_worker_counts(1,0,1,0); SELECT (wait_event IS DISTINCT FROM 'virtualxid') wait_event_changed FROM pg_stat_activity -WHERE application_name = 'TimescaleDB Background Worker Scheduler' +WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME_2'; -- Test stop @@ -68,7 +68,7 @@ SELECT wait_worker_counts(1,0,1,0); -- make sure start is idempotent SELECT backend_start as orig_backend_start FROM pg_stat_activity -WHERE application_name = 'TimescaleDB Background Worker Scheduler' +WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME_2' \gset -- Since we're doing idempotency tests, we're also going to exercise our queue and start 20 times @@ -85,7 +85,7 @@ FOR i in 1..5 LOOP SELECT (backend_start = $1::timestamptz) backend_start_unchanged FROM pg_stat_activity -WHERE application_name = 'TimescaleDB Background Worker Scheduler' +WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = $2 into r; if(r) THEN PERFORM pg_sleep(0.1); @@ -109,7 +109,7 @@ SELECT wait_worker_counts(1,0,1,0); -- Now let's restart the scheduler and make sure our backend_start changed SELECT backend_start as orig_backend_start FROM pg_stat_activity -WHERE application_name = 'TimescaleDB Background Worker Scheduler' +WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME_2' \gset BEGIN; @@ -126,7 +126,7 @@ FOR i in 1..10 LOOP SELECT (backend_start > $1::timestamptz) backend_start_changed FROM pg_stat_activity -WHERE application_name = 'TimescaleDB Background Worker Scheduler' +WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = $2 into r; if(NOT r) THEN PERFORM pg_sleep(0.1); @@ -143,10 +143,10 @@ SELECT wait_greater(:'orig_backend_start',:'TEST_DBNAME_2'); -- Make sure canceling the launcher backend causes a restart of schedulers SELECT backend_start as orig_backend_start FROM pg_stat_activity -WHERE application_name = 'TimescaleDB Background Worker Scheduler' +WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME_2' \gset -SELECT pg_cancel_backend(pid) FROM pg_stat_activity WHERE application_name = 'TimescaleDB Background Worker Launcher'; +SELECT pg_cancel_backend(pid) FROM pg_stat_activity WHERE backend_type = 'TimescaleDB Background Worker Launcher'; SELECT wait_worker_counts(1,0,1,0); @@ -259,7 +259,7 @@ SELECT wait_for_bgw_scheduler(:'TEST_DBNAME'); -- Connect to TEST_DBNAME (_timescaledb_functions.stop_background_workers() is not available in TEST_DBNAME_2) \c :TEST_DBNAME :ROLE_SUPERUSER SELECT _timescaledb_functions.stop_background_workers(); -SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE application_name = 'TimescaleDB Background Worker Launcher'; +SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE backend_type = 'TimescaleDB Background Worker Launcher'; \c :TEST_DBNAME_2 :ROLE_SUPERUSER -- make sure nobody is using it diff --git a/test/sql/include/bgw_launcher_utils.sql b/test/sql/include/bgw_launcher_utils.sql index de06a853814..3e52778fc02 100644 --- a/test/sql/include/bgw_launcher_utils.sql +++ b/test/sql/include/bgw_launcher_utils.sql @@ -8,11 +8,12 @@ -- we can still test its interactions with its children, but can't test some of the things specific to the launcher. -- So we've added some bits about the version number as needed. -CREATE VIEW worker_counts as SELECT count(*) filter (WHERE application_name = 'TimescaleDB Background Worker Launcher') as launcher, -count(*) filter (WHERE application_name = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME') as single_scheduler, -count(*) filter (WHERE application_name = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME_2') as single_2_scheduler, -count(*) filter (WHERE application_name = 'TimescaleDB Background Worker Scheduler' AND datname = 'template1') as template1_scheduler -FROM pg_stat_activity; +CREATE VIEW worker_counts as +SELECT count(*) filter (WHERE backend_type = 'TimescaleDB Background Worker Launcher') as launcher, + count(*) filter (WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME') as single_scheduler, + count(*) filter (WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME_2') as single_2_scheduler, + count(*) filter (WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = 'template1') as template1_scheduler + FROM pg_stat_activity; CREATE FUNCTION wait_worker_counts(launcher_ct INTEGER, scheduler1_ct INTEGER, scheduler2_ct INTEGER, template1_ct INTEGER) RETURNS BOOLEAN LANGUAGE PLPGSQL AS $BODY$ diff --git a/tsl/test/expected/bgw_scheduler_control.out b/tsl/test/expected/bgw_scheduler_control.out index 8ed411971f4..bb55cb2771b 100644 --- a/tsl/test/expected/bgw_scheduler_control.out +++ b/tsl/test/expected/bgw_scheduler_control.out @@ -99,7 +99,7 @@ SELECT * FROM cleaned_bgw_log; 4 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | test_job_1b | Execute job 1 1 | test_job_1b | job 1000 (test_job_1b) exiting with success: execution time (RANDOM) ms - 5 | DB Scheduler | database scheduler for database (RANDOM) exiting + 5 | DB Scheduler | scheduler for database (RANDOM) exiting with exit status 0 (8 rows) ALTER DATABASE :TEST_DBNAME RESET timescaledb.bgw_log_level; diff --git a/tsl/test/expected/bgw_scheduler_restart.out b/tsl/test/expected/bgw_scheduler_restart.out new file mode 100644 index 00000000000..98245c40c4c --- /dev/null +++ b/tsl/test/expected/bgw_scheduler_restart.out @@ -0,0 +1,191 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. +\c :TEST_DBNAME :ROLE_SUPERUSER +CREATE VIEW tsdb_bgw AS + SELECT datname, pid, backend_type, application_name + FROM pg_stat_activity + WHERE backend_type LIKE '%TimescaleDB%' + ORDER BY datname, backend_type, application_name; +-- Wait for at least one background worker matching pattern to have +-- started. +CREATE PROCEDURE wait_for_some_started( + min_time double precision, + timeout double precision, + pattern text +) AS $$ +DECLARE + backend_count int; +BEGIN + FOR i IN 0..(timeout / min_time)::int + LOOP + PERFORM pg_sleep(min_time); + SELECT count(*) INTO backend_count FROM tsdb_bgw WHERE backend_type LIKE pattern; + IF backend_count > 0 THEN + RETURN; + END IF; + END LOOP; + RAISE EXCEPTION 'backend matching % did not start before timeout', pattern; +END; +$$ LANGUAGE plpgsql; +-- Wait for the number of background workers matching pattern to be +-- zero. +CREATE PROCEDURE wait_for_all_stopped( + min_time double precision, + timeout double precision, + pattern text +) AS $$ +DECLARE + backend_count int; +BEGIN + FOR i IN 0..(timeout / min_time)::int + LOOP + PERFORM pg_sleep(min_time); + SELECT count(*) INTO backend_count FROM tsdb_bgw WHERE backend_type LIKE pattern; + IF backend_count = 0 THEN + RETURN; + END IF; + END LOOP; + RAISE EXCEPTION 'backend matching % did not start before timeout', pattern; +END; +$$ LANGUAGE plpgsql; +-- Show the default scheduler restart time +SHOW timescaledb.bgw_scheduler_restart_time; + timescaledb.bgw_scheduler_restart_time +---------------------------------------- + 1min +(1 row) + +ALTER SYSTEM SET timescaledb.bgw_scheduler_restart_time TO '10s'; +ALTER SYSTEM SET timescaledb.debug_bgw_scheduler_exit_status TO 1; +SELECT pg_reload_conf(); + pg_reload_conf +---------------- + t +(1 row) + +SHOW timescaledb.bgw_scheduler_restart_time; + timescaledb.bgw_scheduler_restart_time +---------------------------------------- + 1min +(1 row) + +-- Launcher is running, so we need to restart it for the scheduler +-- restart time to take effect. +SELECT datname, application_name FROM tsdb_bgw; + datname | application_name +---------+---------------------------------------- + | TimescaleDB Background Worker Launcher +(1 row) + +SELECT pg_terminate_backend(pid) FROM tsdb_bgw + WHERE backend_type LIKE '%Launcher%'; + pg_terminate_backend +---------------------- + t +(1 row) + +-- It will restart automatically, but we wait for it to start. +CALL wait_for_some_started(1, 50, '%Launcher%'); +-- Verify that launcher is running. If it is not, the rest of the test +-- will fail. +SELECT datname, application_name FROM tsdb_bgw; + datname | application_name +--------------------------+----------------------------------------- + db_bgw_scheduler_restart | TimescaleDB Background Worker Scheduler + | TimescaleDB Background Worker Launcher +(2 rows) + +-- Now we can start the background workers. +SELECT _timescaledb_functions.start_background_workers(); + start_background_workers +-------------------------- + t +(1 row) + +-- They should start immediately, but let's wait for them to start. +CALL wait_for_some_started(1, 50, '%Scheduler%'); +-- Check that the schedulers are running. If they are not, the rest of +-- the test is meaningless. +SELECT datname, application_name FROM tsdb_bgw; + datname | application_name +--------------------------+----------------------------------------- + db_bgw_scheduler_restart | TimescaleDB Background Worker Scheduler + | TimescaleDB Background Worker Launcher +(2 rows) + +-- Kill the schedulers and check that they restart. +SELECT pg_terminate_backend(pid) FROM tsdb_bgw + WHERE datname = :'TEST_DBNAME' AND backend_type LIKE '%Scheduler%'; + pg_terminate_backend +---------------------- + t +(1 row) + +-- Wait for scheduler to exit, they should exit immediately. +CALL wait_for_all_stopped(1, 50, '%Scheduler%'); +-- Check that the schedulers really exited. +SELECT datname, application_name FROM tsdb_bgw; + datname | application_name +---------+---------------------------------------- + | TimescaleDB Background Worker Launcher +(1 row) + +-- Wait for scheduler to restart. +CALL wait_for_some_started(10, 100, '%Scheduler%'); +-- Make sure that the launcher and schedulers are running. Otherwise +-- the test will fail. +SELECT datname, application_name FROM tsdb_bgw; + datname | application_name +--------------------------+----------------------------------------- + db_bgw_scheduler_restart | TimescaleDB Background Worker Scheduler + | TimescaleDB Background Worker Launcher +(2 rows) + +-- Now, we had a previous bug where killing the launcher at this point +-- would leave the schedulers running (because the launcher did not +-- have a handle for them) and when launcher is restarting, it would +-- start more schedulers, leaving two schedulers per database. +-- Get the PID of the launcher to be able to compare it after the restart +SELECT pid AS orig_pid FROM tsdb_bgw WHERE backend_type LIKE '%Launcher%' \gset +-- Kill the launcher. Since there are new restarted schedulers, the +-- handle could not be used to terminate them, and they would be left +-- running. +SELECT pg_terminate_backend(pid) FROM tsdb_bgw + WHERE backend_type LIKE '%Launcher%'; + pg_terminate_backend +---------------------- + t +(1 row) + +-- Launcher will restart immediately, but we wait one second to give +-- it a chance to start. +CALL wait_for_some_started(1, 50, '%Launcher%'); +-- Check that the launcher is running and that there are exactly one +-- scheduler per database. Here the old schedulers are killed, so it +-- will be schedulers with a different PID than the ones before the +-- launcher was killed, but we are not showing this here. +SELECT (pid != :orig_pid) AS different_pid, + datname, + application_name + FROM tsdb_bgw; + different_pid | datname | application_name +---------------+--------------------------+----------------------------------------- + t | db_bgw_scheduler_restart | TimescaleDB Background Worker Scheduler + t | | TimescaleDB Background Worker Launcher +(2 rows) + +ALTER SYSTEM RESET timescaledb.bgw_scheduler_restart_time; +ALTER SYSTEM RESET timescaledb.debug_bgw_scheduler_exit_status; +SELECT pg_reload_conf(); + pg_reload_conf +---------------- + t +(1 row) + +SELECT _timescaledb_functions.stop_background_workers(); + stop_background_workers +------------------------- + t +(1 row) + diff --git a/tsl/test/sql/CMakeLists.txt b/tsl/test/sql/CMakeLists.txt index 2b3a007d7a2..cc00f329d4a 100644 --- a/tsl/test/sql/CMakeLists.txt +++ b/tsl/test/sql/CMakeLists.txt @@ -73,6 +73,7 @@ if(CMAKE_BUILD_TYPE MATCHES Debug) bgw_job_stat_history_errors.sql bgw_job_stat_history_errors_permissions.sql bgw_db_scheduler_fixed.sql + bgw_scheduler_restart.sql bgw_reorder_drop_chunks.sql scheduler_fixed.sql compress_bgw_reorder_drop_chunks.sql @@ -185,6 +186,7 @@ set(SOLO_TESTS # log level. bgw_custom bgw_scheduler_control + bgw_scheduler_restart bgw_db_scheduler bgw_job_stat_history_errors_permissions bgw_job_stat_history_errors diff --git a/tsl/test/sql/bgw_scheduler_restart.sql b/tsl/test/sql/bgw_scheduler_restart.sql new file mode 100644 index 00000000000..e762a33827e --- /dev/null +++ b/tsl/test/sql/bgw_scheduler_restart.sql @@ -0,0 +1,135 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. + +\c :TEST_DBNAME :ROLE_SUPERUSER + +CREATE VIEW tsdb_bgw AS + SELECT datname, pid, backend_type, application_name + FROM pg_stat_activity + WHERE backend_type LIKE '%TimescaleDB%' + ORDER BY datname, backend_type, application_name; + +-- Wait for at least one background worker matching pattern to have +-- started. +CREATE PROCEDURE wait_for_some_started( + min_time double precision, + timeout double precision, + pattern text +) AS $$ +DECLARE + backend_count int; +BEGIN + FOR i IN 0..(timeout / min_time)::int + LOOP + PERFORM pg_sleep(min_time); + SELECT count(*) INTO backend_count FROM tsdb_bgw WHERE backend_type LIKE pattern; + IF backend_count > 0 THEN + RETURN; + END IF; + END LOOP; + RAISE EXCEPTION 'backend matching % did not start before timeout', pattern; +END; +$$ LANGUAGE plpgsql; + +-- Wait for the number of background workers matching pattern to be +-- zero. +CREATE PROCEDURE wait_for_all_stopped( + min_time double precision, + timeout double precision, + pattern text +) AS $$ +DECLARE + backend_count int; +BEGIN + FOR i IN 0..(timeout / min_time)::int + LOOP + PERFORM pg_sleep(min_time); + SELECT count(*) INTO backend_count FROM tsdb_bgw WHERE backend_type LIKE pattern; + IF backend_count = 0 THEN + RETURN; + END IF; + END LOOP; + RAISE EXCEPTION 'backend matching % did not start before timeout', pattern; +END; +$$ LANGUAGE plpgsql; + +-- Show the default scheduler restart time +SHOW timescaledb.bgw_scheduler_restart_time; +ALTER SYSTEM SET timescaledb.bgw_scheduler_restart_time TO '10s'; +ALTER SYSTEM SET timescaledb.debug_bgw_scheduler_exit_status TO 1; +SELECT pg_reload_conf(); +SHOW timescaledb.bgw_scheduler_restart_time; + +-- Launcher is running, so we need to restart it for the scheduler +-- restart time to take effect. +SELECT datname, application_name FROM tsdb_bgw; +SELECT pg_terminate_backend(pid) FROM tsdb_bgw + WHERE backend_type LIKE '%Launcher%'; + +-- It will restart automatically, but we wait for it to start. +CALL wait_for_some_started(1, 50, '%Launcher%'); + +-- Verify that launcher is running. If it is not, the rest of the test +-- will fail. +SELECT datname, application_name FROM tsdb_bgw; + +-- Now we can start the background workers. +SELECT _timescaledb_functions.start_background_workers(); + +-- They should start immediately, but let's wait for them to start. +CALL wait_for_some_started(1, 50, '%Scheduler%'); + +-- Check that the schedulers are running. If they are not, the rest of +-- the test is meaningless. +SELECT datname, application_name FROM tsdb_bgw; + +-- Kill the schedulers and check that they restart. +SELECT pg_terminate_backend(pid) FROM tsdb_bgw + WHERE datname = :'TEST_DBNAME' AND backend_type LIKE '%Scheduler%'; + +-- Wait for scheduler to exit, they should exit immediately. +CALL wait_for_all_stopped(1, 50, '%Scheduler%'); + +-- Check that the schedulers really exited. +SELECT datname, application_name FROM tsdb_bgw; + +-- Wait for scheduler to restart. +CALL wait_for_some_started(10, 100, '%Scheduler%'); + +-- Make sure that the launcher and schedulers are running. Otherwise +-- the test will fail. +SELECT datname, application_name FROM tsdb_bgw; + +-- Now, we had a previous bug where killing the launcher at this point +-- would leave the schedulers running (because the launcher did not +-- have a handle for them) and when launcher is restarting, it would +-- start more schedulers, leaving two schedulers per database. + +-- Get the PID of the launcher to be able to compare it after the restart +SELECT pid AS orig_pid FROM tsdb_bgw WHERE backend_type LIKE '%Launcher%' \gset + +-- Kill the launcher. Since there are new restarted schedulers, the +-- handle could not be used to terminate them, and they would be left +-- running. +SELECT pg_terminate_backend(pid) FROM tsdb_bgw + WHERE backend_type LIKE '%Launcher%'; + +-- Launcher will restart immediately, but we wait one second to give +-- it a chance to start. +CALL wait_for_some_started(1, 50, '%Launcher%'); + +-- Check that the launcher is running and that there are exactly one +-- scheduler per database. Here the old schedulers are killed, so it +-- will be schedulers with a different PID than the ones before the +-- launcher was killed, but we are not showing this here. +SELECT (pid != :orig_pid) AS different_pid, + datname, + application_name + FROM tsdb_bgw; + +ALTER SYSTEM RESET timescaledb.bgw_scheduler_restart_time; +ALTER SYSTEM RESET timescaledb.debug_bgw_scheduler_exit_status; +SELECT pg_reload_conf(); + +SELECT _timescaledb_functions.stop_background_workers(); From 24a3a31e6b1392e31aff73338c31c52458760297 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Nordstr=C3=B6m?= Date: Mon, 13 Jan 2025 10:55:24 +0100 Subject: [PATCH 02/32] Transfer visibility when copying arrow tuple slot When creating a heap tuple from an arrow slot, it is necessary to also transfer the visibility information from the compressed tuple. Otherwise, the new tuple, if written into another relation, might not have the correct visibility. --- tsl/src/hypercore/arrow_tts.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tsl/src/hypercore/arrow_tts.c b/tsl/src/hypercore/arrow_tts.c index 7628e8b4240..b6e91e874e9 100644 --- a/tsl/src/hypercore/arrow_tts.c +++ b/tsl/src/hypercore/arrow_tts.c @@ -5,6 +5,7 @@ */ #include #include +#include #include #include #include @@ -719,9 +720,19 @@ tts_arrow_copy_heap_tuple(TupleTableSlot *slot) tuple = ExecCopySlotHeapTuple(aslot->noncompressed_slot); ItemPointerCopy(&slot->tts_tid, &tuple->t_self); - /* Clean up if the non-compressed slot was "borrowed" */ if (aslot->child_slot == aslot->compressed_slot) + { + BufferHeapTupleTableSlot *hslot = (BufferHeapTupleTableSlot *) aslot->compressed_slot; + Assert(TTS_IS_BUFFERTUPLE(aslot->compressed_slot)); + + /* Copy visibility information from the compressed relation tuple */ + memcpy(&tuple->t_data->t_choice, + &hslot->base.tuple->t_data->t_choice, + sizeof(tuple->t_data->t_choice)); + + /* Clean up the "borrowed" non-compressed slot */ ExecClearTuple(aslot->noncompressed_slot); + } return tuple; } From d43b56200eecc937cf4721a9f5cb12f9238e1b94 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Mon, 13 Jan 2025 15:58:01 +0100 Subject: [PATCH 03/32] Apply sort transform optimizations to compressed chunks (#7528) Currently we transform sorting by `time_bucket` to sorting by `time` only for uncompressed chunks. Implement it for the compressed chunks as well. --- .unreleased/compressed-sort-transform | 1 + src/planner/planner.c | 42 +- src/sort_transform.c | 95 +++-- src/sort_transform.h | 8 + tsl/src/nodes/decompress_chunk/planner.c | 64 ++- tsl/test/expected/compress_sort_transform.out | 385 ++++++++++++++++++ .../shared/expected/ordered_append-14.out | 264 ++++++++---- .../shared/expected/ordered_append-15.out | 342 +++++++++++----- .../shared/expected/ordered_append-16.out | 342 +++++++++++----- .../shared/expected/ordered_append-17.out | 342 +++++++++++----- tsl/test/sql/CMakeLists.txt | 1 + tsl/test/sql/compress_sort_transform.sql | 76 ++++ 12 files changed, 1509 insertions(+), 453 deletions(-) create mode 100644 .unreleased/compressed-sort-transform create mode 100644 tsl/test/expected/compress_sort_transform.out create mode 100644 tsl/test/sql/compress_sort_transform.sql diff --git a/.unreleased/compressed-sort-transform b/.unreleased/compressed-sort-transform new file mode 100644 index 00000000000..5e5e91ed0ff --- /dev/null +++ b/.unreleased/compressed-sort-transform @@ -0,0 +1 @@ +Implements: #7528 Transform sorting on `time_bucket` to sorting on time for compressed chunks in some cases. diff --git a/src/planner/planner.c b/src/planner/planner.c index 9f9c9133dc7..e901658fc8c 100644 --- a/src/planner/planner.c +++ b/src/planner/planner.c @@ -56,6 +56,7 @@ #include "partitioning.h" #include "planner/partialize.h" #include "planner/planner.h" +#include "sort_transform.h" #include "utils.h" #include "compat/compat.h" @@ -935,8 +936,6 @@ ts_classify_relation(const PlannerInfo *root, const RelOptInfo *rel, Hypertable return TS_REL_OTHER; } -extern void ts_sort_transform_optimization(PlannerInfo *root, RelOptInfo *rel); - static inline bool should_chunk_append(Hypertable *ht, PlannerInfo *root, RelOptInfo *rel, Path *path, bool ordered, int order_attno) @@ -1183,14 +1182,47 @@ apply_optimizations(PlannerInfo *root, TsRelType reltype, RelOptInfo *rel, Range break; case TS_REL_CHUNK_STANDALONE: case TS_REL_CHUNK_CHILD: - ts_sort_transform_optimization(root, rel); + { /* * Since the sort optimization adds new paths to the rel it has * to happen before any optimizations that replace pathlist. */ - if (ts_cm_functions->set_rel_pathlist_query != NULL) - ts_cm_functions->set_rel_pathlist_query(root, rel, rel->relid, rte, ht); + List *transformed_query_pathkeys = ts_sort_transform_get_pathkeys(root, rel, rte, ht); + if (transformed_query_pathkeys != NIL) + { + List *orig_query_pathkeys = root->query_pathkeys; + root->query_pathkeys = transformed_query_pathkeys; + + /* Create index paths with transformed pathkeys */ + create_index_paths(root, rel); + + /* + * Call the TSL hooks with the transformed pathkeys as well, so + * that the decompression paths also use this optimization. + */ + if (ts_cm_functions->set_rel_pathlist_query != NULL) + ts_cm_functions->set_rel_pathlist_query(root, rel, rel->relid, rte, ht); + + root->query_pathkeys = orig_query_pathkeys; + + /* + * change returned paths to use original pathkeys. have to go through + * all paths since create_index_paths might have modified existing + * pathkey. Always safe to do transform since ordering of + * transformed_query_pathkey implements ordering of + * orig_query_pathkeys. + */ + ts_sort_transform_replace_pathkeys(rel->pathlist, + transformed_query_pathkeys, + orig_query_pathkeys); + } + else + { + if (ts_cm_functions->set_rel_pathlist_query != NULL) + ts_cm_functions->set_rel_pathlist_query(root, rel, rel->relid, rte, ht); + } break; + } default: break; } diff --git a/src/sort_transform.c b/src/sort_transform.c index 60db458fe67..619db10f95a 100644 --- a/src/sort_transform.c +++ b/src/sort_transform.c @@ -16,7 +16,10 @@ #include #include "compat/compat.h" +#include "cross_module_fn.h" #include "func_cache.h" +#include "hypertable.h" +#include "import/allpaths.h" #include "sort_transform.h" /* This optimizations allows GROUP BY clauses that transform time in @@ -29,8 +32,6 @@ * to an ordering on time. */ -extern void ts_sort_transform_optimization(PlannerInfo *root, RelOptInfo *rel); - static Expr * transform_timestamp_cast(FuncExpr *func) { @@ -384,8 +385,9 @@ sort_transform_ec(PlannerInfo *root, EquivalenceClass *orig) * For example: an ORDER BY date_trunc('minute', time) can be implemented by * an ordering of time. */ -void -ts_sort_transform_optimization(PlannerInfo *root, RelOptInfo *rel) +List * +ts_sort_transform_get_pathkeys(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte, + Hypertable *ht) { /* * We attack this problem in three steps: @@ -399,8 +401,7 @@ ts_sort_transform_optimization(PlannerInfo *root, RelOptInfo *rel) * */ ListCell *lc; - List *transformed_query_pathkey = NIL; - List *orig_query_pathkeys = root->query_pathkeys; + List *transformed_query_pathkeys = NIL; PathKey *last_pk; PathKey *new_pk; EquivalenceClass *transformed; @@ -408,8 +409,8 @@ ts_sort_transform_optimization(PlannerInfo *root, RelOptInfo *rel) /* * nothing to do for empty pathkeys */ - if (orig_query_pathkeys == NIL) - return; + if (root->query_pathkeys == NIL) + return NIL; /* * These sort transformations are only safe for single member ORDER BY @@ -420,7 +421,7 @@ ts_sort_transform_optimization(PlannerInfo *root, RelOptInfo *rel) transformed = sort_transform_ec(root, last_pk->pk_eclass); if (transformed == NULL) - return; + return NIL; new_pk = make_canonical_pathkey(root, transformed, @@ -434,30 +435,72 @@ ts_sort_transform_optimization(PlannerInfo *root, RelOptInfo *rel) foreach (lc, root->query_pathkeys) { if (lfirst(lc) != last_pk) - transformed_query_pathkey = lappend(transformed_query_pathkey, lfirst(lc)); + transformed_query_pathkeys = lappend(transformed_query_pathkeys, lfirst(lc)); else - transformed_query_pathkey = lappend(transformed_query_pathkey, new_pk); + transformed_query_pathkeys = lappend(transformed_query_pathkeys, new_pk); } - /* search for indexes on transformed pathkeys */ - root->query_pathkeys = transformed_query_pathkey; - create_index_paths(root, rel); - root->query_pathkeys = orig_query_pathkeys; + return transformed_query_pathkeys; +} - /* - * change returned paths to use original pathkeys. have to go through - * all paths since create_index_paths might have modified existing - * pathkey. Always safe to do transform since ordering of - * transformed_query_pathkey implements ordering of - * orig_query_pathkeys. - */ - foreach (lc, rel->pathlist) +/* + * After we have created new paths with transformed pathkeys, replace them back + * with the original pathkeys. + */ +void +ts_sort_transform_replace_pathkeys(void *node, List *transformed_pathkeys, List *original_pathkeys) +{ + if (node == NULL) { - Path *path = lfirst(lc); + return; + } - if (compare_pathkeys(path->pathkeys, transformed_query_pathkey) == PATHKEYS_EQUAL) + if (IsA(node, List)) + { + List *list = castNode(List, node); + ListCell *lc; + foreach (lc, list) { - path->pathkeys = orig_query_pathkeys; + ts_sort_transform_replace_pathkeys(lfirst(lc), transformed_pathkeys, original_pathkeys); } + return; + } + + Path *path = (Path *) node; + if (compare_pathkeys(path->pathkeys, transformed_pathkeys) == PATHKEYS_EQUAL) + { + path->pathkeys = original_pathkeys; + } + + if (IsA(path, CustomPath)) + { + /* + * We should only see ChunkAppend here. + */ + CustomPath *custom = castNode(CustomPath, path); + ts_sort_transform_replace_pathkeys(custom->custom_paths, + transformed_pathkeys, + original_pathkeys); + } + else if (IsA(path, MergeAppendPath)) + { + MergeAppendPath *append = castNode(MergeAppendPath, path); + ts_sort_transform_replace_pathkeys(append->subpaths, + transformed_pathkeys, + original_pathkeys); + } + else if (IsA(path, AppendPath)) + { + AppendPath *append = castNode(AppendPath, path); + ts_sort_transform_replace_pathkeys(append->subpaths, + transformed_pathkeys, + original_pathkeys); + } + else if (IsA(path, ProjectionPath)) + { + ProjectionPath *projection = castNode(ProjectionPath, path); + ts_sort_transform_replace_pathkeys(projection->subpath, + transformed_pathkeys, + original_pathkeys); } } diff --git a/src/sort_transform.h b/src/sort_transform.h index f3395d353a6..8aefed5dbed 100644 --- a/src/sort_transform.h +++ b/src/sort_transform.h @@ -7,4 +7,12 @@ #include +#include "hypertable.h" + extern Expr *ts_sort_transform_expr(Expr *expr); + +extern List *ts_sort_transform_get_pathkeys(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte, + Hypertable *ht); + +extern void ts_sort_transform_replace_pathkeys(void *node, List *transformed_pathkeys, + List *original_pathkeys); diff --git a/tsl/src/nodes/decompress_chunk/planner.c b/tsl/src/nodes/decompress_chunk/planner.c index 785b846e583..bbb96b03e01 100644 --- a/tsl/src/nodes/decompress_chunk/planner.c +++ b/tsl/src/nodes/decompress_chunk/planner.c @@ -955,6 +955,50 @@ ts_label_sort_with_costsize(PlannerInfo *root, Sort *plan, double limit_tuples) plan->plan.parallel_safe = lefttree->parallel_safe; } +/* + * Find a variable of the given relation somewhere in the expression tree. + * Currently we use this to find the Var argument of time_bucket, when we prepare + * the batch sorted merge parameters after using the monotonous sorting transform + * optimization. + */ +static Var * +find_var_subexpression(void *expr, Index varno) +{ + if (IsA(expr, Var)) + { + Var *var = castNode(Var, expr); + if ((Index) var->varno == (Index) varno) + { + return var; + } + + return NULL; + } + + if (IsA(expr, List)) + { + List *list = castNode(List, expr); + ListCell *lc; + foreach (lc, list) + { + Var *var = find_var_subexpression(lfirst(lc), varno); + if (var != NULL) + { + return var; + } + } + + return NULL; + } + + if (IsA(expr, FuncExpr)) + { + return find_var_subexpression(castNode(FuncExpr, expr)->args, varno); + } + + return NULL; +} + Plan * decompress_chunk_plan_create(PlannerInfo *root, RelOptInfo *rel, CustomPath *path, List *output_targetlist, List *clauses, List *custom_plans) @@ -1130,18 +1174,22 @@ decompress_chunk_plan_create(PlannerInfo *root, RelOptInfo *rel, CustomPath *pat continue; } - Ensure(IsA(em->em_expr, Var), - "non-Var pathkey not expected for compressed batch sorted merge"); - /* - * We found a Var equivalence member that belongs to the - * decompressed relation. We have to convert its varattno which - * is the varattno of the uncompressed chunk tuple, to the - * decompressed scan tuple varattno. + * The equivalence member expression might be a monotonous + * expression of the decompressed relation Var, so recurse to + * find it. */ - Var *var = castNode(Var, em->em_expr); + Var *var = find_var_subexpression(em->em_expr, em_relid); + Ensure(var != NULL, + "non-Var pathkey not expected for compressed batch sorted merge"); + Assert((Index) var->varno == (Index) em_relid); + /* + * Convert its varattno which is the varattno of the + * uncompressed chunk tuple, to the decompressed scan tuple + * varattno. + */ const int decompressed_scan_attno = context.uncompressed_attno_info[var->varattno].custom_scan_attno; Assert(decompressed_scan_attno > 0); diff --git a/tsl/test/expected/compress_sort_transform.out b/tsl/test/expected/compress_sort_transform.out new file mode 100644 index 00000000000..6aeebde6b63 --- /dev/null +++ b/tsl/test/expected/compress_sort_transform.out @@ -0,0 +1,385 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. +-- this test checks the validity of the produced plans for partially compressed chunks +-- when injecting query_pathkeys on top of the append +-- path that combines the uncompressed and compressed parts of a chunk. +set work_mem to '64MB'; +set enable_hashagg to off; +\set PREFIX 'EXPLAIN (analyze, costs off, timing off, summary off)' +CREATE TABLE ht_metrics_partially_compressed(time timestamptz, device int, value float); +SELECT create_hypertable('ht_metrics_partially_compressed','time'); +NOTICE: adding not-null constraint to column "time" + create_hypertable +---------------------------------------------- + (1,public,ht_metrics_partially_compressed,t) +(1 row) + +ALTER TABLE ht_metrics_partially_compressed SET (timescaledb.compress, + timescaledb.compress_segmentby='device', timescaledb.compress_orderby='time'); +INSERT INTO ht_metrics_partially_compressed +SELECT time, device, device * 0.1 +FROM generate_series('2020-01-02'::timestamptz,'2020-01-18'::timestamptz,'20 minute') time, +generate_series(1,3) device; +SELECT compress_chunk(c) FROM show_chunks('ht_metrics_partially_compressed') c; + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk +(3 rows) + +-- make them partially compressed +INSERT INTO ht_metrics_partially_compressed +SELECT time, device, device * 0.1 +FROM generate_series('2020-01-02'::timestamptz,'2020-01-18'::timestamptz,'30 minute') time, +generate_series(1,3) device; +VACUUM ANALYZE ht_metrics_partially_compressed; +-- sort transform +-- Grouping can use compressed data order. +:PREFIX +select device, time_bucket('1 minute', time), count(*) +from ht_metrics_partially_compressed +group by 1, 2 order by 1, 2 limit 1; + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------- + Limit (actual rows=1 loops=1) + -> Finalize GroupAggregate (actual rows=1 loops=1) + Group Key: _hyper_1_1_chunk.device, (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time")) + -> Merge Append (actual rows=3 loops=1) + Sort Key: _hyper_1_1_chunk.device, (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time")) + -> Partial GroupAggregate (actual rows=2 loops=1) + Group Key: _hyper_1_1_chunk.device, time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time") + -> Custom Scan (DecompressChunk) on _hyper_1_1_chunk (actual rows=3 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: compress_hyper_2_4_chunk.device, compress_hyper_2_4_chunk._ts_meta_min_1, compress_hyper_2_4_chunk._ts_meta_max_1 + Sort Method: quicksort + -> Seq Scan on compress_hyper_2_4_chunk (actual rows=3 loops=1) + -> Partial GroupAggregate (actual rows=2 loops=1) + Group Key: _hyper_1_1_chunk.device, (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time")) + -> Sort (actual rows=3 loops=1) + Sort Key: _hyper_1_1_chunk.device, (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time")) + Sort Method: quicksort + -> Seq Scan on _hyper_1_1_chunk (actual rows=960 loops=1) + -> Partial GroupAggregate (actual rows=1 loops=1) + Group Key: _hyper_1_2_chunk.device, time_bucket('@ 1 min'::interval, _hyper_1_2_chunk."time") + -> Custom Scan (DecompressChunk) on _hyper_1_2_chunk (actual rows=2 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: compress_hyper_2_5_chunk.device, compress_hyper_2_5_chunk._ts_meta_min_1, compress_hyper_2_5_chunk._ts_meta_max_1 + Sort Method: quicksort + -> Seq Scan on compress_hyper_2_5_chunk (actual rows=3 loops=1) + -> Partial GroupAggregate (actual rows=1 loops=1) + Group Key: _hyper_1_2_chunk.device, (time_bucket('@ 1 min'::interval, _hyper_1_2_chunk."time")) + -> Sort (actual rows=2 loops=1) + Sort Key: _hyper_1_2_chunk.device, (time_bucket('@ 1 min'::interval, _hyper_1_2_chunk."time")) + Sort Method: quicksort + -> Seq Scan on _hyper_1_2_chunk (actual rows=1008 loops=1) + -> Partial GroupAggregate (actual rows=1 loops=1) + Group Key: _hyper_1_3_chunk.device, time_bucket('@ 1 min'::interval, _hyper_1_3_chunk."time") + -> Custom Scan (DecompressChunk) on _hyper_1_3_chunk (actual rows=2 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: compress_hyper_2_6_chunk.device, compress_hyper_2_6_chunk._ts_meta_min_1, compress_hyper_2_6_chunk._ts_meta_max_1 + Sort Method: quicksort + -> Seq Scan on compress_hyper_2_6_chunk (actual rows=3 loops=1) + -> Partial GroupAggregate (actual rows=1 loops=1) + Group Key: _hyper_1_3_chunk.device, (time_bucket('@ 1 min'::interval, _hyper_1_3_chunk."time")) + -> Sort (actual rows=2 loops=1) + Sort Key: _hyper_1_3_chunk.device, (time_bucket('@ 1 min'::interval, _hyper_1_3_chunk."time")) + Sort Method: quicksort + -> Seq Scan on _hyper_1_3_chunk (actual rows=339 loops=1) +(44 rows) + +-- Batch sorted merge. +:PREFIX +select time_bucket('1 minute', time), count(*) +from ht_metrics_partially_compressed +group by 1 order by 1 limit 1; + QUERY PLAN +--------------------------------------------------------------------------------------------------------- + Limit (actual rows=1 loops=1) + -> Finalize GroupAggregate (actual rows=1 loops=1) + Group Key: (time_bucket('@ 1 min'::interval, ht_metrics_partially_compressed."time")) + -> Custom Scan (ChunkAppend) on ht_metrics_partially_compressed (actual rows=3 loops=1) + Order: time_bucket('@ 1 min'::interval, ht_metrics_partially_compressed."time") + -> Merge Append (actual rows=3 loops=1) + Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time")) + -> Partial GroupAggregate (actual rows=2 loops=1) + Group Key: time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time") + -> Custom Scan (DecompressChunk) on _hyper_1_1_chunk (actual rows=7 loops=1) + -> Sort (actual rows=3 loops=1) + Sort Key: compress_hyper_2_4_chunk._ts_meta_min_1 + Sort Method: quicksort + -> Seq Scan on compress_hyper_2_4_chunk (actual rows=3 loops=1) + -> Partial GroupAggregate (actual rows=2 loops=1) + Group Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time")) + -> Sort (actual rows=7 loops=1) + Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time")) + Sort Method: quicksort + -> Seq Scan on _hyper_1_1_chunk (actual rows=960 loops=1) + -> Merge Append (never executed) + Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_2_chunk."time")) + -> Partial GroupAggregate (never executed) + Group Key: time_bucket('@ 1 min'::interval, _hyper_1_2_chunk."time") + -> Custom Scan (DecompressChunk) on _hyper_1_2_chunk (never executed) + -> Sort (never executed) + Sort Key: compress_hyper_2_5_chunk._ts_meta_min_1 + -> Seq Scan on compress_hyper_2_5_chunk (never executed) + -> Partial GroupAggregate (never executed) + Group Key: (time_bucket('@ 1 min'::interval, _hyper_1_2_chunk."time")) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_2_chunk."time")) + -> Seq Scan on _hyper_1_2_chunk (never executed) + -> Merge Append (never executed) + Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_3_chunk."time")) + -> Partial GroupAggregate (never executed) + Group Key: time_bucket('@ 1 min'::interval, _hyper_1_3_chunk."time") + -> Custom Scan (DecompressChunk) on _hyper_1_3_chunk (never executed) + -> Sort (never executed) + Sort Key: compress_hyper_2_6_chunk._ts_meta_min_1 + -> Seq Scan on compress_hyper_2_6_chunk (never executed) + -> Partial GroupAggregate (never executed) + Group Key: (time_bucket('@ 1 min'::interval, _hyper_1_3_chunk."time")) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_3_chunk."time")) + -> Seq Scan on _hyper_1_3_chunk (never executed) +(46 rows) + +-- Batch sorted merge with different order in SELECT list. +:PREFIX +select count(*), time_bucket('1 minute', time) +from ht_metrics_partially_compressed +group by 2 order by 2 limit 1; + QUERY PLAN +--------------------------------------------------------------------------------------------------------- + Limit (actual rows=1 loops=1) + -> Finalize GroupAggregate (actual rows=1 loops=1) + Group Key: (time_bucket('@ 1 min'::interval, ht_metrics_partially_compressed."time")) + -> Custom Scan (ChunkAppend) on ht_metrics_partially_compressed (actual rows=3 loops=1) + Order: time_bucket('@ 1 min'::interval, ht_metrics_partially_compressed."time") + -> Merge Append (actual rows=3 loops=1) + Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time")) + -> Partial GroupAggregate (actual rows=2 loops=1) + Group Key: time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time") + -> Custom Scan (DecompressChunk) on _hyper_1_1_chunk (actual rows=7 loops=1) + -> Sort (actual rows=3 loops=1) + Sort Key: compress_hyper_2_4_chunk._ts_meta_min_1 + Sort Method: quicksort + -> Seq Scan on compress_hyper_2_4_chunk (actual rows=3 loops=1) + -> Partial GroupAggregate (actual rows=2 loops=1) + Group Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time")) + -> Sort (actual rows=7 loops=1) + Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time")) + Sort Method: quicksort + -> Seq Scan on _hyper_1_1_chunk (actual rows=960 loops=1) + -> Merge Append (never executed) + Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_2_chunk."time")) + -> Partial GroupAggregate (never executed) + Group Key: time_bucket('@ 1 min'::interval, _hyper_1_2_chunk."time") + -> Custom Scan (DecompressChunk) on _hyper_1_2_chunk (never executed) + -> Sort (never executed) + Sort Key: compress_hyper_2_5_chunk._ts_meta_min_1 + -> Seq Scan on compress_hyper_2_5_chunk (never executed) + -> Partial GroupAggregate (never executed) + Group Key: (time_bucket('@ 1 min'::interval, _hyper_1_2_chunk."time")) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_2_chunk."time")) + -> Seq Scan on _hyper_1_2_chunk (never executed) + -> Merge Append (never executed) + Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_3_chunk."time")) + -> Partial GroupAggregate (never executed) + Group Key: time_bucket('@ 1 min'::interval, _hyper_1_3_chunk."time") + -> Custom Scan (DecompressChunk) on _hyper_1_3_chunk (never executed) + -> Sort (never executed) + Sort Key: compress_hyper_2_6_chunk._ts_meta_min_1 + -> Seq Scan on compress_hyper_2_6_chunk (never executed) + -> Partial GroupAggregate (never executed) + Group Key: (time_bucket('@ 1 min'::interval, _hyper_1_3_chunk."time")) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_3_chunk."time")) + -> Seq Scan on _hyper_1_3_chunk (never executed) +(46 rows) + +-- Batch sorted merge with grouping column not in SELECT list. +:PREFIX +select count(*) +from ht_metrics_partially_compressed +group by time_bucket('1 minute', time) limit 1; + QUERY PLAN +--------------------------------------------------------------------------------------------------- + Limit (actual rows=1 loops=1) + -> Finalize GroupAggregate (actual rows=1 loops=1) + Group Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time")) + -> Merge Append (actual rows=3 loops=1) + Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time")) + -> Partial GroupAggregate (actual rows=2 loops=1) + Group Key: time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time") + -> Custom Scan (DecompressChunk) on _hyper_1_1_chunk (actual rows=7 loops=1) + -> Sort (actual rows=3 loops=1) + Sort Key: compress_hyper_2_4_chunk._ts_meta_min_1 + Sort Method: quicksort + -> Seq Scan on compress_hyper_2_4_chunk (actual rows=3 loops=1) + -> Partial GroupAggregate (actual rows=2 loops=1) + Group Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time")) + -> Sort (actual rows=7 loops=1) + Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time")) + Sort Method: quicksort + -> Seq Scan on _hyper_1_1_chunk (actual rows=960 loops=1) + -> Partial GroupAggregate (actual rows=1 loops=1) + Group Key: time_bucket('@ 1 min'::interval, _hyper_1_2_chunk."time") + -> Custom Scan (DecompressChunk) on _hyper_1_2_chunk (actual rows=4 loops=1) + -> Sort (actual rows=3 loops=1) + Sort Key: compress_hyper_2_5_chunk._ts_meta_min_1 + Sort Method: quicksort + -> Seq Scan on compress_hyper_2_5_chunk (actual rows=3 loops=1) + -> Partial GroupAggregate (actual rows=1 loops=1) + Group Key: (time_bucket('@ 1 min'::interval, _hyper_1_2_chunk."time")) + -> Sort (actual rows=4 loops=1) + Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_2_chunk."time")) + Sort Method: quicksort + -> Seq Scan on _hyper_1_2_chunk (actual rows=1008 loops=1) + -> Partial GroupAggregate (actual rows=1 loops=1) + Group Key: time_bucket('@ 1 min'::interval, _hyper_1_3_chunk."time") + -> Custom Scan (DecompressChunk) on _hyper_1_3_chunk (actual rows=4 loops=1) + -> Sort (actual rows=3 loops=1) + Sort Key: compress_hyper_2_6_chunk._ts_meta_min_1 + Sort Method: quicksort + -> Seq Scan on compress_hyper_2_6_chunk (actual rows=3 loops=1) + -> Partial GroupAggregate (actual rows=1 loops=1) + Group Key: (time_bucket('@ 1 min'::interval, _hyper_1_3_chunk."time")) + -> Sort (actual rows=4 loops=1) + Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_3_chunk."time")) + Sort Method: quicksort + -> Seq Scan on _hyper_1_3_chunk (actual rows=339 loops=1) +(44 rows) + +-- Ordering by time_bucket. +:PREFIX +select time_bucket('1 minute', time), * +from ht_metrics_partially_compressed +order by 1 limit 1; + QUERY PLAN +--------------------------------------------------------------------------------------------------------- + Limit (actual rows=1 loops=1) + -> Result (actual rows=1 loops=1) + -> Custom Scan (ChunkAppend) on ht_metrics_partially_compressed (actual rows=1 loops=1) + Order: time_bucket('@ 1 min'::interval, ht_metrics_partially_compressed."time") + -> Merge Append (actual rows=1 loops=1) + Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time")) + -> Result (actual rows=1 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_1_1_chunk (actual rows=1 loops=1) + -> Sort (actual rows=3 loops=1) + Sort Key: compress_hyper_2_4_chunk._ts_meta_min_1 + Sort Method: quicksort + -> Seq Scan on compress_hyper_2_4_chunk (actual rows=3 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time")) + Sort Method: top-N heapsort + -> Seq Scan on _hyper_1_1_chunk (actual rows=960 loops=1) + -> Merge Append (never executed) + Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_2_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_1_2_chunk (never executed) + -> Sort (never executed) + Sort Key: compress_hyper_2_5_chunk._ts_meta_min_1 + -> Seq Scan on compress_hyper_2_5_chunk (never executed) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_2_chunk."time")) + -> Seq Scan on _hyper_1_2_chunk (never executed) + -> Merge Append (never executed) + Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_3_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_1_3_chunk (never executed) + -> Sort (never executed) + Sort Key: compress_hyper_2_6_chunk._ts_meta_min_1 + -> Seq Scan on compress_hyper_2_6_chunk (never executed) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_3_chunk."time")) + -> Seq Scan on _hyper_1_3_chunk (never executed) +(36 rows) + +-- Ordering by time_bucket, but it's not in the SELECT list. +:PREFIX +select * from ht_metrics_partially_compressed +order by time_bucket('1 minute', time) limit 1; + QUERY PLAN +--------------------------------------------------------------------------------------------------- + Limit (actual rows=1 loops=1) + -> Custom Scan (ChunkAppend) on ht_metrics_partially_compressed (actual rows=1 loops=1) + Order: time_bucket('@ 1 min'::interval, ht_metrics_partially_compressed."time") + -> Merge Append (actual rows=1 loops=1) + Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time")) + -> Result (actual rows=1 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_1_1_chunk (actual rows=1 loops=1) + -> Sort (actual rows=3 loops=1) + Sort Key: compress_hyper_2_4_chunk._ts_meta_min_1 + Sort Method: quicksort + -> Seq Scan on compress_hyper_2_4_chunk (actual rows=3 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time")) + Sort Method: top-N heapsort + -> Seq Scan on _hyper_1_1_chunk (actual rows=960 loops=1) + -> Merge Append (never executed) + Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_2_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_1_2_chunk (never executed) + -> Sort (never executed) + Sort Key: compress_hyper_2_5_chunk._ts_meta_min_1 + -> Seq Scan on compress_hyper_2_5_chunk (never executed) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_2_chunk."time")) + -> Seq Scan on _hyper_1_2_chunk (never executed) + -> Merge Append (never executed) + Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_3_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_1_3_chunk (never executed) + -> Sort (never executed) + Sort Key: compress_hyper_2_6_chunk._ts_meta_min_1 + -> Seq Scan on compress_hyper_2_6_chunk (never executed) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_3_chunk."time")) + -> Seq Scan on _hyper_1_3_chunk (never executed) +(35 rows) + +-- Ordering in compressed data order. +:PREFIX +select * from ht_metrics_partially_compressed +order by device, time_bucket('1 minute', time) limit 1; + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------- + Limit (actual rows=1 loops=1) + -> Merge Append (actual rows=1 loops=1) + Sort Key: _hyper_1_1_chunk.device, (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time")) + -> Result (actual rows=1 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_1_1_chunk (actual rows=1 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: compress_hyper_2_4_chunk.device, compress_hyper_2_4_chunk._ts_meta_min_1, compress_hyper_2_4_chunk._ts_meta_max_1 + Sort Method: quicksort + -> Seq Scan on compress_hyper_2_4_chunk (actual rows=3 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: _hyper_1_1_chunk.device, (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time")) + Sort Method: top-N heapsort + -> Seq Scan on _hyper_1_1_chunk (actual rows=960 loops=1) + -> Result (actual rows=1 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_1_2_chunk (actual rows=1 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: compress_hyper_2_5_chunk.device, compress_hyper_2_5_chunk._ts_meta_min_1, compress_hyper_2_5_chunk._ts_meta_max_1 + Sort Method: quicksort + -> Seq Scan on compress_hyper_2_5_chunk (actual rows=3 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: _hyper_1_2_chunk.device, (time_bucket('@ 1 min'::interval, _hyper_1_2_chunk."time")) + Sort Method: top-N heapsort + -> Seq Scan on _hyper_1_2_chunk (actual rows=1008 loops=1) + -> Result (actual rows=1 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_1_3_chunk (actual rows=1 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: compress_hyper_2_6_chunk.device, compress_hyper_2_6_chunk._ts_meta_min_1, compress_hyper_2_6_chunk._ts_meta_max_1 + Sort Method: quicksort + -> Seq Scan on compress_hyper_2_6_chunk (actual rows=3 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: _hyper_1_3_chunk.device, (time_bucket('@ 1 min'::interval, _hyper_1_3_chunk."time")) + Sort Method: top-N heapsort + -> Seq Scan on _hyper_1_3_chunk (actual rows=339 loops=1) +(33 rows) + +reset work_mem; +reset enable_hashagg; diff --git a/tsl/test/shared/expected/ordered_append-14.out b/tsl/test/shared/expected/ordered_append-14.out index b7fab79f296..b46e15c0c79 100644 --- a/tsl/test/shared/expected/ordered_append-14.out +++ b/tsl/test/shared/expected/ordered_append-14.out @@ -2344,18 +2344,22 @@ ORDER BY 1 LIMIT 1; QUERY PLAN Limit (actual rows=1 loops=1) - -> Sort (actual rows=1 loops=1) - Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) - Sort Method: top-N heapsort - -> Result (actual rows=68370 loops=1) - -> Append (actual rows=68370 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=17990 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=20 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=25190 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=30 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=25190 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=30 loops=1) -(12 rows) + -> Custom Scan (ChunkAppend) on metrics_compressed (actual rows=1 loops=1) + Order: time_bucket('@ 1 day'::interval, metrics_compressed."time") + -> Sort (actual rows=1 loops=1) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=17990 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=20 loops=1) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) +(16 rows) -- test query with ORDER BY time_bucket, device_id -- must not use ordered append @@ -2394,18 +2398,22 @@ ORDER BY date_trunc('day', time) LIMIT 1; QUERY PLAN Limit (actual rows=1 loops=1) - -> Sort (actual rows=1 loops=1) - Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) - Sort Method: top-N heapsort - -> Result (actual rows=68370 loops=1) - -> Append (actual rows=68370 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=17990 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=20 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=25190 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=30 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=25190 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=30 loops=1) -(12 rows) + -> Custom Scan (ChunkAppend) on metrics_compressed (actual rows=1 loops=1) + Order: date_trunc('day'::text, metrics_compressed."time") + -> Sort (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=17990 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=20 loops=1) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) +(16 rows) -- test query with ORDER BY date_trunc :PREFIX @@ -2415,18 +2423,22 @@ ORDER BY 1 LIMIT 1; QUERY PLAN Limit (actual rows=1 loops=1) - -> Sort (actual rows=1 loops=1) - Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) - Sort Method: top-N heapsort - -> Result (actual rows=68370 loops=1) - -> Append (actual rows=68370 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=17990 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=20 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=25190 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=30 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=25190 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=30 loops=1) -(12 rows) + -> Custom Scan (ChunkAppend) on metrics_compressed (actual rows=1 loops=1) + Order: date_trunc('day'::text, metrics_compressed."time") + -> Sort (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=17990 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=20 loops=1) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) +(16 rows) -- test query with ORDER BY date_trunc, device_id -- must not use ordered append @@ -3922,30 +3934,54 @@ ORDER BY 1 LIMIT 1; QUERY PLAN Limit (actual rows=1 loops=1) - -> Sort (actual rows=1 loops=1) - Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) - Sort Method: top-N heapsort - -> Result (actual rows=68370 loops=1) - -> Append (actual rows=68370 loops=1) + -> Custom Scan (ChunkAppend) on metrics_space_compressed (actual rows=1 loops=1) + Order: time_bucket('@ 1 day'::interval, metrics_space_compressed."time") + -> Merge Append (actual rows=1 loops=1) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Sort (actual rows=1 loops=1) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=10794 loops=1) -> Seq Scan on compress_hyper_X_X_chunk (actual rows=12 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=15114 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=18 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=15114 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=18 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) -(24 rows) + -> Merge Append (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Merge Append (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) +(48 rows) -- test query with ORDER BY time_bucket, device_id -- must not use ordered append @@ -3993,30 +4029,54 @@ ORDER BY date_trunc('day', time) LIMIT 1; QUERY PLAN Limit (actual rows=1 loops=1) - -> Sort (actual rows=1 loops=1) - Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) - Sort Method: top-N heapsort - -> Result (actual rows=68370 loops=1) - -> Append (actual rows=68370 loops=1) + -> Custom Scan (ChunkAppend) on metrics_space_compressed (actual rows=1 loops=1) + Order: date_trunc('day'::text, metrics_space_compressed."time") + -> Merge Append (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Sort (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=10794 loops=1) -> Seq Scan on compress_hyper_X_X_chunk (actual rows=12 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=15114 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=18 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=15114 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=18 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) -(24 rows) + -> Merge Append (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Merge Append (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) +(48 rows) -- test query with ORDER BY date_trunc :PREFIX @@ -4026,30 +4086,54 @@ ORDER BY 1 LIMIT 1; QUERY PLAN Limit (actual rows=1 loops=1) - -> Sort (actual rows=1 loops=1) - Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) - Sort Method: top-N heapsort - -> Result (actual rows=68370 loops=1) - -> Append (actual rows=68370 loops=1) + -> Custom Scan (ChunkAppend) on metrics_space_compressed (actual rows=1 loops=1) + Order: date_trunc('day'::text, metrics_space_compressed."time") + -> Merge Append (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Sort (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=10794 loops=1) -> Seq Scan on compress_hyper_X_X_chunk (actual rows=12 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=15114 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=18 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=15114 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=18 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) -(24 rows) + -> Merge Append (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Merge Append (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) +(48 rows) -- test query with ORDER BY date_trunc, device_id -- must not use ordered append diff --git a/tsl/test/shared/expected/ordered_append-15.out b/tsl/test/shared/expected/ordered_append-15.out index e617dc1936a..f98e54dc87a 100644 --- a/tsl/test/shared/expected/ordered_append-15.out +++ b/tsl/test/shared/expected/ordered_append-15.out @@ -2368,18 +2368,26 @@ ORDER BY 1 LIMIT 1; QUERY PLAN Limit (actual rows=1 loops=1) - -> Sort (actual rows=1 loops=1) - Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) - Sort Method: top-N heapsort - -> Result (actual rows=68370 loops=1) - -> Append (actual rows=68370 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=17990 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=20 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=25190 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=30 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=25190 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=30 loops=1) -(12 rows) + -> Result (actual rows=1 loops=1) + -> Custom Scan (ChunkAppend) on metrics_compressed (actual rows=1 loops=1) + Order: time_bucket('@ 1 day'::interval, metrics_compressed."time") + -> Sort (actual rows=1 loops=1) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=17990 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=17990 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=20 loops=1) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) +(20 rows) -- test query with ORDER BY time_bucket, device_id -- must not use ordered append @@ -2418,18 +2426,26 @@ ORDER BY date_trunc('day', time) LIMIT 1; QUERY PLAN Limit (actual rows=1 loops=1) - -> Sort (actual rows=1 loops=1) - Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) - Sort Method: top-N heapsort - -> Result (actual rows=68370 loops=1) - -> Append (actual rows=68370 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=17990 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=20 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=25190 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=30 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=25190 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=30 loops=1) -(12 rows) + -> Result (actual rows=1 loops=1) + -> Custom Scan (ChunkAppend) on metrics_compressed (actual rows=1 loops=1) + Order: date_trunc('day'::text, metrics_compressed."time") + -> Sort (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=17990 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=17990 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=20 loops=1) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) +(20 rows) -- test query with ORDER BY date_trunc :PREFIX @@ -2439,18 +2455,26 @@ ORDER BY 1 LIMIT 1; QUERY PLAN Limit (actual rows=1 loops=1) - -> Sort (actual rows=1 loops=1) - Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) - Sort Method: top-N heapsort - -> Result (actual rows=68370 loops=1) - -> Append (actual rows=68370 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=17990 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=20 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=25190 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=30 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=25190 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=30 loops=1) -(12 rows) + -> Result (actual rows=1 loops=1) + -> Custom Scan (ChunkAppend) on metrics_compressed (actual rows=1 loops=1) + Order: date_trunc('day'::text, metrics_compressed."time") + -> Sort (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=17990 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=17990 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=20 loops=1) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) +(20 rows) -- test query with ORDER BY date_trunc, device_id -- must not use ordered append @@ -3952,30 +3976,64 @@ ORDER BY 1 LIMIT 1; QUERY PLAN Limit (actual rows=1 loops=1) - -> Sort (actual rows=1 loops=1) - Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) - Sort Method: top-N heapsort - -> Result (actual rows=68370 loops=1) - -> Append (actual rows=68370 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=10794 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=12 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=15114 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=18 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=15114 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=18 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) -(24 rows) + -> Result (actual rows=1 loops=1) + -> Custom Scan (ChunkAppend) on metrics_space_compressed (actual rows=1 loops=1) + Order: time_bucket('@ 1 day'::interval, metrics_space_compressed."time") + -> Merge Append (actual rows=1 loops=1) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Sort (actual rows=1 loops=1) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=3598 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=10794 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=10794 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=12 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=3598 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) + -> Merge Append (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Merge Append (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) +(58 rows) -- test query with ORDER BY time_bucket, device_id -- must not use ordered append @@ -4023,30 +4081,64 @@ ORDER BY date_trunc('day', time) LIMIT 1; QUERY PLAN Limit (actual rows=1 loops=1) - -> Sort (actual rows=1 loops=1) - Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) - Sort Method: top-N heapsort - -> Result (actual rows=68370 loops=1) - -> Append (actual rows=68370 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=10794 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=12 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=15114 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=18 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=15114 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=18 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) -(24 rows) + -> Result (actual rows=1 loops=1) + -> Custom Scan (ChunkAppend) on metrics_space_compressed (actual rows=1 loops=1) + Order: date_trunc('day'::text, metrics_space_compressed."time") + -> Merge Append (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Sort (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=3598 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=10794 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=10794 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=12 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=3598 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) + -> Merge Append (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Merge Append (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) +(58 rows) -- test query with ORDER BY date_trunc :PREFIX @@ -4056,30 +4148,64 @@ ORDER BY 1 LIMIT 1; QUERY PLAN Limit (actual rows=1 loops=1) - -> Sort (actual rows=1 loops=1) - Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) - Sort Method: top-N heapsort - -> Result (actual rows=68370 loops=1) - -> Append (actual rows=68370 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=10794 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=12 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=15114 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=18 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=15114 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=18 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) -(24 rows) + -> Result (actual rows=1 loops=1) + -> Custom Scan (ChunkAppend) on metrics_space_compressed (actual rows=1 loops=1) + Order: date_trunc('day'::text, metrics_space_compressed."time") + -> Merge Append (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Sort (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=3598 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=10794 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=10794 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=12 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=3598 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) + -> Merge Append (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Merge Append (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) +(58 rows) -- test query with ORDER BY date_trunc, device_id -- must not use ordered append diff --git a/tsl/test/shared/expected/ordered_append-16.out b/tsl/test/shared/expected/ordered_append-16.out index e617dc1936a..f98e54dc87a 100644 --- a/tsl/test/shared/expected/ordered_append-16.out +++ b/tsl/test/shared/expected/ordered_append-16.out @@ -2368,18 +2368,26 @@ ORDER BY 1 LIMIT 1; QUERY PLAN Limit (actual rows=1 loops=1) - -> Sort (actual rows=1 loops=1) - Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) - Sort Method: top-N heapsort - -> Result (actual rows=68370 loops=1) - -> Append (actual rows=68370 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=17990 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=20 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=25190 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=30 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=25190 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=30 loops=1) -(12 rows) + -> Result (actual rows=1 loops=1) + -> Custom Scan (ChunkAppend) on metrics_compressed (actual rows=1 loops=1) + Order: time_bucket('@ 1 day'::interval, metrics_compressed."time") + -> Sort (actual rows=1 loops=1) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=17990 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=17990 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=20 loops=1) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) +(20 rows) -- test query with ORDER BY time_bucket, device_id -- must not use ordered append @@ -2418,18 +2426,26 @@ ORDER BY date_trunc('day', time) LIMIT 1; QUERY PLAN Limit (actual rows=1 loops=1) - -> Sort (actual rows=1 loops=1) - Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) - Sort Method: top-N heapsort - -> Result (actual rows=68370 loops=1) - -> Append (actual rows=68370 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=17990 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=20 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=25190 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=30 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=25190 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=30 loops=1) -(12 rows) + -> Result (actual rows=1 loops=1) + -> Custom Scan (ChunkAppend) on metrics_compressed (actual rows=1 loops=1) + Order: date_trunc('day'::text, metrics_compressed."time") + -> Sort (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=17990 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=17990 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=20 loops=1) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) +(20 rows) -- test query with ORDER BY date_trunc :PREFIX @@ -2439,18 +2455,26 @@ ORDER BY 1 LIMIT 1; QUERY PLAN Limit (actual rows=1 loops=1) - -> Sort (actual rows=1 loops=1) - Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) - Sort Method: top-N heapsort - -> Result (actual rows=68370 loops=1) - -> Append (actual rows=68370 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=17990 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=20 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=25190 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=30 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=25190 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=30 loops=1) -(12 rows) + -> Result (actual rows=1 loops=1) + -> Custom Scan (ChunkAppend) on metrics_compressed (actual rows=1 loops=1) + Order: date_trunc('day'::text, metrics_compressed."time") + -> Sort (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=17990 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=17990 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=20 loops=1) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) +(20 rows) -- test query with ORDER BY date_trunc, device_id -- must not use ordered append @@ -3952,30 +3976,64 @@ ORDER BY 1 LIMIT 1; QUERY PLAN Limit (actual rows=1 loops=1) - -> Sort (actual rows=1 loops=1) - Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) - Sort Method: top-N heapsort - -> Result (actual rows=68370 loops=1) - -> Append (actual rows=68370 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=10794 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=12 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=15114 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=18 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=15114 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=18 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) -(24 rows) + -> Result (actual rows=1 loops=1) + -> Custom Scan (ChunkAppend) on metrics_space_compressed (actual rows=1 loops=1) + Order: time_bucket('@ 1 day'::interval, metrics_space_compressed."time") + -> Merge Append (actual rows=1 loops=1) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Sort (actual rows=1 loops=1) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=3598 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=10794 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=10794 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=12 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=3598 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) + -> Merge Append (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Merge Append (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) +(58 rows) -- test query with ORDER BY time_bucket, device_id -- must not use ordered append @@ -4023,30 +4081,64 @@ ORDER BY date_trunc('day', time) LIMIT 1; QUERY PLAN Limit (actual rows=1 loops=1) - -> Sort (actual rows=1 loops=1) - Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) - Sort Method: top-N heapsort - -> Result (actual rows=68370 loops=1) - -> Append (actual rows=68370 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=10794 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=12 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=15114 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=18 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=15114 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=18 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) -(24 rows) + -> Result (actual rows=1 loops=1) + -> Custom Scan (ChunkAppend) on metrics_space_compressed (actual rows=1 loops=1) + Order: date_trunc('day'::text, metrics_space_compressed."time") + -> Merge Append (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Sort (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=3598 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=10794 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=10794 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=12 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=3598 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) + -> Merge Append (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Merge Append (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) +(58 rows) -- test query with ORDER BY date_trunc :PREFIX @@ -4056,30 +4148,64 @@ ORDER BY 1 LIMIT 1; QUERY PLAN Limit (actual rows=1 loops=1) - -> Sort (actual rows=1 loops=1) - Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) - Sort Method: top-N heapsort - -> Result (actual rows=68370 loops=1) - -> Append (actual rows=68370 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=10794 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=12 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=15114 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=18 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=15114 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=18 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) -(24 rows) + -> Result (actual rows=1 loops=1) + -> Custom Scan (ChunkAppend) on metrics_space_compressed (actual rows=1 loops=1) + Order: date_trunc('day'::text, metrics_space_compressed."time") + -> Merge Append (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Sort (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=3598 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=10794 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=10794 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=12 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=3598 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) + -> Merge Append (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Merge Append (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) +(58 rows) -- test query with ORDER BY date_trunc, device_id -- must not use ordered append diff --git a/tsl/test/shared/expected/ordered_append-17.out b/tsl/test/shared/expected/ordered_append-17.out index 77f78ff6839..9e07dc3f206 100644 --- a/tsl/test/shared/expected/ordered_append-17.out +++ b/tsl/test/shared/expected/ordered_append-17.out @@ -2311,18 +2311,26 @@ ORDER BY 1 LIMIT 1; QUERY PLAN Limit (actual rows=1 loops=1) - -> Sort (actual rows=1 loops=1) - Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) - Sort Method: top-N heapsort - -> Result (actual rows=68370 loops=1) - -> Append (actual rows=68370 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=17990 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=20 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=25190 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=30 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=25190 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=30 loops=1) -(12 rows) + -> Result (actual rows=1 loops=1) + -> Custom Scan (ChunkAppend) on metrics_compressed (actual rows=1 loops=1) + Order: time_bucket('@ 1 day'::interval, metrics_compressed."time") + -> Sort (actual rows=1 loops=1) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=17990 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=17990 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=20 loops=1) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) +(20 rows) -- test query with ORDER BY time_bucket, device_id -- must not use ordered append @@ -2361,18 +2369,26 @@ ORDER BY date_trunc('day', time) LIMIT 1; QUERY PLAN Limit (actual rows=1 loops=1) - -> Sort (actual rows=1 loops=1) - Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) - Sort Method: top-N heapsort - -> Result (actual rows=68370 loops=1) - -> Append (actual rows=68370 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=17990 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=20 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=25190 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=30 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=25190 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=30 loops=1) -(12 rows) + -> Result (actual rows=1 loops=1) + -> Custom Scan (ChunkAppend) on metrics_compressed (actual rows=1 loops=1) + Order: date_trunc('day'::text, metrics_compressed."time") + -> Sort (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=17990 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=17990 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=20 loops=1) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) +(20 rows) -- test query with ORDER BY date_trunc :PREFIX @@ -2382,18 +2398,26 @@ ORDER BY 1 LIMIT 1; QUERY PLAN Limit (actual rows=1 loops=1) - -> Sort (actual rows=1 loops=1) - Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) - Sort Method: top-N heapsort - -> Result (actual rows=68370 loops=1) - -> Append (actual rows=68370 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=17990 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=20 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=25190 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=30 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=25190 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=30 loops=1) -(12 rows) + -> Result (actual rows=1 loops=1) + -> Custom Scan (ChunkAppend) on metrics_compressed (actual rows=1 loops=1) + Order: date_trunc('day'::text, metrics_compressed."time") + -> Sort (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=17990 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=17990 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=20 loops=1) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) +(20 rows) -- test query with ORDER BY date_trunc, device_id -- must not use ordered append @@ -3865,30 +3889,64 @@ ORDER BY 1 LIMIT 1; QUERY PLAN Limit (actual rows=1 loops=1) - -> Sort (actual rows=1 loops=1) - Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) - Sort Method: top-N heapsort - -> Result (actual rows=68370 loops=1) - -> Append (actual rows=68370 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=10794 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=12 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=15114 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=18 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=15114 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=18 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) -(24 rows) + -> Result (actual rows=1 loops=1) + -> Custom Scan (ChunkAppend) on metrics_space_compressed (actual rows=1 loops=1) + Order: time_bucket('@ 1 day'::interval, metrics_space_compressed."time") + -> Merge Append (actual rows=1 loops=1) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Sort (actual rows=1 loops=1) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=3598 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=10794 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=10794 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=12 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=3598 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) + -> Merge Append (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Merge Append (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (time_bucket('@ 1 day'::interval, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) +(58 rows) -- test query with ORDER BY time_bucket, device_id -- must not use ordered append @@ -3936,30 +3994,64 @@ ORDER BY date_trunc('day', time) LIMIT 1; QUERY PLAN Limit (actual rows=1 loops=1) - -> Sort (actual rows=1 loops=1) - Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) - Sort Method: top-N heapsort - -> Result (actual rows=68370 loops=1) - -> Append (actual rows=68370 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=10794 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=12 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=15114 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=18 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=15114 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=18 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) -(24 rows) + -> Result (actual rows=1 loops=1) + -> Custom Scan (ChunkAppend) on metrics_space_compressed (actual rows=1 loops=1) + Order: date_trunc('day'::text, metrics_space_compressed."time") + -> Merge Append (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Sort (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=3598 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=10794 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=10794 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=12 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=3598 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) + -> Merge Append (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Merge Append (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) +(58 rows) -- test query with ORDER BY date_trunc :PREFIX @@ -3969,30 +4061,64 @@ ORDER BY 1 LIMIT 1; QUERY PLAN Limit (actual rows=1 loops=1) - -> Sort (actual rows=1 loops=1) - Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) - Sort Method: top-N heapsort - -> Result (actual rows=68370 loops=1) - -> Append (actual rows=68370 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=10794 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=12 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=15114 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=18 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=15114 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=18 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=5038 loops=1) - -> Seq Scan on compress_hyper_X_X_chunk (actual rows=6 loops=1) -(24 rows) + -> Result (actual rows=1 loops=1) + -> Custom Scan (ChunkAppend) on metrics_space_compressed (actual rows=1 loops=1) + Order: date_trunc('day'::text, metrics_space_compressed."time") + -> Merge Append (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Sort (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=3598 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=10794 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=10794 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=12 loops=1) + -> Sort (actual rows=1 loops=1) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + Sort Method: top-N heapsort + -> Result (actual rows=3598 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (actual rows=3598 loops=1) + -> Seq Scan on compress_hyper_X_X_chunk (actual rows=4 loops=1) + -> Merge Append (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Merge Append (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) + -> Sort (never executed) + Sort Key: (date_trunc('day'::text, _hyper_X_X_chunk."time")) + -> Result (never executed) + -> Custom Scan (DecompressChunk) on _hyper_X_X_chunk (never executed) + -> Seq Scan on compress_hyper_X_X_chunk (never executed) +(58 rows) -- test query with ORDER BY date_trunc, device_id -- must not use ordered append diff --git a/tsl/test/sql/CMakeLists.txt b/tsl/test/sql/CMakeLists.txt index cc00f329d4a..e71fbdcffec 100644 --- a/tsl/test/sql/CMakeLists.txt +++ b/tsl/test/sql/CMakeLists.txt @@ -145,6 +145,7 @@ if((${PG_VERSION_MAJOR} GREATER_EQUAL "15")) APPEND TEST_FILES cagg_refresh_using_merge.sql + compress_sort_transform.sql hypercore_columnar.sql hypercore_constraints.sql hypercore_copy.sql diff --git a/tsl/test/sql/compress_sort_transform.sql b/tsl/test/sql/compress_sort_transform.sql new file mode 100644 index 00000000000..00bc7a64c74 --- /dev/null +++ b/tsl/test/sql/compress_sort_transform.sql @@ -0,0 +1,76 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. + +-- this test checks the validity of the produced plans for partially compressed chunks +-- when injecting query_pathkeys on top of the append +-- path that combines the uncompressed and compressed parts of a chunk. + +set work_mem to '64MB'; +set enable_hashagg to off; + +\set PREFIX 'EXPLAIN (analyze, costs off, timing off, summary off)' + +CREATE TABLE ht_metrics_partially_compressed(time timestamptz, device int, value float); +SELECT create_hypertable('ht_metrics_partially_compressed','time'); +ALTER TABLE ht_metrics_partially_compressed SET (timescaledb.compress, + timescaledb.compress_segmentby='device', timescaledb.compress_orderby='time'); + +INSERT INTO ht_metrics_partially_compressed +SELECT time, device, device * 0.1 +FROM generate_series('2020-01-02'::timestamptz,'2020-01-18'::timestamptz,'20 minute') time, +generate_series(1,3) device; + +SELECT compress_chunk(c) FROM show_chunks('ht_metrics_partially_compressed') c; +-- make them partially compressed +INSERT INTO ht_metrics_partially_compressed +SELECT time, device, device * 0.1 +FROM generate_series('2020-01-02'::timestamptz,'2020-01-18'::timestamptz,'30 minute') time, +generate_series(1,3) device; + +VACUUM ANALYZE ht_metrics_partially_compressed; + +-- sort transform + +-- Grouping can use compressed data order. +:PREFIX +select device, time_bucket('1 minute', time), count(*) +from ht_metrics_partially_compressed +group by 1, 2 order by 1, 2 limit 1; + +-- Batch sorted merge. +:PREFIX +select time_bucket('1 minute', time), count(*) +from ht_metrics_partially_compressed +group by 1 order by 1 limit 1; + +-- Batch sorted merge with different order in SELECT list. +:PREFIX +select count(*), time_bucket('1 minute', time) +from ht_metrics_partially_compressed +group by 2 order by 2 limit 1; + +-- Batch sorted merge with grouping column not in SELECT list. +:PREFIX +select count(*) +from ht_metrics_partially_compressed +group by time_bucket('1 minute', time) limit 1; + +-- Ordering by time_bucket. +:PREFIX +select time_bucket('1 minute', time), * +from ht_metrics_partially_compressed +order by 1 limit 1; + +-- Ordering by time_bucket, but it's not in the SELECT list. +:PREFIX +select * from ht_metrics_partially_compressed +order by time_bucket('1 minute', time) limit 1; + +-- Ordering in compressed data order. +:PREFIX +select * from ht_metrics_partially_compressed +order by device, time_bucket('1 minute', time) limit 1; + +reset work_mem; +reset enable_hashagg; From 6623eb65cf0a446da8c748a52d2c135af34734fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Nordstr=C3=B6m?= Date: Mon, 13 Jan 2025 16:12:10 +0100 Subject: [PATCH 04/32] Fix ABI issue when upgrading PG from 15.3 to 15.4+ A previous change (commit 8d29760) added a workaround for a PostgreSQL bug related to table access methods that was fixed in PG 15.4. The workaround does a build-time check for the PG version to apply the fix, but this won't work across PG server upgrades. To avoid raising an error when running an extension compiled on PG < 15.4 on PG 15.4+, add also a runtime check. --- tsl/src/compression/api.c | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/tsl/src/compression/api.c b/tsl/src/compression/api.c index fda985b31ed..12c8a174fd1 100644 --- a/tsl/src/compression/api.c +++ b/tsl/src/compression/api.c @@ -786,15 +786,30 @@ set_access_method(Oid relid, const char *amname) AlterTableInternal(relid, list_make1(&cmd), false); #if (PG_VERSION_NUM < 150004) - /* Fix for PostgreSQL bug where pg_depend was not updated to reflect the - * new dependency between AM and relation. See related PG fix here: - * https://github.com/postgres/postgres/commit/97d89101045fac8cb36f4ef6c08526ea0841a596 */ - if (changeDependencyFor(RelationRelationId, relid, AccessMethodRelationId, amoid, new_amoid) != - 1) - elog(ERROR, - "could not change access method dependency for relation \"%s.%s\"", - get_namespace_name(get_rel_namespace(relid)), - get_rel_name(relid)); + + /* + * Also do a runtime check in order to be ABI compatible with PG server + * upgrades, e.g., upgrading from 15.3 to 15.4 without updating the + * extension. + */ + const char *version_num_str = GetConfigOption("server_version_num", false, false); + int server_version_num; + + if (parse_int(version_num_str, &server_version_num, 0, NULL) && server_version_num < 150004) + { + /* Fix for PostgreSQL bug where pg_depend was not updated to reflect the + * new dependency between AM and relation. See related PG fix here: + * https://github.com/postgres/postgres/commit/97d89101045fac8cb36f4ef6c08526ea0841a596 */ + if (changeDependencyFor(RelationRelationId, + relid, + AccessMethodRelationId, + amoid, + new_amoid) != 1) + elog(ERROR, + "could not change access method dependency for relation \"%s.%s\"", + get_namespace_name(get_rel_namespace(relid)), + get_rel_name(relid)); + } #endif hypercore_alter_access_method_finish(relid, !to_hypercore); From 1718120fc05f8d59f59c5543d2fadabf35b4c58b Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Thu, 5 Dec 2024 17:58:24 +0100 Subject: [PATCH 05/32] Add optional `force` argument to `refresh_continuous_aggregate` Add optional `force` parameter to the `refresh_continuous_aggregate` procedure that would allow user to partially re-materialize cagg within a time window that has been previously materialized. --- .unreleased/pr_7521 | 1 + sql/ddl_api.sql | 3 +- sql/updates/latest-dev.sql | 11 ++ sql/updates/reverse-dev.sql | 9 ++ tsl/src/bgw_policy/job.c | 3 +- tsl/src/continuous_aggs/create.c | 7 +- tsl/src/continuous_aggs/invalidation.c | 23 ++- tsl/src/continuous_aggs/invalidation.h | 2 +- tsl/src/continuous_aggs/refresh.c | 20 ++- tsl/src/continuous_aggs/refresh.h | 3 +- tsl/test/expected/cagg_refresh.out | 122 +++++++++++----- .../expected/cagg_refresh_using_merge.out | 120 +++++++++++----- tsl/test/expected/chunk_utils_internal.out | 136 +++++++++++------- tsl/test/shared/expected/extension.out | 2 +- tsl/test/sql/chunk_utils_internal.sql | 21 +++ tsl/test/sql/include/cagg_refresh_common.sql | 43 ++++-- 16 files changed, 376 insertions(+), 150 deletions(-) create mode 100644 .unreleased/pr_7521 diff --git a/.unreleased/pr_7521 b/.unreleased/pr_7521 new file mode 100644 index 00000000000..a230550a94c --- /dev/null +++ b/.unreleased/pr_7521 @@ -0,0 +1 @@ +Implements: #7521 Add optional `force` argument to `refresh_continuous_aggregate` diff --git a/sql/ddl_api.sql b/sql/ddl_api.sql index 7f07a1fd432..5048ddadfee 100644 --- a/sql/ddl_api.sql +++ b/sql/ddl_api.sql @@ -212,6 +212,7 @@ AS '@MODULE_PATHNAME@', 'ts_tablespace_show' LANGUAGE C VOLATILE STRICT; CREATE OR REPLACE PROCEDURE @extschema@.refresh_continuous_aggregate( continuous_aggregate REGCLASS, window_start "any", - window_end "any" + window_end "any", + force BOOLEAN = FALSE ) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_continuous_agg_refresh'; diff --git a/sql/updates/latest-dev.sql b/sql/updates/latest-dev.sql index d233b0ae5f8..5255e9e3903 100644 --- a/sql/updates/latest-dev.sql +++ b/sql/updates/latest-dev.sql @@ -114,3 +114,14 @@ CREATE FUNCTION @extschema@.hypertable_columnstore_stats (hypertable REGCLASS) STABLE STRICT AS 'SELECT * FROM @extschema@.hypertable_compression_stats($1)' SET search_path TO pg_catalog, pg_temp; + +-- Recreate `refresh_continuous_aggregate` procedure to add `force` argument +DROP PROCEDURE IF EXISTS @extschema@.refresh_continuous_aggregate (continuous_aggregate REGCLASS, window_start "any", window_end "any"); + +CREATE PROCEDURE @extschema@.refresh_continuous_aggregate( + continuous_aggregate REGCLASS, + window_start "any", + window_end "any", + force BOOLEAN = FALSE +) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_update_placeholder'; + diff --git a/sql/updates/reverse-dev.sql b/sql/updates/reverse-dev.sql index 19fea4c8a05..bff21e0a960 100644 --- a/sql/updates/reverse-dev.sql +++ b/sql/updates/reverse-dev.sql @@ -58,3 +58,12 @@ DROP VIEW timescaledb_information.hypertable_columnstore_settings; DROP VIEW timescaledb_information.chunk_columnstore_settings; DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_update_watermark(INTEGER); + +-- Recreate `refresh_continuous_aggregate` procedure to remove the `force` argument +DROP PROCEDURE IF EXISTS @extschema@.refresh_continuous_aggregate (continuous_aggregate REGCLASS, window_start "any", window_end "any", force BOOLEAN); + +CREATE PROCEDURE @extschema@.refresh_continuous_aggregate( + continuous_aggregate REGCLASS, + window_start "any", + window_end "any" +) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_continuous_agg_refresh'; diff --git a/tsl/src/bgw_policy/job.c b/tsl/src/bgw_policy/job.c index 706789e748f..f4a9dfdd966 100644 --- a/tsl/src/bgw_policy/job.c +++ b/tsl/src/bgw_policy/job.c @@ -377,7 +377,8 @@ policy_refresh_cagg_execute(int32 job_id, Jsonb *config) &policy_data.refresh_window, CAGG_REFRESH_POLICY, policy_data.start_is_null, - policy_data.end_is_null); + policy_data.end_is_null, + false); return true; } diff --git a/tsl/src/continuous_aggs/create.c b/tsl/src/continuous_aggs/create.c index f290cd38956..7c7bd044010 100644 --- a/tsl/src/continuous_aggs/create.c +++ b/tsl/src/continuous_aggs/create.c @@ -940,7 +940,12 @@ tsl_process_continuous_agg_viewstmt(Node *node, const char *query_string, void * refresh_window.start = cagg_get_time_min(cagg); refresh_window.end = ts_time_get_noend_or_max(refresh_window.type); - continuous_agg_refresh_internal(cagg, &refresh_window, CAGG_REFRESH_CREATION, true, true); + continuous_agg_refresh_internal(cagg, + &refresh_window, + CAGG_REFRESH_CREATION, + true, + true, + false); } return DDL_DONE; diff --git a/tsl/src/continuous_aggs/invalidation.c b/tsl/src/continuous_aggs/invalidation.c index 339e6997b89..1d5d08dd55e 100644 --- a/tsl/src/continuous_aggs/invalidation.c +++ b/tsl/src/continuous_aggs/invalidation.c @@ -140,7 +140,8 @@ static Invalidation cut_cagg_invalidation_and_compute_remainder( const CaggInvalidationState *state, const InternalTimeRange *refresh_window, const Invalidation *mergedentry, const Invalidation *current_remainder); static void clear_cagg_invalidations_for_refresh(const CaggInvalidationState *state, - const InternalTimeRange *refresh_window); + const InternalTimeRange *refresh_window, + bool force); static void invalidation_state_init(CaggInvalidationState *state, const ContinuousAgg *cagg, Oid dimtype, const CaggsInfo *all_caggs); static void invalidation_state_cleanup(const CaggInvalidationState *state); @@ -878,7 +879,7 @@ cut_cagg_invalidation_and_compute_remainder(const CaggInvalidationState *state, */ static void clear_cagg_invalidations_for_refresh(const CaggInvalidationState *state, - const InternalTimeRange *refresh_window) + const InternalTimeRange *refresh_window, bool force) { ScanIterator iterator; int32 cagg_hyper_id = state->mat_hypertable_id; @@ -892,6 +893,20 @@ clear_cagg_invalidations_for_refresh(const CaggInvalidationState *state, MemoryContextReset(state->per_tuple_mctx); + /* Force refresh within the entire window */ + if (force) + { + Invalidation logentry; + + logentry.hyper_id = cagg_hyper_id; + logentry.lowest_modified_value = refresh_window->start; + logentry.greatest_modified_value = refresh_window->end; + logentry.is_modified = false; + ItemPointerSet(&logentry.tid, InvalidBlockNumber, 0); + + save_invalidation_for_refresh(state, &logentry); + } + /* Process all invalidations for the continuous aggregate */ ts_scanner_foreach(&iterator) { @@ -981,7 +996,7 @@ InvalidationStore * invalidation_process_cagg_log(const ContinuousAgg *cagg, const InternalTimeRange *refresh_window, const CaggsInfo *all_caggs_info, const long max_materializations, bool *do_merged_refresh, InternalTimeRange *ret_merged_refresh_window, - const CaggRefreshCallContext callctx) + const CaggRefreshCallContext callctx, bool force) { CaggInvalidationState state; InvalidationStore *store = NULL; @@ -991,7 +1006,7 @@ invalidation_process_cagg_log(const ContinuousAgg *cagg, const InternalTimeRange invalidation_state_init(&state, cagg, refresh_window->type, all_caggs_info); state.invalidations = tuplestore_begin_heap(false, false, work_mem); - clear_cagg_invalidations_for_refresh(&state, refresh_window); + clear_cagg_invalidations_for_refresh(&state, refresh_window, force); count = tuplestore_tuple_count(state.invalidations); if (count == 0) diff --git a/tsl/src/continuous_aggs/invalidation.h b/tsl/src/continuous_aggs/invalidation.h index 71b8b07f7e5..ed0529d2a73 100644 --- a/tsl/src/continuous_aggs/invalidation.h +++ b/tsl/src/continuous_aggs/invalidation.h @@ -49,6 +49,6 @@ extern InvalidationStore * invalidation_process_cagg_log(const ContinuousAgg *cagg, const InternalTimeRange *refresh_window, const CaggsInfo *all_caggs_info, const long max_materializations, bool *do_merged_refresh, InternalTimeRange *ret_merged_refresh_window, - const CaggRefreshCallContext callctx); + const CaggRefreshCallContext callctx, bool force); extern void invalidation_store_free(InvalidationStore *store); diff --git a/tsl/src/continuous_aggs/refresh.c b/tsl/src/continuous_aggs/refresh.c index b6097dbc8c5..d5a9783e003 100644 --- a/tsl/src/continuous_aggs/refresh.c +++ b/tsl/src/continuous_aggs/refresh.c @@ -77,7 +77,7 @@ static void emit_up_to_date_notice(const ContinuousAgg *cagg, const CaggRefreshC static bool process_cagg_invalidations_and_refresh(const ContinuousAgg *cagg, const InternalTimeRange *refresh_window, const CaggRefreshCallContext callctx, - int32 chunk_id); + int32 chunk_id, bool force); static void fill_bucket_offset_origin(const ContinuousAgg *cagg, const InternalTimeRange *const refresh_window, NullableDatum *offset, NullableDatum *origin); @@ -628,6 +628,7 @@ Datum continuous_agg_refresh(PG_FUNCTION_ARGS) { Oid cagg_relid = PG_ARGISNULL(0) ? InvalidOid : PG_GETARG_OID(0); + bool force = PG_ARGISNULL(3) ? false : PG_GETARG_BOOL(3); ContinuousAgg *cagg; InternalTimeRange refresh_window = { .type = InvalidOid, @@ -659,7 +660,8 @@ continuous_agg_refresh(PG_FUNCTION_ARGS) &refresh_window, CAGG_REFRESH_WINDOW, PG_ARGISNULL(1), - PG_ARGISNULL(2)); + PG_ARGISNULL(2), + force); PG_RETURN_VOID(); } @@ -703,7 +705,8 @@ continuous_agg_calculate_merged_refresh_window(const ContinuousAgg *cagg, static bool process_cagg_invalidations_and_refresh(const ContinuousAgg *cagg, const InternalTimeRange *refresh_window, - const CaggRefreshCallContext callctx, int32 chunk_id) + const CaggRefreshCallContext callctx, int32 chunk_id, + bool force) { InvalidationStore *invalidations; Oid hyper_relid = ts_hypertable_id_to_relid(cagg->data.mat_hypertable_id, false); @@ -727,7 +730,8 @@ process_cagg_invalidations_and_refresh(const ContinuousAgg *cagg, ts_guc_cagg_max_individual_materializations, &do_merged_refresh, &merged_refresh_window, - callctx); + callctx, + force); if (invalidations != NULL || do_merged_refresh) { @@ -759,7 +763,7 @@ void continuous_agg_refresh_internal(const ContinuousAgg *cagg, const InternalTimeRange *refresh_window_arg, const CaggRefreshCallContext callctx, const bool start_isnull, - const bool end_isnull) + const bool end_isnull, bool force) { int32 mat_id = cagg->data.mat_hypertable_id; InternalTimeRange refresh_window = *refresh_window_arg; @@ -881,7 +885,11 @@ continuous_agg_refresh_internal(const ContinuousAgg *cagg, cagg = ts_continuous_agg_find_by_mat_hypertable_id(mat_id, false); - if (!process_cagg_invalidations_and_refresh(cagg, &refresh_window, callctx, INVALID_CHUNK_ID)) + if (!process_cagg_invalidations_and_refresh(cagg, + &refresh_window, + callctx, + INVALID_CHUNK_ID, + force)) emit_up_to_date_notice(cagg, callctx); /* Restore search_path */ diff --git a/tsl/src/continuous_aggs/refresh.h b/tsl/src/continuous_aggs/refresh.h index c050dfefb32..6032861a851 100644 --- a/tsl/src/continuous_aggs/refresh.h +++ b/tsl/src/continuous_aggs/refresh.h @@ -20,4 +20,5 @@ extern void continuous_agg_calculate_merged_refresh_window( extern void continuous_agg_refresh_internal(const ContinuousAgg *cagg, const InternalTimeRange *refresh_window, const CaggRefreshCallContext callctx, - const bool start_isnull, const bool end_isnull); + const bool start_isnull, const bool end_isnull, + bool force); diff --git a/tsl/test/expected/cagg_refresh.out b/tsl/test/expected/cagg_refresh.out index 291ce335272..b25f4884990 100644 --- a/tsl/test/expected/cagg_refresh.out +++ b/tsl/test/expected/cagg_refresh.out @@ -13,6 +13,18 @@ SELECT create_hypertable('conditions', 'time'); (1,public,conditions,t) (1 row) +-- Test refresh on a cagg built on an empty table +CREATE MATERIALIZED VIEW daily_temp +WITH (timescaledb.continuous, + timescaledb.materialized_only=true) +AS +SELECT time_bucket('1 day', time) AS day, device, avg(temp) AS avg_temp +FROM conditions +GROUP BY 1,2 WITH NO DATA; +CALL refresh_continuous_aggregate('daily_temp', NULL, NULL); +psql:include/cagg_refresh_common.sql:17: NOTICE: continuous aggregate "daily_temp" is already up-to-date +CALL refresh_continuous_aggregate('daily_temp', NULL, NULL, force => true); +psql:include/cagg_refresh_common.sql:18: NOTICE: continuous aggregate "daily_temp" is already up-to-date SELECT setseed(.12); setseed --------- @@ -40,13 +52,6 @@ LIMIT 10; Mon May 04 22:30:00 2020 PDT | 0 | 8 (10 rows) -CREATE MATERIALIZED VIEW daily_temp -WITH (timescaledb.continuous, - timescaledb.materialized_only=true) -AS -SELECT time_bucket('1 day', time) AS day, device, avg(temp) AS avg_temp -FROM conditions -GROUP BY 1,2 WITH NO DATA; -- The continuous aggregate should be empty SELECT * FROM daily_temp ORDER BY day DESC, device; @@ -71,16 +76,16 @@ CALL refresh_continuous_aggregate('daily_temp', '2020-05-03 17:00 PDT', '2020-05 -- These refreshes will fail since they don't align with the bucket's -- time zone CALL refresh_continuous_aggregate('daily_temp', '2020-05-03', '2020-05-04'); -psql:include/cagg_refresh_common.sql:43: ERROR: refresh window too small +psql:include/cagg_refresh_common.sql:47: ERROR: refresh window too small DETAIL: The refresh window must cover at least one bucket of data. HINT: Align the refresh window with the bucket time zone or use at least two buckets. CALL refresh_continuous_aggregate('daily_temp', '2020-05-03 00:00 PDT', '2020-05-04 00:00 PDT'); -psql:include/cagg_refresh_common.sql:44: ERROR: refresh window too small +psql:include/cagg_refresh_common.sql:48: ERROR: refresh window too small DETAIL: The refresh window must cover at least one bucket of data. HINT: Align the refresh window with the bucket time zone or use at least two buckets. -- Refresh window less than one bucket CALL refresh_continuous_aggregate('daily_temp', '2020-05-03 00:00 UTC', '2020-05-03 23:59 UTC'); -psql:include/cagg_refresh_common.sql:47: ERROR: refresh window too small +psql:include/cagg_refresh_common.sql:51: ERROR: refresh window too small DETAIL: The refresh window must cover at least one bucket of data. HINT: Align the refresh window with the bucket time zone or use at least two buckets. -- Refresh window bigger than one bucket, but failing since it is not @@ -89,7 +94,7 @@ HINT: Align the refresh window with the bucket time zone or use at least two bu -- Refresh window: [----------) -- Buckets: [------|------] CALL refresh_continuous_aggregate('daily_temp', '2020-05-03 01:00 UTC', '2020-05-04 08:00 UTC'); -psql:include/cagg_refresh_common.sql:53: ERROR: refresh window too small +psql:include/cagg_refresh_common.sql:57: ERROR: refresh window too small DETAIL: The refresh window must cover at least one bucket of data. HINT: Align the refresh window with the bucket time zone or use at least two buckets. \set VERBOSITY terse @@ -117,14 +122,14 @@ ORDER BY day DESC, device; -- Refresh the rest (and try DEBUG output) SET client_min_messages TO DEBUG1; CALL refresh_continuous_aggregate('daily_temp', '2020-04-30', '2020-05-04'); -psql:include/cagg_refresh_common.sql:65: LOG: statement: CALL refresh_continuous_aggregate('daily_temp', '2020-04-30', '2020-05-04'); -psql:include/cagg_refresh_common.sql:65: DEBUG: hypertable 1 existing watermark >= new invalidation threshold 1588723200000000 1588550400000000 -psql:include/cagg_refresh_common.sql:65: DEBUG: continuous aggregate refresh (individual invalidation) on "daily_temp" in window [ Thu Apr 30 17:00:00 2020 PDT, Sat May 02 17:00:00 2020 PDT ] -psql:include/cagg_refresh_common.sql:65: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" -psql:include/cagg_refresh_common.sql:65: LOG: inserted 8 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" -psql:include/cagg_refresh_common.sql:65: DEBUG: hypertable 2 existing watermark >= new watermark 1588723200000000 1588723200000000 +psql:include/cagg_refresh_common.sql:69: LOG: statement: CALL refresh_continuous_aggregate('daily_temp', '2020-04-30', '2020-05-04'); +psql:include/cagg_refresh_common.sql:69: DEBUG: hypertable 1 existing watermark >= new invalidation threshold 1588723200000000 1588550400000000 +psql:include/cagg_refresh_common.sql:69: DEBUG: continuous aggregate refresh (individual invalidation) on "daily_temp" in window [ Thu Apr 30 17:00:00 2020 PDT, Sat May 02 17:00:00 2020 PDT ] +psql:include/cagg_refresh_common.sql:69: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +psql:include/cagg_refresh_common.sql:69: LOG: inserted 8 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +psql:include/cagg_refresh_common.sql:69: DEBUG: hypertable 2 existing watermark >= new watermark 1588723200000000 1588723200000000 RESET client_min_messages; -psql:include/cagg_refresh_common.sql:66: LOG: statement: RESET client_min_messages; +psql:include/cagg_refresh_common.sql:70: LOG: statement: RESET client_min_messages; -- Compare the aggregate to the equivalent query on the source table SELECT * FROM daily_temp ORDER BY day DESC, device; @@ -182,47 +187,88 @@ ORDER BY 1 DESC,2; -- Test unusual, but valid input CALL refresh_continuous_aggregate('daily_temp', '2020-05-01'::timestamptz, '2020-05-03'::date); -psql:include/cagg_refresh_common.sql:78: NOTICE: continuous aggregate "daily_temp" is already up-to-date +psql:include/cagg_refresh_common.sql:82: NOTICE: continuous aggregate "daily_temp" is already up-to-date CALL refresh_continuous_aggregate('daily_temp', '2020-05-01'::date, '2020-05-03'::date); -psql:include/cagg_refresh_common.sql:79: NOTICE: continuous aggregate "daily_temp" is already up-to-date +psql:include/cagg_refresh_common.sql:83: NOTICE: continuous aggregate "daily_temp" is already up-to-date -- Unbounded window forward in time CALL refresh_continuous_aggregate('daily_temp', '2020-05-03', NULL); -psql:include/cagg_refresh_common.sql:82: NOTICE: continuous aggregate "daily_temp" is already up-to-date +psql:include/cagg_refresh_common.sql:86: NOTICE: continuous aggregate "daily_temp" is already up-to-date CALL refresh_continuous_aggregate('daily_temp', NULL, NULL); -- Unbounded window back in time CALL refresh_continuous_aggregate('daily_temp', NULL, '2020-05-01'); -psql:include/cagg_refresh_common.sql:86: NOTICE: continuous aggregate "daily_temp" is already up-to-date +psql:include/cagg_refresh_common.sql:90: NOTICE: continuous aggregate "daily_temp" is already up-to-date -- Test bad input \set ON_ERROR_STOP 0 -- Bad continuous aggregate name CALL refresh_continuous_aggregate(NULL, '2020-05-03', '2020-05-05'); -psql:include/cagg_refresh_common.sql:91: ERROR: invalid continuous aggregate +psql:include/cagg_refresh_common.sql:95: ERROR: invalid continuous aggregate CALL refresh_continuous_aggregate('xyz', '2020-05-03', '2020-05-05'); -psql:include/cagg_refresh_common.sql:92: ERROR: relation "xyz" does not exist at character 35 +psql:include/cagg_refresh_common.sql:96: ERROR: relation "xyz" does not exist at character 35 -- Valid object, but not a continuous aggregate CALL refresh_continuous_aggregate('conditions', '2020-05-03', '2020-05-05'); -psql:include/cagg_refresh_common.sql:94: ERROR: relation "conditions" is not a continuous aggregate +psql:include/cagg_refresh_common.sql:98: ERROR: relation "conditions" is not a continuous aggregate -- Object ID with no object CALL refresh_continuous_aggregate(1, '2020-05-03', '2020-05-05'); -psql:include/cagg_refresh_common.sql:96: ERROR: continuous aggregate does not exist +psql:include/cagg_refresh_common.sql:100: ERROR: continuous aggregate does not exist -- Lacking arguments CALL refresh_continuous_aggregate('daily_temp'); -psql:include/cagg_refresh_common.sql:98: ERROR: procedure refresh_continuous_aggregate(unknown) does not exist at character 6 +psql:include/cagg_refresh_common.sql:102: ERROR: procedure refresh_continuous_aggregate(unknown) does not exist at character 6 CALL refresh_continuous_aggregate('daily_temp', '2020-05-03'); -psql:include/cagg_refresh_common.sql:99: ERROR: procedure refresh_continuous_aggregate(unknown, unknown) does not exist at character 6 +psql:include/cagg_refresh_common.sql:103: ERROR: procedure refresh_continuous_aggregate(unknown, unknown) does not exist at character 6 -- Bad time ranges CALL refresh_continuous_aggregate('daily_temp', 'xyz', '2020-05-05'); -psql:include/cagg_refresh_common.sql:101: ERROR: invalid input syntax for type timestamp with time zone: "xyz" +psql:include/cagg_refresh_common.sql:105: ERROR: invalid input syntax for type timestamp with time zone: "xyz" CALL refresh_continuous_aggregate('daily_temp', '2020-05-03', 'xyz'); -psql:include/cagg_refresh_common.sql:102: ERROR: invalid input syntax for type timestamp with time zone: "xyz" +psql:include/cagg_refresh_common.sql:106: ERROR: invalid input syntax for type timestamp with time zone: "xyz" CALL refresh_continuous_aggregate('daily_temp', '2020-05-03', '2020-05-01'); -psql:include/cagg_refresh_common.sql:103: ERROR: refresh window too small +psql:include/cagg_refresh_common.sql:107: ERROR: refresh window too small -- Bad time input CALL refresh_continuous_aggregate('daily_temp', '2020-05-01'::text, '2020-05-03'::text); -psql:include/cagg_refresh_common.sql:105: ERROR: invalid time argument type "text" +psql:include/cagg_refresh_common.sql:109: ERROR: invalid time argument type "text" CALL refresh_continuous_aggregate('daily_temp', 0, '2020-05-01'); -psql:include/cagg_refresh_common.sql:106: ERROR: invalid time argument type "integer" +psql:include/cagg_refresh_common.sql:110: ERROR: invalid time argument type "integer" \set ON_ERROR_STOP 1 +-- Test forceful refreshment. Here we simulate the situation that we've seen +-- with tiered data when `timescaledb.enable_tiered_reads` were disabled on the +-- server level. In that case we would not see materialized tiered data and +-- we wouldn't be able to re-materialize the data using a normal refresh call +-- because it would skip previously materialized ranges, but it should be +-- possible with `force=>true` parameter. To simulate this use-case we clear +-- the materialization hypertable and forefully re-materialize it. +SELECT format('%I.%I', ht.schema_name, ht.table_name) AS mat_ht, mat_hypertable_id FROM _timescaledb_catalog.continuous_agg cagg +JOIN _timescaledb_catalog.hypertable ht ON cagg.mat_hypertable_id = ht.id +WHERE user_view_name = 'daily_temp' \gset +-- Delete the data from the materialization hypertable +DELETE FROM :mat_ht; +-- Run regular refresh, it should not touch previously materialized range +CALL refresh_continuous_aggregate('daily_temp', '2020-05-02', '2020-05-05 17:00'); +psql:include/cagg_refresh_common.sql:128: NOTICE: continuous aggregate "daily_temp" is already up-to-date +SELECT * FROM daily_temp +ORDER BY day DESC, device; + day | device | avg_temp +-----+--------+---------- +(0 rows) + +-- Run it again with force=>true, the data should be rematerialized +CALL refresh_continuous_aggregate('daily_temp', '2020-05-02', '2020-05-05 17:00', force=>true); +SELECT * FROM daily_temp +ORDER BY day DESC, device; + day | device | avg_temp +------------------------------+--------+------------------ + Mon May 04 17:00:00 2020 PDT | 0 | 19.3846153846154 + Mon May 04 17:00:00 2020 PDT | 1 | 16.5555555555556 + Mon May 04 17:00:00 2020 PDT | 2 | 18.5714285714286 + Mon May 04 17:00:00 2020 PDT | 3 | 23.5714285714286 + Sun May 03 17:00:00 2020 PDT | 0 | 15.7647058823529 + Sun May 03 17:00:00 2020 PDT | 1 | 24.3142857142857 + Sun May 03 17:00:00 2020 PDT | 2 | 14.8205128205128 + Sun May 03 17:00:00 2020 PDT | 3 | 18.1111111111111 + Sat May 02 17:00:00 2020 PDT | 0 | 17 + Sat May 02 17:00:00 2020 PDT | 1 | 18.75 + Sat May 02 17:00:00 2020 PDT | 2 | 20 + Sat May 02 17:00:00 2020 PDT | 3 | 21.5217391304348 +(12 rows) + -- Test different time types CREATE TABLE conditions_date (time date NOT NULL, device int, temp float); SELECT create_hypertable('conditions_date', 'time'); @@ -268,7 +314,7 @@ AS SELECT time_bucket(SMALLINT '20', time) AS bucket, device, avg(temp) AS avg_temp FROM conditions_smallint c GROUP BY 1,2 WITH NO DATA; -psql:include/cagg_refresh_common.sql:150: ERROR: custom time function required on hypertable "conditions_smallint" +psql:include/cagg_refresh_common.sql:177: ERROR: custom time function required on hypertable "conditions_smallint" \set ON_ERROR_STOP 1 SELECT set_integer_now_func('conditions_smallint', 'smallint_now'); set_integer_now_func @@ -423,7 +469,7 @@ AS SELECT time_bucket('7 days', time) AS day, device, avg(temp) AS avg_temp FROM conditions GROUP BY 1,2 WITH DATA; -psql:include/cagg_refresh_common.sql:255: NOTICE: refreshing continuous aggregate "weekly_temp_with_data" +psql:include/cagg_refresh_common.sql:282: NOTICE: refreshing continuous aggregate "weekly_temp_with_data" SELECT * FROM weekly_temp_without_data; day | device | avg_temp -----+--------+---------- @@ -445,7 +491,7 @@ SELECT * FROM weekly_temp_with_data ORDER BY 1,2; \set ON_ERROR_STOP 0 -- REFRESH MATERIALIZED VIEW is blocked on continuous aggregates REFRESH MATERIALIZED VIEW weekly_temp_without_data; -psql:include/cagg_refresh_common.sql:262: ERROR: operation not supported on continuous aggregate +psql:include/cagg_refresh_common.sql:289: ERROR: operation not supported on continuous aggregate -- These should fail since we do not allow refreshing inside a -- transaction, not even as part of CREATE MATERIALIZED VIEW. DO LANGUAGE PLPGSQL $$ BEGIN @@ -457,7 +503,7 @@ SELECT time_bucket('7 days', time) AS day, device, avg(temp) AS avg_temp FROM conditions GROUP BY 1,2 WITH DATA; END $$; -psql:include/cagg_refresh_common.sql:274: ERROR: CREATE MATERIALIZED VIEW ... WITH DATA cannot be executed from a function +psql:include/cagg_refresh_common.sql:301: ERROR: CREATE MATERIALIZED VIEW ... WITH DATA cannot be executed from a function BEGIN; CREATE MATERIALIZED VIEW weekly_conditions WITH (timescaledb.continuous, @@ -466,7 +512,7 @@ AS SELECT time_bucket('7 days', time) AS day, device, avg(temp) AS avg_temp FROM conditions GROUP BY 1,2 WITH DATA; -psql:include/cagg_refresh_common.sql:283: ERROR: CREATE MATERIALIZED VIEW ... WITH DATA cannot run inside a transaction block +psql:include/cagg_refresh_common.sql:310: ERROR: CREATE MATERIALIZED VIEW ... WITH DATA cannot run inside a transaction block COMMIT; \set ON_ERROR_STOP 1 -- This should not fail since we do not refresh the continuous diff --git a/tsl/test/expected/cagg_refresh_using_merge.out b/tsl/test/expected/cagg_refresh_using_merge.out index 6ad8fa3a46b..ca5481bbc93 100644 --- a/tsl/test/expected/cagg_refresh_using_merge.out +++ b/tsl/test/expected/cagg_refresh_using_merge.out @@ -15,6 +15,18 @@ SELECT create_hypertable('conditions', 'time'); (1,public,conditions,t) (1 row) +-- Test refresh on a cagg built on an empty table +CREATE MATERIALIZED VIEW daily_temp +WITH (timescaledb.continuous, + timescaledb.materialized_only=true) +AS +SELECT time_bucket('1 day', time) AS day, device, avg(temp) AS avg_temp +FROM conditions +GROUP BY 1,2 WITH NO DATA; +CALL refresh_continuous_aggregate('daily_temp', NULL, NULL); +psql:include/cagg_refresh_common.sql:17: NOTICE: continuous aggregate "daily_temp" is already up-to-date +CALL refresh_continuous_aggregate('daily_temp', NULL, NULL, force => true); +psql:include/cagg_refresh_common.sql:18: NOTICE: continuous aggregate "daily_temp" is already up-to-date SELECT setseed(.12); setseed --------- @@ -42,13 +54,6 @@ LIMIT 10; Mon May 04 22:30:00 2020 PDT | 0 | 8 (10 rows) -CREATE MATERIALIZED VIEW daily_temp -WITH (timescaledb.continuous, - timescaledb.materialized_only=true) -AS -SELECT time_bucket('1 day', time) AS day, device, avg(temp) AS avg_temp -FROM conditions -GROUP BY 1,2 WITH NO DATA; -- The continuous aggregate should be empty SELECT * FROM daily_temp ORDER BY day DESC, device; @@ -73,16 +78,16 @@ CALL refresh_continuous_aggregate('daily_temp', '2020-05-03 17:00 PDT', '2020-05 -- These refreshes will fail since they don't align with the bucket's -- time zone CALL refresh_continuous_aggregate('daily_temp', '2020-05-03', '2020-05-04'); -psql:include/cagg_refresh_common.sql:43: ERROR: refresh window too small +psql:include/cagg_refresh_common.sql:47: ERROR: refresh window too small DETAIL: The refresh window must cover at least one bucket of data. HINT: Align the refresh window with the bucket time zone or use at least two buckets. CALL refresh_continuous_aggregate('daily_temp', '2020-05-03 00:00 PDT', '2020-05-04 00:00 PDT'); -psql:include/cagg_refresh_common.sql:44: ERROR: refresh window too small +psql:include/cagg_refresh_common.sql:48: ERROR: refresh window too small DETAIL: The refresh window must cover at least one bucket of data. HINT: Align the refresh window with the bucket time zone or use at least two buckets. -- Refresh window less than one bucket CALL refresh_continuous_aggregate('daily_temp', '2020-05-03 00:00 UTC', '2020-05-03 23:59 UTC'); -psql:include/cagg_refresh_common.sql:47: ERROR: refresh window too small +psql:include/cagg_refresh_common.sql:51: ERROR: refresh window too small DETAIL: The refresh window must cover at least one bucket of data. HINT: Align the refresh window with the bucket time zone or use at least two buckets. -- Refresh window bigger than one bucket, but failing since it is not @@ -91,7 +96,7 @@ HINT: Align the refresh window with the bucket time zone or use at least two bu -- Refresh window: [----------) -- Buckets: [------|------] CALL refresh_continuous_aggregate('daily_temp', '2020-05-03 01:00 UTC', '2020-05-04 08:00 UTC'); -psql:include/cagg_refresh_common.sql:53: ERROR: refresh window too small +psql:include/cagg_refresh_common.sql:57: ERROR: refresh window too small DETAIL: The refresh window must cover at least one bucket of data. HINT: Align the refresh window with the bucket time zone or use at least two buckets. \set VERBOSITY terse @@ -119,13 +124,13 @@ ORDER BY day DESC, device; -- Refresh the rest (and try DEBUG output) SET client_min_messages TO DEBUG1; CALL refresh_continuous_aggregate('daily_temp', '2020-04-30', '2020-05-04'); -psql:include/cagg_refresh_common.sql:65: LOG: statement: CALL refresh_continuous_aggregate('daily_temp', '2020-04-30', '2020-05-04'); -psql:include/cagg_refresh_common.sql:65: DEBUG: hypertable 1 existing watermark >= new invalidation threshold 1588723200000000 1588550400000000 -psql:include/cagg_refresh_common.sql:65: DEBUG: continuous aggregate refresh (individual invalidation) on "daily_temp" in window [ Thu Apr 30 17:00:00 2020 PDT, Sat May 02 17:00:00 2020 PDT ] -psql:include/cagg_refresh_common.sql:65: LOG: inserted 8 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" -psql:include/cagg_refresh_common.sql:65: DEBUG: hypertable 2 existing watermark >= new watermark 1588723200000000 1588723200000000 +psql:include/cagg_refresh_common.sql:69: LOG: statement: CALL refresh_continuous_aggregate('daily_temp', '2020-04-30', '2020-05-04'); +psql:include/cagg_refresh_common.sql:69: DEBUG: hypertable 1 existing watermark >= new invalidation threshold 1588723200000000 1588550400000000 +psql:include/cagg_refresh_common.sql:69: DEBUG: continuous aggregate refresh (individual invalidation) on "daily_temp" in window [ Thu Apr 30 17:00:00 2020 PDT, Sat May 02 17:00:00 2020 PDT ] +psql:include/cagg_refresh_common.sql:69: LOG: inserted 8 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +psql:include/cagg_refresh_common.sql:69: DEBUG: hypertable 2 existing watermark >= new watermark 1588723200000000 1588723200000000 RESET client_min_messages; -psql:include/cagg_refresh_common.sql:66: LOG: statement: RESET client_min_messages; +psql:include/cagg_refresh_common.sql:70: LOG: statement: RESET client_min_messages; -- Compare the aggregate to the equivalent query on the source table SELECT * FROM daily_temp ORDER BY day DESC, device; @@ -183,47 +188,88 @@ ORDER BY 1 DESC,2; -- Test unusual, but valid input CALL refresh_continuous_aggregate('daily_temp', '2020-05-01'::timestamptz, '2020-05-03'::date); -psql:include/cagg_refresh_common.sql:78: NOTICE: continuous aggregate "daily_temp" is already up-to-date +psql:include/cagg_refresh_common.sql:82: NOTICE: continuous aggregate "daily_temp" is already up-to-date CALL refresh_continuous_aggregate('daily_temp', '2020-05-01'::date, '2020-05-03'::date); -psql:include/cagg_refresh_common.sql:79: NOTICE: continuous aggregate "daily_temp" is already up-to-date +psql:include/cagg_refresh_common.sql:83: NOTICE: continuous aggregate "daily_temp" is already up-to-date -- Unbounded window forward in time CALL refresh_continuous_aggregate('daily_temp', '2020-05-03', NULL); -psql:include/cagg_refresh_common.sql:82: NOTICE: continuous aggregate "daily_temp" is already up-to-date +psql:include/cagg_refresh_common.sql:86: NOTICE: continuous aggregate "daily_temp" is already up-to-date CALL refresh_continuous_aggregate('daily_temp', NULL, NULL); -- Unbounded window back in time CALL refresh_continuous_aggregate('daily_temp', NULL, '2020-05-01'); -psql:include/cagg_refresh_common.sql:86: NOTICE: continuous aggregate "daily_temp" is already up-to-date +psql:include/cagg_refresh_common.sql:90: NOTICE: continuous aggregate "daily_temp" is already up-to-date -- Test bad input \set ON_ERROR_STOP 0 -- Bad continuous aggregate name CALL refresh_continuous_aggregate(NULL, '2020-05-03', '2020-05-05'); -psql:include/cagg_refresh_common.sql:91: ERROR: invalid continuous aggregate +psql:include/cagg_refresh_common.sql:95: ERROR: invalid continuous aggregate CALL refresh_continuous_aggregate('xyz', '2020-05-03', '2020-05-05'); -psql:include/cagg_refresh_common.sql:92: ERROR: relation "xyz" does not exist at character 35 +psql:include/cagg_refresh_common.sql:96: ERROR: relation "xyz" does not exist at character 35 -- Valid object, but not a continuous aggregate CALL refresh_continuous_aggregate('conditions', '2020-05-03', '2020-05-05'); -psql:include/cagg_refresh_common.sql:94: ERROR: relation "conditions" is not a continuous aggregate +psql:include/cagg_refresh_common.sql:98: ERROR: relation "conditions" is not a continuous aggregate -- Object ID with no object CALL refresh_continuous_aggregate(1, '2020-05-03', '2020-05-05'); -psql:include/cagg_refresh_common.sql:96: ERROR: continuous aggregate does not exist +psql:include/cagg_refresh_common.sql:100: ERROR: continuous aggregate does not exist -- Lacking arguments CALL refresh_continuous_aggregate('daily_temp'); -psql:include/cagg_refresh_common.sql:98: ERROR: procedure refresh_continuous_aggregate(unknown) does not exist at character 6 +psql:include/cagg_refresh_common.sql:102: ERROR: procedure refresh_continuous_aggregate(unknown) does not exist at character 6 CALL refresh_continuous_aggregate('daily_temp', '2020-05-03'); -psql:include/cagg_refresh_common.sql:99: ERROR: procedure refresh_continuous_aggregate(unknown, unknown) does not exist at character 6 +psql:include/cagg_refresh_common.sql:103: ERROR: procedure refresh_continuous_aggregate(unknown, unknown) does not exist at character 6 -- Bad time ranges CALL refresh_continuous_aggregate('daily_temp', 'xyz', '2020-05-05'); -psql:include/cagg_refresh_common.sql:101: ERROR: invalid input syntax for type timestamp with time zone: "xyz" +psql:include/cagg_refresh_common.sql:105: ERROR: invalid input syntax for type timestamp with time zone: "xyz" CALL refresh_continuous_aggregate('daily_temp', '2020-05-03', 'xyz'); -psql:include/cagg_refresh_common.sql:102: ERROR: invalid input syntax for type timestamp with time zone: "xyz" +psql:include/cagg_refresh_common.sql:106: ERROR: invalid input syntax for type timestamp with time zone: "xyz" CALL refresh_continuous_aggregate('daily_temp', '2020-05-03', '2020-05-01'); -psql:include/cagg_refresh_common.sql:103: ERROR: refresh window too small +psql:include/cagg_refresh_common.sql:107: ERROR: refresh window too small -- Bad time input CALL refresh_continuous_aggregate('daily_temp', '2020-05-01'::text, '2020-05-03'::text); -psql:include/cagg_refresh_common.sql:105: ERROR: invalid time argument type "text" +psql:include/cagg_refresh_common.sql:109: ERROR: invalid time argument type "text" CALL refresh_continuous_aggregate('daily_temp', 0, '2020-05-01'); -psql:include/cagg_refresh_common.sql:106: ERROR: invalid time argument type "integer" +psql:include/cagg_refresh_common.sql:110: ERROR: invalid time argument type "integer" \set ON_ERROR_STOP 1 +-- Test forceful refreshment. Here we simulate the situation that we've seen +-- with tiered data when `timescaledb.enable_tiered_reads` were disabled on the +-- server level. In that case we would not see materialized tiered data and +-- we wouldn't be able to re-materialize the data using a normal refresh call +-- because it would skip previously materialized ranges, but it should be +-- possible with `force=>true` parameter. To simulate this use-case we clear +-- the materialization hypertable and forefully re-materialize it. +SELECT format('%I.%I', ht.schema_name, ht.table_name) AS mat_ht, mat_hypertable_id FROM _timescaledb_catalog.continuous_agg cagg +JOIN _timescaledb_catalog.hypertable ht ON cagg.mat_hypertable_id = ht.id +WHERE user_view_name = 'daily_temp' \gset +-- Delete the data from the materialization hypertable +DELETE FROM :mat_ht; +-- Run regular refresh, it should not touch previously materialized range +CALL refresh_continuous_aggregate('daily_temp', '2020-05-02', '2020-05-05 17:00'); +psql:include/cagg_refresh_common.sql:128: NOTICE: continuous aggregate "daily_temp" is already up-to-date +SELECT * FROM daily_temp +ORDER BY day DESC, device; + day | device | avg_temp +-----+--------+---------- +(0 rows) + +-- Run it again with force=>true, the data should be rematerialized +CALL refresh_continuous_aggregate('daily_temp', '2020-05-02', '2020-05-05 17:00', force=>true); +SELECT * FROM daily_temp +ORDER BY day DESC, device; + day | device | avg_temp +------------------------------+--------+------------------ + Mon May 04 17:00:00 2020 PDT | 0 | 19.3846153846154 + Mon May 04 17:00:00 2020 PDT | 1 | 16.5555555555556 + Mon May 04 17:00:00 2020 PDT | 2 | 18.5714285714286 + Mon May 04 17:00:00 2020 PDT | 3 | 23.5714285714286 + Sun May 03 17:00:00 2020 PDT | 0 | 15.7647058823529 + Sun May 03 17:00:00 2020 PDT | 1 | 24.3142857142857 + Sun May 03 17:00:00 2020 PDT | 2 | 14.8205128205128 + Sun May 03 17:00:00 2020 PDT | 3 | 18.1111111111111 + Sat May 02 17:00:00 2020 PDT | 0 | 17 + Sat May 02 17:00:00 2020 PDT | 1 | 18.75 + Sat May 02 17:00:00 2020 PDT | 2 | 20 + Sat May 02 17:00:00 2020 PDT | 3 | 21.5217391304348 +(12 rows) + -- Test different time types CREATE TABLE conditions_date (time date NOT NULL, device int, temp float); SELECT create_hypertable('conditions_date', 'time'); @@ -269,7 +315,7 @@ AS SELECT time_bucket(SMALLINT '20', time) AS bucket, device, avg(temp) AS avg_temp FROM conditions_smallint c GROUP BY 1,2 WITH NO DATA; -psql:include/cagg_refresh_common.sql:150: ERROR: custom time function required on hypertable "conditions_smallint" +psql:include/cagg_refresh_common.sql:177: ERROR: custom time function required on hypertable "conditions_smallint" \set ON_ERROR_STOP 1 SELECT set_integer_now_func('conditions_smallint', 'smallint_now'); set_integer_now_func @@ -424,7 +470,7 @@ AS SELECT time_bucket('7 days', time) AS day, device, avg(temp) AS avg_temp FROM conditions GROUP BY 1,2 WITH DATA; -psql:include/cagg_refresh_common.sql:255: NOTICE: refreshing continuous aggregate "weekly_temp_with_data" +psql:include/cagg_refresh_common.sql:282: NOTICE: refreshing continuous aggregate "weekly_temp_with_data" SELECT * FROM weekly_temp_without_data; day | device | avg_temp -----+--------+---------- @@ -446,7 +492,7 @@ SELECT * FROM weekly_temp_with_data ORDER BY 1,2; \set ON_ERROR_STOP 0 -- REFRESH MATERIALIZED VIEW is blocked on continuous aggregates REFRESH MATERIALIZED VIEW weekly_temp_without_data; -psql:include/cagg_refresh_common.sql:262: ERROR: operation not supported on continuous aggregate +psql:include/cagg_refresh_common.sql:289: ERROR: operation not supported on continuous aggregate -- These should fail since we do not allow refreshing inside a -- transaction, not even as part of CREATE MATERIALIZED VIEW. DO LANGUAGE PLPGSQL $$ BEGIN @@ -458,7 +504,7 @@ SELECT time_bucket('7 days', time) AS day, device, avg(temp) AS avg_temp FROM conditions GROUP BY 1,2 WITH DATA; END $$; -psql:include/cagg_refresh_common.sql:274: ERROR: CREATE MATERIALIZED VIEW ... WITH DATA cannot be executed from a function +psql:include/cagg_refresh_common.sql:301: ERROR: CREATE MATERIALIZED VIEW ... WITH DATA cannot be executed from a function BEGIN; CREATE MATERIALIZED VIEW weekly_conditions WITH (timescaledb.continuous, @@ -467,7 +513,7 @@ AS SELECT time_bucket('7 days', time) AS day, device, avg(temp) AS avg_temp FROM conditions GROUP BY 1,2 WITH DATA; -psql:include/cagg_refresh_common.sql:283: ERROR: CREATE MATERIALIZED VIEW ... WITH DATA cannot run inside a transaction block +psql:include/cagg_refresh_common.sql:310: ERROR: CREATE MATERIALIZED VIEW ... WITH DATA cannot run inside a transaction block COMMIT; \set ON_ERROR_STOP 1 -- This should not fail since we do not refresh the continuous diff --git a/tsl/test/expected/chunk_utils_internal.out b/tsl/test/expected/chunk_utils_internal.out index bcaead5d1eb..520484a8651 100644 --- a/tsl/test/expected/chunk_utils_internal.out +++ b/tsl/test/expected/chunk_utils_internal.out @@ -785,6 +785,40 @@ SET timescaledb.enable_tiered_reads=true; Index Cond: (timec < 'Sun Jan 01 01:00:00 2023 PST'::timestamp with time zone) (4 rows) +-- Test forceful refreshment. Here we simulate the situation that we've seen +-- with tiered data when `timescaledb.enable_tiered_reads` were disabled on the +-- server level. In that case we would not see materialized tiered data and +-- we wouldn't be able to re-materialize the data using a normal refresh call +-- because it would skip previously materialized ranges, but it should be +-- possible with `force=>true` parameter. +CREATE MATERIALIZED VIEW ht_try_weekly +WITH (timescaledb.continuous) AS +SELECT time_bucket(interval '1 week', timec) AS ts_bucket, avg(value) +FROM ht_try +GROUP BY 1 +WITH NO DATA; +SELECT * FROM ht_try_weekly; + ts_bucket | avg +-----------+----- +(0 rows) + +SET timescaledb.enable_tiered_reads=false; +CALL refresh_continuous_aggregate('ht_try_weekly', '2019-12-29', '2020-01-10', force=>false); +SELECT * FROM ht_try_weekly; + ts_bucket | avg +-----------+----- +(0 rows) + +SET timescaledb.enable_tiered_reads=true; +CALL refresh_continuous_aggregate('ht_try_weekly', '2019-12-29', '2020-01-10', force=>true); +SELECT * FROM ht_try_weekly; + ts_bucket | avg +------------------------------+----------------------- + Sun Dec 29 16:00:00 2019 PST | 1000.0000000000000000 +(1 row) + +DROP MATERIALIZED VIEW ht_try_weekly; +NOTICE: drop cascades to table _timescaledb_internal._hyper_6_12_chunk -- This test verifies that a bugfix regarding the way `ROWID_VAR`s are adjusted -- in the chunks' targetlists on DELETE/UPDATE works (including partially -- compressed chunks) @@ -797,7 +831,7 @@ SELECT compress_chunk(show_chunks('ht_try', newer_than => '2021-01-01'::timestam compress_chunk ----------------------------------------- _timescaledb_internal._hyper_5_10_chunk - _timescaledb_internal._hyper_5_12_chunk + _timescaledb_internal._hyper_5_13_chunk (2 rows) INSERT INTO ht_try VALUES ('2021-06-05 01:00', 10, 222); @@ -900,7 +934,7 @@ Indexes: Triggers: ts_insert_blocker BEFORE INSERT ON ht_try FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker() Child tables: _timescaledb_internal._hyper_5_10_chunk, - _timescaledb_internal._hyper_5_12_chunk + _timescaledb_internal._hyper_5_13_chunk -- verify that still can read from the table after catalog manipulations EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) SELECT * FROM ht_try; @@ -908,10 +942,10 @@ EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) SELECT * FROM ht_try; ---------------------------------------------------------------------------------- Append (actual rows=3 loops=1) -> Custom Scan (DecompressChunk) on _hyper_5_10_chunk (actual rows=1 loops=1) - -> Seq Scan on compress_hyper_6_13_chunk (actual rows=1 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_5_12_chunk (actual rows=1 loops=1) - -> Seq Scan on compress_hyper_6_14_chunk (actual rows=1 loops=1) - -> Seq Scan on _hyper_5_12_chunk (actual rows=1 loops=1) + -> Seq Scan on compress_hyper_7_14_chunk (actual rows=1 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_5_13_chunk (actual rows=1 loops=1) + -> Seq Scan on compress_hyper_7_15_chunk (actual rows=1 loops=1) + -> Seq Scan on _hyper_5_13_chunk (actual rows=1 loops=1) (6 rows) ROLLBACK; @@ -964,7 +998,7 @@ RESTRICT SELECT create_hypertable('hyper_constr', 'time', chunk_time_interval => 10); create_hypertable --------------------------- - (7,public,hyper_constr,t) + (8,public,hyper_constr,t) (1 row) INSERT INTO hyper_constr VALUES( 10, 200, 22, 1, 111, 44); @@ -997,7 +1031,7 @@ WHERE hypertable_id IN (SELECT id from _timescaledb_catalog.hypertable ORDER BY table_name; table_name | status | osm_chunk --------------------+--------+----------- - _hyper_7_15_chunk | 0 | f + _hyper_8_16_chunk | 0 | f child_hyper_constr | 0 | t (2 rows) @@ -1085,15 +1119,15 @@ where hypertable_id = (Select id from _timescaledb_catalog.hypertable where tabl ORDER BY id; id | table_name ----+-------------------- - 15 | _hyper_7_15_chunk - 16 | child_hyper_constr + 16 | _hyper_8_16_chunk + 17 | child_hyper_constr (2 rows) -- show_chunks will not show the OSM chunk which is visible via the above query SELECT show_chunks('hyper_constr'); show_chunks ----------------------------------------- - _timescaledb_internal._hyper_7_15_chunk + _timescaledb_internal._hyper_8_16_chunk (1 row) ROLLBACK; @@ -1125,7 +1159,7 @@ CREATE TABLE test1.copy_test ( SELECT create_hypertable('test1.copy_test', 'time', chunk_time_interval => interval '1 day'); create_hypertable ----------------------- - (8,test1,copy_test,t) + (9,test1,copy_test,t) (1 row) COPY test1.copy_test FROM STDIN DELIMITER ','; @@ -1146,13 +1180,13 @@ SELECT table_name, status FROM _timescaledb_catalog.chunk WHERE table_name = :'COPY_CHUNK_NAME'; table_name | status -------------------+-------- - _hyper_8_17_chunk | 4 + _hyper_9_18_chunk | 4 (1 row) \set ON_ERROR_STOP 0 -- Copy should fail because one of che chunks is frozen COPY test1.copy_test FROM STDIN DELIMITER ','; -ERROR: cannot INSERT into frozen chunk "_hyper_8_17_chunk" +ERROR: cannot INSERT into frozen chunk "_hyper_9_18_chunk" \set ON_ERROR_STOP 1 -- Count existing rows SELECT COUNT(*) FROM test1.copy_test; @@ -1166,13 +1200,13 @@ SELECT table_name, status FROM _timescaledb_catalog.chunk WHERE table_name = :'COPY_CHUNK_NAME'; table_name | status -------------------+-------- - _hyper_8_17_chunk | 4 + _hyper_9_18_chunk | 4 (1 row) \set ON_ERROR_STOP 0 -- Copy should fail because one of che chunks is frozen COPY test1.copy_test FROM STDIN DELIMITER ','; -ERROR: cannot INSERT into frozen chunk "_hyper_8_17_chunk" +ERROR: cannot INSERT into frozen chunk "_hyper_9_18_chunk" \set ON_ERROR_STOP 1 -- Count existing rows SELECT COUNT(*) FROM test1.copy_test; @@ -1193,7 +1227,7 @@ SELECT table_name, status FROM _timescaledb_catalog.chunk WHERE table_name = :'COPY_CHUNK_NAME'; table_name | status -------------------+-------- - _hyper_8_17_chunk | 0 + _hyper_9_18_chunk | 0 (1 row) -- Copy should work now @@ -1296,12 +1330,12 @@ WHERE ht.table_name LIKE 'osm%' ORDER BY 2,3; table_name | id | dimension_id | range_start | range_end ------------+----+--------------+---------------------+--------------------- - osm_int2 | 16 | 7 | 9223372036854775806 | 9223372036854775807 - osm_int4 | 17 | 8 | 9223372036854775806 | 9223372036854775807 - osm_int8 | 18 | 9 | 9223372036854775806 | 9223372036854775807 - osm_date | 19 | 10 | 9223372036854775806 | 9223372036854775807 - osm_ts | 20 | 11 | 9223372036854775806 | 9223372036854775807 - osm_tstz | 21 | 12 | 9223372036854775806 | 9223372036854775807 + osm_int2 | 17 | 8 | 9223372036854775806 | 9223372036854775807 + osm_int4 | 18 | 9 | 9223372036854775806 | 9223372036854775807 + osm_int8 | 19 | 10 | 9223372036854775806 | 9223372036854775807 + osm_date | 20 | 11 | 9223372036854775806 | 9223372036854775807 + osm_ts | 21 | 12 | 9223372036854775806 | 9223372036854775807 + osm_tstz | 22 | 13 | 9223372036854775806 | 9223372036854775807 (6 rows) -- test that correct slice is found and updated for table with multiple chunk constraints @@ -1314,8 +1348,8 @@ _timescaledb_catalog.chunk c, _timescaledb_catalog.chunk_constraint cc WHERE c.h AND c.id = cc.chunk_id; id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk | chunk_id | dimension_slice_id | constraint_name | hypertable_constraint_name ----+---------------+-----------------------+--------------------+---------------------+---------+--------+-----------+----------+--------------------+-----------------------------+---------------------------- - 25 | 15 | _timescaledb_internal | _hyper_15_25_chunk | | f | 0 | f | 25 | | 25_5_test_multicon_time_key | test_multicon_time_key - 25 | 15 | _timescaledb_internal | _hyper_15_25_chunk | | f | 0 | f | 25 | 22 | constraint_22 | + 26 | 16 | _timescaledb_internal | _hyper_16_26_chunk | | f | 0 | f | 26 | | 26_5_test_multicon_time_key | test_multicon_time_key + 26 | 16 | _timescaledb_internal | _hyper_16_26_chunk | | f | 0 | f | 26 | 23 | constraint_23 | (2 rows) \c :TEST_DBNAME :ROLE_SUPERUSER ; @@ -1333,7 +1367,7 @@ FROM _timescaledb_catalog.chunk c, _timescaledb_catalog.chunk_constraint cc, _ti WHERE c.hypertable_id = :htid AND cc.chunk_id = c.id AND ds.id = cc.dimension_slice_id; chunk_id | table_name | status | osm_chunk | dimension_slice_id | range_start | range_end ----------+--------------------+--------+-----------+--------------------+------------------+------------------ - 25 | _hyper_15_25_chunk | 0 | t | 22 | 1577955600000000 | 1578128400000000 + 26 | _hyper_16_26_chunk | 0 | t | 23 | 1577955600000000 | 1578128400000000 (1 row) -- check that range was reset to default - infinity @@ -1361,7 +1395,7 @@ FROM _timescaledb_catalog.chunk c, _timescaledb_catalog.chunk_constraint cc, _ti WHERE c.hypertable_id = :htid AND cc.chunk_id = c.id AND ds.id = cc.dimension_slice_id ORDER BY cc.chunk_id; chunk_id | table_name | status | osm_chunk | dimension_slice_id | range_start | range_end ----------+--------------------+--------+-----------+--------------------+---------------------+--------------------- - 25 | _hyper_15_25_chunk | 0 | t | 22 | 9223372036854775806 | 9223372036854775807 + 26 | _hyper_16_26_chunk | 0 | t | 23 | 9223372036854775806 | 9223372036854775807 (1 row) -- TEST for orderedappend that depends on hypertable_osm_range_update functionality @@ -1386,9 +1420,9 @@ FROM _timescaledb_catalog.chunk c, _timescaledb_catalog.chunk_constraint cc, _ti WHERE c.hypertable_id = :htid AND cc.chunk_id = c.id AND ds.id = cc.dimension_slice_id ORDER BY cc.chunk_id; chunk_id | table_name | status | osm_chunk | dimension_slice_id | range_start | range_end ----------+-------------------------+--------+-----------+--------------------+---------------------+--------------------- - 26 | _hyper_16_26_chunk | 0 | f | 23 | 1577836800000000 | 1577923200000000 - 27 | _hyper_16_27_chunk | 0 | f | 24 | 1577923200000000 | 1578009600000000 - 28 | test_chunkapp_fdw_child | 0 | t | 25 | 9223372036854775806 | 9223372036854775807 + 27 | _hyper_17_27_chunk | 0 | f | 24 | 1577836800000000 | 1577923200000000 + 28 | _hyper_17_28_chunk | 0 | f | 25 | 1577923200000000 | 1578009600000000 + 29 | test_chunkapp_fdw_child | 0 | t | 26 | 9223372036854775806 | 9223372036854775807 (3 rows) -- attempt to update overlapping range, should fail @@ -1409,9 +1443,9 @@ FROM _timescaledb_catalog.chunk c, _timescaledb_catalog.chunk_constraint cc, _ti WHERE c.hypertable_id = :htid AND cc.chunk_id = c.id AND ds.id = cc.dimension_slice_id ORDER BY cc.chunk_id; chunk_id | table_name | status | osm_chunk | dimension_slice_id | range_start | range_end ----------+-------------------------+--------+-----------+--------------------+------------------+------------------ - 26 | _hyper_16_26_chunk | 0 | f | 23 | 1577836800000000 | 1577923200000000 - 27 | _hyper_16_27_chunk | 0 | f | 24 | 1577923200000000 | 1578009600000000 - 28 | test_chunkapp_fdw_child | 0 | t | 25 | 1578038400000000 | 1578124800000000 + 27 | _hyper_17_27_chunk | 0 | f | 24 | 1577836800000000 | 1577923200000000 + 28 | _hyper_17_28_chunk | 0 | f | 25 | 1577923200000000 | 1578009600000000 + 29 | test_chunkapp_fdw_child | 0 | t | 26 | 1578038400000000 | 1578124800000000 (3 rows) -- ordered append should be possible as ranges do not overlap @@ -1420,8 +1454,8 @@ WHERE c.hypertable_id = :htid AND cc.chunk_id = c.id AND ds.id = cc.dimension_sl ------------------------------------------------------------------------------------------------- Custom Scan (ChunkAppend) on test_chunkapp Order: test_chunkapp."time" - -> Index Scan Backward using _hyper_16_26_chunk_test_chunkapp_time_idx on _hyper_16_26_chunk - -> Index Scan Backward using _hyper_16_27_chunk_test_chunkapp_time_idx on _hyper_16_27_chunk + -> Index Scan Backward using _hyper_17_27_chunk_test_chunkapp_time_idx on _hyper_17_27_chunk + -> Index Scan Backward using _hyper_17_28_chunk_test_chunkapp_time_idx on _hyper_17_28_chunk -> Foreign Scan on test_chunkapp_fdw_child (5 rows) @@ -1462,9 +1496,9 @@ SELECT _timescaledb_functions.hypertable_osm_range_update('test_chunkapp',empty: QUERY PLAN ------------------------------------------------------------------------------------------------- Merge Append - Sort Key: _hyper_16_26_chunk."time" - -> Index Scan Backward using _hyper_16_26_chunk_test_chunkapp_time_idx on _hyper_16_26_chunk - -> Index Scan Backward using _hyper_16_27_chunk_test_chunkapp_time_idx on _hyper_16_27_chunk + Sort Key: _hyper_17_27_chunk."time" + -> Index Scan Backward using _hyper_17_27_chunk_test_chunkapp_time_idx on _hyper_17_27_chunk + -> Index Scan Backward using _hyper_17_28_chunk_test_chunkapp_time_idx on _hyper_17_28_chunk -> Foreign Scan on test_chunkapp_fdw_child (5 rows) @@ -1481,9 +1515,9 @@ FROM _timescaledb_catalog.chunk c, _timescaledb_catalog.chunk_constraint cc, _ti WHERE c.hypertable_id = :htid AND cc.chunk_id = c.id AND ds.id = cc.dimension_slice_id ORDER BY cc.chunk_id; chunk_id | table_name | status | osm_chunk | dimension_slice_id | range_start | range_end ----------+-------------------------+--------+-----------+--------------------+---------------------+--------------------- - 26 | _hyper_16_26_chunk | 0 | f | 23 | 1577836800000000 | 1577923200000000 - 27 | _hyper_16_27_chunk | 0 | f | 24 | 1577923200000000 | 1578009600000000 - 28 | test_chunkapp_fdw_child | 0 | t | 25 | 9223372036854775806 | 9223372036854775807 + 27 | _hyper_17_27_chunk | 0 | f | 24 | 1577836800000000 | 1577923200000000 + 28 | _hyper_17_28_chunk | 0 | f | 25 | 1577923200000000 | 1578009600000000 + 29 | test_chunkapp_fdw_child | 0 | t | 26 | 9223372036854775806 | 9223372036854775807 (3 rows) -- but also, OSM chunk should be included in the scan, since range is invalid and chunk is not empty @@ -1491,10 +1525,10 @@ WHERE c.hypertable_id = :htid AND cc.chunk_id = c.id AND ds.id = cc.dimension_sl QUERY PLAN ------------------------------------------------------------------------------------------------- Merge Append - Sort Key: _hyper_16_26_chunk."time" - -> Index Scan Backward using _hyper_16_26_chunk_test_chunkapp_time_idx on _hyper_16_26_chunk + Sort Key: _hyper_17_27_chunk."time" + -> Index Scan Backward using _hyper_17_27_chunk_test_chunkapp_time_idx on _hyper_17_27_chunk Index Cond: ("time" < 'Sun Jan 01 00:00:00 2023 PST'::timestamp with time zone) - -> Index Scan Backward using _hyper_16_27_chunk_test_chunkapp_time_idx on _hyper_16_27_chunk + -> Index Scan Backward using _hyper_17_28_chunk_test_chunkapp_time_idx on _hyper_17_28_chunk Index Cond: ("time" < 'Sun Jan 01 00:00:00 2023 PST'::timestamp with time zone) -> Foreign Scan on test_chunkapp_fdw_child (7 rows) @@ -1522,8 +1556,8 @@ SELECT _timescaledb_functions.hypertable_osm_range_update('test_chunkapp', NULL: ------------------------------------------------------------------------------------------------- Custom Scan (ChunkAppend) on test_chunkapp Order: test_chunkapp."time" - -> Index Scan Backward using _hyper_16_26_chunk_test_chunkapp_time_idx on _hyper_16_26_chunk - -> Index Scan Backward using _hyper_16_27_chunk_test_chunkapp_time_idx on _hyper_16_27_chunk + -> Index Scan Backward using _hyper_17_27_chunk_test_chunkapp_time_idx on _hyper_17_27_chunk + -> Index Scan Backward using _hyper_17_28_chunk_test_chunkapp_time_idx on _hyper_17_28_chunk -> Foreign Scan on test_chunkapp_fdw_child (5 rows) @@ -1540,9 +1574,9 @@ SELECT * FROM test_chunkapp ORDER BY 1; ------------------------------------------------------------------------------------------------- Custom Scan (ChunkAppend) on test_chunkapp Order: test_chunkapp."time" - -> Index Scan Backward using _hyper_16_26_chunk_test_chunkapp_time_idx on _hyper_16_26_chunk + -> Index Scan Backward using _hyper_17_27_chunk_test_chunkapp_time_idx on _hyper_17_27_chunk Index Cond: ("time" < 'Sun Jan 01 00:00:00 2023 PST'::timestamp with time zone) - -> Index Scan Backward using _hyper_16_27_chunk_test_chunkapp_time_idx on _hyper_16_27_chunk + -> Index Scan Backward using _hyper_17_28_chunk_test_chunkapp_time_idx on _hyper_17_28_chunk Index Cond: ("time" < 'Sun Jan 01 00:00:00 2023 PST'::timestamp with time zone) (6 rows) @@ -1579,7 +1613,7 @@ CREATE TABLE test2(time timestamptz not null, a int); SELECT create_hypertable('test2', 'time'); create_hypertable --------------------- - (17,public,test2,t) + (18,public,test2,t) (1 row) INSERT INTO test2 VALUES ('2020-01-01'::timestamptz, 1); @@ -1590,7 +1624,7 @@ psql:include/chunk_utils_internal_orderedappend.sql:138: NOTICE: default order SELECT compress_chunk(show_chunks('test2')); compress_chunk ------------------------------------------ - _timescaledb_internal._hyper_17_29_chunk + _timescaledb_internal._hyper_18_30_chunk (1 row) -- find internal compression table, call API function on it @@ -1599,7 +1633,7 @@ FROM _timescaledb_catalog.hypertable ht, _timescaledb_catalog.hypertable cht WHERE ht.table_name = 'test2' and cht.id = ht.compressed_hypertable_id \gset \set ON_ERROR_STOP 0 SELECT _timescaledb_functions.hypertable_osm_range_update(:'COMPRESSION_TBLNM'::regclass, '2020-01-01'::timestamptz); -psql:include/chunk_utils_internal_orderedappend.sql:145: ERROR: could not find time dimension for hypertable _timescaledb_internal._compressed_hypertable_18 +psql:include/chunk_utils_internal_orderedappend.sql:145: ERROR: could not find time dimension for hypertable _timescaledb_internal._compressed_hypertable_19 \set ON_ERROR_STOP 1 -- test wrong/incompatible data types with hypertable time dimension -- update range of int2 with int4 diff --git a/tsl/test/shared/expected/extension.out b/tsl/test/shared/expected/extension.out index db64fd94b2f..08e275bd2e2 100644 --- a/tsl/test/shared/expected/extension.out +++ b/tsl/test/shared/expected/extension.out @@ -258,7 +258,7 @@ ORDER BY pronamespace::regnamespace::text COLLATE "C", p.oid::regprocedure::text locf(anyelement,anyelement,boolean) move_chunk(regclass,name,name,regclass,boolean) recompress_chunk(regclass,boolean) - refresh_continuous_aggregate(regclass,"any","any") + refresh_continuous_aggregate(regclass,"any","any",boolean) remove_columnstore_policy(regclass,boolean) remove_compression_policy(regclass,boolean) remove_continuous_aggregate_policy(regclass,boolean,boolean) diff --git a/tsl/test/sql/chunk_utils_internal.sql b/tsl/test/sql/chunk_utils_internal.sql index 1ac6ed7c817..feeb83bc54b 100644 --- a/tsl/test/sql/chunk_utils_internal.sql +++ b/tsl/test/sql/chunk_utils_internal.sql @@ -419,6 +419,27 @@ SET timescaledb.enable_tiered_reads=true; :EXPLAIN SELECT * from ht_try WHERE timec > '2022-01-01 01:00'; :EXPLAIN SELECT * from ht_try WHERE timec < '2023-01-01 01:00'; +-- Test forceful refreshment. Here we simulate the situation that we've seen +-- with tiered data when `timescaledb.enable_tiered_reads` were disabled on the +-- server level. In that case we would not see materialized tiered data and +-- we wouldn't be able to re-materialize the data using a normal refresh call +-- because it would skip previously materialized ranges, but it should be +-- possible with `force=>true` parameter. +CREATE MATERIALIZED VIEW ht_try_weekly +WITH (timescaledb.continuous) AS +SELECT time_bucket(interval '1 week', timec) AS ts_bucket, avg(value) +FROM ht_try +GROUP BY 1 +WITH NO DATA; +SELECT * FROM ht_try_weekly; +SET timescaledb.enable_tiered_reads=false; +CALL refresh_continuous_aggregate('ht_try_weekly', '2019-12-29', '2020-01-10', force=>false); +SELECT * FROM ht_try_weekly; +SET timescaledb.enable_tiered_reads=true; +CALL refresh_continuous_aggregate('ht_try_weekly', '2019-12-29', '2020-01-10', force=>true); +SELECT * FROM ht_try_weekly; +DROP MATERIALIZED VIEW ht_try_weekly; + -- This test verifies that a bugfix regarding the way `ROWID_VAR`s are adjusted -- in the chunks' targetlists on DELETE/UPDATE works (including partially -- compressed chunks) diff --git a/tsl/test/sql/include/cagg_refresh_common.sql b/tsl/test/sql/include/cagg_refresh_common.sql index 83400d9a245..b7192a83d0c 100644 --- a/tsl/test/sql/include/cagg_refresh_common.sql +++ b/tsl/test/sql/include/cagg_refresh_common.sql @@ -5,6 +5,18 @@ CREATE TABLE conditions (time timestamptz NOT NULL, device int, temp float); SELECT create_hypertable('conditions', 'time'); +-- Test refresh on a cagg built on an empty table +CREATE MATERIALIZED VIEW daily_temp +WITH (timescaledb.continuous, + timescaledb.materialized_only=true) +AS +SELECT time_bucket('1 day', time) AS day, device, avg(temp) AS avg_temp +FROM conditions +GROUP BY 1,2 WITH NO DATA; + +CALL refresh_continuous_aggregate('daily_temp', NULL, NULL); +CALL refresh_continuous_aggregate('daily_temp', NULL, NULL, force => true); + SELECT setseed(.12); INSERT INTO conditions @@ -16,14 +28,6 @@ SELECT * FROM conditions ORDER BY time DESC, device LIMIT 10; -CREATE MATERIALIZED VIEW daily_temp -WITH (timescaledb.continuous, - timescaledb.materialized_only=true) -AS -SELECT time_bucket('1 day', time) AS day, device, avg(temp) AS avg_temp -FROM conditions -GROUP BY 1,2 WITH NO DATA; - -- The continuous aggregate should be empty SELECT * FROM daily_temp ORDER BY day DESC, device; @@ -106,6 +110,29 @@ CALL refresh_continuous_aggregate('daily_temp', '2020-05-01'::text, '2020-05-03' CALL refresh_continuous_aggregate('daily_temp', 0, '2020-05-01'); \set ON_ERROR_STOP 1 +-- Test forceful refreshment. Here we simulate the situation that we've seen +-- with tiered data when `timescaledb.enable_tiered_reads` were disabled on the +-- server level. In that case we would not see materialized tiered data and +-- we wouldn't be able to re-materialize the data using a normal refresh call +-- because it would skip previously materialized ranges, but it should be +-- possible with `force=>true` parameter. To simulate this use-case we clear +-- the materialization hypertable and forefully re-materialize it. +SELECT format('%I.%I', ht.schema_name, ht.table_name) AS mat_ht, mat_hypertable_id FROM _timescaledb_catalog.continuous_agg cagg +JOIN _timescaledb_catalog.hypertable ht ON cagg.mat_hypertable_id = ht.id +WHERE user_view_name = 'daily_temp' \gset + +-- Delete the data from the materialization hypertable +DELETE FROM :mat_ht; + +-- Run regular refresh, it should not touch previously materialized range +CALL refresh_continuous_aggregate('daily_temp', '2020-05-02', '2020-05-05 17:00'); +SELECT * FROM daily_temp +ORDER BY day DESC, device; +-- Run it again with force=>true, the data should be rematerialized +CALL refresh_continuous_aggregate('daily_temp', '2020-05-02', '2020-05-05 17:00', force=>true); +SELECT * FROM daily_temp +ORDER BY day DESC, device; + -- Test different time types CREATE TABLE conditions_date (time date NOT NULL, device int, temp float); SELECT create_hypertable('conditions_date', 'time'); From d1348af0ca72b872148c775d81b2ae2c7e25d417 Mon Sep 17 00:00:00 2001 From: Keyur Panchal Date: Tue, 14 Jan 2025 08:08:12 -0700 Subject: [PATCH 06/32] Add index creation on orderby columns (#7436) Previously, for chunks without a segmentby column specified in the compression settings, no indexes were created. This change allows indexes to be created on orderby columns only in this scenario. Since there is no segmentby in the compression settings for these chunks we also disable segmentwise recompression for them as it is more optimal to do a full recompression in this case. --- .unreleased/pr_7436 | 1 + tsl/src/compression/api.c | 12 + tsl/src/compression/compression_storage.c | 5 - tsl/test/expected/compression_bgw.out | 12 +- tsl/test/expected/compression_ddl.out | 210 ++++++------------ tsl/test/expected/compression_indexcreate.out | 124 +++++++++++ tsl/test/expected/compression_insert.out | 44 ++-- .../expected/compression_qualpushdown.out | 4 +- .../compression_sequence_num_removal.out | 38 +--- .../expected/compression_update_delete-14.out | 54 ++--- .../expected/compression_update_delete-15.out | 54 ++--- .../expected/compression_update_delete-16.out | 54 ++--- .../expected/compression_update_delete-17.out | 54 ++--- tsl/test/expected/hypercore_vacuum.out | 9 +- tsl/test/expected/telemetry_stats.out | 4 +- tsl/test/expected/vector_agg_default.out | 14 +- tsl/test/sql/CMakeLists.txt | 1 + tsl/test/sql/compression_indexcreate.sql | 44 ++++ 18 files changed, 371 insertions(+), 367 deletions(-) create mode 100644 .unreleased/pr_7436 create mode 100644 tsl/test/expected/compression_indexcreate.out create mode 100644 tsl/test/sql/compression_indexcreate.sql diff --git a/.unreleased/pr_7436 b/.unreleased/pr_7436 new file mode 100644 index 00000000000..8f7b34d56bb --- /dev/null +++ b/.unreleased/pr_7436 @@ -0,0 +1 @@ +Implements: #7436 Add index creation on orderby columns diff --git a/tsl/src/compression/api.c b/tsl/src/compression/api.c index 12c8a174fd1..e2a370a6ef5 100644 --- a/tsl/src/compression/api.c +++ b/tsl/src/compression/api.c @@ -1066,6 +1066,18 @@ get_compressed_chunk_index_for_recompression(Chunk *uncompressed_chunk) CompressionSettings *settings = ts_compression_settings_get(compressed_chunk->table_id); + // For chunks with no segmentby, we don't want to do segmentwise recompression as it is less + // performant than a full recompression. This is temporary; once we optimize recompression + // code for chunks with no segments we should remove this check. + int num_segmentby = ts_array_length(settings->fd.segmentby); + + if (num_segmentby == 0) + { + table_close(compressed_chunk_rel, NoLock); + table_close(uncompressed_chunk_rel, NoLock); + return InvalidOid; + } + CatalogIndexState indstate = CatalogOpenIndexes(compressed_chunk_rel); Oid index_oid = get_compressed_chunk_index(indstate, settings); CatalogCloseIndexes(indstate); diff --git a/tsl/src/compression/compression_storage.c b/tsl/src/compression/compression_storage.c index c9c75cbd0a7..58fb2233133 100644 --- a/tsl/src/compression/compression_storage.c +++ b/tsl/src/compression/compression_storage.c @@ -308,11 +308,6 @@ create_compressed_chunk_indexes(Chunk *chunk, CompressionSettings *settings) } } - if (list_length(indexcols) == 0) - { - return; - } - SortByDir ordering; SortByNulls nulls_ordering; diff --git a/tsl/test/expected/compression_bgw.out b/tsl/test/expected/compression_bgw.out index fddf7b28022..a5694442f78 100644 --- a/tsl/test/expected/compression_bgw.out +++ b/tsl/test/expected/compression_bgw.out @@ -175,8 +175,8 @@ WHERE compression_status LIKE 'Compressed' ORDER BY chunk_name; chunk_name | before_compression_total_bytes | after_compression_total_bytes ------------------+--------------------------------+------------------------------- - _hyper_3_5_chunk | 24576 | 24576 - _hyper_3_6_chunk | 24576 | 24576 + _hyper_3_5_chunk | 24576 | 40960 + _hyper_3_6_chunk | 24576 | 40960 (2 rows) --integer tests @@ -215,8 +215,8 @@ WHERE compression_status LIKE 'Compressed' ORDER BY chunk_name; chunk_name | before_compression_total_bytes | after_compression_total_bytes -------------------+--------------------------------+------------------------------- - _hyper_5_12_chunk | 24576 | 24576 - _hyper_5_13_chunk | 24576 | 24576 + _hyper_5_12_chunk | 24576 | 40960 + _hyper_5_13_chunk | 24576 | 40960 (2 rows) --bigint test @@ -255,8 +255,8 @@ WHERE compression_status LIKE 'Compressed' ORDER BY chunk_name; chunk_name | before_compression_total_bytes | after_compression_total_bytes -------------------+--------------------------------+------------------------------- - _hyper_7_19_chunk | 24576 | 24576 - _hyper_7_20_chunk | 24576 | 24576 + _hyper_7_19_chunk | 24576 | 40960 + _hyper_7_20_chunk | 24576 | 40960 (2 rows) --TEST 8 diff --git a/tsl/test/expected/compression_ddl.out b/tsl/test/expected/compression_ddl.out index edb7008c317..8d7e9d836fc 100644 --- a/tsl/test/expected/compression_ddl.out +++ b/tsl/test/expected/compression_ddl.out @@ -1899,49 +1899,37 @@ SELECT compress_chunk(show_chunks('test_partials')); VACUUM ANALYZE test_partials; -- fully compressed EXPLAIN (costs off) SELECT * FROM test_partials ORDER BY time; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------ Custom Scan (ChunkAppend) on test_partials Order: test_partials."time" -> Custom Scan (DecompressChunk) on _hyper_33_120_chunk - -> Sort - Sort Key: compress_hyper_34_123_chunk._ts_meta_min_1, compress_hyper_34_123_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_34_123_chunk + -> Index Scan Backward using compress_hyper_34_123_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_34_123_chunk -> Custom Scan (DecompressChunk) on _hyper_33_121_chunk - -> Sort - Sort Key: compress_hyper_34_124_chunk._ts_meta_min_1, compress_hyper_34_124_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_34_124_chunk + -> Index Scan Backward using compress_hyper_34_124_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_34_124_chunk -> Custom Scan (DecompressChunk) on _hyper_33_122_chunk - -> Sort - Sort Key: compress_hyper_34_125_chunk._ts_meta_min_1, compress_hyper_34_125_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_34_125_chunk -(14 rows) + -> Index Scan Backward using compress_hyper_34_125_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_34_125_chunk +(8 rows) -- test P, F, F INSERT INTO test_partials VALUES ('2020-01-01 00:03', 1, 2); EXPLAIN (costs off) SELECT * FROM test_partials ORDER BY time; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------ Custom Scan (ChunkAppend) on test_partials Order: test_partials."time" -> Merge Append Sort Key: _hyper_33_120_chunk."time" -> Custom Scan (DecompressChunk) on _hyper_33_120_chunk - -> Sort - Sort Key: compress_hyper_34_123_chunk._ts_meta_min_1, compress_hyper_34_123_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_34_123_chunk + -> Index Scan Backward using compress_hyper_34_123_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_34_123_chunk -> Sort Sort Key: _hyper_33_120_chunk."time" -> Seq Scan on _hyper_33_120_chunk -> Custom Scan (DecompressChunk) on _hyper_33_121_chunk - -> Sort - Sort Key: compress_hyper_34_124_chunk._ts_meta_min_1, compress_hyper_34_124_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_34_124_chunk + -> Index Scan Backward using compress_hyper_34_124_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_34_124_chunk -> Custom Scan (DecompressChunk) on _hyper_33_122_chunk - -> Sort - Sort Key: compress_hyper_34_125_chunk._ts_meta_min_1, compress_hyper_34_125_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_34_125_chunk -(19 rows) + -> Index Scan Backward using compress_hyper_34_125_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_34_125_chunk +(13 rows) -- verify correct results SELECT * FROM test_partials ORDER BY time; @@ -1961,33 +1949,27 @@ SELECT * FROM test_partials ORDER BY time; -- P, P, F INSERT INTO test_partials VALUES ('2021-01-01 00:03', 1, 2); EXPLAIN (costs off) SELECT * FROM test_partials ORDER BY time; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------ Custom Scan (ChunkAppend) on test_partials Order: test_partials."time" -> Merge Append Sort Key: _hyper_33_120_chunk."time" -> Custom Scan (DecompressChunk) on _hyper_33_120_chunk - -> Sort - Sort Key: compress_hyper_34_123_chunk._ts_meta_min_1, compress_hyper_34_123_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_34_123_chunk + -> Index Scan Backward using compress_hyper_34_123_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_34_123_chunk -> Sort Sort Key: _hyper_33_120_chunk."time" -> Seq Scan on _hyper_33_120_chunk -> Merge Append Sort Key: _hyper_33_121_chunk."time" -> Custom Scan (DecompressChunk) on _hyper_33_121_chunk - -> Sort - Sort Key: compress_hyper_34_124_chunk._ts_meta_min_1, compress_hyper_34_124_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_34_124_chunk + -> Index Scan Backward using compress_hyper_34_124_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_34_124_chunk -> Sort Sort Key: _hyper_33_121_chunk."time" -> Seq Scan on _hyper_33_121_chunk -> Custom Scan (DecompressChunk) on _hyper_33_122_chunk - -> Sort - Sort Key: compress_hyper_34_125_chunk._ts_meta_min_1, compress_hyper_34_125_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_34_125_chunk -(24 rows) + -> Index Scan Backward using compress_hyper_34_125_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_34_125_chunk +(18 rows) -- verify correct results SELECT * FROM test_partials ORDER BY time; @@ -2009,39 +1991,33 @@ SELECT * FROM test_partials ORDER BY time; INSERT INTO test_partials VALUES ('2022-01-01 00:03', 1, 2); INSERT INTO test_partials VALUES ('2023-01-01 00:03', 1, 2); EXPLAIN (costs off) SELECT * FROM test_partials ORDER BY time; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------ Custom Scan (ChunkAppend) on test_partials Order: test_partials."time" -> Merge Append Sort Key: _hyper_33_120_chunk."time" -> Custom Scan (DecompressChunk) on _hyper_33_120_chunk - -> Sort - Sort Key: compress_hyper_34_123_chunk._ts_meta_min_1, compress_hyper_34_123_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_34_123_chunk + -> Index Scan Backward using compress_hyper_34_123_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_34_123_chunk -> Sort Sort Key: _hyper_33_120_chunk."time" -> Seq Scan on _hyper_33_120_chunk -> Merge Append Sort Key: _hyper_33_121_chunk."time" -> Custom Scan (DecompressChunk) on _hyper_33_121_chunk - -> Sort - Sort Key: compress_hyper_34_124_chunk._ts_meta_min_1, compress_hyper_34_124_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_34_124_chunk + -> Index Scan Backward using compress_hyper_34_124_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_34_124_chunk -> Sort Sort Key: _hyper_33_121_chunk."time" -> Seq Scan on _hyper_33_121_chunk -> Merge Append Sort Key: _hyper_33_122_chunk."time" -> Custom Scan (DecompressChunk) on _hyper_33_122_chunk - -> Sort - Sort Key: compress_hyper_34_125_chunk._ts_meta_min_1, compress_hyper_34_125_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_34_125_chunk + -> Index Scan Backward using compress_hyper_34_125_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_34_125_chunk -> Sort Sort Key: _hyper_33_122_chunk."time" -> Seq Scan on _hyper_33_122_chunk -> Index Scan Backward using _hyper_33_126_chunk_test_partials_time_idx on _hyper_33_126_chunk -(30 rows) +(24 rows) -- F, F, P, U -- recompress all chunks @@ -2060,29 +2036,23 @@ END $$; INSERT INTO test_partials VALUES ('2022-01-01 00:02', 1, 2); EXPLAIN (COSTS OFF) SELECT * FROM test_partials ORDER BY time; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------ Custom Scan (ChunkAppend) on test_partials Order: test_partials."time" -> Custom Scan (DecompressChunk) on _hyper_33_120_chunk - -> Sort - Sort Key: compress_hyper_34_127_chunk._ts_meta_min_1, compress_hyper_34_127_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_34_127_chunk + -> Index Scan Backward using compress_hyper_34_127_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_34_127_chunk -> Custom Scan (DecompressChunk) on _hyper_33_121_chunk - -> Sort - Sort Key: compress_hyper_34_128_chunk._ts_meta_min_1, compress_hyper_34_128_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_34_128_chunk + -> Index Scan Backward using compress_hyper_34_128_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_34_128_chunk -> Merge Append Sort Key: _hyper_33_122_chunk."time" -> Custom Scan (DecompressChunk) on _hyper_33_122_chunk - -> Sort - Sort Key: compress_hyper_34_129_chunk._ts_meta_min_1, compress_hyper_34_129_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_34_129_chunk + -> Index Scan Backward using compress_hyper_34_129_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_34_129_chunk -> Sort Sort Key: _hyper_33_122_chunk."time" -> Seq Scan on _hyper_33_122_chunk -> Index Scan Backward using _hyper_33_126_chunk_test_partials_time_idx on _hyper_33_126_chunk -(20 rows) +(14 rows) -- F, F, P, F, F INSERT INTO test_partials VALUES ('2024-01-01 00:02', 1, 2); @@ -2094,36 +2064,26 @@ SELECT compress_chunk(c) FROM show_chunks('test_partials', newer_than => '2022-0 (2 rows) EXPLAIN (costs off) SELECT * FROM test_partials ORDER BY time; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------ Custom Scan (ChunkAppend) on test_partials Order: test_partials."time" -> Custom Scan (DecompressChunk) on _hyper_33_120_chunk - -> Sort - Sort Key: compress_hyper_34_127_chunk._ts_meta_min_1, compress_hyper_34_127_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_34_127_chunk + -> Index Scan Backward using compress_hyper_34_127_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_34_127_chunk -> Custom Scan (DecompressChunk) on _hyper_33_121_chunk - -> Sort - Sort Key: compress_hyper_34_128_chunk._ts_meta_min_1, compress_hyper_34_128_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_34_128_chunk + -> Index Scan Backward using compress_hyper_34_128_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_34_128_chunk -> Merge Append Sort Key: _hyper_33_122_chunk."time" -> Custom Scan (DecompressChunk) on _hyper_33_122_chunk - -> Sort - Sort Key: compress_hyper_34_129_chunk._ts_meta_min_1, compress_hyper_34_129_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_34_129_chunk + -> Index Scan Backward using compress_hyper_34_129_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_34_129_chunk -> Sort Sort Key: _hyper_33_122_chunk."time" -> Seq Scan on _hyper_33_122_chunk -> Custom Scan (DecompressChunk) on _hyper_33_126_chunk - -> Sort - Sort Key: compress_hyper_34_131_chunk._ts_meta_min_1, compress_hyper_34_131_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_34_131_chunk + -> Index Scan Backward using compress_hyper_34_131_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_34_131_chunk -> Custom Scan (DecompressChunk) on _hyper_33_130_chunk - -> Sort - Sort Key: compress_hyper_34_132_chunk._ts_meta_min_1, compress_hyper_34_132_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_34_132_chunk -(27 rows) + -> Index Scan Backward using compress_hyper_34_132_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_34_132_chunk +(17 rows) -- verify result correctness SELECT * FROM test_partials ORDER BY time; @@ -2191,24 +2151,20 @@ SELECT add_dimension('space_part', 'a', number_partitions => 5); -- plan is still the same EXPLAIN (COSTS OFF) SELECT * FROM space_part ORDER BY time; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------ Custom Scan (ChunkAppend) on space_part Order: space_part."time" -> Merge Append Sort Key: _hyper_35_133_chunk."time" -> Custom Scan (DecompressChunk) on _hyper_35_133_chunk - -> Sort - Sort Key: compress_hyper_36_135_chunk._ts_meta_min_1, compress_hyper_36_135_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_135_chunk + -> Index Scan Backward using compress_hyper_36_135_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_135_chunk -> Sort Sort Key: _hyper_35_133_chunk."time" -> Seq Scan on _hyper_35_133_chunk -> Custom Scan (DecompressChunk) on _hyper_35_134_chunk - -> Sort - Sort Key: compress_hyper_36_136_chunk._ts_meta_min_1, compress_hyper_36_136_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_136_chunk -(15 rows) + -> Index Scan Backward using compress_hyper_36_136_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_136_chunk +(11 rows) -- now add more chunks that do adhere to the new space partitioning -- chunks 3,4 @@ -2219,28 +2175,24 @@ INSERT INTO space_part VALUES ('2022-01-01 00:03', 2, 1, 1); -- plan still ok EXPLAIN (COSTS OFF) SELECT * FROM space_part ORDER BY time; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------ Custom Scan (ChunkAppend) on space_part Order: space_part."time" -> Merge Append Sort Key: _hyper_35_133_chunk."time" -> Custom Scan (DecompressChunk) on _hyper_35_133_chunk - -> Sort - Sort Key: compress_hyper_36_135_chunk._ts_meta_min_1, compress_hyper_36_135_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_135_chunk + -> Index Scan Backward using compress_hyper_36_135_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_135_chunk -> Sort Sort Key: _hyper_35_133_chunk."time" -> Seq Scan on _hyper_35_133_chunk -> Custom Scan (DecompressChunk) on _hyper_35_134_chunk - -> Sort - Sort Key: compress_hyper_36_136_chunk._ts_meta_min_1, compress_hyper_36_136_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_136_chunk + -> Index Scan Backward using compress_hyper_36_136_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_136_chunk -> Merge Append Sort Key: _hyper_35_137_chunk."time" -> Index Scan Backward using _hyper_35_137_chunk_space_part_time_idx on _hyper_35_137_chunk -> Index Scan Backward using _hyper_35_138_chunk_space_part_time_idx on _hyper_35_138_chunk -(19 rows) +(15 rows) -- compress them SELECT compress_chunk(c, if_not_compressed=>true) FROM show_chunks('space_part') c; @@ -2255,95 +2207,71 @@ NOTICE: chunk "_hyper_35_134_chunk" is already compressed -- plan still ok EXPLAIN (COSTS OFF) SELECT * FROM space_part ORDER BY time; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------ Custom Scan (ChunkAppend) on space_part Order: space_part."time" -> Custom Scan (DecompressChunk) on _hyper_35_133_chunk - -> Sort - Sort Key: compress_hyper_36_139_chunk._ts_meta_min_1, compress_hyper_36_139_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_139_chunk + -> Index Scan Backward using compress_hyper_36_139_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_139_chunk -> Custom Scan (DecompressChunk) on _hyper_35_134_chunk - -> Sort - Sort Key: compress_hyper_36_136_chunk._ts_meta_min_1, compress_hyper_36_136_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_136_chunk + -> Index Scan Backward using compress_hyper_36_136_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_136_chunk -> Merge Append Sort Key: _hyper_35_137_chunk."time" -> Custom Scan (DecompressChunk) on _hyper_35_137_chunk - -> Sort - Sort Key: compress_hyper_36_140_chunk._ts_meta_min_1, compress_hyper_36_140_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_140_chunk + -> Index Scan Backward using compress_hyper_36_140_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_140_chunk -> Custom Scan (DecompressChunk) on _hyper_35_138_chunk - -> Sort - Sort Key: compress_hyper_36_141_chunk._ts_meta_min_1, compress_hyper_36_141_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_141_chunk -(20 rows) + -> Index Scan Backward using compress_hyper_36_141_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_141_chunk +(12 rows) -- make second one of them partial insert into space_part values ('2022-01-01 00:02', 2, 1, 1), ('2022-01-01 00:02', 2, 1, 1); EXPLAIN (COSTS OFF) SELECT * FROM space_part ORDER BY time; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------ Custom Scan (ChunkAppend) on space_part Order: space_part."time" -> Custom Scan (DecompressChunk) on _hyper_35_133_chunk - -> Sort - Sort Key: compress_hyper_36_139_chunk._ts_meta_min_1, compress_hyper_36_139_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_139_chunk + -> Index Scan Backward using compress_hyper_36_139_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_139_chunk -> Custom Scan (DecompressChunk) on _hyper_35_134_chunk - -> Sort - Sort Key: compress_hyper_36_136_chunk._ts_meta_min_1, compress_hyper_36_136_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_136_chunk + -> Index Scan Backward using compress_hyper_36_136_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_136_chunk -> Merge Append Sort Key: _hyper_35_137_chunk."time" -> Custom Scan (DecompressChunk) on _hyper_35_137_chunk - -> Sort - Sort Key: compress_hyper_36_140_chunk._ts_meta_min_1, compress_hyper_36_140_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_140_chunk + -> Index Scan Backward using compress_hyper_36_140_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_140_chunk -> Custom Scan (DecompressChunk) on _hyper_35_138_chunk - -> Sort - Sort Key: compress_hyper_36_141_chunk._ts_meta_min_1, compress_hyper_36_141_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_141_chunk + -> Index Scan Backward using compress_hyper_36_141_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_141_chunk -> Sort Sort Key: _hyper_35_138_chunk."time" -> Seq Scan on _hyper_35_138_chunk -(23 rows) +(15 rows) -- make other one partial too INSERT INTO space_part VALUES ('2022-01-01 00:02', 1, 1, 1); EXPLAIN (COSTS OFF) SELECT * FROM space_part ORDER BY time; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------ Custom Scan (ChunkAppend) on space_part Order: space_part."time" -> Custom Scan (DecompressChunk) on _hyper_35_133_chunk - -> Sort - Sort Key: compress_hyper_36_139_chunk._ts_meta_min_1, compress_hyper_36_139_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_139_chunk + -> Index Scan Backward using compress_hyper_36_139_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_139_chunk -> Custom Scan (DecompressChunk) on _hyper_35_134_chunk - -> Sort - Sort Key: compress_hyper_36_136_chunk._ts_meta_min_1, compress_hyper_36_136_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_136_chunk + -> Index Scan Backward using compress_hyper_36_136_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_136_chunk -> Merge Append Sort Key: _hyper_35_137_chunk."time" -> Custom Scan (DecompressChunk) on _hyper_35_137_chunk - -> Sort - Sort Key: compress_hyper_36_140_chunk._ts_meta_min_1, compress_hyper_36_140_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_140_chunk + -> Index Scan Backward using compress_hyper_36_140_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_140_chunk -> Sort Sort Key: _hyper_35_137_chunk."time" -> Seq Scan on _hyper_35_137_chunk -> Custom Scan (DecompressChunk) on _hyper_35_138_chunk - -> Sort - Sort Key: compress_hyper_36_141_chunk._ts_meta_min_1, compress_hyper_36_141_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_141_chunk + -> Index Scan Backward using compress_hyper_36_141_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_141_chunk -> Sort Sort Key: _hyper_35_138_chunk."time" -> Seq Scan on _hyper_35_138_chunk -(26 rows) +(18 rows) -- test creation of unique expression index does not interfere with enabling compression -- github issue 6205 diff --git a/tsl/test/expected/compression_indexcreate.out b/tsl/test/expected/compression_indexcreate.out new file mode 100644 index 00000000000..5e891c291f8 --- /dev/null +++ b/tsl/test/expected/compression_indexcreate.out @@ -0,0 +1,124 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. +-- force index scan to be used when possible +set enable_seqscan to false; +\set PREFIX 'EXPLAIN (analyze, costs off, summary off, timing off) ' +create table segind(time timestamptz, a int, b int); +select create_hypertable('segind', by_range('time')); +NOTICE: adding not-null constraint to column "time" + create_hypertable +------------------- + (1,t) +(1 row) + +-- enable compression on hypertable with no segment by column +alter table segind set (timescaledb.compress, timescaledb.compress_segmentby='', timescaledb.compress_orderby='time, b'); +insert into segind values('2024-11-08 10:31:28.436014-07', 1, 1), ('2024-11-08 10:32:28.436014-07', 2, 1), ('2024-11-08 10:33:28.436014-07', 3, 1), ('2024-11-08 10:34:28.436014-07', 2, 1), ('2024-11-08 10:35:28.436014-07', 1, 2), ('2024-11-08 10:36:28.436014-07', 4, 1); +-- compress chunk +-- this should create an index using orderby columns +select compress_chunk(show_chunks('segind')); + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk +(1 row) + +-- query using orderby columns should use the index +:PREFIX select * from segind where b = 1; + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------- + Custom Scan (DecompressChunk) on _hyper_1_1_chunk (actual rows=5 loops=1) + Vectorized Filter: (b = 1) + Rows Removed by Filter: 1 + -> Index Scan using compress_hyper_2_2_chunk__ts_meta_min_1__ts_meta_max_1__ts__idx on compress_hyper_2_2_chunk (actual rows=1 loops=1) + Index Cond: ((_ts_meta_min_2 <= 1) AND (_ts_meta_max_2 >= 1)) +(5 rows) + +:PREFIX select * from segind where time = '2024-11-08 10:32:28.436014-07'; + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Custom Scan (DecompressChunk) on _hyper_1_1_chunk (actual rows=1 loops=1) + Vectorized Filter: ("time" = 'Fri Nov 08 09:32:28.436014 2024 PST'::timestamp with time zone) + Rows Removed by Filter: 5 + -> Index Scan using compress_hyper_2_2_chunk__ts_meta_min_1__ts_meta_max_1__ts__idx on compress_hyper_2_2_chunk (actual rows=1 loops=1) + Index Cond: ((_ts_meta_min_1 <= 'Fri Nov 08 09:32:28.436014 2024 PST'::timestamp with time zone) AND (_ts_meta_max_1 >= 'Fri Nov 08 09:32:28.436014 2024 PST'::timestamp with time zone)) +(5 rows) + +:PREFIX select * from segind where b = 1 and time = '2024-11-08 10:32:28.436014-07'; + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Custom Scan (DecompressChunk) on _hyper_1_1_chunk (actual rows=1 loops=1) + Vectorized Filter: ((b = 1) AND ("time" = 'Fri Nov 08 09:32:28.436014 2024 PST'::timestamp with time zone)) + Rows Removed by Filter: 5 + -> Index Scan using compress_hyper_2_2_chunk__ts_meta_min_1__ts_meta_max_1__ts__idx on compress_hyper_2_2_chunk (actual rows=1 loops=1) + Index Cond: ((_ts_meta_min_1 <= 'Fri Nov 08 09:32:28.436014 2024 PST'::timestamp with time zone) AND (_ts_meta_max_1 >= 'Fri Nov 08 09:32:28.436014 2024 PST'::timestamp with time zone) AND (_ts_meta_min_2 <= 1) AND (_ts_meta_max_2 >= 1)) +(5 rows) + +-- a query on another column should perform a seq scan since there is no index on it +:PREFIX select * from segind where a = 1; + QUERY PLAN +--------------------------------------------------------------------------- + Custom Scan (DecompressChunk) on _hyper_1_1_chunk (actual rows=2 loops=1) + Vectorized Filter: (a = 1) + Rows Removed by Filter: 4 + -> Seq Scan on compress_hyper_2_2_chunk (actual rows=1 loops=1) +(4 rows) + +-- decompress the chunk to drop the index +select decompress_chunk(show_chunks('segind')); + decompress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk +(1 row) + +-- change compression settings to use segmentby column +alter table segind set (timescaledb.compress, timescaledb.compress_segmentby='a', timescaledb.compress_orderby='time, b'); +-- compress chunk +-- this should create an index using segmentby and orderby columns +select compress_chunk(show_chunks('segind')); + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk +(1 row) + +-- queries using segmentby or orderby columns should use the index +:PREFIX select * from segind where b = 1; + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------- + Custom Scan (DecompressChunk) on _hyper_1_1_chunk (actual rows=5 loops=1) + Vectorized Filter: (b = 1) + Rows Removed by Filter: 1 + -> Index Scan using compress_hyper_2_3_chunk_a__ts_meta_min_1__ts_meta_max_1__t_idx on compress_hyper_2_3_chunk (actual rows=4 loops=1) + Index Cond: ((_ts_meta_min_2 <= 1) AND (_ts_meta_max_2 >= 1)) +(5 rows) + +:PREFIX select * from segind where time = '2024-11-08 10:32:28.436014-07'; + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Custom Scan (DecompressChunk) on _hyper_1_1_chunk (actual rows=1 loops=1) + Vectorized Filter: ("time" = 'Fri Nov 08 09:32:28.436014 2024 PST'::timestamp with time zone) + Rows Removed by Filter: 3 + -> Index Scan using compress_hyper_2_3_chunk_a__ts_meta_min_1__ts_meta_max_1__t_idx on compress_hyper_2_3_chunk (actual rows=2 loops=1) + Index Cond: ((_ts_meta_min_1 <= 'Fri Nov 08 09:32:28.436014 2024 PST'::timestamp with time zone) AND (_ts_meta_max_1 >= 'Fri Nov 08 09:32:28.436014 2024 PST'::timestamp with time zone)) +(5 rows) + +:PREFIX select * from segind where b = 1 and time = '2024-11-08 10:32:28.436014-07'; + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Custom Scan (DecompressChunk) on _hyper_1_1_chunk (actual rows=1 loops=1) + Vectorized Filter: ((b = 1) AND ("time" = 'Fri Nov 08 09:32:28.436014 2024 PST'::timestamp with time zone)) + Rows Removed by Filter: 3 + -> Index Scan using compress_hyper_2_3_chunk_a__ts_meta_min_1__ts_meta_max_1__t_idx on compress_hyper_2_3_chunk (actual rows=2 loops=1) + Index Cond: ((_ts_meta_min_1 <= 'Fri Nov 08 09:32:28.436014 2024 PST'::timestamp with time zone) AND (_ts_meta_max_1 >= 'Fri Nov 08 09:32:28.436014 2024 PST'::timestamp with time zone) AND (_ts_meta_min_2 <= 1) AND (_ts_meta_max_2 >= 1)) +(5 rows) + +:PREFIX select * from segind where a = 1; + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------- + Custom Scan (DecompressChunk) on _hyper_1_1_chunk (actual rows=2 loops=1) + -> Index Scan using compress_hyper_2_3_chunk_a__ts_meta_min_1__ts_meta_max_1__t_idx on compress_hyper_2_3_chunk (actual rows=1 loops=1) + Index Cond: (a = 1) +(3 rows) + +-- cleanup +RESET enable_seqscan; diff --git a/tsl/test/expected/compression_insert.out b/tsl/test/expected/compression_insert.out index 12da960379e..a2e42c82389 100644 --- a/tsl/test/expected/compression_insert.out +++ b/tsl/test/expected/compression_insert.out @@ -790,13 +790,11 @@ SELECT compress_chunk(format('%I.%I',chunk_schema,chunk_name), true) FROM timesc -- should be ordered append :PREFIX SELECT * FROM test_ordering ORDER BY 1; - QUERY PLAN --------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------- Custom Scan (DecompressChunk) on _hyper_13_20_chunk - -> Sort - Sort Key: compress_hyper_14_21_chunk._ts_meta_min_1, compress_hyper_14_21_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_14_21_chunk -(4 rows) + -> Index Scan Backward using compress_hyper_14_21_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_14_21_chunk +(2 rows) INSERT INTO test_ordering SELECT 1; -- should not be ordered append @@ -807,39 +805,35 @@ INSERT INTO test_ordering SELECT 1; -- It was hard to include a path without pushed down sort for consideration, as `add_path` would reject -- the path with sort pushdown, which is desirable in most cases :PREFIX SELECT * FROM test_ordering ORDER BY 1; - QUERY PLAN --------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------- Custom Scan (ChunkAppend) on test_ordering Order: test_ordering."time" -> Merge Append Sort Key: _hyper_13_20_chunk."time" -> Custom Scan (DecompressChunk) on _hyper_13_20_chunk - -> Sort - Sort Key: compress_hyper_14_21_chunk._ts_meta_min_1, compress_hyper_14_21_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_14_21_chunk + -> Index Scan Backward using compress_hyper_14_21_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_14_21_chunk -> Sort Sort Key: _hyper_13_20_chunk."time" -> Seq Scan on _hyper_13_20_chunk -(11 rows) +(9 rows) INSERT INTO test_ordering VALUES (105),(104),(103); -- should be ordered append :PREFIX SELECT * FROM test_ordering ORDER BY 1; - QUERY PLAN --------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------- Custom Scan (ChunkAppend) on test_ordering Order: test_ordering."time" -> Merge Append Sort Key: _hyper_13_20_chunk."time" -> Custom Scan (DecompressChunk) on _hyper_13_20_chunk - -> Sort - Sort Key: compress_hyper_14_21_chunk._ts_meta_min_1, compress_hyper_14_21_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_14_21_chunk + -> Index Scan Backward using compress_hyper_14_21_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_14_21_chunk -> Sort Sort Key: _hyper_13_20_chunk."time" -> Seq Scan on _hyper_13_20_chunk -> Index Only Scan Backward using _hyper_13_22_chunk_test_ordering_time_idx on _hyper_13_22_chunk -(12 rows) +(10 rows) --insert into compressed + uncompressed chunk INSERT INTO test_ordering VALUES (21), (22),(113); @@ -881,19 +875,15 @@ SELECT compress_chunk(format('%I.%I',chunk_schema,chunk_name), true) FROM timesc -- should be ordered append :PREFIX SELECT * FROM test_ordering ORDER BY 1; - QUERY PLAN --------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------- Custom Scan (ChunkAppend) on test_ordering Order: test_ordering."time" -> Custom Scan (DecompressChunk) on _hyper_13_20_chunk - -> Sort - Sort Key: compress_hyper_14_23_chunk._ts_meta_min_1, compress_hyper_14_23_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_14_23_chunk + -> Index Scan Backward using compress_hyper_14_23_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_14_23_chunk -> Custom Scan (DecompressChunk) on _hyper_13_22_chunk - -> Sort - Sort Key: compress_hyper_14_24_chunk._ts_meta_min_1, compress_hyper_14_24_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_14_24_chunk -(10 rows) + -> Index Scan Backward using compress_hyper_14_24_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_14_24_chunk +(6 rows) SET timescaledb.enable_decompression_sorted_merge = 1; -- TEST cagg triggers with insert into compressed chunk diff --git a/tsl/test/expected/compression_qualpushdown.out b/tsl/test/expected/compression_qualpushdown.out index 7914e82cd06..f1baf41e932 100644 --- a/tsl/test/expected/compression_qualpushdown.out +++ b/tsl/test/expected/compression_qualpushdown.out @@ -135,9 +135,9 @@ order by factorid, end_dt; Output: _hyper_3_4_chunk.factorid, _hyper_3_4_chunk.end_dt, _hyper_3_4_chunk.logret Filter: ((_hyper_3_4_chunk.end_dt >= '12-10-2012'::date) AND (_hyper_3_4_chunk.end_dt <= '12-11-2012'::date)) Vectorized Filter: (_hyper_3_4_chunk.fmid = 56) - -> Seq Scan on _timescaledb_internal.compress_hyper_4_5_chunk + -> Index Scan using compress_hyper_4_5_chunk__ts_meta_min_1__ts_meta_max_1_idx on _timescaledb_internal.compress_hyper_4_5_chunk Output: compress_hyper_4_5_chunk._ts_meta_count, compress_hyper_4_5_chunk.fmid, compress_hyper_4_5_chunk.factorid, compress_hyper_4_5_chunk.start_dt, compress_hyper_4_5_chunk._ts_meta_min_1, compress_hyper_4_5_chunk._ts_meta_max_1, compress_hyper_4_5_chunk.end_dt, compress_hyper_4_5_chunk.interval_number, compress_hyper_4_5_chunk.logret, compress_hyper_4_5_chunk.knowledge_date - Filter: ((compress_hyper_4_5_chunk._ts_meta_max_1 >= '12-10-2012'::date) AND (compress_hyper_4_5_chunk._ts_meta_min_1 <= '12-11-2012'::date)) + Index Cond: ((compress_hyper_4_5_chunk._ts_meta_min_1 <= '12-11-2012'::date) AND (compress_hyper_4_5_chunk._ts_meta_max_1 >= '12-10-2012'::date)) (10 rows) --no pushdown here diff --git a/tsl/test/expected/compression_sequence_num_removal.out b/tsl/test/expected/compression_sequence_num_removal.out index 49c102455cf..b810ae44c0b 100644 --- a/tsl/test/expected/compression_sequence_num_removal.out +++ b/tsl/test/expected/compression_sequence_num_removal.out @@ -310,8 +310,8 @@ SELECT compress_chunk(show_chunks('hyper')); :EXPLAIN SELECT * FROM hyper ORDER BY time; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Custom Scan (ChunkAppend) on public.hyper Output: hyper."time", hyper.device_id, hyper.val Order: hyper."time" @@ -319,33 +319,21 @@ ORDER BY time; Runtime Exclusion: false -> Custom Scan (DecompressChunk) on _timescaledb_internal._hyper_1_10_chunk Output: _hyper_1_10_chunk."time", _hyper_1_10_chunk.device_id, _hyper_1_10_chunk.val - -> Sort + -> Index Scan using compress_hyper_2_14_chunk__ts_meta_min_1__ts_meta_max_1_idx on _timescaledb_internal.compress_hyper_2_14_chunk Output: compress_hyper_2_14_chunk._ts_meta_count, compress_hyper_2_14_chunk._ts_meta_min_1, compress_hyper_2_14_chunk._ts_meta_max_1, compress_hyper_2_14_chunk."time", compress_hyper_2_14_chunk.device_id, compress_hyper_2_14_chunk.val - Sort Key: compress_hyper_2_14_chunk._ts_meta_min_1, compress_hyper_2_14_chunk._ts_meta_max_1 - -> Seq Scan on _timescaledb_internal.compress_hyper_2_14_chunk - Output: compress_hyper_2_14_chunk._ts_meta_count, compress_hyper_2_14_chunk._ts_meta_min_1, compress_hyper_2_14_chunk._ts_meta_max_1, compress_hyper_2_14_chunk."time", compress_hyper_2_14_chunk.device_id, compress_hyper_2_14_chunk.val -> Custom Scan (DecompressChunk) on _timescaledb_internal._hyper_1_11_chunk Output: _hyper_1_11_chunk."time", _hyper_1_11_chunk.device_id, _hyper_1_11_chunk.val - -> Sort + -> Index Scan using compress_hyper_2_15_chunk__ts_meta_min_1__ts_meta_max_1_idx on _timescaledb_internal.compress_hyper_2_15_chunk Output: compress_hyper_2_15_chunk._ts_meta_count, compress_hyper_2_15_chunk._ts_meta_min_1, compress_hyper_2_15_chunk._ts_meta_max_1, compress_hyper_2_15_chunk."time", compress_hyper_2_15_chunk.device_id, compress_hyper_2_15_chunk.val - Sort Key: compress_hyper_2_15_chunk._ts_meta_min_1, compress_hyper_2_15_chunk._ts_meta_max_1 - -> Seq Scan on _timescaledb_internal.compress_hyper_2_15_chunk - Output: compress_hyper_2_15_chunk._ts_meta_count, compress_hyper_2_15_chunk._ts_meta_min_1, compress_hyper_2_15_chunk._ts_meta_max_1, compress_hyper_2_15_chunk."time", compress_hyper_2_15_chunk.device_id, compress_hyper_2_15_chunk.val -> Custom Scan (DecompressChunk) on _timescaledb_internal._hyper_1_12_chunk Output: _hyper_1_12_chunk."time", _hyper_1_12_chunk.device_id, _hyper_1_12_chunk.val - -> Sort + -> Index Scan using compress_hyper_2_16_chunk__ts_meta_min_1__ts_meta_max_1_idx on _timescaledb_internal.compress_hyper_2_16_chunk Output: compress_hyper_2_16_chunk._ts_meta_count, compress_hyper_2_16_chunk._ts_meta_min_1, compress_hyper_2_16_chunk._ts_meta_max_1, compress_hyper_2_16_chunk."time", compress_hyper_2_16_chunk.device_id, compress_hyper_2_16_chunk.val - Sort Key: compress_hyper_2_16_chunk._ts_meta_min_1, compress_hyper_2_16_chunk._ts_meta_max_1 - -> Seq Scan on _timescaledb_internal.compress_hyper_2_16_chunk - Output: compress_hyper_2_16_chunk._ts_meta_count, compress_hyper_2_16_chunk._ts_meta_min_1, compress_hyper_2_16_chunk._ts_meta_max_1, compress_hyper_2_16_chunk."time", compress_hyper_2_16_chunk.device_id, compress_hyper_2_16_chunk.val -> Custom Scan (DecompressChunk) on _timescaledb_internal._hyper_1_13_chunk Output: _hyper_1_13_chunk."time", _hyper_1_13_chunk.device_id, _hyper_1_13_chunk.val - -> Sort + -> Index Scan using compress_hyper_2_17_chunk__ts_meta_min_1__ts_meta_max_1_idx on _timescaledb_internal.compress_hyper_2_17_chunk Output: compress_hyper_2_17_chunk._ts_meta_count, compress_hyper_2_17_chunk._ts_meta_min_1, compress_hyper_2_17_chunk._ts_meta_max_1, compress_hyper_2_17_chunk."time", compress_hyper_2_17_chunk.device_id, compress_hyper_2_17_chunk.val - Sort Key: compress_hyper_2_17_chunk._ts_meta_min_1, compress_hyper_2_17_chunk._ts_meta_max_1 - -> Seq Scan on _timescaledb_internal.compress_hyper_2_17_chunk - Output: compress_hyper_2_17_chunk._ts_meta_count, compress_hyper_2_17_chunk._ts_meta_min_1, compress_hyper_2_17_chunk._ts_meta_max_1, compress_hyper_2_17_chunk."time", compress_hyper_2_17_chunk.device_id, compress_hyper_2_17_chunk.val -(33 rows) +(21 rows) -- modify two chunks by adding sequence number to the segments SELECT comp_ch.table_name AS "CHUNK_NAME", comp_ch.schema_name|| '.' || comp_ch.table_name AS "CHUNK_FULL_NAME" @@ -399,18 +387,12 @@ ORDER BY time; Output: compress_hyper_2_14_chunk._ts_meta_count, compress_hyper_2_14_chunk._ts_meta_min_1, compress_hyper_2_14_chunk._ts_meta_max_1, compress_hyper_2_14_chunk."time", compress_hyper_2_14_chunk.device_id, compress_hyper_2_14_chunk.val, compress_hyper_2_14_chunk._ts_meta_sequence_num -> Custom Scan (DecompressChunk) on _timescaledb_internal._hyper_1_11_chunk Output: _hyper_1_11_chunk."time", _hyper_1_11_chunk.device_id, _hyper_1_11_chunk.val - -> Sort + -> Index Scan using compress_hyper_2_15_chunk__ts_meta_min_1__ts_meta_max_1_idx on _timescaledb_internal.compress_hyper_2_15_chunk Output: compress_hyper_2_15_chunk._ts_meta_count, compress_hyper_2_15_chunk._ts_meta_min_1, compress_hyper_2_15_chunk._ts_meta_max_1, compress_hyper_2_15_chunk."time", compress_hyper_2_15_chunk.device_id, compress_hyper_2_15_chunk.val - Sort Key: compress_hyper_2_15_chunk._ts_meta_min_1, compress_hyper_2_15_chunk._ts_meta_max_1 - -> Seq Scan on _timescaledb_internal.compress_hyper_2_15_chunk - Output: compress_hyper_2_15_chunk._ts_meta_count, compress_hyper_2_15_chunk._ts_meta_min_1, compress_hyper_2_15_chunk._ts_meta_max_1, compress_hyper_2_15_chunk."time", compress_hyper_2_15_chunk.device_id, compress_hyper_2_15_chunk.val -> Custom Scan (DecompressChunk) on _timescaledb_internal._hyper_1_12_chunk Output: _hyper_1_12_chunk."time", _hyper_1_12_chunk.device_id, _hyper_1_12_chunk.val - -> Sort + -> Index Scan using compress_hyper_2_16_chunk__ts_meta_min_1__ts_meta_max_1_idx on _timescaledb_internal.compress_hyper_2_16_chunk Output: compress_hyper_2_16_chunk._ts_meta_count, compress_hyper_2_16_chunk._ts_meta_min_1, compress_hyper_2_16_chunk._ts_meta_max_1, compress_hyper_2_16_chunk."time", compress_hyper_2_16_chunk.device_id, compress_hyper_2_16_chunk.val - Sort Key: compress_hyper_2_16_chunk._ts_meta_min_1, compress_hyper_2_16_chunk._ts_meta_max_1 - -> Seq Scan on _timescaledb_internal.compress_hyper_2_16_chunk - Output: compress_hyper_2_16_chunk._ts_meta_count, compress_hyper_2_16_chunk._ts_meta_min_1, compress_hyper_2_16_chunk._ts_meta_max_1, compress_hyper_2_16_chunk."time", compress_hyper_2_16_chunk.device_id, compress_hyper_2_16_chunk.val -> Custom Scan (DecompressChunk) on _timescaledb_internal._hyper_1_13_chunk Output: _hyper_1_13_chunk."time", _hyper_1_13_chunk.device_id, _hyper_1_13_chunk.val -> Sort @@ -418,7 +400,7 @@ ORDER BY time; Sort Key: compress_hyper_2_17_chunk._ts_meta_sequence_num -> Seq Scan on _timescaledb_internal.compress_hyper_2_17_chunk Output: compress_hyper_2_17_chunk._ts_meta_count, compress_hyper_2_17_chunk._ts_meta_min_1, compress_hyper_2_17_chunk._ts_meta_max_1, compress_hyper_2_17_chunk."time", compress_hyper_2_17_chunk.device_id, compress_hyper_2_17_chunk.val, compress_hyper_2_17_chunk._ts_meta_sequence_num -(33 rows) +(27 rows) -- test rolling up chunks during compression -- this will not work with chunks which have sequence numbers diff --git a/tsl/test/expected/compression_update_delete-14.out b/tsl/test/expected/compression_update_delete-14.out index d31b39d057a..7233b14cd70 100644 --- a/tsl/test/expected/compression_update_delete-14.out +++ b/tsl/test/expected/compression_update_delete-14.out @@ -2779,23 +2779,17 @@ SELECT compress_chunk(show_chunks('test_partials')); -- fully compressed EXPLAIN (costs off) SELECT * FROM test_partials ORDER BY time; - QUERY PLAN --------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------- Custom Scan (ChunkAppend) on test_partials Order: test_partials."time" -> Custom Scan (DecompressChunk) on _hyper_35_68_chunk - -> Sort - Sort Key: compress_hyper_36_71_chunk._ts_meta_min_1, compress_hyper_36_71_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_71_chunk + -> Index Scan Backward using compress_hyper_36_71_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_71_chunk -> Custom Scan (DecompressChunk) on _hyper_35_69_chunk - -> Sort - Sort Key: compress_hyper_36_72_chunk._ts_meta_min_1, compress_hyper_36_72_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_72_chunk + -> Index Scan Backward using compress_hyper_36_72_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_72_chunk -> Custom Scan (DecompressChunk) on _hyper_35_70_chunk - -> Sort - Sort Key: compress_hyper_36_73_chunk._ts_meta_min_1, compress_hyper_36_73_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_73_chunk -(14 rows) + -> Index Scan Backward using compress_hyper_36_73_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_73_chunk +(8 rows) -- verify correct results SELECT * FROM test_partials ORDER BY time; @@ -2844,38 +2838,32 @@ EXPLAIN (costs off) DELETE FROM test_partials WHERE time >= ALL(SELECT time from DELETE FROM test_partials WHERE time >= ALL(SELECT time from test_partials); -- All 3 chunks will now become partially compressed chunks EXPLAIN (costs off) SELECT * FROM test_partials ORDER BY time; - QUERY PLAN --------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------- Custom Scan (ChunkAppend) on test_partials Order: test_partials."time" -> Merge Append Sort Key: _hyper_35_68_chunk."time" -> Custom Scan (DecompressChunk) on _hyper_35_68_chunk - -> Sort - Sort Key: compress_hyper_36_71_chunk._ts_meta_min_1, compress_hyper_36_71_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_71_chunk + -> Index Scan Backward using compress_hyper_36_71_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_71_chunk -> Sort Sort Key: _hyper_35_68_chunk."time" -> Seq Scan on _hyper_35_68_chunk -> Merge Append Sort Key: _hyper_35_69_chunk."time" -> Custom Scan (DecompressChunk) on _hyper_35_69_chunk - -> Sort - Sort Key: compress_hyper_36_72_chunk._ts_meta_min_1, compress_hyper_36_72_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_72_chunk + -> Index Scan Backward using compress_hyper_36_72_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_72_chunk -> Sort Sort Key: _hyper_35_69_chunk."time" -> Seq Scan on _hyper_35_69_chunk -> Merge Append Sort Key: _hyper_35_70_chunk."time" -> Custom Scan (DecompressChunk) on _hyper_35_70_chunk - -> Sort - Sort Key: compress_hyper_36_73_chunk._ts_meta_min_1, compress_hyper_36_73_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_73_chunk + -> Index Scan Backward using compress_hyper_36_73_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_73_chunk -> Sort Sort Key: _hyper_35_70_chunk."time" -> Seq Scan on _hyper_35_70_chunk -(29 rows) +(23 rows) -- verify correct results SELECT * FROM test_partials ORDER BY time; @@ -2899,23 +2887,17 @@ SELECT compress_chunk(show_chunks('test_partials')); -- fully compressed EXPLAIN (costs off) SELECT * FROM test_partials ORDER BY time; - QUERY PLAN --------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------- Custom Scan (ChunkAppend) on test_partials Order: test_partials."time" -> Custom Scan (DecompressChunk) on _hyper_35_68_chunk - -> Sort - Sort Key: compress_hyper_36_74_chunk._ts_meta_min_1, compress_hyper_36_74_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_74_chunk + -> Index Scan Backward using compress_hyper_36_74_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_74_chunk -> Custom Scan (DecompressChunk) on _hyper_35_69_chunk - -> Sort - Sort Key: compress_hyper_36_75_chunk._ts_meta_min_1, compress_hyper_36_75_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_75_chunk + -> Index Scan Backward using compress_hyper_36_75_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_75_chunk -> Custom Scan (DecompressChunk) on _hyper_35_70_chunk - -> Sort - Sort Key: compress_hyper_36_76_chunk._ts_meta_min_1, compress_hyper_36_76_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_76_chunk -(14 rows) + -> Index Scan Backward using compress_hyper_36_76_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_76_chunk +(8 rows) DROP TABLE test_partials; CREATE TABLE test_meta_filters(time timestamptz NOT NULL, device text, metric text, v1 float, v2 float); diff --git a/tsl/test/expected/compression_update_delete-15.out b/tsl/test/expected/compression_update_delete-15.out index d31b39d057a..7233b14cd70 100644 --- a/tsl/test/expected/compression_update_delete-15.out +++ b/tsl/test/expected/compression_update_delete-15.out @@ -2779,23 +2779,17 @@ SELECT compress_chunk(show_chunks('test_partials')); -- fully compressed EXPLAIN (costs off) SELECT * FROM test_partials ORDER BY time; - QUERY PLAN --------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------- Custom Scan (ChunkAppend) on test_partials Order: test_partials."time" -> Custom Scan (DecompressChunk) on _hyper_35_68_chunk - -> Sort - Sort Key: compress_hyper_36_71_chunk._ts_meta_min_1, compress_hyper_36_71_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_71_chunk + -> Index Scan Backward using compress_hyper_36_71_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_71_chunk -> Custom Scan (DecompressChunk) on _hyper_35_69_chunk - -> Sort - Sort Key: compress_hyper_36_72_chunk._ts_meta_min_1, compress_hyper_36_72_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_72_chunk + -> Index Scan Backward using compress_hyper_36_72_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_72_chunk -> Custom Scan (DecompressChunk) on _hyper_35_70_chunk - -> Sort - Sort Key: compress_hyper_36_73_chunk._ts_meta_min_1, compress_hyper_36_73_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_73_chunk -(14 rows) + -> Index Scan Backward using compress_hyper_36_73_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_73_chunk +(8 rows) -- verify correct results SELECT * FROM test_partials ORDER BY time; @@ -2844,38 +2838,32 @@ EXPLAIN (costs off) DELETE FROM test_partials WHERE time >= ALL(SELECT time from DELETE FROM test_partials WHERE time >= ALL(SELECT time from test_partials); -- All 3 chunks will now become partially compressed chunks EXPLAIN (costs off) SELECT * FROM test_partials ORDER BY time; - QUERY PLAN --------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------- Custom Scan (ChunkAppend) on test_partials Order: test_partials."time" -> Merge Append Sort Key: _hyper_35_68_chunk."time" -> Custom Scan (DecompressChunk) on _hyper_35_68_chunk - -> Sort - Sort Key: compress_hyper_36_71_chunk._ts_meta_min_1, compress_hyper_36_71_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_71_chunk + -> Index Scan Backward using compress_hyper_36_71_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_71_chunk -> Sort Sort Key: _hyper_35_68_chunk."time" -> Seq Scan on _hyper_35_68_chunk -> Merge Append Sort Key: _hyper_35_69_chunk."time" -> Custom Scan (DecompressChunk) on _hyper_35_69_chunk - -> Sort - Sort Key: compress_hyper_36_72_chunk._ts_meta_min_1, compress_hyper_36_72_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_72_chunk + -> Index Scan Backward using compress_hyper_36_72_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_72_chunk -> Sort Sort Key: _hyper_35_69_chunk."time" -> Seq Scan on _hyper_35_69_chunk -> Merge Append Sort Key: _hyper_35_70_chunk."time" -> Custom Scan (DecompressChunk) on _hyper_35_70_chunk - -> Sort - Sort Key: compress_hyper_36_73_chunk._ts_meta_min_1, compress_hyper_36_73_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_73_chunk + -> Index Scan Backward using compress_hyper_36_73_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_73_chunk -> Sort Sort Key: _hyper_35_70_chunk."time" -> Seq Scan on _hyper_35_70_chunk -(29 rows) +(23 rows) -- verify correct results SELECT * FROM test_partials ORDER BY time; @@ -2899,23 +2887,17 @@ SELECT compress_chunk(show_chunks('test_partials')); -- fully compressed EXPLAIN (costs off) SELECT * FROM test_partials ORDER BY time; - QUERY PLAN --------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------- Custom Scan (ChunkAppend) on test_partials Order: test_partials."time" -> Custom Scan (DecompressChunk) on _hyper_35_68_chunk - -> Sort - Sort Key: compress_hyper_36_74_chunk._ts_meta_min_1, compress_hyper_36_74_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_74_chunk + -> Index Scan Backward using compress_hyper_36_74_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_74_chunk -> Custom Scan (DecompressChunk) on _hyper_35_69_chunk - -> Sort - Sort Key: compress_hyper_36_75_chunk._ts_meta_min_1, compress_hyper_36_75_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_75_chunk + -> Index Scan Backward using compress_hyper_36_75_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_75_chunk -> Custom Scan (DecompressChunk) on _hyper_35_70_chunk - -> Sort - Sort Key: compress_hyper_36_76_chunk._ts_meta_min_1, compress_hyper_36_76_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_76_chunk -(14 rows) + -> Index Scan Backward using compress_hyper_36_76_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_76_chunk +(8 rows) DROP TABLE test_partials; CREATE TABLE test_meta_filters(time timestamptz NOT NULL, device text, metric text, v1 float, v2 float); diff --git a/tsl/test/expected/compression_update_delete-16.out b/tsl/test/expected/compression_update_delete-16.out index d31b39d057a..7233b14cd70 100644 --- a/tsl/test/expected/compression_update_delete-16.out +++ b/tsl/test/expected/compression_update_delete-16.out @@ -2779,23 +2779,17 @@ SELECT compress_chunk(show_chunks('test_partials')); -- fully compressed EXPLAIN (costs off) SELECT * FROM test_partials ORDER BY time; - QUERY PLAN --------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------- Custom Scan (ChunkAppend) on test_partials Order: test_partials."time" -> Custom Scan (DecompressChunk) on _hyper_35_68_chunk - -> Sort - Sort Key: compress_hyper_36_71_chunk._ts_meta_min_1, compress_hyper_36_71_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_71_chunk + -> Index Scan Backward using compress_hyper_36_71_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_71_chunk -> Custom Scan (DecompressChunk) on _hyper_35_69_chunk - -> Sort - Sort Key: compress_hyper_36_72_chunk._ts_meta_min_1, compress_hyper_36_72_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_72_chunk + -> Index Scan Backward using compress_hyper_36_72_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_72_chunk -> Custom Scan (DecompressChunk) on _hyper_35_70_chunk - -> Sort - Sort Key: compress_hyper_36_73_chunk._ts_meta_min_1, compress_hyper_36_73_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_73_chunk -(14 rows) + -> Index Scan Backward using compress_hyper_36_73_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_73_chunk +(8 rows) -- verify correct results SELECT * FROM test_partials ORDER BY time; @@ -2844,38 +2838,32 @@ EXPLAIN (costs off) DELETE FROM test_partials WHERE time >= ALL(SELECT time from DELETE FROM test_partials WHERE time >= ALL(SELECT time from test_partials); -- All 3 chunks will now become partially compressed chunks EXPLAIN (costs off) SELECT * FROM test_partials ORDER BY time; - QUERY PLAN --------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------- Custom Scan (ChunkAppend) on test_partials Order: test_partials."time" -> Merge Append Sort Key: _hyper_35_68_chunk."time" -> Custom Scan (DecompressChunk) on _hyper_35_68_chunk - -> Sort - Sort Key: compress_hyper_36_71_chunk._ts_meta_min_1, compress_hyper_36_71_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_71_chunk + -> Index Scan Backward using compress_hyper_36_71_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_71_chunk -> Sort Sort Key: _hyper_35_68_chunk."time" -> Seq Scan on _hyper_35_68_chunk -> Merge Append Sort Key: _hyper_35_69_chunk."time" -> Custom Scan (DecompressChunk) on _hyper_35_69_chunk - -> Sort - Sort Key: compress_hyper_36_72_chunk._ts_meta_min_1, compress_hyper_36_72_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_72_chunk + -> Index Scan Backward using compress_hyper_36_72_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_72_chunk -> Sort Sort Key: _hyper_35_69_chunk."time" -> Seq Scan on _hyper_35_69_chunk -> Merge Append Sort Key: _hyper_35_70_chunk."time" -> Custom Scan (DecompressChunk) on _hyper_35_70_chunk - -> Sort - Sort Key: compress_hyper_36_73_chunk._ts_meta_min_1, compress_hyper_36_73_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_73_chunk + -> Index Scan Backward using compress_hyper_36_73_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_73_chunk -> Sort Sort Key: _hyper_35_70_chunk."time" -> Seq Scan on _hyper_35_70_chunk -(29 rows) +(23 rows) -- verify correct results SELECT * FROM test_partials ORDER BY time; @@ -2899,23 +2887,17 @@ SELECT compress_chunk(show_chunks('test_partials')); -- fully compressed EXPLAIN (costs off) SELECT * FROM test_partials ORDER BY time; - QUERY PLAN --------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------- Custom Scan (ChunkAppend) on test_partials Order: test_partials."time" -> Custom Scan (DecompressChunk) on _hyper_35_68_chunk - -> Sort - Sort Key: compress_hyper_36_74_chunk._ts_meta_min_1, compress_hyper_36_74_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_74_chunk + -> Index Scan Backward using compress_hyper_36_74_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_74_chunk -> Custom Scan (DecompressChunk) on _hyper_35_69_chunk - -> Sort - Sort Key: compress_hyper_36_75_chunk._ts_meta_min_1, compress_hyper_36_75_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_75_chunk + -> Index Scan Backward using compress_hyper_36_75_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_75_chunk -> Custom Scan (DecompressChunk) on _hyper_35_70_chunk - -> Sort - Sort Key: compress_hyper_36_76_chunk._ts_meta_min_1, compress_hyper_36_76_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_76_chunk -(14 rows) + -> Index Scan Backward using compress_hyper_36_76_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_76_chunk +(8 rows) DROP TABLE test_partials; CREATE TABLE test_meta_filters(time timestamptz NOT NULL, device text, metric text, v1 float, v2 float); diff --git a/tsl/test/expected/compression_update_delete-17.out b/tsl/test/expected/compression_update_delete-17.out index 5808f1ba88e..4b283e9b8e1 100644 --- a/tsl/test/expected/compression_update_delete-17.out +++ b/tsl/test/expected/compression_update_delete-17.out @@ -2779,23 +2779,17 @@ SELECT compress_chunk(show_chunks('test_partials')); -- fully compressed EXPLAIN (costs off) SELECT * FROM test_partials ORDER BY time; - QUERY PLAN --------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------- Custom Scan (ChunkAppend) on test_partials Order: test_partials."time" -> Custom Scan (DecompressChunk) on _hyper_35_68_chunk - -> Sort - Sort Key: compress_hyper_36_71_chunk._ts_meta_min_1, compress_hyper_36_71_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_71_chunk + -> Index Scan Backward using compress_hyper_36_71_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_71_chunk -> Custom Scan (DecompressChunk) on _hyper_35_69_chunk - -> Sort - Sort Key: compress_hyper_36_72_chunk._ts_meta_min_1, compress_hyper_36_72_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_72_chunk + -> Index Scan Backward using compress_hyper_36_72_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_72_chunk -> Custom Scan (DecompressChunk) on _hyper_35_70_chunk - -> Sort - Sort Key: compress_hyper_36_73_chunk._ts_meta_min_1, compress_hyper_36_73_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_73_chunk -(14 rows) + -> Index Scan Backward using compress_hyper_36_73_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_73_chunk +(8 rows) -- verify correct results SELECT * FROM test_partials ORDER BY time; @@ -2844,38 +2838,32 @@ EXPLAIN (costs off) DELETE FROM test_partials WHERE time >= ALL(SELECT time from DELETE FROM test_partials WHERE time >= ALL(SELECT time from test_partials); -- All 3 chunks will now become partially compressed chunks EXPLAIN (costs off) SELECT * FROM test_partials ORDER BY time; - QUERY PLAN --------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------- Custom Scan (ChunkAppend) on test_partials Order: test_partials."time" -> Merge Append Sort Key: _hyper_35_68_chunk."time" -> Custom Scan (DecompressChunk) on _hyper_35_68_chunk - -> Sort - Sort Key: compress_hyper_36_71_chunk._ts_meta_min_1, compress_hyper_36_71_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_71_chunk + -> Index Scan Backward using compress_hyper_36_71_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_71_chunk -> Sort Sort Key: _hyper_35_68_chunk."time" -> Seq Scan on _hyper_35_68_chunk -> Merge Append Sort Key: _hyper_35_69_chunk."time" -> Custom Scan (DecompressChunk) on _hyper_35_69_chunk - -> Sort - Sort Key: compress_hyper_36_72_chunk._ts_meta_min_1, compress_hyper_36_72_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_72_chunk + -> Index Scan Backward using compress_hyper_36_72_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_72_chunk -> Sort Sort Key: _hyper_35_69_chunk."time" -> Seq Scan on _hyper_35_69_chunk -> Merge Append Sort Key: _hyper_35_70_chunk."time" -> Custom Scan (DecompressChunk) on _hyper_35_70_chunk - -> Sort - Sort Key: compress_hyper_36_73_chunk._ts_meta_min_1, compress_hyper_36_73_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_73_chunk + -> Index Scan Backward using compress_hyper_36_73_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_73_chunk -> Sort Sort Key: _hyper_35_70_chunk."time" -> Seq Scan on _hyper_35_70_chunk -(29 rows) +(23 rows) -- verify correct results SELECT * FROM test_partials ORDER BY time; @@ -2899,23 +2887,17 @@ SELECT compress_chunk(show_chunks('test_partials')); -- fully compressed EXPLAIN (costs off) SELECT * FROM test_partials ORDER BY time; - QUERY PLAN --------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------- Custom Scan (ChunkAppend) on test_partials Order: test_partials."time" -> Custom Scan (DecompressChunk) on _hyper_35_68_chunk - -> Sort - Sort Key: compress_hyper_36_74_chunk._ts_meta_min_1, compress_hyper_36_74_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_74_chunk + -> Index Scan Backward using compress_hyper_36_74_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_74_chunk -> Custom Scan (DecompressChunk) on _hyper_35_69_chunk - -> Sort - Sort Key: compress_hyper_36_75_chunk._ts_meta_min_1, compress_hyper_36_75_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_75_chunk + -> Index Scan Backward using compress_hyper_36_75_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_75_chunk -> Custom Scan (DecompressChunk) on _hyper_35_70_chunk - -> Sort - Sort Key: compress_hyper_36_76_chunk._ts_meta_min_1, compress_hyper_36_76_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_36_76_chunk -(14 rows) + -> Index Scan Backward using compress_hyper_36_76_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_36_76_chunk +(8 rows) DROP TABLE test_partials; CREATE TABLE test_meta_filters(time timestamptz NOT NULL, device text, metric text, v1 float, v2 float); diff --git a/tsl/test/expected/hypercore_vacuum.out b/tsl/test/expected/hypercore_vacuum.out index eab589787e3..502deddcdca 100644 --- a/tsl/test/expected/hypercore_vacuum.out +++ b/tsl/test/expected/hypercore_vacuum.out @@ -414,12 +414,15 @@ inner join pg_am am on (cl.relam = am.oid); select indexrelid::regclass from pg_index i inner join compressed_rels crels on (i.indrelid = crels.compressed_relid); - indexrelid ------------------------------------------------------------------------- + indexrelid +----------------------------------------------------------------------------------- _timescaledb_internal.compress_hyper_5_13_chunk_ts_hypercore_proxy_idx + _timescaledb_internal.compress_hyper_5_13_chunk__ts_meta_min_1__ts_meta_max_1_idx _timescaledb_internal.compress_hyper_5_14_chunk_ts_hypercore_proxy_idx + _timescaledb_internal.compress_hyper_5_14_chunk__ts_meta_min_1__ts_meta_max_1_idx _timescaledb_internal.compress_hyper_5_15_chunk_ts_hypercore_proxy_idx -(3 rows) + _timescaledb_internal.compress_hyper_5_15_chunk__ts_meta_min_1__ts_meta_max_1_idx +(6 rows) -- delete some data to generate garbage delete from hystable where temp > 20; diff --git a/tsl/test/expected/telemetry_stats.out b/tsl/test/expected/telemetry_stats.out index 5ad9ac55903..089cd6c70c5 100644 --- a/tsl/test/expected/telemetry_stats.out +++ b/tsl/test/expected/telemetry_stats.out @@ -328,13 +328,13 @@ SELECT jsonb_pretty(rels) AS relations FROM relations; "num_compressed_chunks": 5, + "uncompressed_heap_size": 221184, + "uncompressed_row_count": 736, + - "compressed_indexes_size": 16384, + + "compressed_indexes_size": 81920, + "uncompressed_toast_size": 0, + "uncompressed_indexes_size": 131072, + "num_compressed_hypertables": 2, + "compressed_row_count_frozen_immediately": 14+ }, + - "indexes_size": 204800, + + "indexes_size": 270336, + "num_children": 11, + "num_relations": 2, + "num_reltuples": 658 + diff --git a/tsl/test/expected/vector_agg_default.out b/tsl/test/expected/vector_agg_default.out index 930d105a2ed..49b629cd239 100644 --- a/tsl/test/expected/vector_agg_default.out +++ b/tsl/test/expected/vector_agg_default.out @@ -185,21 +185,17 @@ select sum(c) from dvagg having sum(c) > 0; -- Some negative cases. set timescaledb.debug_require_vector_agg to 'forbid'; explain (costs off) select sum(c) from dvagg group by grouping sets ((), (a)); - QUERY PLAN ----------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------ MixedAggregate Hash Key: _hyper_1_1_chunk.a Group Key: () -> Append -> Custom Scan (DecompressChunk) on _hyper_1_1_chunk - -> Sort - Sort Key: compress_hyper_2_2_chunk._ts_meta_min_1, compress_hyper_2_2_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_2_2_chunk + -> Index Scan Backward using compress_hyper_2_2_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_2_2_chunk -> Custom Scan (DecompressChunk) on _hyper_1_3_chunk - -> Sort - Sort Key: compress_hyper_2_4_chunk._ts_meta_min_1, compress_hyper_2_4_chunk._ts_meta_max_1 - -> Seq Scan on compress_hyper_2_4_chunk -(12 rows) + -> Index Scan Backward using compress_hyper_2_4_chunk__ts_meta_min_1__ts_meta_max_1_idx on compress_hyper_2_4_chunk +(8 rows) -- As a reference, the result on decompressed table. select decompress_chunk(show_chunks('dvagg')); diff --git a/tsl/test/sql/CMakeLists.txt b/tsl/test/sql/CMakeLists.txt index e71fbdcffec..5b2451fb0d7 100644 --- a/tsl/test/sql/CMakeLists.txt +++ b/tsl/test/sql/CMakeLists.txt @@ -27,6 +27,7 @@ set(TEST_FILES compression_defaults.sql compression_fks.sql compression_insert.sql + compression_indexcreate.sql compression_policy.sql compression_qualpushdown.sql compression_sequence_num_removal.sql diff --git a/tsl/test/sql/compression_indexcreate.sql b/tsl/test/sql/compression_indexcreate.sql new file mode 100644 index 00000000000..b24f4f22117 --- /dev/null +++ b/tsl/test/sql/compression_indexcreate.sql @@ -0,0 +1,44 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. + +-- force index scan to be used when possible +set enable_seqscan to false; +\set PREFIX 'EXPLAIN (analyze, costs off, summary off, timing off) ' +create table segind(time timestamptz, a int, b int); +select create_hypertable('segind', by_range('time')); + +-- enable compression on hypertable with no segment by column +alter table segind set (timescaledb.compress, timescaledb.compress_segmentby='', timescaledb.compress_orderby='time, b'); +insert into segind values('2024-11-08 10:31:28.436014-07', 1, 1), ('2024-11-08 10:32:28.436014-07', 2, 1), ('2024-11-08 10:33:28.436014-07', 3, 1), ('2024-11-08 10:34:28.436014-07', 2, 1), ('2024-11-08 10:35:28.436014-07', 1, 2), ('2024-11-08 10:36:28.436014-07', 4, 1); + +-- compress chunk +-- this should create an index using orderby columns +select compress_chunk(show_chunks('segind')); + +-- query using orderby columns should use the index +:PREFIX select * from segind where b = 1; +:PREFIX select * from segind where time = '2024-11-08 10:32:28.436014-07'; +:PREFIX select * from segind where b = 1 and time = '2024-11-08 10:32:28.436014-07'; + +-- a query on another column should perform a seq scan since there is no index on it +:PREFIX select * from segind where a = 1; + +-- decompress the chunk to drop the index +select decompress_chunk(show_chunks('segind')); + +-- change compression settings to use segmentby column +alter table segind set (timescaledb.compress, timescaledb.compress_segmentby='a', timescaledb.compress_orderby='time, b'); + +-- compress chunk +-- this should create an index using segmentby and orderby columns +select compress_chunk(show_chunks('segind')); + +-- queries using segmentby or orderby columns should use the index +:PREFIX select * from segind where b = 1; +:PREFIX select * from segind where time = '2024-11-08 10:32:28.436014-07'; +:PREFIX select * from segind where b = 1 and time = '2024-11-08 10:32:28.436014-07'; +:PREFIX select * from segind where a = 1; + +-- cleanup +RESET enable_seqscan; From 9de9eecf7331f94e4126a6b2e5cbf8e94032ee38 Mon Sep 17 00:00:00 2001 From: Mats Kindahl Date: Tue, 7 Jan 2025 15:54:36 +0100 Subject: [PATCH 07/32] Add error for DELETE triggers with transition tables DELETE triggers cannot be supported for compressed tables not using the hypercore table access method since it can delete an entire compressed segment and a transition table is not built for this case, so generating an error for this case. This commit also adds a bunch of tests for triggers with transition tables for normal compressed tables and extend the hypercore trigger tests. --- src/hypertable.c | 4 + src/hypertable.h | 1 + src/process_utility.c | 17 ++ tsl/src/compression/create.c | 42 ++++- tsl/test/expected/compression_trigger.out | 193 ++++++++++++++++++++++ tsl/test/expected/hypercore_trigger.out | 85 +++++++++- tsl/test/sql/CMakeLists.txt | 1 + tsl/test/sql/compression_trigger.sql | 143 ++++++++++++++++ tsl/test/sql/hypercore_trigger.sql | 25 ++- 9 files changed, 496 insertions(+), 15 deletions(-) create mode 100644 tsl/test/expected/compression_trigger.out create mode 100644 tsl/test/sql/compression_trigger.sql diff --git a/src/hypertable.c b/src/hypertable.c index ac3c2c5e765..100a163c42b 100644 --- a/src/hypertable.c +++ b/src/hypertable.c @@ -240,6 +240,7 @@ Hypertable * ts_hypertable_from_tupleinfo(const TupleInfo *ti) { Hypertable *h = MemoryContextAllocZero(ti->mctx, sizeof(Hypertable)); + char relkind; ts_hypertable_formdata_fill(&h->fd, ti); h->main_table_relid = @@ -249,6 +250,9 @@ ts_hypertable_from_tupleinfo(const TupleInfo *ti) ts_subspace_store_init(h->space, ti->mctx, ts_guc_max_cached_chunks_per_hypertable); h->chunk_sizing_func = get_chunk_sizing_func_oid(&h->fd); + if (OidIsValid(h->main_table_relid)) + ts_get_rel_info(h->main_table_relid, &h->amoid, &relkind); + if (ts_guc_enable_chunk_skipping) { h->range_space = diff --git a/src/hypertable.h b/src/hypertable.h index 9b961b801e3..77168b94fa3 100644 --- a/src/hypertable.h +++ b/src/hypertable.h @@ -51,6 +51,7 @@ typedef struct Hypertable FormData_hypertable fd; Oid main_table_relid; Oid chunk_sizing_func; + Oid amoid; Hyperspace *space; SubspaceStore *chunk_cache; ChunkRangeSpace *range_space; diff --git a/src/process_utility.c b/src/process_utility.c index 1aee1d67243..7c70f0b3d65 100644 --- a/src/process_utility.c +++ b/src/process_utility.c @@ -4482,6 +4482,23 @@ process_create_trigger_start(ProcessUtilityArgs *args) errmsg("ROW triggers with transition tables are not supported on hypertables"))); } + /* + * We currently cannot support delete triggers with transition tables on + * compressed tables that are not using hypercore table access method + * since deleting a complete segment will not build a transition table for + * the delete. + */ + if (stmt->transitionRels && TRIGGER_FOR_DELETE(tgtype) && + TS_HYPERTABLE_HAS_COMPRESSION_ENABLED(ht) && !ts_is_hypercore_am(ht->amoid)) + { + ts_cache_release(hcache); + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("DELETE triggers with transition tables not supported"), + errdetail("Compressed hypertables not using \"hypercore\" access method are not " + "supported if the trigger use transition tables."))); + } + add_hypertable_to_process_args(args, ht); /* diff --git a/tsl/src/compression/create.c b/tsl/src/compression/create.c index 5e675f0c724..897e2c3d887 100644 --- a/tsl/src/compression/create.c +++ b/tsl/src/compression/create.c @@ -860,7 +860,6 @@ validate_hypertable_for_compression(Hypertable *ht) "cannot compress tables with reserved column prefix '%s'", COMPRESSION_COLUMN_METADATA_PREFIX); } - table_close(rel, AccessShareLock); if (row_size > MaxHeapTupleSize) { @@ -871,6 +870,47 @@ validate_hypertable_for_compression(Hypertable *ht) row_size, MaxHeapTupleSize))); } + + /* + * Check that all triggers are ok for compressed tables. + */ + Relation pg_trigger = table_open(TriggerRelationId, AccessShareLock); + HeapTuple tuple; + + ScanKeyData key; + ScanKeyInit(&key, + Anum_pg_trigger_tgrelid, + BTEqualStrategyNumber, + F_OIDEQ, + ObjectIdGetDatum(ht->main_table_relid)); + + SysScanDesc scan = systable_beginscan(pg_trigger, TriggerRelidNameIndexId, true, NULL, 1, &key); + + while (HeapTupleIsValid(tuple = systable_getnext(scan))) + { + bool oldtable_isnull; + Form_pg_trigger trigrec = (Form_pg_trigger) GETSTRUCT(tuple); + + /* + * We currently cannot support transition tables for DELETE triggers + * on compressed tables that are not using hypercore table access + * method since deleting a complete segment will not build a + * transition table for the delete. + */ + fastgetattr(tuple, Anum_pg_trigger_tgoldtable, pg_trigger->rd_att, &oldtable_isnull); + if (!oldtable_isnull && !TRIGGER_FOR_ROW(trigrec->tgtype) && + TRIGGER_FOR_DELETE(trigrec->tgtype) && !ts_is_hypercore_am(ht->amoid)) + ereport(ERROR, + errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("DELETE triggers with transition tables not supported"), + errdetail( + "Compressed hypertables not using \"hypercore\" access method are not " + "supported if the trigger use transition tables.")); + } + + systable_endscan(scan); + table_close(pg_trigger, AccessShareLock); + table_close(rel, AccessShareLock); } /* diff --git a/tsl/test/expected/compression_trigger.out b/tsl/test/expected/compression_trigger.out new file mode 100644 index 00000000000..c9d7e844ea5 --- /dev/null +++ b/tsl/test/expected/compression_trigger.out @@ -0,0 +1,193 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. +-- This is copied from hypercore_trigger.sql +set client_min_messages to warning; +create table readings( + metric_id serial, + created_at timestamptz not null unique, + location_id smallint, --segmentby attribute with index + owner_id bigint, --segmentby attribute without index + device_id bigint, --non-segmentby attribute + temp float8, + humidity float4 +); +select create_hypertable('readings', by_range('created_at')); + create_hypertable +------------------- + (1,t) +(1 row) + +select setseed(1); + setseed +--------- + +(1 row) + +insert into readings(created_at, location_id, device_id, owner_id, temp, humidity) +select t, ceil(random()*10), ceil(random()*30), ceil(random() * 5), random()*40, random()*100 +from generate_series('2022-06-01'::timestamptz, '2022-07-01', '5m') t; +alter table readings set ( + timescaledb.compress, + timescaledb.compress_orderby = 'created_at', + timescaledb.compress_segmentby = 'location_id, owner_id' +); +select compress_chunk(show_chunks('readings')); + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk + _timescaledb_internal._hyper_1_5_chunk + _timescaledb_internal._hyper_1_6_chunk +(6 rows) + +create table saved_rows (like readings, new_row bool not null, kind text); +create function save_transition_table() returns trigger as $$ +begin + case tg_op + when 'INSERT' then + insert into saved_rows select n.*, true, tg_op from new_table n; + when 'DELETE' then + insert into saved_rows select o.*, false, tg_op from old_table o; + when 'UPDATE' then + insert into saved_rows select n.*, true, tg_op from new_table n; + insert into saved_rows select o.*, false, tg_op from old_table o; + end case; + return null; +end; +$$ language plpgsql; +create trigger save_insert_transition_table_trg + after insert on readings + referencing new table as new_table + for each statement execute function save_transition_table(); +insert into readings(created_at, location_id, device_id, owner_id, temp, humidity) +values ('2022-06-01 00:01:23', 999, 666, 111, 3.14, 3.14), + ('2022-06-01 00:02:23', 999, 666, 112, 3.14, 3.14); +select * from saved_rows order by metric_id; + metric_id | created_at | location_id | owner_id | device_id | temp | humidity | new_row | kind +-----------+------------------------------+-------------+----------+-----------+------+----------+---------+-------- + 8642 | Wed Jun 01 00:01:23 2022 PDT | 999 | 111 | 666 | 3.14 | 3.14 | t | INSERT + 8643 | Wed Jun 01 00:02:23 2022 PDT | 999 | 112 | 666 | 3.14 | 3.14 | t | INSERT +(2 rows) + +truncate saved_rows; +select compress_chunk(show_chunks('readings')); + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk + _timescaledb_internal._hyper_1_5_chunk + _timescaledb_internal._hyper_1_6_chunk +(6 rows) + +copy readings(created_at, location_id, device_id, owner_id, temp, humidity) from stdin with (format csv); +select * from saved_rows order by metric_id; + metric_id | created_at | location_id | owner_id | device_id | temp | humidity | new_row | kind +-----------+------------------------------+-------------+----------+-----------+------+----------+---------+-------- + 8644 | Wed Jun 01 00:01:35 2022 PDT | 999 | 111 | 666 | 3.14 | 3.14 | t | INSERT +(1 row) + +truncate saved_rows; +select compress_chunk(show_chunks('readings')); + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk + _timescaledb_internal._hyper_1_5_chunk + _timescaledb_internal._hyper_1_6_chunk +(6 rows) + +create trigger save_update_transition_table_trg + after update on readings + referencing new table as new_table old table as old_table + for each statement execute function save_transition_table(); +select * from readings where location_id = 999 order by metric_id; + metric_id | created_at | location_id | owner_id | device_id | temp | humidity +-----------+------------------------------+-------------+----------+-----------+------+---------- + 8642 | Wed Jun 01 00:01:23 2022 PDT | 999 | 111 | 666 | 3.14 | 3.14 + 8643 | Wed Jun 01 00:02:23 2022 PDT | 999 | 112 | 666 | 3.14 | 3.14 + 8644 | Wed Jun 01 00:01:35 2022 PDT | 999 | 111 | 666 | 3.14 | 3.14 +(3 rows) + +update readings set humidity = 99.99 where location_id = 999; +select * from saved_rows order by metric_id; + metric_id | created_at | location_id | owner_id | device_id | temp | humidity | new_row | kind +-----------+------------------------------+-------------+----------+-----------+------+----------+---------+-------- + 8642 | Wed Jun 01 00:01:23 2022 PDT | 999 | 111 | 666 | 3.14 | 99.99 | t | UPDATE + 8642 | Wed Jun 01 00:01:23 2022 PDT | 999 | 111 | 666 | 3.14 | 3.14 | f | UPDATE + 8643 | Wed Jun 01 00:02:23 2022 PDT | 999 | 112 | 666 | 3.14 | 99.99 | t | UPDATE + 8643 | Wed Jun 01 00:02:23 2022 PDT | 999 | 112 | 666 | 3.14 | 3.14 | f | UPDATE + 8644 | Wed Jun 01 00:01:35 2022 PDT | 999 | 111 | 666 | 3.14 | 99.99 | t | UPDATE + 8644 | Wed Jun 01 00:01:35 2022 PDT | 999 | 111 | 666 | 3.14 | 3.14 | f | UPDATE +(6 rows) + +truncate saved_rows; +select compress_chunk(show_chunks('readings')); + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk + _timescaledb_internal._hyper_1_5_chunk + _timescaledb_internal._hyper_1_6_chunk +(6 rows) + +-- This is not supported since it is possible to delete an entire +-- segment without executing the trigger. +\set ON_ERROR_STOP 0 +create trigger save_delete_transition_table_trg + after delete on readings + referencing old table as old_table + for each statement execute function save_transition_table(); +ERROR: DELETE triggers with transition tables not supported +\set ON_ERROR_STOP 1 +-- Test that we get an error when enabling compression and have a +-- delete trigger with a transition table. We allow transition tables +-- for update and insert triggers. +create table test2( + created_at timestamptz not null unique, + location_id bigint, + temp float8 +); +select create_hypertable('test2', by_range('created_at')); + create_hypertable +------------------- + (3,t) +(1 row) + +create trigger save_test2_insert_trg + after insert on test2 + referencing new table as new_table + for each statement execute function save_transition_table(); +create trigger save_test2_update_trg + after update on test2 + referencing new table as new_table old table as old_table + for each statement execute function save_transition_table(); +create trigger save_test2_delete_trg + after delete on test2 + referencing old table as old_table + for each statement execute function save_transition_table(); +-- This should fail +\set ON_ERROR_STOP 0 +alter table test2 set ( + timescaledb.compress, + timescaledb.compress_orderby = 'created_at', + timescaledb.compress_segmentby = 'location_id' +); +ERROR: DELETE triggers with transition tables not supported +\set ON_ERROR_STOP 1 +-- drop the delete trigger +drop trigger save_test2_delete_trg on test2; +-- This should now succeed. +alter table test2 set ( + timescaledb.compress, + timescaledb.compress_orderby = 'created_at', + timescaledb.compress_segmentby = 'location_id' +); diff --git a/tsl/test/expected/hypercore_trigger.out b/tsl/test/expected/hypercore_trigger.out index 4ac4daeab42..5b21dd15248 100644 --- a/tsl/test/expected/hypercore_trigger.out +++ b/tsl/test/expected/hypercore_trigger.out @@ -111,6 +111,9 @@ select format('%I.%I', chunk_schema, chunk_name)::regclass as chunk2 where format('%I.%I', hypertable_schema, hypertable_name)::regclass = :'hypertable'::regclass order by chunk2 asc limit 1 offset 1 \gset +-- Set the hypercore access method on the hypertable for transition +-- tables to work properly. +alter table :hypertable set access method hypercore; create table saved_rows (like :chunk1, new_row bool not null, kind text); create table count_stmt (inserts int, updates int, deletes int); create function save_row() returns trigger as $$ @@ -371,7 +374,7 @@ create trigger save_insert_transition_table_trg insert into readings(created_at, location_id, device_id, owner_id, temp, humidity) select created_at, location_id, device_id, owner_id, temp, humidity from sample order by created_at limit 2; -select * from saved_rows; +select * from saved_rows order by metric_id; metric_id | created_at | location_id | owner_id | device_id | temp | humidity | new_row | kind -----------+------------------------------+-------------+----------+-----------+------+----------+---------+-------- 11533 | Wed Jun 01 00:01:23 2022 PDT | 999 | 111 | 666 | 3.14 | 3.14 | t | INSERT @@ -379,19 +382,60 @@ select * from saved_rows; (2 rows) truncate saved_rows; +-- Compress the data again to make sure that it is fully compressed. +select compress_chunk(show_chunks(:'hypertable'), hypercore_use_access_method => true); + compress_chunk +----------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk + _timescaledb_internal._hyper_1_5_chunk + _timescaledb_internal._hyper_1_6_chunk + _timescaledb_internal._hyper_1_13_chunk + _timescaledb_internal._hyper_1_15_chunk + _timescaledb_internal._hyper_1_17_chunk + _timescaledb_internal._hyper_1_19_chunk +(10 rows) + copy readings(created_at, location_id, device_id, owner_id, temp, humidity) from stdin with (format csv); -select * from saved_rows; +select * from saved_rows order by metric_id; metric_id | created_at | location_id | owner_id | device_id | temp | humidity | new_row | kind -----------+------------------------------+-------------+----------+-----------+------+----------+---------+-------- 11535 | Wed Jun 01 00:01:35 2022 PDT | 999 | 111 | 666 | 3.14 | 3.14 | t | INSERT (1 row) truncate saved_rows; +-- Compress the data again to make sure that it is fully compressed. +select compress_chunk(show_chunks(:'hypertable'), hypercore_use_access_method => true); +NOTICE: chunk "_hyper_1_2_chunk" is already compressed +NOTICE: chunk "_hyper_1_3_chunk" is already compressed +NOTICE: chunk "_hyper_1_4_chunk" is already compressed +NOTICE: chunk "_hyper_1_5_chunk" is already compressed +NOTICE: chunk "_hyper_1_6_chunk" is already compressed +NOTICE: chunk "_hyper_1_13_chunk" is already compressed +NOTICE: chunk "_hyper_1_15_chunk" is already compressed +NOTICE: chunk "_hyper_1_17_chunk" is already compressed +NOTICE: chunk "_hyper_1_19_chunk" is already compressed + compress_chunk +----------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk + _timescaledb_internal._hyper_1_5_chunk + _timescaledb_internal._hyper_1_6_chunk + _timescaledb_internal._hyper_1_13_chunk + _timescaledb_internal._hyper_1_15_chunk + _timescaledb_internal._hyper_1_17_chunk + _timescaledb_internal._hyper_1_19_chunk +(10 rows) + create trigger save_update_transition_table_trg after update on readings referencing new table as new_table old table as old_table for each statement execute function save_transition_table(); -select * from readings where location_id = 999; +select * from readings where location_id = 999 order by metric_id; metric_id | created_at | location_id | owner_id | device_id | temp | humidity -----------+------------------------------+-------------+----------+-----------+------+---------- 11533 | Wed Jun 01 00:01:23 2022 PDT | 999 | 111 | 666 | 3.14 | 3.14 @@ -400,23 +444,48 @@ select * from readings where location_id = 999; (3 rows) update readings set humidity = 99.99 where location_id = 999; -select * from saved_rows; +select * from saved_rows order by metric_id; metric_id | created_at | location_id | owner_id | device_id | temp | humidity | new_row | kind -----------+------------------------------+-------------+----------+-----------+------+----------+---------+-------- 11533 | Wed Jun 01 00:01:23 2022 PDT | 999 | 111 | 666 | 3.14 | 99.99 | t | UPDATE - 11534 | Wed Jun 01 00:02:23 2022 PDT | 999 | 112 | 666 | 3.14 | 99.99 | t | UPDATE - 11535 | Wed Jun 01 00:01:35 2022 PDT | 999 | 111 | 666 | 3.14 | 99.99 | t | UPDATE 11533 | Wed Jun 01 00:01:23 2022 PDT | 999 | 111 | 666 | 3.14 | 3.14 | f | UPDATE + 11534 | Wed Jun 01 00:02:23 2022 PDT | 999 | 112 | 666 | 3.14 | 99.99 | t | UPDATE 11534 | Wed Jun 01 00:02:23 2022 PDT | 999 | 112 | 666 | 3.14 | 3.14 | f | UPDATE + 11535 | Wed Jun 01 00:01:35 2022 PDT | 999 | 111 | 666 | 3.14 | 99.99 | t | UPDATE 11535 | Wed Jun 01 00:01:35 2022 PDT | 999 | 111 | 666 | 3.14 | 3.14 | f | UPDATE (6 rows) truncate saved_rows; +-- Compress the data again to make sure that it is fully compressed. +select compress_chunk(show_chunks(:'hypertable'), hypercore_use_access_method => true); +NOTICE: chunk "_hyper_1_2_chunk" is already compressed +NOTICE: chunk "_hyper_1_3_chunk" is already compressed +NOTICE: chunk "_hyper_1_4_chunk" is already compressed +NOTICE: chunk "_hyper_1_5_chunk" is already compressed +NOTICE: chunk "_hyper_1_6_chunk" is already compressed +NOTICE: chunk "_hyper_1_13_chunk" is already compressed +NOTICE: chunk "_hyper_1_15_chunk" is already compressed +NOTICE: chunk "_hyper_1_17_chunk" is already compressed +NOTICE: chunk "_hyper_1_19_chunk" is already compressed + compress_chunk +----------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk + _timescaledb_internal._hyper_1_5_chunk + _timescaledb_internal._hyper_1_6_chunk + _timescaledb_internal._hyper_1_13_chunk + _timescaledb_internal._hyper_1_15_chunk + _timescaledb_internal._hyper_1_17_chunk + _timescaledb_internal._hyper_1_19_chunk +(10 rows) + create trigger save_delete_transition_table_trg after delete on readings referencing old table as old_table for each statement execute function save_transition_table(); -select * from readings where location_id = 999; +select * from readings where location_id = 999 order by metric_id; metric_id | created_at | location_id | owner_id | device_id | temp | humidity -----------+------------------------------+-------------+----------+-----------+------+---------- 11533 | Wed Jun 01 00:01:23 2022 PDT | 999 | 111 | 666 | 3.14 | 99.99 @@ -425,7 +494,7 @@ select * from readings where location_id = 999; (3 rows) delete from readings where location_id = 999; -select * from saved_rows; +select * from saved_rows order by metric_id; metric_id | created_at | location_id | owner_id | device_id | temp | humidity | new_row | kind -----------+------------------------------+-------------+----------+-----------+------+----------+---------+-------- 11533 | Wed Jun 01 00:01:23 2022 PDT | 999 | 111 | 666 | 3.14 | 99.99 | f | DELETE diff --git a/tsl/test/sql/CMakeLists.txt b/tsl/test/sql/CMakeLists.txt index 5b2451fb0d7..6401bf70e8f 100644 --- a/tsl/test/sql/CMakeLists.txt +++ b/tsl/test/sql/CMakeLists.txt @@ -34,6 +34,7 @@ set(TEST_FILES compression_settings.sql compression_sorted_merge_columns.sql compression_sorted_merge_distinct.sql + compression_trigger.sql decompress_index.sql foreign_keys.sql move.sql diff --git a/tsl/test/sql/compression_trigger.sql b/tsl/test/sql/compression_trigger.sql new file mode 100644 index 00000000000..6c021660cd2 --- /dev/null +++ b/tsl/test/sql/compression_trigger.sql @@ -0,0 +1,143 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. + +-- This is copied from hypercore_trigger.sql + +set client_min_messages to warning; + +create table readings( + metric_id serial, + created_at timestamptz not null unique, + location_id smallint, --segmentby attribute with index + owner_id bigint, --segmentby attribute without index + device_id bigint, --non-segmentby attribute + temp float8, + humidity float4 +); + +select create_hypertable('readings', by_range('created_at')); + +select setseed(1); + +insert into readings(created_at, location_id, device_id, owner_id, temp, humidity) +select t, ceil(random()*10), ceil(random()*30), ceil(random() * 5), random()*40, random()*100 +from generate_series('2022-06-01'::timestamptz, '2022-07-01', '5m') t; + +alter table readings set ( + timescaledb.compress, + timescaledb.compress_orderby = 'created_at', + timescaledb.compress_segmentby = 'location_id, owner_id' +); + +select compress_chunk(show_chunks('readings')); + +create table saved_rows (like readings, new_row bool not null, kind text); + +create function save_transition_table() returns trigger as $$ +begin + case tg_op + when 'INSERT' then + insert into saved_rows select n.*, true, tg_op from new_table n; + when 'DELETE' then + insert into saved_rows select o.*, false, tg_op from old_table o; + when 'UPDATE' then + insert into saved_rows select n.*, true, tg_op from new_table n; + insert into saved_rows select o.*, false, tg_op from old_table o; + end case; + return null; +end; +$$ language plpgsql; + +create trigger save_insert_transition_table_trg + after insert on readings + referencing new table as new_table + for each statement execute function save_transition_table(); + +insert into readings(created_at, location_id, device_id, owner_id, temp, humidity) +values ('2022-06-01 00:01:23', 999, 666, 111, 3.14, 3.14), + ('2022-06-01 00:02:23', 999, 666, 112, 3.14, 3.14); + +select * from saved_rows order by metric_id; + +truncate saved_rows; + +select compress_chunk(show_chunks('readings')); + +copy readings(created_at, location_id, device_id, owner_id, temp, humidity) from stdin with (format csv); +"2022-06-01 00:01:35",999,666,111,3.14,3.14 +\. + +select * from saved_rows order by metric_id; + +truncate saved_rows; + +select compress_chunk(show_chunks('readings')); + +create trigger save_update_transition_table_trg + after update on readings + referencing new table as new_table old table as old_table + for each statement execute function save_transition_table(); + +select * from readings where location_id = 999 order by metric_id; + +update readings set humidity = 99.99 where location_id = 999; + +select * from saved_rows order by metric_id; + +truncate saved_rows; + +select compress_chunk(show_chunks('readings')); + +-- This is not supported since it is possible to delete an entire +-- segment without executing the trigger. +\set ON_ERROR_STOP 0 +create trigger save_delete_transition_table_trg + after delete on readings + referencing old table as old_table + for each statement execute function save_transition_table(); +\set ON_ERROR_STOP 1 + +-- Test that we get an error when enabling compression and have a +-- delete trigger with a transition table. We allow transition tables +-- for update and insert triggers. +create table test2( + created_at timestamptz not null unique, + location_id bigint, + temp float8 +); + +select create_hypertable('test2', by_range('created_at')); + +create trigger save_test2_insert_trg + after insert on test2 + referencing new table as new_table + for each statement execute function save_transition_table(); +create trigger save_test2_update_trg + after update on test2 + referencing new table as new_table old table as old_table + for each statement execute function save_transition_table(); +create trigger save_test2_delete_trg + after delete on test2 + referencing old table as old_table + for each statement execute function save_transition_table(); + + +-- This should fail +\set ON_ERROR_STOP 0 +alter table test2 set ( + timescaledb.compress, + timescaledb.compress_orderby = 'created_at', + timescaledb.compress_segmentby = 'location_id' +); +\set ON_ERROR_STOP 1 + +-- drop the delete trigger +drop trigger save_test2_delete_trg on test2; + +-- This should now succeed. +alter table test2 set ( + timescaledb.compress, + timescaledb.compress_orderby = 'created_at', + timescaledb.compress_segmentby = 'location_id' +); diff --git a/tsl/test/sql/hypercore_trigger.sql b/tsl/test/sql/hypercore_trigger.sql index 7c590961ae8..27132835ac5 100644 --- a/tsl/test/sql/hypercore_trigger.sql +++ b/tsl/test/sql/hypercore_trigger.sql @@ -4,6 +4,10 @@ \ir include/setup_hypercore.sql +-- Set the hypercore access method on the hypertable for transition +-- tables to work properly. +alter table :hypertable set access method hypercore; + create table saved_rows (like :chunk1, new_row bool not null, kind text); create table count_stmt (inserts int, updates int, deletes int); @@ -203,41 +207,50 @@ insert into readings(created_at, location_id, device_id, owner_id, temp, humidit select created_at, location_id, device_id, owner_id, temp, humidity from sample order by created_at limit 2; -select * from saved_rows; +select * from saved_rows order by metric_id; truncate saved_rows; +-- Compress the data again to make sure that it is fully compressed. +select compress_chunk(show_chunks(:'hypertable'), hypercore_use_access_method => true); + copy readings(created_at, location_id, device_id, owner_id, temp, humidity) from stdin with (format csv); "2022-06-01 00:01:35",999,666,111,3.14,3.14 \. -select * from saved_rows; +select * from saved_rows order by metric_id; truncate saved_rows; +-- Compress the data again to make sure that it is fully compressed. +select compress_chunk(show_chunks(:'hypertable'), hypercore_use_access_method => true); + create trigger save_update_transition_table_trg after update on readings referencing new table as new_table old table as old_table for each statement execute function save_transition_table(); -select * from readings where location_id = 999; +select * from readings where location_id = 999 order by metric_id; update readings set humidity = 99.99 where location_id = 999; -select * from saved_rows; +select * from saved_rows order by metric_id; truncate saved_rows; +-- Compress the data again to make sure that it is fully compressed. +select compress_chunk(show_chunks(:'hypertable'), hypercore_use_access_method => true); + create trigger save_delete_transition_table_trg after delete on readings referencing old table as old_table for each statement execute function save_transition_table(); -select * from readings where location_id = 999; +select * from readings where location_id = 999 order by metric_id; delete from readings where location_id = 999; -select * from saved_rows; +select * from saved_rows order by metric_id; truncate saved_rows; From 3e021795b58aad6cebea69503817bc1f9c9a4476 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Wed, 15 Jan 2025 09:51:03 +0100 Subject: [PATCH 08/32] Add chunk ranges to constify_now test and remove costs --- tsl/test/shared/expected/constify_now.out | 78 ++++++++++++----------- tsl/test/shared/sql/constify_now.sql | 20 +++--- 2 files changed, 51 insertions(+), 47 deletions(-) diff --git a/tsl/test/shared/expected/constify_now.out b/tsl/test/shared/expected/constify_now.out index 233654ed6c7..fd697248e66 100644 --- a/tsl/test/shared/expected/constify_now.out +++ b/tsl/test/shared/expected/constify_now.out @@ -662,45 +662,49 @@ VALUES (2, '2023-01-15'::timestamptz, 13, 4), (5, '2023-01-20'::timestamptz, 13, 1); -- four chunks, all of them should be excluded at plantime -SELECT COUNT(*) FROM show_chunks('logged_data'); - count - 4 -(1 row) +SELECT row_number() OVER (), range_start, range_end FROM timescaledb_information.chunks WHERE hypertable_schema = 'public' AND hypertable_name = 'logged_data' ORDER BY 2,3; + row_number | range_start | range_end +------------+------------------------------+------------------------------ + 1 | Thu Dec 29 01:00:00 2022 UTC | Thu Jan 05 01:00:00 2023 UTC + 2 | Thu Jan 05 01:00:00 2023 UTC | Thu Jan 12 01:00:00 2023 UTC + 3 | Thu Jan 12 01:00:00 2023 UTC | Thu Jan 19 01:00:00 2023 UTC + 4 | Thu Jan 19 01:00:00 2023 UTC | Thu Jan 26 01:00:00 2023 UTC +(4 rows) SET timescaledb.current_timestamp_mock TO '2024-01-01 0:30:00+00'; SET timescaledb.enable_chunk_append TO true; SET timescaledb.enable_constraint_aware_append TO true; -- for all the queries below, exclusion should be happening at plantime -EXPLAIN (ANALYZE, SUMMARY OFF, TIMING OFF) SELECT * FROM logged_data WHERE +EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM logged_data WHERE timestamp BETWEEN now() - interval '1 day' AND now() AND rawtag_id = 1 ORDER BY "timestamp" ASC; QUERY PLAN - Sort (cost=0.01..0.02 rows=0 width=20) (actual rows=0 loops=1) + Sort (actual rows=0 loops=1) Sort Key: "timestamp" Sort Method: quicksort - -> Result (cost=0.00..0.00 rows=0 width=0) (actual rows=0 loops=1) + -> Result (actual rows=0 loops=1) One-Time Filter: false (5 rows) -EXPLAIN (ANALYZE, SUMMARY OFF, TIMING OFF) SELECT * FROM logged_data WHERE +EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM logged_data WHERE timestamp <= now() AND timestamp >= now() - interval '1 day' AND rawtag_id = 1 ORDER BY "timestamp" ASC; QUERY PLAN - Sort (cost=0.01..0.02 rows=0 width=20) (actual rows=0 loops=1) + Sort (actual rows=0 loops=1) Sort Key: "timestamp" Sort Method: quicksort - -> Result (cost=0.00..0.00 rows=0 width=0) (actual rows=0 loops=1) + -> Result (actual rows=0 loops=1) One-Time Filter: false (5 rows) -EXPLAIN (ANALYZE, SUMMARY OFF, TIMING OFF) SELECT * FROM logged_data WHERE +EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM logged_data WHERE timestamp <= now() AND timestamp >= now() - interval '1 day' ORDER BY "timestamp" ASC; QUERY PLAN - Sort (cost=0.01..0.02 rows=0 width=20) (actual rows=0 loops=1) + Sort (actual rows=0 loops=1) Sort Key: "timestamp" Sort Method: quicksort - -> Result (cost=0.00..0.00 rows=0 width=0) (actual rows=0 loops=1) + -> Result (actual rows=0 loops=1) One-Time Filter: false (5 rows) @@ -709,12 +713,12 @@ SET timescaledb.current_timestamp_mock TO '2023-01-11'; PREPARE pbtw AS SELECT * FROM logged_data WHERE timestamp BETWEEN now() - interval '5 day' AND now() AND rawtag_id = 1 ORDER BY "timestamp" ASC; -EXPLAIN (SUMMARY OFF, TIMING OFF) EXECUTE pbtw; +EXPLAIN (COSTS OFF, SUMMARY OFF, TIMING OFF) EXECUTE pbtw; QUERY PLAN - Custom Scan (ChunkAppend) on logged_data (cost=0.16..22.08 rows=3 width=20) + Custom Scan (ChunkAppend) on logged_data Order: logged_data."timestamp" Chunks excluded during startup: 2 - -> Index Scan Backward using _hyper_X_X_chunk_logged_data_timestamp_idx on _hyper_X_X_chunk (cost=0.16..7.36 rows=1 width=20) + -> Index Scan Backward using _hyper_X_X_chunk_logged_data_timestamp_idx on _hyper_X_X_chunk Index Cond: (("timestamp" >= (ts_now_mock() - '@ 5 days'::interval)) AND ("timestamp" <= ts_now_mock())) Filter: (rawtag_id = 1) (6 rows) @@ -728,15 +732,15 @@ EXECUTE pbtw; -- now move mock_now() to the future SET timescaledb.current_timestamp_mock TO '2023-01-21 0:30:00+00'; -EXPLAIN (SUMMARY OFF, TIMING OFF) EXECUTE pbtw; +EXPLAIN (COSTS OFF, SUMMARY OFF, TIMING OFF) EXECUTE pbtw; QUERY PLAN - Custom Scan (ChunkAppend) on logged_data (cost=0.16..22.08 rows=3 width=20) + Custom Scan (ChunkAppend) on logged_data Order: logged_data."timestamp" Chunks excluded during startup: 1 - -> Index Scan Backward using _hyper_X_X_chunk_logged_data_timestamp_idx on _hyper_X_X_chunk (cost=0.16..7.36 rows=1 width=20) + -> Index Scan Backward using _hyper_X_X_chunk_logged_data_timestamp_idx on _hyper_X_X_chunk Index Cond: (("timestamp" >= (ts_now_mock() - '@ 5 days'::interval)) AND ("timestamp" <= ts_now_mock())) Filter: (rawtag_id = 1) - -> Index Scan Backward using _hyper_X_X_chunk_logged_data_timestamp_idx on _hyper_X_X_chunk (cost=0.16..7.36 rows=1 width=20) + -> Index Scan Backward using _hyper_X_X_chunk_logged_data_timestamp_idx on _hyper_X_X_chunk Index Cond: (("timestamp" >= (ts_now_mock() - '@ 5 days'::interval)) AND ("timestamp" <= ts_now_mock())) Filter: (rawtag_id = 1) (9 rows) @@ -749,9 +753,9 @@ EXECUTE pbtw; -- much further into the future, no rows should be returned SET timescaledb.current_timestamp_mock TO '2024-01-21 0:30:00+00'; -EXPLAIN (SUMMARY OFF, TIMING OFF) EXECUTE pbtw; +EXPLAIN (COSTS OFF, SUMMARY OFF, TIMING OFF) EXECUTE pbtw; QUERY PLAN - Custom Scan (ChunkAppend) on logged_data (cost=0.16..22.08 rows=3 width=20) + Custom Scan (ChunkAppend) on logged_data Order: logged_data."timestamp" Chunks excluded during startup: 3 (3 rows) @@ -769,17 +773,17 @@ SET timescaledb.current_timestamp_mock TO '2023-01-11'; PREPARE pbtw AS SELECT * FROM logged_data WHERE timestamp BETWEEN now() - interval '5 day' AND now() AND rawtag_id = 1 ORDER BY "timestamp" ASC; -EXPLAIN (SUMMARY OFF, TIMING OFF) EXECUTE pbtw; +EXPLAIN (COSTS OFF, SUMMARY OFF, TIMING OFF) EXECUTE pbtw; QUERY PLAN - Merge Append (cost=0.50..22.14 rows=3 width=20) + Merge Append Sort Key: _hyper_X_X_chunk."timestamp" - -> Index Scan Backward using _hyper_X_X_chunk_logged_data_timestamp_idx on _hyper_X_X_chunk (cost=0.16..7.36 rows=1 width=20) + -> Index Scan Backward using _hyper_X_X_chunk_logged_data_timestamp_idx on _hyper_X_X_chunk Index Cond: (("timestamp" >= (ts_now_mock() - '@ 5 days'::interval)) AND ("timestamp" <= ts_now_mock())) Filter: (rawtag_id = 1) - -> Index Scan Backward using _hyper_X_X_chunk_logged_data_timestamp_idx on _hyper_X_X_chunk (cost=0.16..7.36 rows=1 width=20) + -> Index Scan Backward using _hyper_X_X_chunk_logged_data_timestamp_idx on _hyper_X_X_chunk Index Cond: (("timestamp" >= (ts_now_mock() - '@ 5 days'::interval)) AND ("timestamp" <= ts_now_mock())) Filter: (rawtag_id = 1) - -> Index Scan Backward using _hyper_X_X_chunk_logged_data_timestamp_idx on _hyper_X_X_chunk (cost=0.16..7.36 rows=1 width=20) + -> Index Scan Backward using _hyper_X_X_chunk_logged_data_timestamp_idx on _hyper_X_X_chunk Index Cond: (("timestamp" >= (ts_now_mock() - '@ 5 days'::interval)) AND ("timestamp" <= ts_now_mock())) Filter: (rawtag_id = 1) (11 rows) @@ -793,17 +797,17 @@ EXECUTE pbtw; -- now move mock_now() to the future SET timescaledb.current_timestamp_mock TO '2023-01-21 0:30:00+00'; -EXPLAIN (SUMMARY OFF, TIMING OFF) EXECUTE pbtw; +EXPLAIN (COSTS OFF, SUMMARY OFF, TIMING OFF) EXECUTE pbtw; QUERY PLAN - Merge Append (cost=0.50..22.14 rows=3 width=20) + Merge Append Sort Key: _hyper_X_X_chunk."timestamp" - -> Index Scan Backward using _hyper_X_X_chunk_logged_data_timestamp_idx on _hyper_X_X_chunk (cost=0.16..7.36 rows=1 width=20) + -> Index Scan Backward using _hyper_X_X_chunk_logged_data_timestamp_idx on _hyper_X_X_chunk Index Cond: (("timestamp" >= (ts_now_mock() - '@ 5 days'::interval)) AND ("timestamp" <= ts_now_mock())) Filter: (rawtag_id = 1) - -> Index Scan Backward using _hyper_X_X_chunk_logged_data_timestamp_idx on _hyper_X_X_chunk (cost=0.16..7.36 rows=1 width=20) + -> Index Scan Backward using _hyper_X_X_chunk_logged_data_timestamp_idx on _hyper_X_X_chunk Index Cond: (("timestamp" >= (ts_now_mock() - '@ 5 days'::interval)) AND ("timestamp" <= ts_now_mock())) Filter: (rawtag_id = 1) - -> Index Scan Backward using _hyper_X_X_chunk_logged_data_timestamp_idx on _hyper_X_X_chunk (cost=0.16..7.36 rows=1 width=20) + -> Index Scan Backward using _hyper_X_X_chunk_logged_data_timestamp_idx on _hyper_X_X_chunk Index Cond: (("timestamp" >= (ts_now_mock() - '@ 5 days'::interval)) AND ("timestamp" <= ts_now_mock())) Filter: (rawtag_id = 1) (11 rows) @@ -816,17 +820,17 @@ EXECUTE pbtw; -- much further into the future, no rows should be returned SET timescaledb.current_timestamp_mock TO '2024-01-21 0:30:00+00'; -EXPLAIN (SUMMARY OFF, TIMING OFF) EXECUTE pbtw; +EXPLAIN (COSTS OFF, SUMMARY OFF, TIMING OFF) EXECUTE pbtw; QUERY PLAN - Merge Append (cost=0.50..22.14 rows=3 width=20) + Merge Append Sort Key: _hyper_X_X_chunk."timestamp" - -> Index Scan Backward using _hyper_X_X_chunk_logged_data_timestamp_idx on _hyper_X_X_chunk (cost=0.16..7.36 rows=1 width=20) + -> Index Scan Backward using _hyper_X_X_chunk_logged_data_timestamp_idx on _hyper_X_X_chunk Index Cond: (("timestamp" >= (ts_now_mock() - '@ 5 days'::interval)) AND ("timestamp" <= ts_now_mock())) Filter: (rawtag_id = 1) - -> Index Scan Backward using _hyper_X_X_chunk_logged_data_timestamp_idx on _hyper_X_X_chunk (cost=0.16..7.36 rows=1 width=20) + -> Index Scan Backward using _hyper_X_X_chunk_logged_data_timestamp_idx on _hyper_X_X_chunk Index Cond: (("timestamp" >= (ts_now_mock() - '@ 5 days'::interval)) AND ("timestamp" <= ts_now_mock())) Filter: (rawtag_id = 1) - -> Index Scan Backward using _hyper_X_X_chunk_logged_data_timestamp_idx on _hyper_X_X_chunk (cost=0.16..7.36 rows=1 width=20) + -> Index Scan Backward using _hyper_X_X_chunk_logged_data_timestamp_idx on _hyper_X_X_chunk Index Cond: (("timestamp" >= (ts_now_mock() - '@ 5 days'::interval)) AND ("timestamp" <= ts_now_mock())) Filter: (rawtag_id = 1) (11 rows) diff --git a/tsl/test/shared/sql/constify_now.sql b/tsl/test/shared/sql/constify_now.sql index 4b15e011cb6..91669868618 100644 --- a/tsl/test/shared/sql/constify_now.sql +++ b/tsl/test/shared/sql/constify_now.sql @@ -228,7 +228,7 @@ VALUES (5, '2023-01-20'::timestamptz, 13, 1); -- four chunks, all of them should be excluded at plantime -SELECT COUNT(*) FROM show_chunks('logged_data'); +SELECT row_number() OVER (), range_start, range_end FROM timescaledb_information.chunks WHERE hypertable_schema = 'public' AND hypertable_name = 'logged_data' ORDER BY 2,3; SET timescaledb.current_timestamp_mock TO '2024-01-01 0:30:00+00'; @@ -236,15 +236,15 @@ SET timescaledb.enable_chunk_append TO true; SET timescaledb.enable_constraint_aware_append TO true; -- for all the queries below, exclusion should be happening at plantime -EXPLAIN (ANALYZE, SUMMARY OFF, TIMING OFF) SELECT * FROM logged_data WHERE +EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM logged_data WHERE timestamp BETWEEN now() - interval '1 day' AND now() AND rawtag_id = 1 ORDER BY "timestamp" ASC; -EXPLAIN (ANALYZE, SUMMARY OFF, TIMING OFF) SELECT * FROM logged_data WHERE +EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM logged_data WHERE timestamp <= now() AND timestamp >= now() - interval '1 day' AND rawtag_id = 1 ORDER BY "timestamp" ASC; -EXPLAIN (ANALYZE, SUMMARY OFF, TIMING OFF) SELECT * FROM logged_data WHERE +EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM logged_data WHERE timestamp <= now() AND timestamp >= now() - interval '1 day' ORDER BY "timestamp" ASC; @@ -254,15 +254,15 @@ PREPARE pbtw AS SELECT * FROM logged_data WHERE timestamp BETWEEN now() - interval '5 day' AND now() AND rawtag_id = 1 ORDER BY "timestamp" ASC; -EXPLAIN (SUMMARY OFF, TIMING OFF) EXECUTE pbtw; +EXPLAIN (COSTS OFF, SUMMARY OFF, TIMING OFF) EXECUTE pbtw; EXECUTE pbtw; -- now move mock_now() to the future SET timescaledb.current_timestamp_mock TO '2023-01-21 0:30:00+00'; -EXPLAIN (SUMMARY OFF, TIMING OFF) EXECUTE pbtw; +EXPLAIN (COSTS OFF, SUMMARY OFF, TIMING OFF) EXECUTE pbtw; EXECUTE pbtw; -- much further into the future, no rows should be returned SET timescaledb.current_timestamp_mock TO '2024-01-21 0:30:00+00'; -EXPLAIN (SUMMARY OFF, TIMING OFF) EXECUTE pbtw; +EXPLAIN (COSTS OFF, SUMMARY OFF, TIMING OFF) EXECUTE pbtw; EXECUTE pbtw; DEALLOCATE pbtw; @@ -274,15 +274,15 @@ PREPARE pbtw AS SELECT * FROM logged_data WHERE timestamp BETWEEN now() - interval '5 day' AND now() AND rawtag_id = 1 ORDER BY "timestamp" ASC; -EXPLAIN (SUMMARY OFF, TIMING OFF) EXECUTE pbtw; +EXPLAIN (COSTS OFF, SUMMARY OFF, TIMING OFF) EXECUTE pbtw; EXECUTE pbtw; -- now move mock_now() to the future SET timescaledb.current_timestamp_mock TO '2023-01-21 0:30:00+00'; -EXPLAIN (SUMMARY OFF, TIMING OFF) EXECUTE pbtw; +EXPLAIN (COSTS OFF, SUMMARY OFF, TIMING OFF) EXECUTE pbtw; EXECUTE pbtw; -- much further into the future, no rows should be returned SET timescaledb.current_timestamp_mock TO '2024-01-21 0:30:00+00'; -EXPLAIN (SUMMARY OFF, TIMING OFF) EXECUTE pbtw; +EXPLAIN (COSTS OFF, SUMMARY OFF, TIMING OFF) EXECUTE pbtw; EXECUTE pbtw; DEALLOCATE pbtw; From 60ecd4605212acdf4c2b643a415a8f1d0b3b5ea7 Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Tue, 14 Jan 2025 17:41:46 +0100 Subject: [PATCH 09/32] Add `include_tiered_data` option to cagg policy The `include_tiered_data` option allows user to override the value of `timescaledb.enable_tiered_reads` defined on instance level for a particular continuous aggregate policy. --- .unreleased/pr_7587 | 1 + sql/policy_api.sql | 3 +- sql/updates/latest-dev.sql | 19 ++ sql/updates/reverse-dev.sql | 20 ++ src/guc.h | 2 +- tsl/src/bgw_policy/continuous_aggregate_api.c | 23 +- tsl/src/bgw_policy/continuous_aggregate_api.h | 4 +- tsl/src/bgw_policy/job.c | 30 +++ tsl/src/bgw_policy/job.h | 3 +- tsl/src/bgw_policy/policies_v2.c | 5 +- tsl/src/bgw_policy/policies_v2.h | 1 + tsl/test/expected/chunk_utils_internal.out | 207 ++++++++++++------ tsl/test/shared/expected/extension.out | 2 +- tsl/test/sql/chunk_utils_internal.sql | 48 ++++ 14 files changed, 297 insertions(+), 71 deletions(-) create mode 100644 .unreleased/pr_7587 diff --git a/.unreleased/pr_7587 b/.unreleased/pr_7587 new file mode 100644 index 00000000000..bd14aaf1c90 --- /dev/null +++ b/.unreleased/pr_7587 @@ -0,0 +1 @@ +Implements: #7587 Add `include_tiered_data` parameter to `add_continuous_aggregate_policy` API diff --git a/sql/policy_api.sql b/sql/policy_api.sql index 83a8a39049f..be8cc33cce1 100644 --- a/sql/policy_api.sql +++ b/sql/policy_api.sql @@ -85,7 +85,8 @@ CREATE OR REPLACE FUNCTION @extschema@.add_continuous_aggregate_policy( end_offset "any", schedule_interval INTERVAL, if_not_exists BOOL = false, initial_start TIMESTAMPTZ = NULL, - timezone TEXT = NULL + timezone TEXT = NULL, + include_tiered_data BOOL = NULL ) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_policy_refresh_cagg_add' diff --git a/sql/updates/latest-dev.sql b/sql/updates/latest-dev.sql index 5255e9e3903..33d290e374a 100644 --- a/sql/updates/latest-dev.sql +++ b/sql/updates/latest-dev.sql @@ -125,3 +125,22 @@ CREATE PROCEDURE @extschema@.refresh_continuous_aggregate( force BOOLEAN = FALSE ) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_update_placeholder'; +-- Add `include_tiered_data` argument to `add_continuous_aggregate_policy` +DROP FUNCTION @extschema@.add_continuous_aggregate_policy( + continuous_aggregate REGCLASS, start_offset "any", + end_offset "any", schedule_interval INTERVAL, + if_not_exists BOOL, + initial_start TIMESTAMPTZ, + timezone TEXT +); +CREATE FUNCTION @extschema@.add_continuous_aggregate_policy( + continuous_aggregate REGCLASS, start_offset "any", + end_offset "any", schedule_interval INTERVAL, + if_not_exists BOOL = false, + initial_start TIMESTAMPTZ = NULL, + timezone TEXT = NULL, + include_tiered_data BOOL = NULL +) +RETURNS INTEGER +AS '@MODULE_PATHNAME@', 'ts_update_placeholder' +LANGUAGE C VOLATILE; diff --git a/sql/updates/reverse-dev.sql b/sql/updates/reverse-dev.sql index bff21e0a960..37f3659dd6f 100644 --- a/sql/updates/reverse-dev.sql +++ b/sql/updates/reverse-dev.sql @@ -67,3 +67,23 @@ CREATE PROCEDURE @extschema@.refresh_continuous_aggregate( window_start "any", window_end "any" ) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_continuous_agg_refresh'; + +-- Remove `include_tiered_data` argument from `add_continuous_aggregate_policy` +DROP FUNCTION @extschema@.add_continuous_aggregate_policy( + continuous_aggregate REGCLASS, start_offset "any", + end_offset "any", schedule_interval INTERVAL, + if_not_exists BOOL, + initial_start TIMESTAMPTZ, + timezone TEXT, + include_tiered_data BOOL +); +CREATE FUNCTION @extschema@.add_continuous_aggregate_policy( + continuous_aggregate REGCLASS, start_offset "any", + end_offset "any", schedule_interval INTERVAL, + if_not_exists BOOL = false, + initial_start TIMESTAMPTZ = NULL, + timezone TEXT = NULL +) +RETURNS INTEGER +AS '@MODULE_PATHNAME@', 'ts_policy_refresh_cagg_add' +LANGUAGE C VOLATILE; diff --git a/src/guc.h b/src/guc.h index 54a341b7a3b..7691be19850 100644 --- a/src/guc.h +++ b/src/guc.h @@ -44,7 +44,7 @@ extern bool ts_guc_enable_cagg_reorder_groupby; extern TSDLLEXPORT int ts_guc_cagg_max_individual_materializations; extern bool ts_guc_enable_now_constify; extern bool ts_guc_enable_foreign_key_propagation; -extern bool ts_guc_enable_osm_reads; +extern TSDLLEXPORT bool ts_guc_enable_osm_reads; #if PG16_GE extern TSDLLEXPORT bool ts_guc_enable_cagg_sort_pushdown; #endif diff --git a/tsl/src/bgw_policy/continuous_aggregate_api.c b/tsl/src/bgw_policy/continuous_aggregate_api.c index 245c120060c..e400e4ccf13 100644 --- a/tsl/src/bgw_policy/continuous_aggregate_api.c +++ b/tsl/src/bgw_policy/continuous_aggregate_api.c @@ -136,6 +136,16 @@ policy_refresh_cagg_get_refresh_end(const Dimension *dim, const Jsonb *config, b return res; } +bool +policy_refresh_cagg_get_include_tiered_data(const Jsonb *config, bool *isnull) +{ + bool found; + bool res = ts_jsonb_get_bool_field(config, POL_REFRESH_CONF_KEY_INCLUDE_TIERED_DATA, &found); + + *isnull = !found; + return res; +} + /* returns false if a policy could not be found */ bool policy_refresh_cagg_exists(int32 materialization_id) @@ -519,7 +529,8 @@ Datum policy_refresh_cagg_add_internal(Oid cagg_oid, Oid start_offset_type, NullableDatum start_offset, Oid end_offset_type, NullableDatum end_offset, Interval refresh_interval, bool if_not_exists, bool fixed_schedule, - TimestampTz initial_start, const char *timezone) + TimestampTz initial_start, const char *timezone, + NullableDatum include_tiered_data) { NameData application_name; NameData proc_name, proc_schema, check_name, check_schema, owner; @@ -630,6 +641,10 @@ policy_refresh_cagg_add_internal(Oid cagg_oid, Oid start_offset_type, NullableDa policyconf.offset_end.value); else ts_jsonb_add_null(parse_state, POL_REFRESH_CONF_KEY_END_OFFSET); + if (!include_tiered_data.isnull) + ts_jsonb_add_bool(parse_state, + POL_REFRESH_CONF_KEY_INCLUDE_TIERED_DATA, + include_tiered_data.value); JsonbValue *result = pushJsonbValue(&parse_state, WJB_END_OBJECT, NULL); Jsonb *config = JsonbValueToJsonb(result); @@ -660,6 +675,7 @@ policy_refresh_cagg_add(PG_FUNCTION_ARGS) Interval refresh_interval; bool if_not_exists; NullableDatum start_offset, end_offset; + NullableDatum include_tiered_data; ts_feature_flag_check(FEATURE_POLICY); @@ -682,6 +698,8 @@ policy_refresh_cagg_add(PG_FUNCTION_ARGS) bool fixed_schedule = !PG_ARGISNULL(5); text *timezone = PG_ARGISNULL(6) ? NULL : PG_GETARG_TEXT_PP(6); char *valid_timezone = NULL; + include_tiered_data.value = PG_GETARG_DATUM(7); + include_tiered_data.isnull = PG_ARGISNULL(7); Datum retval; /* if users pass in -infinity for initial_start, then use the current_timestamp instead */ @@ -704,7 +722,8 @@ policy_refresh_cagg_add(PG_FUNCTION_ARGS) if_not_exists, fixed_schedule, initial_start, - valid_timezone); + valid_timezone, + include_tiered_data); if (!TIMESTAMP_NOT_FINITE(initial_start)) { int32 job_id = DatumGetInt32(retval); diff --git a/tsl/src/bgw_policy/continuous_aggregate_api.h b/tsl/src/bgw_policy/continuous_aggregate_api.h index 160d7bd67d4..50588b424d8 100644 --- a/tsl/src/bgw_policy/continuous_aggregate_api.h +++ b/tsl/src/bgw_policy/continuous_aggregate_api.h @@ -20,6 +20,7 @@ int64 policy_refresh_cagg_get_refresh_start(const ContinuousAgg *cagg, const Dim const Jsonb *config, bool *start_isnull); int64 policy_refresh_cagg_get_refresh_end(const Dimension *dim, const Jsonb *config, bool *end_isnull); +bool policy_refresh_cagg_get_include_tiered_data(const Jsonb *config, bool *isnull); bool policy_refresh_cagg_refresh_start_lt(int32 materialization_id, Oid cmp_type, Datum cmp_interval); bool policy_refresh_cagg_exists(int32 materialization_id); @@ -28,5 +29,6 @@ Datum policy_refresh_cagg_add_internal(Oid cagg_oid, Oid start_offset_type, NullableDatum start_offset, Oid end_offset_type, NullableDatum end_offset, Interval refresh_interval, bool if_not_exists, bool fixed_schedule, - TimestampTz initial_start, const char *timezone); + TimestampTz initial_start, const char *timezone, + NullableDatum include_tiered_data); Datum policy_refresh_cagg_remove_internal(Oid cagg_oid, bool if_exists); diff --git a/tsl/src/bgw_policy/job.c b/tsl/src/bgw_policy/job.c index f4a9dfdd966..2e2dc547336 100644 --- a/tsl/src/bgw_policy/job.c +++ b/tsl/src/bgw_policy/job.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -51,6 +52,7 @@ #include "dimension_slice.h" #include "dimension_vector.h" #include "errors.h" +#include "guc.h" #include "job.h" #include "reorder.h" #include "utils.h" @@ -372,7 +374,21 @@ policy_refresh_cagg_execute(int32 job_id, Jsonb *config) { PolicyContinuousAggData policy_data; + StringInfo str = makeStringInfo(); + JsonbToCStringIndent(str, &config->root, VARSIZE(config)); + policy_refresh_cagg_read_and_validate_config(config, &policy_data); + + bool enable_osm_reads_old = ts_guc_enable_osm_reads; + + if (!policy_data.include_tiered_data_isnull) + { + SetConfigOption("timescaledb.enable_tiered_reads", + policy_data.include_tiered_data ? "on" : "off", + PGC_USERSET, + PGC_S_SESSION); + } + continuous_agg_refresh_internal(policy_data.cagg, &policy_data.refresh_window, CAGG_REFRESH_POLICY, @@ -380,6 +396,14 @@ policy_refresh_cagg_execute(int32 job_id, Jsonb *config) policy_data.end_is_null, false); + if (!policy_data.include_tiered_data_isnull) + { + SetConfigOption("timescaledb.enable_tiered_reads", + enable_osm_reads_old ? "on" : "off", + PGC_USERSET, + PGC_S_SESSION); + } + return true; } @@ -392,6 +416,7 @@ policy_refresh_cagg_read_and_validate_config(Jsonb *config, PolicyContinuousAggD Oid dim_type; int64 refresh_start, refresh_end; bool start_isnull, end_isnull; + bool include_tiered_data, include_tiered_data_isnull; materialization_id = policy_continuous_aggregate_get_mat_hypertable_id(config); mat_ht = ts_hypertable_get_by_id(materialization_id); @@ -418,6 +443,9 @@ policy_refresh_cagg_read_and_validate_config(Jsonb *config, PolicyContinuousAggD ts_internal_to_time_string(refresh_end, dim_type)), errhint("The start of the window must be before the end."))); + include_tiered_data = + policy_refresh_cagg_get_include_tiered_data(config, &include_tiered_data_isnull); + if (policy_data) { policy_data->refresh_window.type = dim_type; @@ -426,6 +454,8 @@ policy_refresh_cagg_read_and_validate_config(Jsonb *config, PolicyContinuousAggD policy_data->cagg = cagg; policy_data->start_is_null = start_isnull; policy_data->end_is_null = end_isnull; + policy_data->include_tiered_data = include_tiered_data; + policy_data->include_tiered_data_isnull = include_tiered_data_isnull; } } diff --git a/tsl/src/bgw_policy/job.h b/tsl/src/bgw_policy/job.h index 70e488cd475..404d9764046 100644 --- a/tsl/src/bgw_policy/job.h +++ b/tsl/src/bgw_policy/job.h @@ -36,7 +36,8 @@ typedef struct PolicyContinuousAggData { InternalTimeRange refresh_window; ContinuousAgg *cagg; - bool start_is_null, end_is_null; + bool include_tiered_data; + bool start_is_null, end_is_null, include_tiered_data_isnull; } PolicyContinuousAggData; typedef struct PolicyCompressionData diff --git a/tsl/src/bgw_policy/policies_v2.c b/tsl/src/bgw_policy/policies_v2.c index 9bfbe9e3a55..eeb1643abcf 100644 --- a/tsl/src/bgw_policy/policies_v2.c +++ b/tsl/src/bgw_policy/policies_v2.c @@ -206,6 +206,8 @@ validate_and_create_policies(policies_info all_policies, bool if_exists) /* Create policies as required, delete the old ones if coming from alter */ if (all_policies.refresh && all_policies.refresh->create_policy) { + NullableDatum include_tiered_data = { .isnull = true }; + if (all_policies.is_alter_policy) policy_refresh_cagg_remove_internal(all_policies.rel_oid, if_exists); refresh_job_id = policy_refresh_cagg_add_internal(all_policies.rel_oid, @@ -217,7 +219,8 @@ validate_and_create_policies(policies_info all_policies, bool if_exists) false, false, DT_NOBEGIN, - NULL); + NULL, + include_tiered_data); } if (all_policies.compress && all_policies.compress->create_policy) { diff --git a/tsl/src/bgw_policy/policies_v2.h b/tsl/src/bgw_policy/policies_v2.h index 4a5eeb852d8..03535adb4e4 100644 --- a/tsl/src/bgw_policy/policies_v2.h +++ b/tsl/src/bgw_policy/policies_v2.h @@ -19,6 +19,7 @@ #define POL_REFRESH_CONF_KEY_MAT_HYPERTABLE_ID "mat_hypertable_id" #define POL_REFRESH_CONF_KEY_START_OFFSET "start_offset" #define POL_REFRESH_CONF_KEY_END_OFFSET "end_offset" +#define POL_REFRESH_CONF_KEY_INCLUDE_TIERED_DATA "include_tiered_data" #define POLICY_COMPRESSION_PROC_NAME "policy_compression" #define POLICY_COMPRESSION_CHECK_NAME "policy_compression_check" diff --git a/tsl/test/expected/chunk_utils_internal.out b/tsl/test/expected/chunk_utils_internal.out index 520484a8651..d03c131a74d 100644 --- a/tsl/test/expected/chunk_utils_internal.out +++ b/tsl/test/expected/chunk_utils_internal.out @@ -819,6 +819,87 @@ SELECT * FROM ht_try_weekly; DROP MATERIALIZED VIEW ht_try_weekly; NOTICE: drop cascades to table _timescaledb_internal._hyper_6_12_chunk +-- Test refresh policy with different settings of `include_tiered_data` parameter +CREATE FUNCTION create_test_cagg(include_tiered_data BOOL) +RETURNS INTEGER AS +$$ +DECLARE + cfg jsonb; + job_id INTEGER; +BEGIN + CREATE MATERIALIZED VIEW ht_try_weekly + WITH (timescaledb.continuous) AS + SELECT time_bucket(interval '1 week', timec) AS ts_bucket, avg(value) + FROM ht_try + GROUP BY 1 + WITH NO DATA; + + job_id := add_continuous_aggregate_policy( + 'ht_try_weekly', + start_offset => NULL, + end_offset => INTERVAL '1 hour', + schedule_interval => INTERVAL '1 hour', + include_tiered_data => include_tiered_data + ); + + cfg := config FROM _timescaledb_config.bgw_job WHERE id = job_id; + RAISE NOTICE 'config: %', jsonb_pretty(cfg); + + RETURN job_id; +END +$$ LANGUAGE plpgsql; +-- include tiered data +SELECT create_test_cagg(true) AS job_id \gset +NOTICE: config: { + "end_offset": "@ 1 hour", + "start_offset": null, + "mat_hypertable_id": 7, + "include_tiered_data": true +} +CALL run_job(:job_id); +SELECT * FROM ht_try_weekly ORDER BY 1; + ts_bucket | avg +------------------------------+----------------------- + Sun Dec 29 16:00:00 2019 PST | 1000.0000000000000000 + Sun May 01 17:00:00 2022 PDT | 222.0000000000000000 +(2 rows) + +DROP MATERIALIZED VIEW ht_try_weekly; +NOTICE: drop cascades to 2 other objects +-- exclude tiered data +SELECT create_test_cagg(false) AS job_id \gset +NOTICE: config: { + "end_offset": "@ 1 hour", + "start_offset": null, + "mat_hypertable_id": 8, + "include_tiered_data": false +} +CALL run_job(:job_id); +SELECT * FROM ht_try_weekly ORDER BY 1; + ts_bucket | avg +------------------------------+---------------------- + Sun May 01 17:00:00 2022 PDT | 222.0000000000000000 +(1 row) + +DROP MATERIALIZED VIEW ht_try_weekly; +NOTICE: drop cascades to table _timescaledb_internal._hyper_8_15_chunk +-- default behavior: use instance-wide GUC value +SELECT create_test_cagg(null) AS job_id \gset +NOTICE: config: { + "end_offset": "@ 1 hour", + "start_offset": null, + "mat_hypertable_id": 9 +} +CALL run_job(:job_id); +SELECT * FROM ht_try_weekly ORDER BY 1; + ts_bucket | avg +------------------------------+----------------------- + Sun Dec 29 16:00:00 2019 PST | 1000.0000000000000000 + Sun May 01 17:00:00 2022 PDT | 222.0000000000000000 +(2 rows) + +DROP MATERIALIZED VIEW ht_try_weekly; +NOTICE: drop cascades to 2 other objects -- This test verifies that a bugfix regarding the way `ROWID_VAR`s are adjusted -- in the chunks' targetlists on DELETE/UPDATE works (including partially -- compressed chunks) @@ -831,7 +912,7 @@ SELECT compress_chunk(show_chunks('ht_try', newer_than => '2021-01-01'::timestam compress_chunk ----------------------------------------- _timescaledb_internal._hyper_5_10_chunk - _timescaledb_internal._hyper_5_13_chunk + _timescaledb_internal._hyper_5_18_chunk (2 rows) INSERT INTO ht_try VALUES ('2021-06-05 01:00', 10, 222); @@ -934,7 +1015,7 @@ Indexes: Triggers: ts_insert_blocker BEFORE INSERT ON ht_try FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker() Child tables: _timescaledb_internal._hyper_5_10_chunk, - _timescaledb_internal._hyper_5_13_chunk + _timescaledb_internal._hyper_5_18_chunk -- verify that still can read from the table after catalog manipulations EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) SELECT * FROM ht_try; @@ -942,10 +1023,10 @@ EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) SELECT * FROM ht_try; ---------------------------------------------------------------------------------- Append (actual rows=3 loops=1) -> Custom Scan (DecompressChunk) on _hyper_5_10_chunk (actual rows=1 loops=1) - -> Seq Scan on compress_hyper_7_14_chunk (actual rows=1 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_5_13_chunk (actual rows=1 loops=1) - -> Seq Scan on compress_hyper_7_15_chunk (actual rows=1 loops=1) - -> Seq Scan on _hyper_5_13_chunk (actual rows=1 loops=1) + -> Seq Scan on compress_hyper_10_19_chunk (actual rows=1 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_5_18_chunk (actual rows=1 loops=1) + -> Seq Scan on compress_hyper_10_20_chunk (actual rows=1 loops=1) + -> Seq Scan on _hyper_5_18_chunk (actual rows=1 loops=1) (6 rows) ROLLBACK; @@ -996,9 +1077,9 @@ RESTRICT ,CHECK ( temp > 10) ); SELECT create_hypertable('hyper_constr', 'time', chunk_time_interval => 10); - create_hypertable ---------------------------- - (8,public,hyper_constr,t) + create_hypertable +---------------------------- + (11,public,hyper_constr,t) (1 row) INSERT INTO hyper_constr VALUES( 10, 200, 22, 1, 111, 44); @@ -1031,7 +1112,7 @@ WHERE hypertable_id IN (SELECT id from _timescaledb_catalog.hypertable ORDER BY table_name; table_name | status | osm_chunk --------------------+--------+----------- - _hyper_8_16_chunk | 0 | f + _hyper_11_21_chunk | 0 | f child_hyper_constr | 0 | t (2 rows) @@ -1119,15 +1200,15 @@ where hypertable_id = (Select id from _timescaledb_catalog.hypertable where tabl ORDER BY id; id | table_name ----+-------------------- - 16 | _hyper_8_16_chunk - 17 | child_hyper_constr + 21 | _hyper_11_21_chunk + 22 | child_hyper_constr (2 rows) -- show_chunks will not show the OSM chunk which is visible via the above query SELECT show_chunks('hyper_constr'); - show_chunks ------------------------------------------ - _timescaledb_internal._hyper_8_16_chunk + show_chunks +------------------------------------------ + _timescaledb_internal._hyper_11_21_chunk (1 row) ROLLBACK; @@ -1157,9 +1238,9 @@ CREATE TABLE test1.copy_test ( "value" double precision NOT NULL ); SELECT create_hypertable('test1.copy_test', 'time', chunk_time_interval => interval '1 day'); - create_hypertable ------------------------ - (9,test1,copy_test,t) + create_hypertable +------------------------ + (12,test1,copy_test,t) (1 row) COPY test1.copy_test FROM STDIN DELIMITER ','; @@ -1178,15 +1259,15 @@ SELECT _timescaledb_functions.freeze_chunk( :'COPY_CHNAME'); -- Check state SELECT table_name, status FROM _timescaledb_catalog.chunk WHERE table_name = :'COPY_CHUNK_NAME'; - table_name | status --------------------+-------- - _hyper_9_18_chunk | 4 + table_name | status +--------------------+-------- + _hyper_12_23_chunk | 4 (1 row) \set ON_ERROR_STOP 0 -- Copy should fail because one of che chunks is frozen COPY test1.copy_test FROM STDIN DELIMITER ','; -ERROR: cannot INSERT into frozen chunk "_hyper_9_18_chunk" +ERROR: cannot INSERT into frozen chunk "_hyper_12_23_chunk" \set ON_ERROR_STOP 1 -- Count existing rows SELECT COUNT(*) FROM test1.copy_test; @@ -1198,15 +1279,15 @@ SELECT COUNT(*) FROM test1.copy_test; -- Check state SELECT table_name, status FROM _timescaledb_catalog.chunk WHERE table_name = :'COPY_CHUNK_NAME'; - table_name | status --------------------+-------- - _hyper_9_18_chunk | 4 + table_name | status +--------------------+-------- + _hyper_12_23_chunk | 4 (1 row) \set ON_ERROR_STOP 0 -- Copy should fail because one of che chunks is frozen COPY test1.copy_test FROM STDIN DELIMITER ','; -ERROR: cannot INSERT into frozen chunk "_hyper_9_18_chunk" +ERROR: cannot INSERT into frozen chunk "_hyper_12_23_chunk" \set ON_ERROR_STOP 1 -- Count existing rows SELECT COUNT(*) FROM test1.copy_test; @@ -1225,9 +1306,9 @@ SELECT _timescaledb_functions.unfreeze_chunk( :'COPY_CHNAME'); -- Check state SELECT table_name, status FROM _timescaledb_catalog.chunk WHERE table_name = :'COPY_CHUNK_NAME'; - table_name | status --------------------+-------- - _hyper_9_18_chunk | 0 + table_name | status +--------------------+-------- + _hyper_12_23_chunk | 0 (1 row) -- Copy should work now @@ -1330,12 +1411,12 @@ WHERE ht.table_name LIKE 'osm%' ORDER BY 2,3; table_name | id | dimension_id | range_start | range_end ------------+----+--------------+---------------------+--------------------- - osm_int2 | 17 | 8 | 9223372036854775806 | 9223372036854775807 - osm_int4 | 18 | 9 | 9223372036854775806 | 9223372036854775807 - osm_int8 | 19 | 10 | 9223372036854775806 | 9223372036854775807 - osm_date | 20 | 11 | 9223372036854775806 | 9223372036854775807 - osm_ts | 21 | 12 | 9223372036854775806 | 9223372036854775807 - osm_tstz | 22 | 13 | 9223372036854775806 | 9223372036854775807 + osm_int2 | 22 | 11 | 9223372036854775806 | 9223372036854775807 + osm_int4 | 23 | 12 | 9223372036854775806 | 9223372036854775807 + osm_int8 | 24 | 13 | 9223372036854775806 | 9223372036854775807 + osm_date | 25 | 14 | 9223372036854775806 | 9223372036854775807 + osm_ts | 26 | 15 | 9223372036854775806 | 9223372036854775807 + osm_tstz | 27 | 16 | 9223372036854775806 | 9223372036854775807 (6 rows) -- test that correct slice is found and updated for table with multiple chunk constraints @@ -1348,8 +1429,8 @@ _timescaledb_catalog.chunk c, _timescaledb_catalog.chunk_constraint cc WHERE c.h AND c.id = cc.chunk_id; id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk | chunk_id | dimension_slice_id | constraint_name | hypertable_constraint_name ----+---------------+-----------------------+--------------------+---------------------+---------+--------+-----------+----------+--------------------+-----------------------------+---------------------------- - 26 | 16 | _timescaledb_internal | _hyper_16_26_chunk | | f | 0 | f | 26 | | 26_5_test_multicon_time_key | test_multicon_time_key - 26 | 16 | _timescaledb_internal | _hyper_16_26_chunk | | f | 0 | f | 26 | 23 | constraint_23 | + 31 | 19 | _timescaledb_internal | _hyper_19_31_chunk | | f | 0 | f | 31 | | 31_5_test_multicon_time_key | test_multicon_time_key + 31 | 19 | _timescaledb_internal | _hyper_19_31_chunk | | f | 0 | f | 31 | 28 | constraint_28 | (2 rows) \c :TEST_DBNAME :ROLE_SUPERUSER ; @@ -1367,7 +1448,7 @@ FROM _timescaledb_catalog.chunk c, _timescaledb_catalog.chunk_constraint cc, _ti WHERE c.hypertable_id = :htid AND cc.chunk_id = c.id AND ds.id = cc.dimension_slice_id; chunk_id | table_name | status | osm_chunk | dimension_slice_id | range_start | range_end ----------+--------------------+--------+-----------+--------------------+------------------+------------------ - 26 | _hyper_16_26_chunk | 0 | t | 23 | 1577955600000000 | 1578128400000000 + 31 | _hyper_19_31_chunk | 0 | t | 28 | 1577955600000000 | 1578128400000000 (1 row) -- check that range was reset to default - infinity @@ -1395,7 +1476,7 @@ FROM _timescaledb_catalog.chunk c, _timescaledb_catalog.chunk_constraint cc, _ti WHERE c.hypertable_id = :htid AND cc.chunk_id = c.id AND ds.id = cc.dimension_slice_id ORDER BY cc.chunk_id; chunk_id | table_name | status | osm_chunk | dimension_slice_id | range_start | range_end ----------+--------------------+--------+-----------+--------------------+---------------------+--------------------- - 26 | _hyper_16_26_chunk | 0 | t | 23 | 9223372036854775806 | 9223372036854775807 + 31 | _hyper_19_31_chunk | 0 | t | 28 | 9223372036854775806 | 9223372036854775807 (1 row) -- TEST for orderedappend that depends on hypertable_osm_range_update functionality @@ -1420,9 +1501,9 @@ FROM _timescaledb_catalog.chunk c, _timescaledb_catalog.chunk_constraint cc, _ti WHERE c.hypertable_id = :htid AND cc.chunk_id = c.id AND ds.id = cc.dimension_slice_id ORDER BY cc.chunk_id; chunk_id | table_name | status | osm_chunk | dimension_slice_id | range_start | range_end ----------+-------------------------+--------+-----------+--------------------+---------------------+--------------------- - 27 | _hyper_17_27_chunk | 0 | f | 24 | 1577836800000000 | 1577923200000000 - 28 | _hyper_17_28_chunk | 0 | f | 25 | 1577923200000000 | 1578009600000000 - 29 | test_chunkapp_fdw_child | 0 | t | 26 | 9223372036854775806 | 9223372036854775807 + 32 | _hyper_20_32_chunk | 0 | f | 29 | 1577836800000000 | 1577923200000000 + 33 | _hyper_20_33_chunk | 0 | f | 30 | 1577923200000000 | 1578009600000000 + 34 | test_chunkapp_fdw_child | 0 | t | 31 | 9223372036854775806 | 9223372036854775807 (3 rows) -- attempt to update overlapping range, should fail @@ -1443,9 +1524,9 @@ FROM _timescaledb_catalog.chunk c, _timescaledb_catalog.chunk_constraint cc, _ti WHERE c.hypertable_id = :htid AND cc.chunk_id = c.id AND ds.id = cc.dimension_slice_id ORDER BY cc.chunk_id; chunk_id | table_name | status | osm_chunk | dimension_slice_id | range_start | range_end ----------+-------------------------+--------+-----------+--------------------+------------------+------------------ - 27 | _hyper_17_27_chunk | 0 | f | 24 | 1577836800000000 | 1577923200000000 - 28 | _hyper_17_28_chunk | 0 | f | 25 | 1577923200000000 | 1578009600000000 - 29 | test_chunkapp_fdw_child | 0 | t | 26 | 1578038400000000 | 1578124800000000 + 32 | _hyper_20_32_chunk | 0 | f | 29 | 1577836800000000 | 1577923200000000 + 33 | _hyper_20_33_chunk | 0 | f | 30 | 1577923200000000 | 1578009600000000 + 34 | test_chunkapp_fdw_child | 0 | t | 31 | 1578038400000000 | 1578124800000000 (3 rows) -- ordered append should be possible as ranges do not overlap @@ -1454,8 +1535,8 @@ WHERE c.hypertable_id = :htid AND cc.chunk_id = c.id AND ds.id = cc.dimension_sl ------------------------------------------------------------------------------------------------- Custom Scan (ChunkAppend) on test_chunkapp Order: test_chunkapp."time" - -> Index Scan Backward using _hyper_17_27_chunk_test_chunkapp_time_idx on _hyper_17_27_chunk - -> Index Scan Backward using _hyper_17_28_chunk_test_chunkapp_time_idx on _hyper_17_28_chunk + -> Index Scan Backward using _hyper_20_32_chunk_test_chunkapp_time_idx on _hyper_20_32_chunk + -> Index Scan Backward using _hyper_20_33_chunk_test_chunkapp_time_idx on _hyper_20_33_chunk -> Foreign Scan on test_chunkapp_fdw_child (5 rows) @@ -1496,9 +1577,9 @@ SELECT _timescaledb_functions.hypertable_osm_range_update('test_chunkapp',empty: QUERY PLAN ------------------------------------------------------------------------------------------------- Merge Append - Sort Key: _hyper_17_27_chunk."time" - -> Index Scan Backward using _hyper_17_27_chunk_test_chunkapp_time_idx on _hyper_17_27_chunk - -> Index Scan Backward using _hyper_17_28_chunk_test_chunkapp_time_idx on _hyper_17_28_chunk + Sort Key: _hyper_20_32_chunk."time" + -> Index Scan Backward using _hyper_20_32_chunk_test_chunkapp_time_idx on _hyper_20_32_chunk + -> Index Scan Backward using _hyper_20_33_chunk_test_chunkapp_time_idx on _hyper_20_33_chunk -> Foreign Scan on test_chunkapp_fdw_child (5 rows) @@ -1515,9 +1596,9 @@ FROM _timescaledb_catalog.chunk c, _timescaledb_catalog.chunk_constraint cc, _ti WHERE c.hypertable_id = :htid AND cc.chunk_id = c.id AND ds.id = cc.dimension_slice_id ORDER BY cc.chunk_id; chunk_id | table_name | status | osm_chunk | dimension_slice_id | range_start | range_end ----------+-------------------------+--------+-----------+--------------------+---------------------+--------------------- - 27 | _hyper_17_27_chunk | 0 | f | 24 | 1577836800000000 | 1577923200000000 - 28 | _hyper_17_28_chunk | 0 | f | 25 | 1577923200000000 | 1578009600000000 - 29 | test_chunkapp_fdw_child | 0 | t | 26 | 9223372036854775806 | 9223372036854775807 + 32 | _hyper_20_32_chunk | 0 | f | 29 | 1577836800000000 | 1577923200000000 + 33 | _hyper_20_33_chunk | 0 | f | 30 | 1577923200000000 | 1578009600000000 + 34 | test_chunkapp_fdw_child | 0 | t | 31 | 9223372036854775806 | 9223372036854775807 (3 rows) -- but also, OSM chunk should be included in the scan, since range is invalid and chunk is not empty @@ -1525,10 +1606,10 @@ WHERE c.hypertable_id = :htid AND cc.chunk_id = c.id AND ds.id = cc.dimension_sl QUERY PLAN ------------------------------------------------------------------------------------------------- Merge Append - Sort Key: _hyper_17_27_chunk."time" - -> Index Scan Backward using _hyper_17_27_chunk_test_chunkapp_time_idx on _hyper_17_27_chunk + Sort Key: _hyper_20_32_chunk."time" + -> Index Scan Backward using _hyper_20_32_chunk_test_chunkapp_time_idx on _hyper_20_32_chunk Index Cond: ("time" < 'Sun Jan 01 00:00:00 2023 PST'::timestamp with time zone) - -> Index Scan Backward using _hyper_17_28_chunk_test_chunkapp_time_idx on _hyper_17_28_chunk + -> Index Scan Backward using _hyper_20_33_chunk_test_chunkapp_time_idx on _hyper_20_33_chunk Index Cond: ("time" < 'Sun Jan 01 00:00:00 2023 PST'::timestamp with time zone) -> Foreign Scan on test_chunkapp_fdw_child (7 rows) @@ -1556,8 +1637,8 @@ SELECT _timescaledb_functions.hypertable_osm_range_update('test_chunkapp', NULL: ------------------------------------------------------------------------------------------------- Custom Scan (ChunkAppend) on test_chunkapp Order: test_chunkapp."time" - -> Index Scan Backward using _hyper_17_27_chunk_test_chunkapp_time_idx on _hyper_17_27_chunk - -> Index Scan Backward using _hyper_17_28_chunk_test_chunkapp_time_idx on _hyper_17_28_chunk + -> Index Scan Backward using _hyper_20_32_chunk_test_chunkapp_time_idx on _hyper_20_32_chunk + -> Index Scan Backward using _hyper_20_33_chunk_test_chunkapp_time_idx on _hyper_20_33_chunk -> Foreign Scan on test_chunkapp_fdw_child (5 rows) @@ -1574,9 +1655,9 @@ SELECT * FROM test_chunkapp ORDER BY 1; ------------------------------------------------------------------------------------------------- Custom Scan (ChunkAppend) on test_chunkapp Order: test_chunkapp."time" - -> Index Scan Backward using _hyper_17_27_chunk_test_chunkapp_time_idx on _hyper_17_27_chunk + -> Index Scan Backward using _hyper_20_32_chunk_test_chunkapp_time_idx on _hyper_20_32_chunk Index Cond: ("time" < 'Sun Jan 01 00:00:00 2023 PST'::timestamp with time zone) - -> Index Scan Backward using _hyper_17_28_chunk_test_chunkapp_time_idx on _hyper_17_28_chunk + -> Index Scan Backward using _hyper_20_33_chunk_test_chunkapp_time_idx on _hyper_20_33_chunk Index Cond: ("time" < 'Sun Jan 01 00:00:00 2023 PST'::timestamp with time zone) (6 rows) @@ -1613,7 +1694,7 @@ CREATE TABLE test2(time timestamptz not null, a int); SELECT create_hypertable('test2', 'time'); create_hypertable --------------------- - (18,public,test2,t) + (21,public,test2,t) (1 row) INSERT INTO test2 VALUES ('2020-01-01'::timestamptz, 1); @@ -1624,7 +1705,7 @@ psql:include/chunk_utils_internal_orderedappend.sql:138: NOTICE: default order SELECT compress_chunk(show_chunks('test2')); compress_chunk ------------------------------------------ - _timescaledb_internal._hyper_18_30_chunk + _timescaledb_internal._hyper_21_35_chunk (1 row) -- find internal compression table, call API function on it @@ -1633,7 +1714,7 @@ FROM _timescaledb_catalog.hypertable ht, _timescaledb_catalog.hypertable cht WHERE ht.table_name = 'test2' and cht.id = ht.compressed_hypertable_id \gset \set ON_ERROR_STOP 0 SELECT _timescaledb_functions.hypertable_osm_range_update(:'COMPRESSION_TBLNM'::regclass, '2020-01-01'::timestamptz); -psql:include/chunk_utils_internal_orderedappend.sql:145: ERROR: could not find time dimension for hypertable _timescaledb_internal._compressed_hypertable_19 +psql:include/chunk_utils_internal_orderedappend.sql:145: ERROR: could not find time dimension for hypertable _timescaledb_internal._compressed_hypertable_22 \set ON_ERROR_STOP 1 -- test wrong/incompatible data types with hypertable time dimension -- update range of int2 with int4 diff --git a/tsl/test/shared/expected/extension.out b/tsl/test/shared/expected/extension.out index 08e275bd2e2..fc8c707689d 100644 --- a/tsl/test/shared/expected/extension.out +++ b/tsl/test/shared/expected/extension.out @@ -213,7 +213,7 @@ ORDER BY pronamespace::regnamespace::text COLLATE "C", p.oid::regprocedure::text ts_now_mock() add_columnstore_policy(regclass,"any",boolean,interval,timestamp with time zone,text,interval,boolean) add_compression_policy(regclass,"any",boolean,interval,timestamp with time zone,text,interval,boolean) - add_continuous_aggregate_policy(regclass,"any","any",interval,boolean,timestamp with time zone,text) + add_continuous_aggregate_policy(regclass,"any","any",interval,boolean,timestamp with time zone,text,boolean) add_dimension(regclass,_timescaledb_internal.dimension_info,boolean) add_dimension(regclass,name,integer,anyelement,regproc,boolean) add_job(regproc,interval,jsonb,timestamp with time zone,boolean,regproc,boolean,text) diff --git a/tsl/test/sql/chunk_utils_internal.sql b/tsl/test/sql/chunk_utils_internal.sql index feeb83bc54b..ebb234e13ab 100644 --- a/tsl/test/sql/chunk_utils_internal.sql +++ b/tsl/test/sql/chunk_utils_internal.sql @@ -440,6 +440,54 @@ CALL refresh_continuous_aggregate('ht_try_weekly', '2019-12-29', '2020-01-10', f SELECT * FROM ht_try_weekly; DROP MATERIALIZED VIEW ht_try_weekly; +-- Test refresh policy with different settings of `include_tiered_data` parameter +CREATE FUNCTION create_test_cagg(include_tiered_data BOOL) +RETURNS INTEGER AS +$$ +DECLARE + cfg jsonb; + job_id INTEGER; +BEGIN + CREATE MATERIALIZED VIEW ht_try_weekly + WITH (timescaledb.continuous) AS + SELECT time_bucket(interval '1 week', timec) AS ts_bucket, avg(value) + FROM ht_try + GROUP BY 1 + WITH NO DATA; + + job_id := add_continuous_aggregate_policy( + 'ht_try_weekly', + start_offset => NULL, + end_offset => INTERVAL '1 hour', + schedule_interval => INTERVAL '1 hour', + include_tiered_data => include_tiered_data + ); + + cfg := config FROM _timescaledb_config.bgw_job WHERE id = job_id; + RAISE NOTICE 'config: %', jsonb_pretty(cfg); + + RETURN job_id; +END +$$ LANGUAGE plpgsql; + +-- include tiered data +SELECT create_test_cagg(true) AS job_id \gset +CALL run_job(:job_id); +SELECT * FROM ht_try_weekly ORDER BY 1; +DROP MATERIALIZED VIEW ht_try_weekly; + +-- exclude tiered data +SELECT create_test_cagg(false) AS job_id \gset +CALL run_job(:job_id); +SELECT * FROM ht_try_weekly ORDER BY 1; +DROP MATERIALIZED VIEW ht_try_weekly; + +-- default behavior: use instance-wide GUC value +SELECT create_test_cagg(null) AS job_id \gset +CALL run_job(:job_id); +SELECT * FROM ht_try_weekly ORDER BY 1; +DROP MATERIALIZED VIEW ht_try_weekly; + -- This test verifies that a bugfix regarding the way `ROWID_VAR`s are adjusted -- in the chunks' targetlists on DELETE/UPDATE works (including partially -- compressed chunks) From 85437c0b2f25d9cab0d5c263f248a6c59572afb6 Mon Sep 17 00:00:00 2001 From: Alex Kasko Date: Wed, 1 Jan 2025 16:36:36 +0000 Subject: [PATCH 10/32] Improve transaction check in CAgg refresh Procedures that use multiple transactions cannot be run in a transaction block (from a function, from dynamic SQL) or in a subtransaction (from a procedure block with an EXCEPTION clause). Such procedures use PreventInTransactionBlock function to check whether they can be run. Though currently such checks are incompete, because PreventInTransactionBlock requires isTopLevel argument to throw a consistent error when the call originates from a function. This isTopLevel flag (that is a bit poorly named - see below) is not readily available inside C procedures. The source of truth for it - ProcessUtilityContext parameter is passed to ProcessUtility hooks, but is not included with the function calls. There is an undocumented SPI_inside_nonatomic_context function, that would have been sufficient for isTopLevel flag, but it currently returns false when SPI connection is absent (that is a valid scenario when C procedures are called from top-lelev SQL instead of PLPG procedures or DO blocks) so it cannot be used. To work around this the value of ProcessUtilityContext parameter is saved when TS ProcessUtility hook is entered and can be accessed from C procedures using new ts_process_utility_is_context_nonatomic function. The result is called "non-atomic" instead of "top-level" because the way how isTopLevel flag is determined from the ProcessUtilityContext value in standard_ProcessUtility is insufficient for C procedures - it excludes PROCESS_UTILITY_QUERY_NONATOMIC value (used when called from PLPG procedure without an EXCEPTION clause) that is a valid use case for C procedures with transactions. See details in the description of ExecuteCallStmt function. It is expected that calls to C procedures are done with CALL and always pass though the ProcessUtility hook. The ProcessUtilityContext parameter is set to PROCESS_UTILITY_TOPLEVEL value by default. In unlikely case when a C procedure is called without passing through ProcessUtility hook and the call is done in atomic context, then PreventInTransactionBlock checks will pass, but SPI_commit will fail when checking that all current active snapshots are portal-owned snapshots (the same behaviour that was observed before this change). In atomic context there will be an additional snapshot set in _SPI_execute_plan, see the snapshot handling invariants description in that function. Closes #6533. --- .unreleased/pr_7566 | 2 + src/process_utility.c | 23 ++++++ src/process_utility.h | 53 +++++++++++++ tsl/src/continuous_aggs/refresh.c | 30 +++++--- tsl/test/expected/cagg_refresh.out | 65 ++++++++++++++++ .../expected/cagg_refresh_using_merge.out | 65 ++++++++++++++++ tsl/test/sql/include/cagg_refresh_common.sql | 74 +++++++++++++++++++ 7 files changed, 301 insertions(+), 11 deletions(-) create mode 100644 .unreleased/pr_7566 diff --git a/.unreleased/pr_7566 b/.unreleased/pr_7566 new file mode 100644 index 00000000000..ca3c51d8392 --- /dev/null +++ b/.unreleased/pr_7566 @@ -0,0 +1,2 @@ +Fixes: #7566 Improve transaction check in CAgg refresh +Thanks: @staticlibs for sending PR to improve transaction check in CAgg refresh diff --git a/src/process_utility.c b/src/process_utility.c index 7c70f0b3d65..239feebbf49 100644 --- a/src/process_utility.c +++ b/src/process_utility.c @@ -93,6 +93,7 @@ void _process_utility_fini(void); static ProcessUtility_hook_type prev_ProcessUtility_hook; static bool expect_chunk_modification = false; +static ProcessUtilityContext last_process_utility_context = PROCESS_UTILITY_TOPLEVEL; static DDLResult process_altertable_set_options(AlterTableCmd *cmd, Hypertable *ht); static DDLResult process_altertable_reset_options(AlterTableCmd *cmd, Hypertable *ht); @@ -111,6 +112,13 @@ prev_ProcessUtility(ProcessUtilityArgs *args) args->queryEnv, args->dest, args->completion_tag); + + /* + * Reset the last_process_utility_context value that is saved at the + * entrance of the TS ProcessUtility hook and can be used for transaction + * checks inside refresh_cagg and other procedures. + */ + ts_process_utility_context_reset(); } static void @@ -5048,6 +5056,8 @@ timescaledb_ddl_command_start(PlannedStmt *pstmt, const char *query_string, bool QueryEnvironment *queryEnv, DestReceiver *dest, QueryCompletion *completion_tag) { + last_process_utility_context = context; + ProcessUtilityArgs args = { .query_string = query_string, .context = context, .params = params, @@ -5173,6 +5183,19 @@ ts_process_utility_set_expect_chunk_modification(bool expect) expect_chunk_modification = expect; } +bool +ts_process_utility_is_context_nonatomic(void) +{ + ProcessUtilityContext context = last_process_utility_context; + return context == PROCESS_UTILITY_TOPLEVEL || context == PROCESS_UTILITY_QUERY_NONATOMIC; +} + +void +ts_process_utility_context_reset(void) +{ + last_process_utility_context = PROCESS_UTILITY_TOPLEVEL; +} + static void process_utility_xact_abort(XactEvent event, void *arg) { diff --git a/src/process_utility.h b/src/process_utility.h index 49f019abd85..cfa16f8ab15 100644 --- a/src/process_utility.h +++ b/src/process_utility.h @@ -36,3 +36,56 @@ typedef enum typedef DDLResult (*ts_process_utility_handler_t)(ProcessUtilityArgs *args); extern void ts_process_utility_set_expect_chunk_modification(bool expect); + +/* + * Procedures that use multiple transactions cannot be run in a transaction + * block (from a function, from dynamic SQL) or in a subtransaction (from a + * procedure block with an EXCEPTION clause). Such procedures use + * PreventInTransactionBlock function to check whether they can be run. + * + * Though currently such checks are incomplete, because + * PreventInTransactionBlock requires isTopLevel argument to throw a + * consistent error when the call originates from a function. This + * isTopLevel flag (that is a bit poorly named - see below) is not readily + * available inside C procedures. The source of truth for it - + * ProcessUtilityContext parameter is passed to ProcessUtility hooks, but + * is not included with the function calls. There is an undocumented + * SPI_inside_nonatomic_context function, that would have been sufficient + * for isTopLevel flag, but it currently returns false when SPI connection + * is absent (that is a valid scenario when C procedures are called from + * top-lelev SQL instead of PLPG procedures or DO blocks) so it cannot be + * used. + * + * To work around this the value of ProcessUtilityContext parameter is + * saved when TS ProcessUtility hook is entered and can be accessed from + * C procedures using new ts_process_utility_is_context_nonatomic function. + * The result is called "non-atomic" instead of "top-level" because the way + * how isTopLevel flag is determined from the ProcessUtilityContext value + * in standard_ProcessUtility is insufficient for C procedures - it + * excludes PROCESS_UTILITY_QUERY_NONATOMIC value (used when called from + * PLPG procedure without an EXCEPTION clause) that is a valid use case for + * C procedures with transactions. See details in the description of + * ExecuteCallStmt function. + * + * It is expected that calls to C procedures are done with CALL and always + * pass though the ProcessUtility hook. The ProcessUtilityContext + * parameter is set to PROCESS_UTILITY_TOPLEVEL value by default. In + * unlikely case when a C procedure is called without passing through + * ProcessUtility hook and the call is done in atomic context, then + * PreventInTransactionBlock checks will pass, but SPI_commit will fail + * when checking that all current active snapshots are portal-owned + * snapshots (the same behaviour that was observed before this change). + * In atomic context there will be an additional snapshot set in + * _SPI_execute_plan, see the snapshot handling invariants description + * in that function. + */ +extern TSDLLEXPORT bool ts_process_utility_is_context_nonatomic(void); + +/* + * Currently in TS ProcessUtility hook the saved ProcessUtilityContext + * value is reset back to PROCESS_UTILITY_TOPLEVEL on normal exit but + * is NOT reset in case of ereport exit. C procedures can call this + * function to reset the saved value before doing the checks that can + * result in ereport exit. + */ +extern TSDLLEXPORT void ts_process_utility_context_reset(void); diff --git a/tsl/src/continuous_aggs/refresh.c b/tsl/src/continuous_aggs/refresh.c index d5a9783e003..60d9d50c038 100644 --- a/tsl/src/continuous_aggs/refresh.c +++ b/tsl/src/continuous_aggs/refresh.c @@ -31,6 +31,7 @@ #include "invalidation.h" #include "invalidation_threshold.h" #include "materialize.h" +#include "process_utility.h" #include "refresh.h" #define CAGG_REFRESH_LOG_LEVEL (callctx == CAGG_REFRESH_POLICY ? LOG : DEBUG1) @@ -768,6 +769,24 @@ continuous_agg_refresh_internal(const ContinuousAgg *cagg, int32 mat_id = cagg->data.mat_hypertable_id; InternalTimeRange refresh_window = *refresh_window_arg; int64 invalidation_threshold; + bool nonatomic = ts_process_utility_is_context_nonatomic(); + + /* Reset the saved ProcessUtilityContext value promptly before + * calling Prevent* checks so the potential unsupported (atomic) + * value won't linger there in case of ereport exit. + */ + ts_process_utility_context_reset(); + + PreventCommandIfReadOnly(REFRESH_FUNCTION_NAME); + + /* Prevent running refresh if we're in a transaction block since a refresh + * can run two transactions and might take a long time to release locks if + * there's a lot to materialize. Strictly, it is optional to prohibit + * transaction blocks since there will be only one transaction if the + * invalidation threshold needs no update. However, materialization might + * still take a long time and it is probably best for consistency to always + * prevent transaction blocks. */ + PreventInTransactionBlock(nonatomic, REFRESH_FUNCTION_NAME); /* Connect to SPI manager due to the underlying SPI calls */ int rc = SPI_connect_ext(SPI_OPT_NONATOMIC); @@ -784,17 +803,6 @@ continuous_agg_refresh_internal(const ContinuousAgg *cagg, get_relkind_objtype(get_rel_relkind(cagg->relid)), get_rel_name(cagg->relid)); - PreventCommandIfReadOnly(REFRESH_FUNCTION_NAME); - - /* Prevent running refresh if we're in a transaction block since a refresh - * can run two transactions and might take a long time to release locks if - * there's a lot to materialize. Strictly, it is optional to prohibit - * transaction blocks since there will be only one transaction if the - * invalidation threshold needs no update. However, materialization might - * still take a long time and it is probably best for consistency to always - * prevent transaction blocks. */ - PreventInTransactionBlock(true, REFRESH_FUNCTION_NAME); - /* No bucketing when open ended */ if (!(start_isnull && end_isnull)) { diff --git a/tsl/test/expected/cagg_refresh.out b/tsl/test/expected/cagg_refresh.out index b25f4884990..e77cdc70a68 100644 --- a/tsl/test/expected/cagg_refresh.out +++ b/tsl/test/expected/cagg_refresh.out @@ -535,3 +535,68 @@ SELECT time_bucket('7 days', time) AS day, device, avg(temp) AS avg_temp FROM conditions GROUP BY 1,2 WITH NO DATA; COMMIT; +-- refresh_continuous_aggregate can run two transactions, thus it cannot be +-- called in a transaction block (from a function, from dynamic SQL) or in a +-- subtransaction (from a procedure block with an EXCEPTION clause). Though it +-- does NOT require a top level context and can be called from a procedure +-- block without an EXCEPTION clause. +-- DO block +DO $$ +BEGIN + CALL refresh_continuous_aggregate('daily_temp', '2020-05-03 00:00 UTC', '2020-05-04 00:00 UTC'); +END; $$; +psql:include/cagg_refresh_common.sql:347: NOTICE: continuous aggregate "daily_temp" is already up-to-date +-- Procedure without subtransaction +CREATE OR REPLACE PROCEDURE refresh_cagg_proc_normal() +LANGUAGE PLPGSQL AS +$$ +BEGIN + CALL refresh_continuous_aggregate('daily_temp', '2020-05-03 00:00 UTC', '2020-05-04 00:00 UTC'); +END; $$; +CALL refresh_cagg_proc_normal(); +psql:include/cagg_refresh_common.sql:357: NOTICE: continuous aggregate "daily_temp" is already up-to-date +\set ON_ERROR_STOP 0 +-- Procedure with subtransaction +CREATE OR REPLACE PROCEDURE refresh_cagg_proc_subtransaction() +LANGUAGE PLPGSQL AS +$$ +DECLARE + errmsg TEXT; +BEGIN + CALL refresh_continuous_aggregate('daily_temp', '2020-05-03 00:00 UTC', '2020-05-04 00:00 UTC'); +EXCEPTION WHEN OTHERS THEN + GET STACKED DIAGNOSTICS errmsg = MESSAGE_TEXT; + RAISE EXCEPTION '%', errmsg; +END; $$; +CALL refresh_cagg_proc_subtransaction(); +psql:include/cagg_refresh_common.sql:374: ERROR: refresh_continuous_aggregate() cannot run inside a transaction block +-- Function +CREATE OR REPLACE FUNCTION refresh_cagg_fun() +RETURNS INT LANGUAGE PLPGSQL AS +$$ +BEGIN + CALL refresh_continuous_aggregate('daily_temp', '2020-05-03 00:00 UTC', '2020-05-04 00:00 UTC'); + RETURN 1; +END; $$; +SELECT * from refresh_cagg_fun(); +psql:include/cagg_refresh_common.sql:385: ERROR: refresh_continuous_aggregate() cannot be executed from a function +-- Dynamic SQL +DO $$ +BEGIN + EXECUTE $inner$ + CALL refresh_continuous_aggregate('daily_temp', '2020-05-03 00:00 UTC', '2020-05-04 00:00 UTC'); + $inner$; +END; $$; +psql:include/cagg_refresh_common.sql:393: ERROR: refresh_continuous_aggregate() cannot be executed from a function +-- Trigger +CREATE TABLE refresh_cagg_trigger_table(a int); +CREATE FUNCTION refresh_cagg_trigger_fun() +RETURNS TRIGGER LANGUAGE PLPGSQL AS $$ +BEGIN + CALL refresh_continuous_aggregate('daily_temp', '2020-05-03 00:00 UTC', '2020-05-04 00:00 UTC'); +END; $$; +CREATE TRIGGER refresh_cagg_trigger AFTER INSERT ON refresh_cagg_trigger_table +EXECUTE FUNCTION refresh_cagg_trigger_fun(); +INSERT INTO refresh_cagg_trigger_table VALUES(1); +psql:include/cagg_refresh_common.sql:407: ERROR: refresh_continuous_aggregate() cannot be executed from a function +\set ON_ERROR_STOP 1 diff --git a/tsl/test/expected/cagg_refresh_using_merge.out b/tsl/test/expected/cagg_refresh_using_merge.out index ca5481bbc93..68d493b56d9 100644 --- a/tsl/test/expected/cagg_refresh_using_merge.out +++ b/tsl/test/expected/cagg_refresh_using_merge.out @@ -536,6 +536,71 @@ SELECT time_bucket('7 days', time) AS day, device, avg(temp) AS avg_temp FROM conditions GROUP BY 1,2 WITH NO DATA; COMMIT; +-- refresh_continuous_aggregate can run two transactions, thus it cannot be +-- called in a transaction block (from a function, from dynamic SQL) or in a +-- subtransaction (from a procedure block with an EXCEPTION clause). Though it +-- does NOT require a top level context and can be called from a procedure +-- block without an EXCEPTION clause. +-- DO block +DO $$ +BEGIN + CALL refresh_continuous_aggregate('daily_temp', '2020-05-03 00:00 UTC', '2020-05-04 00:00 UTC'); +END; $$; +psql:include/cagg_refresh_common.sql:347: NOTICE: continuous aggregate "daily_temp" is already up-to-date +-- Procedure without subtransaction +CREATE OR REPLACE PROCEDURE refresh_cagg_proc_normal() +LANGUAGE PLPGSQL AS +$$ +BEGIN + CALL refresh_continuous_aggregate('daily_temp', '2020-05-03 00:00 UTC', '2020-05-04 00:00 UTC'); +END; $$; +CALL refresh_cagg_proc_normal(); +psql:include/cagg_refresh_common.sql:357: NOTICE: continuous aggregate "daily_temp" is already up-to-date +\set ON_ERROR_STOP 0 +-- Procedure with subtransaction +CREATE OR REPLACE PROCEDURE refresh_cagg_proc_subtransaction() +LANGUAGE PLPGSQL AS +$$ +DECLARE + errmsg TEXT; +BEGIN + CALL refresh_continuous_aggregate('daily_temp', '2020-05-03 00:00 UTC', '2020-05-04 00:00 UTC'); +EXCEPTION WHEN OTHERS THEN + GET STACKED DIAGNOSTICS errmsg = MESSAGE_TEXT; + RAISE EXCEPTION '%', errmsg; +END; $$; +CALL refresh_cagg_proc_subtransaction(); +psql:include/cagg_refresh_common.sql:374: ERROR: refresh_continuous_aggregate() cannot run inside a transaction block +-- Function +CREATE OR REPLACE FUNCTION refresh_cagg_fun() +RETURNS INT LANGUAGE PLPGSQL AS +$$ +BEGIN + CALL refresh_continuous_aggregate('daily_temp', '2020-05-03 00:00 UTC', '2020-05-04 00:00 UTC'); + RETURN 1; +END; $$; +SELECT * from refresh_cagg_fun(); +psql:include/cagg_refresh_common.sql:385: ERROR: refresh_continuous_aggregate() cannot be executed from a function +-- Dynamic SQL +DO $$ +BEGIN + EXECUTE $inner$ + CALL refresh_continuous_aggregate('daily_temp', '2020-05-03 00:00 UTC', '2020-05-04 00:00 UTC'); + $inner$; +END; $$; +psql:include/cagg_refresh_common.sql:393: ERROR: refresh_continuous_aggregate() cannot be executed from a function +-- Trigger +CREATE TABLE refresh_cagg_trigger_table(a int); +CREATE FUNCTION refresh_cagg_trigger_fun() +RETURNS TRIGGER LANGUAGE PLPGSQL AS $$ +BEGIN + CALL refresh_continuous_aggregate('daily_temp', '2020-05-03 00:00 UTC', '2020-05-04 00:00 UTC'); +END; $$; +CREATE TRIGGER refresh_cagg_trigger AFTER INSERT ON refresh_cagg_trigger_table +EXECUTE FUNCTION refresh_cagg_trigger_fun(); +INSERT INTO refresh_cagg_trigger_table VALUES(1); +psql:include/cagg_refresh_common.sql:407: ERROR: refresh_continuous_aggregate() cannot be executed from a function +\set ON_ERROR_STOP 1 -- Additional tests for MERGE refresh DROP TABLE conditions CASCADE; NOTICE: drop cascades to 10 other objects diff --git a/tsl/test/sql/include/cagg_refresh_common.sql b/tsl/test/sql/include/cagg_refresh_common.sql index b7192a83d0c..efe5fec8244 100644 --- a/tsl/test/sql/include/cagg_refresh_common.sql +++ b/tsl/test/sql/include/cagg_refresh_common.sql @@ -333,3 +333,77 @@ SELECT time_bucket('7 days', time) AS day, device, avg(temp) AS avg_temp FROM conditions GROUP BY 1,2 WITH NO DATA; COMMIT; + +-- refresh_continuous_aggregate can run two transactions, thus it cannot be +-- called in a transaction block (from a function, from dynamic SQL) or in a +-- subtransaction (from a procedure block with an EXCEPTION clause). Though it +-- does NOT require a top level context and can be called from a procedure +-- block without an EXCEPTION clause. + +-- DO block +DO $$ +BEGIN + CALL refresh_continuous_aggregate('daily_temp', '2020-05-03 00:00 UTC', '2020-05-04 00:00 UTC'); +END; $$; + +-- Procedure without subtransaction +CREATE OR REPLACE PROCEDURE refresh_cagg_proc_normal() +LANGUAGE PLPGSQL AS +$$ +BEGIN + CALL refresh_continuous_aggregate('daily_temp', '2020-05-03 00:00 UTC', '2020-05-04 00:00 UTC'); +END; $$; + +CALL refresh_cagg_proc_normal(); + +\set ON_ERROR_STOP 0 + +-- Procedure with subtransaction +CREATE OR REPLACE PROCEDURE refresh_cagg_proc_subtransaction() +LANGUAGE PLPGSQL AS +$$ +DECLARE + errmsg TEXT; +BEGIN + CALL refresh_continuous_aggregate('daily_temp', '2020-05-03 00:00 UTC', '2020-05-04 00:00 UTC'); +EXCEPTION WHEN OTHERS THEN + GET STACKED DIAGNOSTICS errmsg = MESSAGE_TEXT; + RAISE EXCEPTION '%', errmsg; +END; $$; + +CALL refresh_cagg_proc_subtransaction(); + +-- Function +CREATE OR REPLACE FUNCTION refresh_cagg_fun() +RETURNS INT LANGUAGE PLPGSQL AS +$$ +BEGIN + CALL refresh_continuous_aggregate('daily_temp', '2020-05-03 00:00 UTC', '2020-05-04 00:00 UTC'); + RETURN 1; +END; $$; + +SELECT * from refresh_cagg_fun(); + +-- Dynamic SQL +DO $$ +BEGIN + EXECUTE $inner$ + CALL refresh_continuous_aggregate('daily_temp', '2020-05-03 00:00 UTC', '2020-05-04 00:00 UTC'); + $inner$; +END; $$; + +-- Trigger +CREATE TABLE refresh_cagg_trigger_table(a int); + +CREATE FUNCTION refresh_cagg_trigger_fun() +RETURNS TRIGGER LANGUAGE PLPGSQL AS $$ +BEGIN + CALL refresh_continuous_aggregate('daily_temp', '2020-05-03 00:00 UTC', '2020-05-04 00:00 UTC'); +END; $$; + +CREATE TRIGGER refresh_cagg_trigger AFTER INSERT ON refresh_cagg_trigger_table +EXECUTE FUNCTION refresh_cagg_trigger_fun(); + +INSERT INTO refresh_cagg_trigger_table VALUES(1); + +\set ON_ERROR_STOP 1 From 04009b1f070cd1cd8c9c8ed8efaecc768dcb9063 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Nordstr=C3=B6m?= Date: Wed, 15 Jan 2025 15:25:29 +0100 Subject: [PATCH 11/32] Refactor vacuum cutoffs compatibility functions The compatibility layer in TimescaleDB consolidates the API differences around getting the vacuum cutoffs (used by, e.g., TimescaleDB's reorder) based on an old API. Since PostgreSQL 17 introduced a new function API to get the vacuum cutoffs, this change refactors the compitability layer to provide the new API for older PostgreSQL versions. --- src/compat/compat.h | 108 ++++++++++++++++++-------------------------- tsl/src/reorder.c | 37 +++++++++------ 2 files changed, 69 insertions(+), 76 deletions(-) diff --git a/src/compat/compat.h b/src/compat/compat.h index 58654de9f9e..a5e66e47efb 100644 --- a/src/compat/compat.h +++ b/src/compat/compat.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -417,74 +418,54 @@ pg_strtoint64(const char *str) check_index_is_clusterable(rel, indexOid, true, lock) #endif +#if PG16_LT /* * PG15 consolidate VACUUM xid cutoff logic. * * https://github.com/postgres/postgres/commit/efa4a946 + * + * PG16 introduced VacuumCutoffs so define here for previous PG versions. */ -#if PG15_LT -#define vacuum_set_xid_limits_compat(rel, \ - freeze_min_age, \ - freeze_table_age, \ - multixact_freeze_min_age, \ - multixact_freeze_table_age, \ - oldestXmin, \ - freezeLimit, \ - multiXactCutoff) \ - vacuum_set_xid_limits(rel, \ - freeze_min_age, \ - freeze_table_age, \ - multixact_freeze_min_age, \ - multixact_freeze_table_age, \ - oldestXmin, \ - freezeLimit, \ - NULL, \ - multiXactCutoff, \ - NULL) -#elif PG16_LT -#define vacuum_set_xid_limits_compat(rel, \ - freeze_min_age, \ - freeze_table_age, \ - multixact_freeze_min_age, \ - multixact_freeze_table_age, \ - oldestXmin, \ - freezeLimit, \ - multiXactCutoff) \ - do \ - { \ - MultiXactId oldestMxact; \ - vacuum_set_xid_limits(rel, \ - freeze_min_age, \ - freeze_table_age, \ - multixact_freeze_min_age, \ - multixact_freeze_table_age, \ - oldestXmin, \ - &oldestMxact, \ - freezeLimit, \ - multiXactCutoff); \ - } while (0) -#else -#define vacuum_set_xid_limits_compat(rel, \ - freezeMinAge, \ - freezeTableAge, \ - multixactFreezeMinAge, \ - multixactFreezeTableAge, \ - oldestXmin, \ - freezeLimit, \ - multiXactCutoff) \ - do \ - { \ - struct VacuumCutoffs cutoffs; \ - /* vacuum_get_cutoffs uses only the *_age members of the VacuumParams object */ \ - VacuumParams params = { .freeze_min_age = freezeMinAge, \ - .freeze_table_age = freezeTableAge, \ - .multixact_freeze_min_age = multixactFreezeMinAge, \ - .multixact_freeze_table_age = multixactFreezeTableAge }; \ - vacuum_get_cutoffs(rel, ¶ms, &cutoffs); \ - *(oldestXmin) = cutoffs.OldestXmin; \ - *(freezeLimit) = cutoffs.FreezeLimit; \ - *(multiXactCutoff) = cutoffs.MultiXactCutoff; \ - } while (0) +struct VacuumCutoffs +{ + TransactionId relfrozenxid; + MultiXactId relminmxid; + TransactionId OldestXmin; + MultiXactId OldestMxact; + TransactionId FreezeLimit; + MultiXactId MultiXactCutoff; +}; + +static inline bool +vacuum_get_cutoffs(Relation rel, const VacuumParams *params, struct VacuumCutoffs *cutoffs) +{ +#if PG15 + return vacuum_set_xid_limits(rel, + 0, + 0, + 0, + 0, + &cutoffs->OldestXmin, + &cutoffs->OldestMxact, + &cutoffs->FreezeLimit, + &cutoffs->MultiXactCutoff); +#elif PG14 + vacuum_set_xid_limits(rel, + 0, + 0, + 0, + 0, + &cutoffs->OldestXmin, + &cutoffs->FreezeLimit, + NULL, + &cutoffs->MultiXactCutoff, + NULL); + + /* Should aggressive vacuum be done? PG14 doesn't support the return value + * so return false. */ + return false; +#endif +} #endif /* @@ -977,3 +958,4 @@ RestrictSearchPath(void) is_internal, \ constraintId) #endif + diff --git a/tsl/src/reorder.c b/tsl/src/reorder.c index f9c2b4e3c61..87f99bf266c 100644 --- a/tsl/src/reorder.c +++ b/tsl/src/reorder.c @@ -534,9 +534,6 @@ copy_heap_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex, bool verbose, int natts; Datum *values; bool *isnull; - TransactionId OldestXmin; - TransactionId FreezeXid; - MultiXactId MultiXactCutoff; bool use_sort; double num_tuples = 0, tups_vacuumed = 0, tups_recently_dead = 0; BlockNumber num_pages; @@ -626,24 +623,38 @@ copy_heap_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex, bool verbose, * Since we're going to rewrite the whole table anyway, there's no reason * not to be aggressive about this. */ - vacuum_set_xid_limits_compat(OldHeap, 0, 0, 0, 0, &OldestXmin, &FreezeXid, &MultiXactCutoff); + struct VacuumCutoffs cutoffs; + VacuumParams params; + + memset(¶ms, 0, sizeof(VacuumParams)); + vacuum_get_cutoffs(OldHeap, ¶ms, &cutoffs); /* * FreezeXid will become the table's new relfrozenxid, and that mustn't go * backwards, so take the max. */ - if (TransactionIdPrecedes(FreezeXid, OldHeap->rd_rel->relfrozenxid)) - FreezeXid = OldHeap->rd_rel->relfrozenxid; + { + TransactionId relfrozenxid = OldHeap->rd_rel->relfrozenxid; + + if (TransactionIdIsValid(relfrozenxid) && + TransactionIdPrecedes(cutoffs.FreezeLimit, relfrozenxid)) + cutoffs.FreezeLimit = relfrozenxid; + } /* * MultiXactCutoff, similarly, shouldn't go backwards either. */ - if (MultiXactIdPrecedes(MultiXactCutoff, OldHeap->rd_rel->relminmxid)) - MultiXactCutoff = OldHeap->rd_rel->relminmxid; + { + MultiXactId relminmxid = OldHeap->rd_rel->relminmxid; + + if (MultiXactIdIsValid(relminmxid) && + MultiXactIdPrecedes(cutoffs.MultiXactCutoff, relminmxid)) + cutoffs.MultiXactCutoff = relminmxid; + } /* return selected values to caller */ - *pFreezeXid = FreezeXid; - *pCutoffMulti = MultiXactCutoff; + *pFreezeXid = cutoffs.FreezeLimit; + *pCutoffMulti = cutoffs.MultiXactCutoff; /* * We know how to use a sort to duplicate the ordering of a btree index, @@ -677,9 +688,9 @@ copy_heap_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex, bool verbose, NewHeap, OldIndex, use_sort, - OldestXmin, - &FreezeXid, - &MultiXactCutoff, + cutoffs.OldestXmin, + &cutoffs.FreezeLimit, + &cutoffs.MultiXactCutoff, &num_tuples, &tups_vacuumed, &tups_recently_dead); From 08050fb9d6170ec43c3cfa18f59355b299f5ea55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Nordstr=C3=B6m?= Date: Wed, 9 Oct 2024 13:41:35 +0200 Subject: [PATCH 12/32] Add support for merging chunks New procedures to `merge_chunks` are introduced that can merge an arbitrary number of chunks if the right conditions apply. Basic checks are done to ensure that the chunks can be merged from a partitioning perspective. Some more advanced cases that are potentially mergeable are not supported at this time (e.g., chunks with non-adjacent partitioning) and merging of compressed chunks. Merging compressed chunks requires additional work, although the same basic rewrite approach should work also on the internal compressed relations. Still, one needs to handle merges of a compressed chunk and a non-compressed chunk, or two compressed chunks with different compression settings, partially compressed chunks, and so forth. This is left for a future enhancement. Currently, the merge defaults to taking an AccessExclusive lock on the merged chunks to prevent deadlocks and concurrent modifications. Weaker locking is supported via an anonymous settings variable, and it is used in tests to illustrate various deadlock scenarios. Alternative locking approaches, including multi-transactional merges, can be considered in the future. The actual merging is done by rewriting all the data from multiple chunks into a (temporary) merged heap using the same approach as that implemented to support VACUUM FULL and CLUSTER. Then this new heap is swapped into one of the original relations while the rest are dropped. This approach is MVCC compliant and implements correct visibility under higher isolation levels, while also cleaning up garbage tuples. --- .unreleased/pr_7433 | 1 + sql/maintenance_utils.sql | 8 + sql/updates/latest-dev.sql | 9 + sql/updates/reverse-dev.sql | 4 + src/chunk.c | 13 + src/chunk.h | 3 +- src/chunk_constraint.c | 9 +- src/chunk_constraint.h | 9 +- src/compat/compat.h | 12 + src/cross_module_fn.c | 2 + src/cross_module_fn.h | 1 + src/dimension.h | 3 +- src/dimension_slice.h | 12 +- src/hypercube.h | 4 +- tsl/src/chunk.c | 902 +++++++++++++++++- tsl/src/chunk.h | 1 + tsl/src/init.c | 1 + tsl/test/expected/merge_chunks.out | 490 ++++++++++ .../expected/merge_chunks_concurrent.out | 611 ++++++++++++ tsl/test/isolation/specs/CMakeLists.txt | 3 +- .../specs/merge_chunks_concurrent.spec | 174 ++++ tsl/test/shared/expected/extension.out | 2 + tsl/test/sql/CMakeLists.txt | 1 + tsl/test/sql/merge_chunks.sql | 247 +++++ 24 files changed, 2494 insertions(+), 28 deletions(-) create mode 100644 .unreleased/pr_7433 create mode 100644 tsl/test/expected/merge_chunks.out create mode 100644 tsl/test/isolation/expected/merge_chunks_concurrent.out create mode 100644 tsl/test/isolation/specs/merge_chunks_concurrent.spec create mode 100644 tsl/test/sql/merge_chunks.sql diff --git a/.unreleased/pr_7433 b/.unreleased/pr_7433 new file mode 100644 index 00000000000..c733d50d429 --- /dev/null +++ b/.unreleased/pr_7433 @@ -0,0 +1 @@ +Implements: #7433 Add support for merging chunks diff --git a/sql/maintenance_utils.sql b/sql/maintenance_utils.sql index 339ea07beeb..7edcbf9e3dc 100644 --- a/sql/maintenance_utils.sql +++ b/sql/maintenance_utils.sql @@ -57,6 +57,14 @@ CREATE OR REPLACE PROCEDURE @extschema@.convert_to_rowstore( if_columnstore BOOLEAN = true ) AS '@MODULE_PATHNAME@', 'ts_decompress_chunk' LANGUAGE C; +CREATE OR REPLACE PROCEDURE @extschema@.merge_chunks( + chunk1 REGCLASS, chunk2 REGCLASS +) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_merge_two_chunks'; + +CREATE OR REPLACE PROCEDURE @extschema@.merge_chunks( + chunks REGCLASS[] +) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_merge_chunks'; + CREATE OR REPLACE FUNCTION _timescaledb_functions.recompress_chunk_segmentwise( uncompressed_chunk REGCLASS, if_compressed BOOLEAN = true diff --git a/sql/updates/latest-dev.sql b/sql/updates/latest-dev.sql index 33d290e374a..5863834971c 100644 --- a/sql/updates/latest-dev.sql +++ b/sql/updates/latest-dev.sql @@ -144,3 +144,12 @@ CREATE FUNCTION @extschema@.add_continuous_aggregate_policy( RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C VOLATILE; + +-- Merge chunks +CREATE PROCEDURE @extschema@.merge_chunks( + chunk1 REGCLASS, chunk2 REGCLASS +) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_update_placeholder'; + +CREATE PROCEDURE @extschema@.merge_chunks( + chunks REGCLASS[] +) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_update_placeholder'; diff --git a/sql/updates/reverse-dev.sql b/sql/updates/reverse-dev.sql index 37f3659dd6f..89c7935ebbf 100644 --- a/sql/updates/reverse-dev.sql +++ b/sql/updates/reverse-dev.sql @@ -87,3 +87,7 @@ CREATE FUNCTION @extschema@.add_continuous_aggregate_policy( RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_policy_refresh_cagg_add' LANGUAGE C VOLATILE; + +-- Merge chunks +DROP PROCEDURE IF EXISTS @extschema@.merge_chunks(chunk1 REGCLASS, chunk2 REGCLASS); +DROP PROCEDURE IF EXISTS @extschema@.merge_chunks(chunks REGCLASS[]); diff --git a/src/chunk.c b/src/chunk.c index 565cc6f87bd..f2d620548b0 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -31,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -5158,3 +5160,14 @@ ts_chunk_drop_osm_chunk(PG_FUNCTION_ARGS) ts_cache_release(hcache); PG_RETURN_BOOL(true); } + +TS_FUNCTION_INFO_V1(ts_merge_two_chunks); + +Datum +ts_merge_two_chunks(PG_FUNCTION_ARGS) +{ + Datum chunks[2] = { PG_GETARG_DATUM(0), PG_GETARG_DATUM(1) }; + ArrayType *chunk_array = + construct_array(chunks, 2, REGCLASSOID, sizeof(Oid), true, TYPALIGN_INT); + return DirectFunctionCall1(ts_cm_functions->merge_chunks, PointerGetDatum(chunk_array)); +} diff --git a/src/chunk.h b/src/chunk.h index aaa93026e14..690fff8ca2c 100644 --- a/src/chunk.h +++ b/src/chunk.h @@ -188,7 +188,8 @@ extern bool ts_chunk_exists_relid(Oid relid); extern TSDLLEXPORT bool ts_chunk_exists_with_compression(int32 hypertable_id); extern void ts_chunk_recreate_all_constraints_for_dimension(Hypertable *ht, int32 dimension_id); extern int ts_chunk_delete_by_hypertable_id(int32 hypertable_id); -extern int ts_chunk_delete_by_name(const char *schema, const char *table, DropBehavior behavior); +extern TSDLLEXPORT int ts_chunk_delete_by_name(const char *schema, const char *table, + DropBehavior behavior); extern bool ts_chunk_set_name(Chunk *chunk, const char *newname); extern bool ts_chunk_set_schema(Chunk *chunk, const char *newschema); extern TSDLLEXPORT List *ts_chunk_get_window(int32 dimension_id, int64 point, int count, diff --git a/src/chunk_constraint.c b/src/chunk_constraint.c index 260ef6a17f7..de0d9e6b5d0 100644 --- a/src/chunk_constraint.c +++ b/src/chunk_constraint.c @@ -272,9 +272,9 @@ ts_chunk_constraints_add_from_tuple(ChunkConstraints *ccs, const TupleInfo *ti) /* * Create a dimensional CHECK constraint for a partitioning dimension. */ -static Constraint * -create_dimension_check_constraint(const Dimension *dim, const DimensionSlice *slice, - const char *name) +Constraint * +ts_chunk_constraint_dimensional_create(const Dimension *dim, const DimensionSlice *slice, + const char *name) { Constraint *constr = NULL; bool isvarlena; @@ -489,7 +489,8 @@ ts_chunk_constraints_create(const Hypertable *ht, const Chunk *chunk) dim = ts_hyperspace_get_dimension_by_id(ht->space, slice->fd.dimension_id); Assert(dim); - constr = create_dimension_check_constraint(dim, slice, NameStr(cc->fd.constraint_name)); + constr = + ts_chunk_constraint_dimensional_create(dim, slice, NameStr(cc->fd.constraint_name)); /* In some cases, a CHECK constraint is not needed. For instance, * if the range is -INF to +INF. */ diff --git a/src/chunk_constraint.h b/src/chunk_constraint.h index 585e243ffdc..50dcda84ca4 100644 --- a/src/chunk_constraint.h +++ b/src/chunk_constraint.h @@ -42,9 +42,9 @@ extern int ts_chunk_constraint_scan_by_dimension_slice(const DimensionSlice *sli ChunkScanCtx *ctx, MemoryContext mctx); extern int ts_chunk_constraint_scan_by_dimension_slice_to_list(const DimensionSlice *slice, List **list, MemoryContext mctx); -extern int ts_chunk_constraint_scan_by_dimension_slice_id(int32 dimension_slice_id, - ChunkConstraints *ccs, - MemoryContext mctx); +extern int TSDLLEXPORT ts_chunk_constraint_scan_by_dimension_slice_id(int32 dimension_slice_id, + ChunkConstraints *ccs, + MemoryContext mctx); extern ChunkConstraint *ts_chunk_constraints_add(ChunkConstraints *ccs, int32 chunk_id, int32 dimension_slice_id, const char *constraint_name, @@ -58,6 +58,9 @@ extern TSDLLEXPORT int ts_chunk_constraints_add_inheritable_constraints(ChunkCon extern TSDLLEXPORT int ts_chunk_constraints_add_inheritable_check_constraints( ChunkConstraints *ccs, int32 chunk_id, const char chunk_relkind, Oid hypertable_oid); extern TSDLLEXPORT void ts_chunk_constraints_insert_metadata(const ChunkConstraints *ccs); +extern TSDLLEXPORT Constraint *ts_chunk_constraint_dimensional_create(const Dimension *dim, + const DimensionSlice *slice, + const char *name); extern TSDLLEXPORT void ts_chunk_constraints_create(const Hypertable *ht, const Chunk *chunk); extern void ts_chunk_constraint_create_on_chunk(const Hypertable *ht, const Chunk *chunk, Oid constraint_oid); diff --git a/src/compat/compat.h b/src/compat/compat.h index a5e66e47efb..6e73f19961b 100644 --- a/src/compat/compat.h +++ b/src/compat/compat.h @@ -959,3 +959,15 @@ RestrictSearchPath(void) constraintId) #endif +#if PG17_LT +/* + * Overflow-aware comparison functions to be used in qsort. Introduced in PG + * 17 and included here for older PG versions. + */ +static inline int +pg_cmp_u32(uint32 a, uint32 b) +{ + return (a > b) - (a < b); +} + +#endif diff --git a/src/cross_module_fn.c b/src/cross_module_fn.c index 5f1b5a0290e..2985cf32fc2 100644 --- a/src/cross_module_fn.c +++ b/src/cross_module_fn.c @@ -98,6 +98,7 @@ CROSSMODULE_WRAPPER(chunk_create_empty_table); CROSSMODULE_WRAPPER(recompress_chunk_segmentwise); CROSSMODULE_WRAPPER(get_compressed_chunk_index_for_recompression); +CROSSMODULE_WRAPPER(merge_chunks); /* hypercore */ CROSSMODULE_WRAPPER(is_compressed_tid); @@ -407,6 +408,7 @@ TSDLLEXPORT CrossModuleFunctions ts_cm_functions_default = { .recompress_chunk_segmentwise = error_no_default_fn_pg_community, .get_compressed_chunk_index_for_recompression = error_no_default_fn_pg_community, .preprocess_query_tsl = preprocess_query_tsl_default_fn_community, + .merge_chunks = error_no_default_fn_pg_community, }; TSDLLEXPORT CrossModuleFunctions *ts_cm_functions = &ts_cm_functions_default; diff --git a/src/cross_module_fn.h b/src/cross_module_fn.h index da2c520b260..90fb7c6523a 100644 --- a/src/cross_module_fn.h +++ b/src/cross_module_fn.h @@ -163,6 +163,7 @@ typedef struct CrossModuleFunctions PGFunction recompress_chunk_segmentwise; PGFunction get_compressed_chunk_index_for_recompression; void (*preprocess_query_tsl)(Query *parse, int *cursor_opts); + PGFunction merge_chunks; } CrossModuleFunctions; extern TSDLLEXPORT CrossModuleFunctions *ts_cm_functions; diff --git a/src/dimension.h b/src/dimension.h index be84046b888..44d9d1a8000 100644 --- a/src/dimension.h +++ b/src/dimension.h @@ -125,7 +125,8 @@ extern Hyperspace *ts_dimension_scan(int32 hypertable_id, Oid main_table_relid, extern DimensionSlice *ts_dimension_calculate_default_slice(const Dimension *dim, int64 value); extern TSDLLEXPORT Point *ts_hyperspace_calculate_point(const Hyperspace *h, TupleTableSlot *slot); extern int ts_dimension_get_slice_ordinal(const Dimension *dim, const DimensionSlice *slice); -extern const Dimension *ts_hyperspace_get_dimension_by_id(const Hyperspace *hs, int32 id); +extern TSDLLEXPORT const Dimension *ts_hyperspace_get_dimension_by_id(const Hyperspace *hs, + int32 id); extern TSDLLEXPORT const Dimension *ts_hyperspace_get_dimension(const Hyperspace *hs, DimensionType type, Index n); extern TSDLLEXPORT Dimension *ts_hyperspace_get_mutable_dimension(Hyperspace *hs, diff --git a/src/dimension_slice.h b/src/dimension_slice.h index 04122048f79..0ad985aabfa 100644 --- a/src/dimension_slice.h +++ b/src/dimension_slice.h @@ -58,8 +58,8 @@ ts_dimension_slice_scan_range_limit(int32 dimension_id, StrategyNumber start_str int limit, const ScanTupLock *tuplock); extern DimensionVec *ts_dimension_slice_collision_scan_limit(int32 dimension_id, int64 range_start, int64 range_end, int limit); -extern bool ts_dimension_slice_scan_for_existing(const DimensionSlice *slice, - const ScanTupLock *tuplock); +extern TSDLLEXPORT bool ts_dimension_slice_scan_for_existing(const DimensionSlice *slice, + const ScanTupLock *tuplock); extern DimensionSlice *ts_dimension_slice_scan_by_id_and_lock(int32 dimension_slice_id, const ScanTupLock *tuplock, MemoryContext mctx, @@ -70,18 +70,20 @@ extern DimensionVec *ts_dimension_slice_scan_by_dimension_before_point(int32 dim ScanDirection scandir, MemoryContext mctx); extern int ts_dimension_slice_delete_by_dimension_id(int32 dimension_id, bool delete_constraints); -extern int ts_dimension_slice_delete_by_id(int32 dimension_slice_id, bool delete_constraints); +extern TSDLLEXPORT int ts_dimension_slice_delete_by_id(int32 dimension_slice_id, + bool delete_constraints); extern TSDLLEXPORT DimensionSlice *ts_dimension_slice_create(int dimension_id, int64 range_start, int64 range_end); extern TSDLLEXPORT DimensionSlice *ts_dimension_slice_copy(const DimensionSlice *original); extern TSDLLEXPORT bool ts_dimension_slices_collide(const DimensionSlice *slice1, const DimensionSlice *slice2); -extern bool ts_dimension_slices_equal(const DimensionSlice *slice1, const DimensionSlice *slice2); +extern TSDLLEXPORT bool ts_dimension_slices_equal(const DimensionSlice *slice1, + const DimensionSlice *slice2); extern bool ts_dimension_slice_cut(DimensionSlice *to_cut, const DimensionSlice *other, int64 coord); extern void ts_dimension_slice_free(DimensionSlice *slice); extern int ts_dimension_slice_insert_multi(DimensionSlice **slice, Size num_slices); -extern void ts_dimension_slice_insert(DimensionSlice *slice); +extern TSDLLEXPORT void ts_dimension_slice_insert(DimensionSlice *slice); extern int ts_dimension_slice_cmp(const DimensionSlice *left, const DimensionSlice *right); extern int ts_dimension_slice_cmp_coordinate(const DimensionSlice *slice, int64 coord); diff --git a/src/hypercube.h b/src/hypercube.h index 1190c9af04a..63cb3cc9a4f 100644 --- a/src/hypercube.h +++ b/src/hypercube.h @@ -27,7 +27,7 @@ typedef struct Hypercube (sizeof(Hypercube) + (sizeof(DimensionSlice *) * (num_dimensions))) extern TSDLLEXPORT Hypercube *ts_hypercube_alloc(int16 num_dimensions); -extern void ts_hypercube_free(Hypercube *hc); +extern TSDLLEXPORT void ts_hypercube_free(Hypercube *hc); extern TSDLLEXPORT DimensionSlice * ts_hypercube_add_slice_from_range(Hypercube *hc, int32 dimension_id, int64 start, int64 end); @@ -41,6 +41,6 @@ extern Hypercube *ts_hypercube_calculate_from_point(const Hyperspace *hs, const extern bool ts_hypercubes_collide(const Hypercube *cube1, const Hypercube *cube2); extern TSDLLEXPORT const DimensionSlice *ts_hypercube_get_slice_by_dimension_id(const Hypercube *hc, int32 dimension_id); -extern Hypercube *ts_hypercube_copy(const Hypercube *hc); +extern TSDLLEXPORT Hypercube *ts_hypercube_copy(const Hypercube *hc); extern bool ts_hypercube_equal(const Hypercube *hc1, const Hypercube *hc2); extern void ts_hypercube_slice_sort(Hypercube *hc); diff --git a/tsl/src/chunk.c b/tsl/src/chunk.c index d9ebf1444a1..6ad0a406832 100644 --- a/tsl/src/chunk.c +++ b/tsl/src/chunk.c @@ -3,44 +3,65 @@ * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ - #include +#include #include +#include +#include +#include +#include +#include #include +#include #include +#include #include +#include +#include +#include +#include +#include #include #include +#include +#include +#include +#include #include #include #include #include #include +#include #include +#include #include #include +#include +#include #include +#include +#include #include #include +#include +#include #include +#include #include #include #include #include #include -#ifdef USE_ASSERT_CHECKING -#endif - -#include -#include "hypercube.h" -#include -#include -#include -#include +#include "annotations.h" +#include "cache.h" #include "chunk.h" -#include "chunk_api.h" #include "debug_point.h" +#include "extension.h" +#include "hypercube.h" +#include "hypertable.h" +#include "hypertable_cache.h" #include "utils.h" /* Data in a frozen chunk cannot be modified. So any operation @@ -186,3 +207,862 @@ chunk_invoke_drop_chunks(Oid relid, Datum older_than, Datum older_than_type, boo return num_results; } + +typedef struct RelationMergeInfo +{ + Oid relid; + struct VacuumCutoffs cutoffs; + const Chunk *chunk; + Relation rel; +} RelationMergeInfo; + +typedef enum MergeLockUpgrade +{ + MERGE_LOCK_UPGRADE, + MERGE_LOCK_CONDITIONAL_UPGRADE, + MERGE_LOCK_ACCESS_EXCLUSIVE, +} MergeLockUpgrade; + +static void +compute_rel_vacuum_cutoffs(Relation rel, struct VacuumCutoffs *cutoffs) +{ + VacuumParams params; + + memset(¶ms, 0, sizeof(VacuumParams)); + vacuum_get_cutoffs(rel, ¶ms, cutoffs); + + /* Frozen Id should not go backwards */ + TransactionId relfrozenxid = rel->rd_rel->relfrozenxid; + + if (TransactionIdIsValid(relfrozenxid) && + TransactionIdPrecedes(cutoffs->FreezeLimit, relfrozenxid)) + cutoffs->FreezeLimit = relfrozenxid; + + MultiXactId relminmxid = rel->rd_rel->relminmxid; + + if (MultiXactIdIsValid(relminmxid) && MultiXactIdPrecedes(cutoffs->MultiXactCutoff, relminmxid)) + cutoffs->MultiXactCutoff = relminmxid; +} + +static void +merge_chunks_finish(Oid new_relid, RelationMergeInfo *relinfos, int nrelids, + TransactionId freeze_limit, MultiXactId multi_cutoff, char relpersistence, + MergeLockUpgrade lock_upgrade) +{ + /* + * The relations being merged are currently locked in ExclusiveLock, which + * means other readers can have locks. To delete the relations, we first + * need to upgrade to an exclusive lock. However, this might lead to + * deadlocks so we need to bail out if we cannot get the lock immediately. + */ + for (int i = 0; i < nrelids; i++) + { + Oid relid = relinfos[i].relid; + + switch (lock_upgrade) + { + case MERGE_LOCK_CONDITIONAL_UPGRADE: + if (!ConditionalLockRelationOid(relid, AccessExclusiveLock)) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("could not lock relation \"%s\" for merge", + get_rel_name(relid)))); + break; + case MERGE_LOCK_UPGRADE: + LockRelationOid(relid, AccessExclusiveLock); + break; + case MERGE_LOCK_ACCESS_EXCLUSIVE: + /* We should already hold AccessExclusivelock. Could preventively + * take it or assert the lock is taken, but it would require + * opening the relation again. */ + break; + } + } + + finish_heap_swap(relinfos[0].relid, + new_relid, + false, /* system catalog */ + false /* swap toast by content */, + false, /* check constraints */ + true, /* internal? */ + freeze_limit, + multi_cutoff, + relpersistence); + + /* + * Delete all the merged relations except the first one, since we are + * keeping it for the heap swap. + */ + ObjectAddresses *objects = new_object_addresses(); + + for (int i = 1; i < nrelids; i++) + { + Oid relid = relinfos[i].relid; + ObjectAddress object = { + .classId = RelationRelationId, + .objectId = relid, + }; + + /* Cannot drop if relation is still open */ + Assert(relinfos[i].rel == NULL); + + if (relinfos[i].chunk) + { + const Oid namespaceid = get_rel_namespace(relid); + const char *schemaname = get_namespace_name(namespaceid); + const char *tablename = get_rel_name(relid); + + ts_chunk_delete_by_name(schemaname, tablename, DROP_RESTRICT); + } + + add_exact_object_address(&object, objects); + } + + performMultipleDeletions(objects, DROP_RESTRICT, PERFORM_DELETION_INTERNAL); + free_object_addresses(objects); +} + +static int +cmp_relations(const void *left, const void *right) +{ + const RelationMergeInfo *linfo = ((RelationMergeInfo *) left); + const RelationMergeInfo *rinfo = ((RelationMergeInfo *) right); + + if (linfo->chunk && rinfo->chunk) + { + const Hypercube *lcube = linfo->chunk->cube; + const Hypercube *rcube = rinfo->chunk->cube; + + Assert(lcube->num_slices == rcube->num_slices); + + for (int i = 0; i < lcube->num_slices; i++) + { + const DimensionSlice *lslice = lcube->slices[i]; + const DimensionSlice *rslice = rcube->slices[i]; + + Assert(lslice->fd.dimension_id == rslice->fd.dimension_id); + + /* Compare start of range for the dimension */ + if (lslice->fd.range_start < rslice->fd.range_start) + return -1; + + if (lslice->fd.range_start > rslice->fd.range_start) + return 1; + + /* If start of range is equal, compare by end of range */ + if (lslice->fd.range_end < rslice->fd.range_end) + return -1; + + if (lslice->fd.range_end > rslice->fd.range_end) + return 1; + } + + /* Should only reach here if partitioning is equal across all + * dimensions. Fall back to comparing relids. */ + } + + return pg_cmp_u32(linfo->relid, rinfo->relid); +} + +/* + * Check that the partition boundaries of two chunks align so that a new valid + * hypercube can be formed if the chunks are merged. This check assumes that + * the hypercubes are sorted so that cube2 "follows" cube1. + * + * The algorithm is simple and only allows merging along a single dimension in + * the same merge. For example, these two cases are mergeable: + * + * ' ____ + * ' |__| + * ' |__| + * + * ' _______ + * ' |__|__| + * + * while these cases are not mergeable: + * ' ____ + * ' __|__| + * ' |__| + * + * ' ______ + * ' |____| + * ' |__| + * + * + * The validation can handle merges of many chunks at once if they are + * "naively" aligned and this function is called on chunk hypercubes in + * "partition order": + * + * ' _____________ + * ' |__|__|__|__| + * + * However, the validation currently won't accept merges of multiple + * dimensions at once: + * + * ' _____________ + * ' |__|__|__|__| + * ' |__|__|__|__| + * + * It also cannot handle complicated merges of multi-dimensional partitioning + * schemes like the one below. + * + * ' _________ + * ' |__a____| + * ' |_b_|_c_| + * + * Merging a,b,c, should be possible but the validation currently cannot + * handle such cases. Instead, it is necessary to first merge b,c. Then merge + * a with the result (b,c) in a separate merge. Note that it is not possible + * to merge only a,b or a,c. + * + * A future, more advanced, validation needs to handle corner-cases like the + * one below that has gaps: + * + * ' _____________ + * ' |__|__|__|__| + * ' |____| |___| + * ' + */ +static void +validate_merge_possible(const Hypercube *cube1, const Hypercube *cube2) +{ + int follow_edges = 0; + int equal_edges = 0; + + Assert(cube1->num_slices == cube2->num_slices); + + for (int i = 0; i < cube1->num_slices; i++) + { + const DimensionSlice *slice1 = cube1->slices[i]; + const DimensionSlice *slice2 = cube2->slices[i]; + + if (ts_dimension_slices_equal(slice1, slice2)) + equal_edges++; + + if (slice1->fd.range_end == slice2->fd.range_start) + follow_edges++; + } + + if (follow_edges != 1 || (cube1->num_slices - equal_edges) != 1) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot create new chunk partition boundaries"), + errhint("Try merging chunks that have adjacent partitions."))); +} + +static const ChunkConstraint * +get_chunk_constraint_by_slice_id(const ChunkConstraints *ccs, int32 slice_id) +{ + for (int i = 0; i < ccs->num_constraints; i++) + { + const ChunkConstraint *cc = &ccs->constraints[i]; + + if (cc->fd.dimension_slice_id == slice_id) + return cc; + } + + return NULL; +} + +static void +chunk_update_constraints(const Chunk *chunk, const Hypercube *new_cube) +{ + Cache *hcache; + const Hypertable *ht = + ts_hypertable_cache_get_cache_and_entry(chunk->hypertable_relid, CACHE_FLAG_NONE, &hcache); + List *new_constraints = NIL; + + for (int i = 0; i < new_cube->num_slices; i++) + { + const DimensionSlice *old_slice = chunk->cube->slices[i]; + DimensionSlice *new_slice = new_cube->slices[i]; + const ChunkConstraint *cc; + ScanTupLock tuplock = { + .waitpolicy = LockWaitBlock, + .lockmode = LockTupleShare, + }; + + /* The new slice has merged range, but still old ID. Should match with + * the old slice. */ + Assert(old_slice->fd.id == new_slice->fd.id); + + /* If nothing changed in this dimension, move on to the next */ + if (ts_dimension_slices_equal(old_slice, new_slice)) + continue; + + cc = get_chunk_constraint_by_slice_id(chunk->constraints, old_slice->fd.id); + + if (cc) + { + ObjectAddress constrobj = { + .classId = ConstraintRelationId, + .objectId = get_relation_constraint_oid(chunk->table_id, + NameStr(cc->fd.constraint_name), + false), + }; + + performDeletion(&constrobj, DROP_RESTRICT, 0); + + /* Create the new check constraint */ + const Dimension *dim = + ts_hyperspace_get_dimension_by_id(ht->space, old_slice->fd.dimension_id); + Constraint *constr = + ts_chunk_constraint_dimensional_create(dim, + new_slice, + NameStr(cc->fd.constraint_name)); + + /* Constraint could be NULL, e.g., if the merged chunk covers the + * entire range in a space dimension it needs no constraint. */ + if (constr != NULL) + new_constraints = lappend(new_constraints, constr); + } + + /* Check if there's already a slice with the new range. If so, avoid + * inserting a new slice. */ + if (!ts_dimension_slice_scan_for_existing(new_slice, &tuplock)) + { + new_slice->fd.id = -1; + ts_dimension_slice_insert(new_slice); + /* A new Id should be assigned */ + Assert(new_slice->fd.id > 0); + } + + /* Update the chunk constraint to point to the new slice ID */ + ts_chunk_constraint_update_slice_id(chunk->fd.id, old_slice->fd.id, new_slice->fd.id); + + /* Delete the old slice if it is orphaned now */ + if (ts_chunk_constraint_scan_by_dimension_slice_id(old_slice->fd.id, + NULL, + CurrentMemoryContext) == 0) + { + ts_dimension_slice_delete_by_id(old_slice->fd.id, false); + } + } + + /* Add new check constraints, if any */ + if (new_constraints != NIL) + { + /* Adding a constraint should require AccessExclusivelock. It should + * already be taken at this point, but specify it to be sure. */ + Relation rel = table_open(chunk->table_id, AccessExclusiveLock); + AddRelationNewConstraints(rel, + NIL /* List *newColDefaults */, + new_constraints, + false /* allow_merge */, + true /* is_local */, + false /* is_internal */, + NULL /* query string */); + table_close(rel, NoLock); + } + + ts_cache_release(hcache); +} + +static void +merge_cubes(Hypercube *merged_cube, const Hypercube *cube) +{ + /* Merge dimension slices */ + for (int i = 0; i < cube->num_slices; i++) + { + const DimensionSlice *slice = cube->slices[i]; + DimensionSlice *merged_slice = merged_cube->slices[i]; + + Assert(slice->fd.dimension_id == merged_slice->fd.dimension_id); + + if (slice->fd.range_start < merged_slice->fd.range_start) + merged_slice->fd.range_start = slice->fd.range_start; + + if (slice->fd.range_end > merged_slice->fd.range_end) + merged_slice->fd.range_end = slice->fd.range_end; + } +} + +/* + * Get the locking mode for merge chunks. + * + * By default, a merge happens with access exclusive locks taken on chunks in + * order to avoid deadlocks. It is possible to use a weaker exclusive lock by + * setting a session variable, thus allowing reads during merges. However, + * that can easily lead to deadlocks as shown in isolation tests. Therefore, + * use the stricter locking settings by default. + */ +static MergeLockUpgrade +merge_chunks_lock_upgrade_mode(void) +{ + const char *lockupgrade = + GetConfigOption("timescaledb.merge_chunks_lock_upgrade_mode", true, false); + + if (lockupgrade == NULL) + return MERGE_LOCK_ACCESS_EXCLUSIVE; + + if (strcmp("upgrade", lockupgrade) == 0) + return MERGE_LOCK_UPGRADE; + + if (strcmp("conditional", lockupgrade) == 0) + return MERGE_LOCK_CONDITIONAL_UPGRADE; + + return MERGE_LOCK_ACCESS_EXCLUSIVE; +} + +#if (PG_VERSION_NUM >= 170000 && PG_VERSION_NUM <= 170002) +/* + * Workaround for changed behavior in the relation rewrite code that appeared + * in PostgreSQL 17.0, but was fixed in 17.3. + * + * Merge chunks uses the relation rewrite functionality from CLUSTER and + * VACUUM FULL. This works for merge because, when writing into a non-empty + * relation, new pages are appended while the existing pages remain the + * same. In PG17.0, however, that changed so that existing pages in the + * relation were zeroed out. The changed behavior was introduced as part of + * this commit: + * + * https://github.com/postgres/postgres/commit/8af256524893987a3e534c6578dd60edfb782a77 + * + * Fortunately, this was fixed in a follow up commit: + * + * https://github.com/postgres/postgres/commit/9695835538c2c8e9cd0048028b8c85e1bbf5c79c + * + * The fix is part of PG 17.3. Howevever, this still leaves PG 17.0 - 17.2 + * with different behavior. + * + * To make the merge chunks code work for the "broken" versions we make PG + * believe the first rewrite operation is the size of the fully merged + * relation so that we reserve the full space needed and then "append" + * backwards into the zeroed space (see illustration below). By doing this, we + * ensure that no valid data is zeroed out. The downside of this approach is + * that there will be a lot of unnecessary writing of zero pages. Below is an + * example of what the rewrite would look like for merging three relations + * with one page each. When writing the first relation, PG believes the merged + * relation already contains two pages when starting the rewrite. These two + * existing pages will be zeroed. When writing the next relation we tell PG + * that there is only one existing page in the merged relation, and so forth. + * + * _____________ + * |_0_|_0_|_x_| + * _________ + * |_0_|_x_| + * _____ + * |_x_| + * + * Result: + * _____________ + * |_x_|_x_|_x_| + * + */ +static BlockNumber merge_rel_nblocks = 0; +static BlockNumber *blockoff = NULL; +static const TableAmRoutine *old_routine = NULL; +static TableAmRoutine routine = {}; + +/* + * TAM relation size function to make PG believe that the merged relation + * contains as specific amount of existing data. + */ +static uint64 +pq17_workaround_merge_relation_size(Relation rel, ForkNumber forkNumber) +{ + if (forkNumber == MAIN_FORKNUM) + return merge_rel_nblocks * BLCKSZ; + + return old_routine->relation_size(rel, forkNumber); +} + +static inline void +pg17_workaround_init(Relation rel, RelationMergeInfo *relinfos, int nrelids) +{ + routine = *rel->rd_tableam; + routine.relation_size = pq17_workaround_merge_relation_size; + old_routine = rel->rd_tableam; + rel->rd_tableam = &routine; + blockoff = palloc(sizeof(BlockNumber) * nrelids); + uint64 totalblocks = 0; + + for (int i = 0; i < nrelids; i++) + { + blockoff[i] = (BlockNumber) totalblocks; + totalblocks += RelationGetNumberOfBlocks(relinfos[i].rel); + + /* Ensure the offsets don't overflow. For the merge itself, it is + * assumed that the write will fail when writing too many blocks */ + Ensure(totalblocks <= MaxBlockNumber, "max number of blocks exceeded for merge"); + } +} + +static inline void +pg17_workaround_cleanup(Relation rel) +{ + pfree(blockoff); + rel->rd_tableam = old_routine; +} + +static inline RelationMergeInfo * +get_relmergeinfo(RelationMergeInfo *relinfos, int nrelids, int i) +{ + RelationMergeInfo *relinfo = &relinfos[nrelids - i - 1]; + merge_rel_nblocks = blockoff[nrelids - i - 1]; + return relinfo; +} + +#else +#define pg17_workaround_init(rel, relinfos, nrelids) +#define pg17_workaround_cleanup(rel) +#define get_relmergeinfo(relinfos, nrelids, i) &relinfos[i] +#endif + +/* + * Merge N chunk relations into one chunk based on Oids. + * + * The input chunk relations are ordered according to partition ranges and the + * "first" relation in that ordered list will be "kept" to hold the merged + * data. The merged chunk will have its partition ranges updated to cover the + * ranges of all the merged chunks. + * + * The merge happens via a heap rewrite, followed by a heap swap, essentially + * the same approach implemented by CLUSTER and VACUUM FULL, but applied on + * several relations in the same operation (many to one). + * + * + * The heap swap approach handles visibility across all PG isolation levels, + * as implemented by the cluster code. + * + * In the first step, all data from each chunk is written to a temporary heap + * (accounting for vacuum, half-dead/visible, and frozen tuples). In the + * second step, a heap swap is performed on one of the chunks and all metadata + * is rewritten to handle, e.g., new partition ranges. Finally, the old chunks + * are dropped, except for the chunk that received the heap swap. + * + * To be able to merge, the function checks that: + * + * - all relations are tables (not, e.g,, views) + * - all relations use same (or compatible) storage on disk + * - all relations are chunks (and not, e.g., foreign/OSM chunks) + * + * A current limitation is that it is not possible to merge compressed chunks + * since this requires additional functionality, such as: + * + * - Handling merge of compressed and non-compressed chunks + * + * - Merging chunks with different compression settings (e.g., different + * orderby or segmentby) + * + * - Merging partial chunks + * + * - Updating additional metadata of the internal compressed relations + */ +Datum +chunk_merge_chunks(PG_FUNCTION_ARGS) +{ + ArrayType *chunks_array = PG_ARGISNULL(0) ? NULL : PG_GETARG_ARRAYTYPE_P(0); + Datum *relids; + bool *nulls; + int nrelids; + RelationMergeInfo *relinfos; + int32 hypertable_id = INVALID_HYPERTABLE_ID; + Hypercube *merged_cube = NULL; + const Hypercube *prev_cube = NULL; + const MergeLockUpgrade lock_upgrade = merge_chunks_lock_upgrade_mode(); + + PreventCommandIfReadOnly("merge_chunks"); + + if (chunks_array == NULL) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("no chunks to merge specified"))); + + deconstruct_array(chunks_array, + REGCLASSOID, + sizeof(Oid), + true, + TYPALIGN_INT, + &relids, + &nulls, + &nrelids); + + if (nrelids < 2) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("must specify at least two chunks to merge"))); + + relinfos = palloc0(sizeof(struct RelationMergeInfo) * nrelids); + + /* Sort relids array in order to find duplicates and lock relations in + * consistent order to avoid deadlocks. It doesn't matter that we don't + * order the nulls array the same since we only care about all relids + * being non-null. */ + qsort(relids, nrelids, sizeof(Datum), oid_cmp); + + /* Step 1: Do sanity checks and then prepare to sort rels in consistent order. */ + for (int i = 0; i < nrelids; i++) + { + Oid relid = DatumGetObjectId(relids[i]); + const Chunk *chunk; + Relation rel; + Oid amoid; + + if (nulls[i] || !OidIsValid(relid)) + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid relation"))); + + if (i > 0 && DatumGetObjectId(relids[i]) == DatumGetObjectId(relids[i - 1])) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("duplicate relation \"%s\" in merge", + get_rel_name(DatumGetObjectId(relids[i]))))); + + /* Lock the relation before doing other checks that can lock dependent + * objects (this can otherwise lead to deadlocks with concurrent + * operations). Note that if we take ExclusiveLock here to allow + * readers while we are rewriting/merging the relations, the lock + * needs to be upgraded to an AccessExclusiveLock later. This can also + * lead to deadlocks. + * + * Ideally, we should probably take locks on all dependent objects as + * well, at least on chunk-related objects that will be + * dropped. Otherwise, that might also cause deadlocks later. For + * example, if doing a concurrent DROP TABLE on one of the chunks will + * lead to deadlock because it grabs locks on all dependencies before + * dropping. + * + * However, for now we won't do that because that requires scanning + * pg_depends and concurrent operations will probably fail anyway if + * we remove the objects. We might as well fail with a deadlock. + */ + LOCKMODE lockmode = + (lock_upgrade == MERGE_LOCK_ACCESS_EXCLUSIVE) ? AccessExclusiveLock : ExclusiveLock; + + rel = try_table_open(relid, lockmode); + + /* Check if the table actually exists. If not, it could have been + * deleted in a concurrent merge. */ + if (rel == NULL) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_TABLE), + errmsg("relation does not exist"), + errdetail("The relation with OID %u might have been removed " + "by a concurrent merge or other operation.", + relid))); + + if (rel->rd_rel->relkind != RELKIND_RELATION) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot merge non-table relations"))); + + /* Only owner is allowed to merge */ + if (!object_ownercheck(RelationRelationId, relid, GetUserId())) + aclcheck_error(ACLCHECK_NOT_OWNER, + get_relkind_objtype(rel->rd_rel->relkind), + get_rel_name(relid)); + + /* Lock toast table to prevent it from being concurrently vacuumed */ + if (rel->rd_rel->reltoastrelid) + LockRelationOid(rel->rd_rel->reltoastrelid, lockmode); + + /* + * Check for active uses of the relation in the current transaction, + * including open scans and pending AFTER trigger events. + */ + CheckTableNotInUse(rel, "merge_chunks"); + + if (IsSystemRelation(rel)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cannot merge system catalog relations"))); + + /* + * Find the chunk corresponding to the relation for final checks. Done + * after locking the chunk relation because scanning for the chunk + * will grab locks on other objects, which might otherwise lead to + * deadlocks during concurrent merges instead of more helpful messages + * (like chunk does not exist because it was merged). + */ + chunk = ts_chunk_get_by_relid(relid, false); + + if (NULL == chunk) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("can only merge hypertable chunks"))); + + if (chunk->fd.osm_chunk) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot merge OSM chunks"))); + + if (chunk->fd.compressed_chunk_id != INVALID_CHUNK_ID) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("merging compressed chunks is not yet supported"), + errhint("Decompress the chunks before merging."))); + + if (hypertable_id == INVALID_HYPERTABLE_ID) + hypertable_id = chunk->fd.hypertable_id; + else if (hypertable_id != chunk->fd.hypertable_id) + { + Assert(i > 0); + + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot merge chunks across different hypertables"), + errdetail("Chunk \"%s\" is part of hypertable \"%s\" while chunk \"%s\" is " + "part of hypertable \"%s\"", + get_rel_name(chunk->table_id), + get_rel_name(chunk->hypertable_relid), + get_rel_name(relinfos[i - 1].chunk->table_id), + get_rel_name(relinfos[i - 1].chunk->hypertable_relid)))); + } + + /* + * It might not be possible to merge two chunks with different + * storage, so better safe than sorry for now. + */ + amoid = rel->rd_rel->relam; + + if (amoid != HEAP_TABLE_AM_OID) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("access method \"%s\" is not supported for merge", + get_am_name(amoid)))); + + relinfos[i].relid = relid; + relinfos[i].rel = rel; + relinfos[i].chunk = chunk; + } + + /* Sort rels in partition order (in case of chunks). This is necessary to + * validate that a merge is possible. */ + qsort(relinfos, nrelids, sizeof(RelationMergeInfo), cmp_relations); + + /* Step 2: Check alignment/mergeability and create the merged hypercube + * (partition ranges). */ + for (int i = 0; i < nrelids; i++) + { + const Chunk *chunk = relinfos[i].chunk; + + Assert(chunk != NULL); + + if (merged_cube == NULL) + { + merged_cube = ts_hypercube_copy(chunk->cube); + Assert(prev_cube == NULL); + } + else + { + Assert(chunk->cube->num_slices == merged_cube->num_slices); + Assert(prev_cube != NULL); + validate_merge_possible(prev_cube, chunk->cube); + merge_cubes(merged_cube, chunk->cube); + } + + prev_cube = chunk->cube; + compute_rel_vacuum_cutoffs(relinfos[i].rel, &relinfos[i].cutoffs); + } + + /* + * Keep the first of the ordered relations. It will receive a heap + * swap. + */ + Relation result_rel = relinfos[0].rel; + /* These will be our final cutoffs for the merged relation */ + struct VacuumCutoffs *cutoffs = &relinfos[0].cutoffs; + + Oid tablespace = result_rel->rd_rel->reltablespace; + char relpersistence = result_rel->rd_rel->relpersistence; + + /* Create the transient heap that will receive the re-ordered data */ + Oid new_relid = make_new_heap_compat(RelationGetRelid(result_rel), + tablespace, + result_rel->rd_rel->relam, + relpersistence, + ExclusiveLock); + Relation new_rel = table_open(new_relid, AccessExclusiveLock); + double total_num_tuples = 0.0; + + pg17_workaround_init(new_rel, relinfos, nrelids); + + /* Step 3: write the data from all the rels into a new merged heap */ + for (int i = 0; i < nrelids; i++) + { + RelationMergeInfo *relinfo = get_relmergeinfo(relinfos, nrelids, i); + struct VacuumCutoffs *cutoffs_i = &relinfo->cutoffs; + Relation rel = relinfo->rel; + + double num_tuples = 0.0; + double tups_vacuumed = 0.0; + double tups_recently_dead = 0.0; + + table_relation_copy_for_cluster(rel, + new_rel, + NULL, + false, + cutoffs_i->OldestXmin, + &cutoffs_i->FreezeLimit, + &cutoffs_i->MultiXactCutoff, + &num_tuples, + &tups_vacuumed, + &tups_recently_dead); + + elog(LOG, + "merged rows from \"%s\" into \"%s\": tuples %lf vacuumed %lf recently dead %lf", + RelationGetRelationName(rel), + RelationGetRelationName(result_rel), + num_tuples, + tups_vacuumed, + tups_recently_dead); + + total_num_tuples += num_tuples; + + if (TransactionIdPrecedes(cutoffs->FreezeLimit, cutoffs_i->FreezeLimit)) + cutoffs->FreezeLimit = cutoffs_i->FreezeLimit; + + if (MultiXactIdPrecedes(cutoffs->MultiXactCutoff, cutoffs_i->MultiXactCutoff)) + cutoffs->MultiXactCutoff = cutoffs_i->MultiXactCutoff; + + /* Close the relations before the heap swap, but keep the locks until + * end of transaction. */ + table_close(rel, NoLock); + relinfo->rel = NULL; + } + + pg17_workaround_cleanup(new_rel); + + /* Update table stats */ + Relation relRelation = table_open(RelationRelationId, RowExclusiveLock); + HeapTuple reltup = SearchSysCacheCopy1(RELOID, ObjectIdGetDatum(new_relid)); + if (!HeapTupleIsValid(reltup)) + elog(ERROR, "cache lookup failed for relation %u", new_relid); + Form_pg_class relform = (Form_pg_class) GETSTRUCT(reltup); + BlockNumber num_pages = RelationGetNumberOfBlocks(new_rel); + relform->relpages = num_pages; + relform->reltuples = total_num_tuples; + + CatalogTupleUpdate(relRelation, &reltup->t_self, reltup); + heap_freetuple(reltup); + table_close(relRelation, RowExclusiveLock); + CommandCounterIncrement(); + + table_close(new_rel, NoLock); + + DEBUG_WAITPOINT("merge_chunks_before_heap_swap"); + + /* Step 4: Keep one of the original rels but transplant the merged heap + * into it using a heap swap. Then close and delete the remaining merged + * rels. */ + merge_chunks_finish(new_relid, + relinfos, + nrelids, + cutoffs->FreezeLimit, + cutoffs->MultiXactCutoff, + relpersistence, + lock_upgrade); + + /* Step 5: Update the dimensional metadata and constraints for the chunk + * we are keeping. */ + if (merged_cube) + { + Assert(relinfos[0].chunk); + chunk_update_constraints(relinfos[0].chunk, merged_cube); + ts_hypercube_free(merged_cube); + } + + pfree(relids); + pfree(nulls); + pfree(relinfos); + + PG_RETURN_VOID(); +} diff --git a/tsl/src/chunk.h b/tsl/src/chunk.h index 496a296df38..5c8db95b77d 100644 --- a/tsl/src/chunk.h +++ b/tsl/src/chunk.h @@ -13,3 +13,4 @@ extern Datum chunk_freeze_chunk(PG_FUNCTION_ARGS); extern Datum chunk_unfreeze_chunk(PG_FUNCTION_ARGS); extern int chunk_invoke_drop_chunks(Oid relid, Datum older_than, Datum older_than_type, bool use_creation_time); +extern Datum chunk_merge_chunks(PG_FUNCTION_ARGS); diff --git a/tsl/src/init.c b/tsl/src/init.c index 50db7ac8c07..594d2277987 100644 --- a/tsl/src/init.c +++ b/tsl/src/init.c @@ -191,6 +191,7 @@ CrossModuleFunctions tsl_cm_functions = { .get_compressed_chunk_index_for_recompression = tsl_get_compressed_chunk_index_for_recompression, .preprocess_query_tsl = tsl_preprocess_query, + .merge_chunks = chunk_merge_chunks, }; static void diff --git a/tsl/test/expected/merge_chunks.out b/tsl/test/expected/merge_chunks.out new file mode 100644 index 00000000000..77c36dd6123 --- /dev/null +++ b/tsl/test/expected/merge_chunks.out @@ -0,0 +1,490 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. +\c :TEST_DBNAME :ROLE_SUPERUSER +CREATE ACCESS METHOD testam TYPE TABLE HANDLER heap_tableam_handler; +set role :ROLE_DEFAULT_PERM_USER; +------------------ +-- Helper views -- +------------------- +create view partitions as +select c.table_name, d.column_name, ds.range_start, ds.range_end +from _timescaledb_catalog.hypertable h +join _timescaledb_catalog.chunk c on (c.hypertable_id = h.id) +join _timescaledb_catalog.dimension d on (d.hypertable_id = h.id) +join _timescaledb_catalog.dimension_slice ds on (d.id = ds.dimension_id) +join _timescaledb_catalog.chunk_constraint cc on (cc.chunk_id = c.id and cc.dimension_slice_id = ds.id) +where h.table_name = 'mergeme' +order by d.id, ds.range_start, ds.range_end; +create view orphaned_slices as +select ds.id, cc.constraint_name from _timescaledb_catalog.dimension_slice ds +left join _timescaledb_catalog.chunk_constraint cc on (ds.id = cc.dimension_slice_id) +where cc.constraint_name is null; +----------------- +-- Setup table -- +----------------- +create table mergeme (time timestamptz not null, device int, temp float); +select create_hypertable('mergeme', 'time', 'device', 3, chunk_time_interval => interval '1 day'); + create_hypertable +---------------------- + (1,public,mergeme,t) +(1 row) + +-- +-- Insert data to create two chunks with same time ranges like this: +-- _______ +-- | | +-- | 1 | +-- |_____| +-- | | +-- | 2 | +-- |_____| +--- +insert into mergeme values ('2024-01-01', 1, 1.0), ('2024-01-01', 2, 2.0); +select "Constraint", "Columns", "Expr" from test.show_constraints('_timescaledb_internal._hyper_1_1_chunk'); + Constraint | Columns | Expr +--------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------ + constraint_1 | {time} | (("time" >= 'Sun Dec 31 16:00:00 2023 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 01 16:00:00 2024 PST'::timestamp with time zone)) + constraint_2 | {device} | (_timescaledb_functions.get_partition_hash(device) < 715827882) +(2 rows) + +-- Show partition layout +select * from partitions; + table_name | column_name | range_start | range_end +------------------+-------------+----------------------+------------------ + _hyper_1_1_chunk | time | 1704067200000000 | 1704153600000000 + _hyper_1_2_chunk | time | 1704067200000000 | 1704153600000000 + _hyper_1_1_chunk | device | -9223372036854775808 | 715827882 + _hyper_1_2_chunk | device | 715827882 | 1431655764 +(4 rows) + +-- Now merge chunk 1 and 2: +begin; +call merge_chunks('_timescaledb_internal._hyper_1_1_chunk', '_timescaledb_internal._hyper_1_2_chunk'); +select * from _timescaledb_internal._hyper_1_1_chunk; + time | device | temp +------------------------------+--------+------ + Mon Jan 01 00:00:00 2024 PST | 1 | 1 + Mon Jan 01 00:00:00 2024 PST | 2 | 2 +(2 rows) + +select reltuples from pg_class where oid='_timescaledb_internal._hyper_1_1_chunk'::regclass; + reltuples +----------- + 2 +(1 row) + +select * from partitions; + table_name | column_name | range_start | range_end +------------------+-------------+----------------------+------------------ + _hyper_1_1_chunk | time | 1704067200000000 | 1704153600000000 + _hyper_1_1_chunk | device | -9223372036854775808 | 1431655764 +(2 rows) + +select "Constraint", "Columns", "Expr" from test.show_constraints('_timescaledb_internal._hyper_1_1_chunk'); + Constraint | Columns | Expr +--------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------ + constraint_1 | {time} | (("time" >= 'Sun Dec 31 16:00:00 2023 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 01 16:00:00 2024 PST'::timestamp with time zone)) + constraint_2 | {device} | (_timescaledb_functions.get_partition_hash(device) < 1431655764) +(2 rows) + +select count(*) as num_orphaned_slices from orphaned_slices; + num_orphaned_slices +--------------------- + 0 +(1 row) + +select * from show_chunks('mergeme'); + show_chunks +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk +(1 row) + +select * from mergeme; + time | device | temp +------------------------------+--------+------ + Mon Jan 01 00:00:00 2024 PST | 1 | 1 + Mon Jan 01 00:00:00 2024 PST | 2 | 2 +(2 rows) + +rollback; +-- create a new chunk as a third space partition +-- _______ +-- | | +-- | 1 | +-- |_____| +-- | | +-- | 2 | +-- |_____| +-- | | +-- | 3 | +-- |_____| +--- +insert into mergeme values ('2024-01-01', 3, 3.0); +-- Test some basic error cases +\set ON_ERROR_STOP 0 +-- Can't merge chunk 1 and 3 +call merge_chunks('_timescaledb_internal._hyper_1_1_chunk', '_timescaledb_internal._hyper_1_3_chunk'); +ERROR: cannot create new chunk partition boundaries +call merge_chunks(NULL); +ERROR: no chunks to merge specified +call merge_chunks(NULL, NULL); +ERROR: invalid relation +call merge_chunks(999999,999991); +ERROR: relation does not exist +call merge_chunks(ARRAY['_timescaledb_internal._hyper_1_1_chunk']); +ERROR: must specify at least two chunks to merge +call merge_chunks('_timescaledb_internal._hyper_1_1_chunk', NULL); +ERROR: invalid relation +call merge_chunks(ARRAY['_timescaledb_internal._hyper_1_1_chunk', NULL]); +ERROR: invalid relation +call merge_chunks('_timescaledb_internal._hyper_1_1_chunk', '_timescaledb_internal._hyper_1_1_chunk'); +ERROR: duplicate relation "_hyper_1_1_chunk" in merge +-- Check permissions +reset role; +set role :ROLE_1; +call merge_chunks('_timescaledb_internal._hyper_1_1_chunk', '_timescaledb_internal._hyper_1_2_chunk'); +ERROR: must be owner of table _hyper_1_1_chunk +reset role; +set role :ROLE_DEFAULT_PERM_USER; +\set ON_ERROR_STOP 1 +-- Show new partition +select * from partitions; + table_name | column_name | range_start | range_end +------------------+-------------+----------------------+--------------------- + _hyper_1_1_chunk | time | 1704067200000000 | 1704153600000000 + _hyper_1_2_chunk | time | 1704067200000000 | 1704153600000000 + _hyper_1_3_chunk | time | 1704067200000000 | 1704153600000000 + _hyper_1_1_chunk | device | -9223372036854775808 | 715827882 + _hyper_1_2_chunk | device | 715827882 | 1431655764 + _hyper_1_3_chunk | device | 1431655764 | 9223372036854775807 +(6 rows) + +begin; +-- Should be able to merge all three chunks +call merge_chunks(ARRAY['_timescaledb_internal._hyper_1_3_chunk', '_timescaledb_internal._hyper_1_1_chunk', '_timescaledb_internal._hyper_1_2_chunk']); +select * from partitions; + table_name | column_name | range_start | range_end +------------------+-------------+----------------------+--------------------- + _hyper_1_1_chunk | time | 1704067200000000 | 1704153600000000 + _hyper_1_1_chunk | device | -9223372036854775808 | 9223372036854775807 +(2 rows) + +-- Note that no space partition CHECK constraint is added because it +-- now covers the entire range from -inf to +inf. +select "Constraint", "Columns", "Expr" from test.show_constraints('_timescaledb_internal._hyper_1_1_chunk'); + Constraint | Columns | Expr +--------------+---------+------------------------------------------------------------------------------------------------------------------------------------------------ + constraint_1 | {time} | (("time" >= 'Sun Dec 31 16:00:00 2023 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 01 16:00:00 2024 PST'::timestamp with time zone)) +(1 row) + +select count(*) as num_orphaned_slices from orphaned_slices; + num_orphaned_slices +--------------------- + 0 +(1 row) + +select * from show_chunks('mergeme'); + show_chunks +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk +(1 row) + +select * from mergeme; + time | device | temp +------------------------------+--------+------ + Mon Jan 01 00:00:00 2024 PST | 1 | 1 + Mon Jan 01 00:00:00 2024 PST | 2 | 2 + Mon Jan 01 00:00:00 2024 PST | 3 | 3 +(3 rows) + +rollback; +-- create two new chunks, 4 and 5, as follows: +-- _____________ _______ +-- | | | | | +-- | 1 | 4 | | 5 | +-- |_____|_____| |_____| +-- | | +-- | 2 | +-- |_____| +-- | | +-- | 3 | +-- |_____| +--- +insert into mergeme values ('2024-01-02', 1, 4.0), ('2024-01-04', 1, 5.0); +-- Show new partitions +select * from partitions; + table_name | column_name | range_start | range_end +------------------+-------------+----------------------+--------------------- + _hyper_1_1_chunk | time | 1704067200000000 | 1704153600000000 + _hyper_1_3_chunk | time | 1704067200000000 | 1704153600000000 + _hyper_1_2_chunk | time | 1704067200000000 | 1704153600000000 + _hyper_1_4_chunk | time | 1704153600000000 | 1704240000000000 + _hyper_1_5_chunk | time | 1704326400000000 | 1704412800000000 + _hyper_1_5_chunk | device | -9223372036854775808 | 715827882 + _hyper_1_4_chunk | device | -9223372036854775808 | 715827882 + _hyper_1_1_chunk | device | -9223372036854775808 | 715827882 + _hyper_1_2_chunk | device | 715827882 | 1431655764 + _hyper_1_3_chunk | device | 1431655764 | 9223372036854775807 +(10 rows) + +\set ON_ERROR_STOP 0 +-- can't merge 3 and 4 +call merge_chunks('_timescaledb_internal._hyper_1_3_chunk', '_timescaledb_internal._hyper_1_4_chunk'); +ERROR: cannot create new chunk partition boundaries +-- can't merge 1 and 5 +call merge_chunks('_timescaledb_internal._hyper_1_1_chunk', '_timescaledb_internal._hyper_1_5_chunk'); +ERROR: cannot create new chunk partition boundaries +-- can't merge 2 and 4 +call merge_chunks('_timescaledb_internal._hyper_1_2_chunk', '_timescaledb_internal._hyper_1_4_chunk'); +ERROR: cannot create new chunk partition boundaries +-- can't merge 4 and 5 +call merge_chunks('_timescaledb_internal._hyper_1_5_chunk', '_timescaledb_internal._hyper_1_4_chunk'); +ERROR: cannot create new chunk partition boundaries +-- currently can't merge 1,2,3,4 due to limitation in how we validate the merge +call merge_chunks(ARRAY['_timescaledb_internal._hyper_1_3_chunk', '_timescaledb_internal._hyper_1_2_chunk', '_timescaledb_internal._hyper_1_4_chunk', '_timescaledb_internal._hyper_1_1_chunk']); +ERROR: cannot create new chunk partition boundaries +begin; +-- Should be able to merge all three chunks 1,2,3 +call merge_chunks(ARRAY['_timescaledb_internal._hyper_1_1_chunk', '_timescaledb_internal._hyper_1_2_chunk', '_timescaledb_internal._hyper_1_3_chunk']); +-- But merging the merged 1,2,3 chunk with 4 is currently not +-- possible, although we chould do it in theory +call merge_chunks('_timescaledb_internal._hyper_1_1_chunk', '_timescaledb_internal._hyper_1_4_chunk'); +ERROR: cannot create new chunk partition boundaries +rollback; +\set ON_ERROR_STOP 1 +alter table mergeme set (timescaledb.compress_orderby='time', timescaledb.compress_segmentby='device'); +select compress_chunk('_timescaledb_internal._hyper_1_1_chunk'); + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk +(1 row) + +select compress_chunk('_timescaledb_internal._hyper_1_3_chunk'); + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_3_chunk +(1 row) + +\set ON_ERROR_STOP 0 +-- Currently cannot merge compressed chunks +call merge_chunks('_timescaledb_internal._hyper_1_1_chunk', '_timescaledb_internal._hyper_1_2_chunk'); +ERROR: merging compressed chunks is not yet supported +call merge_chunks('_timescaledb_internal._hyper_1_2_chunk', '_timescaledb_internal._hyper_1_3_chunk'); +ERROR: merging compressed chunks is not yet supported +\set ON_ERROR_STOP 1 +-- Currently cannot merge chunks using Hypercore TAM +alter table _timescaledb_internal._hyper_1_1_chunk set access method hypercore; +alter table _timescaledb_internal._hyper_1_3_chunk set access method hypercore; +select relname, amname from pg_class cl +join pg_am am on (cl.relam = am.oid) +where cl.oid in ('_timescaledb_internal._hyper_1_1_chunk'::regclass, '_timescaledb_internal._hyper_1_3_chunk'::regclass); + relname | amname +------------------+----------- + _hyper_1_1_chunk | hypercore + _hyper_1_3_chunk | hypercore +(2 rows) + +\set ON_ERROR_STOP 0 +call merge_chunks('_timescaledb_internal._hyper_1_1_chunk', '_timescaledb_internal._hyper_1_2_chunk'); +ERROR: merging compressed chunks is not yet supported +call merge_chunks('_timescaledb_internal._hyper_1_2_chunk', '_timescaledb_internal._hyper_1_3_chunk'); +ERROR: merging compressed chunks is not yet supported +\set ON_ERROR_STOP 1 +--- +-- Test some error cases when merging chunks with non-chunks or chunks +-- from other hypertables +--- +-- Decompress all chunks to ensure we only have non-compressed chunks +select decompress_chunk(ch) from show_chunks('mergeme') ch; +NOTICE: chunk "_hyper_1_2_chunk" is not compressed +NOTICE: chunk "_hyper_1_4_chunk" is not compressed +NOTICE: chunk "_hyper_1_5_chunk" is not compressed + decompress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + + _timescaledb_internal._hyper_1_3_chunk + + +(5 rows) + +-- Create a non-chunk table +create table mergeme_too(time timestamptz not null, device int, temp float); +select create_hypertable('mergeme_too', 'time', 'device', 3, chunk_time_interval => interval '1 day'); + create_hypertable +-------------------------- + (3,public,mergeme_too,t) +(1 row) + +create table mergeme_regular(time timestamptz not null, device int, temp float); +insert into mergeme_too values ('2024-01-01', 1, 1.0); +insert into mergeme_regular select * from mergeme_too; +create materialized view mergeme_mat as +select * from mergeme_too where device=1; +select * from show_chunks('mergeme_too'); + show_chunks +---------------------------------------- + _timescaledb_internal._hyper_3_8_chunk +(1 row) + +\set ON_ERROR_STOP 0 +-- Merge chunk and regular table +call merge_chunks('_timescaledb_internal._hyper_1_1_chunk', 'mergeme_regular'); +ERROR: can only merge hypertable chunks +call merge_chunks('mergeme_regular', '_timescaledb_internal._hyper_1_1_chunk'); +ERROR: can only merge hypertable chunks +call merge_chunks('_timescaledb_internal._hyper_1_1_chunk', 'mergeme_mat'); +ERROR: cannot merge non-table relations +-- Merge chunks from different hypertables +call merge_chunks('_timescaledb_internal._hyper_1_1_chunk', '_timescaledb_internal._hyper_3_8_chunk'); +ERROR: cannot merge chunks across different hypertables +-- Merge with unsupported access method +alter table _timescaledb_internal._hyper_1_1_chunk set access method testam; +call merge_chunks('_timescaledb_internal._hyper_1_1_chunk', '_timescaledb_internal._hyper_1_2_chunk'); +ERROR: access method "testam" is not supported for merge +alter table _timescaledb_internal._hyper_1_1_chunk set access method heap; +-- Merge OSM chunks +reset role; +update _timescaledb_catalog.chunk ch set osm_chunk = true where table_name = '_hyper_1_1_chunk'; +set role :ROLE_DEFAULT_PERM_USER; +call merge_chunks('_timescaledb_internal._hyper_1_1_chunk', '_timescaledb_internal._hyper_1_2_chunk'); +ERROR: cannot merge OSM chunks +reset role; +update _timescaledb_catalog.chunk ch set osm_chunk = false where table_name = '_hyper_1_1_chunk'; +set role :ROLE_DEFAULT_PERM_USER; +\set ON_ERROR_STOP 1 +-- Set seed to consistently generate same data and same set of chunks +select setseed(0.2); + setseed +--------- + +(1 row) + +-- Test merge with bigger data set and chunks with more blocks +insert into mergeme (time, device, temp) +select t, ceil(random()*10), random()*40 +from generate_series('2024-01-01'::timestamptz, '2024-01-04', '0.5s') t; +-- Show partitions before merge +select * from partitions; + table_name | column_name | range_start | range_end +-------------------+-------------+----------------------+--------------------- + _hyper_1_1_chunk | time | 1704067200000000 | 1704153600000000 + _hyper_1_3_chunk | time | 1704067200000000 | 1704153600000000 + _hyper_1_2_chunk | time | 1704067200000000 | 1704153600000000 + _hyper_1_10_chunk | time | 1704153600000000 | 1704240000000000 + _hyper_1_9_chunk | time | 1704153600000000 | 1704240000000000 + _hyper_1_4_chunk | time | 1704153600000000 | 1704240000000000 + _hyper_1_11_chunk | time | 1704240000000000 | 1704326400000000 + _hyper_1_13_chunk | time | 1704240000000000 | 1704326400000000 + _hyper_1_12_chunk | time | 1704240000000000 | 1704326400000000 + _hyper_1_15_chunk | time | 1704326400000000 | 1704412800000000 + _hyper_1_5_chunk | time | 1704326400000000 | 1704412800000000 + _hyper_1_14_chunk | time | 1704326400000000 | 1704412800000000 + _hyper_1_4_chunk | device | -9223372036854775808 | 715827882 + _hyper_1_1_chunk | device | -9223372036854775808 | 715827882 + _hyper_1_5_chunk | device | -9223372036854775808 | 715827882 + _hyper_1_11_chunk | device | -9223372036854775808 | 715827882 + _hyper_1_9_chunk | device | 715827882 | 1431655764 + _hyper_1_2_chunk | device | 715827882 | 1431655764 + _hyper_1_14_chunk | device | 715827882 | 1431655764 + _hyper_1_12_chunk | device | 715827882 | 1431655764 + _hyper_1_15_chunk | device | 1431655764 | 9223372036854775807 + _hyper_1_10_chunk | device | 1431655764 | 9223372036854775807 + _hyper_1_3_chunk | device | 1431655764 | 9223372036854775807 + _hyper_1_13_chunk | device | 1431655764 | 9223372036854775807 +(24 rows) + +-- Merge all chunks until only 1 remains +select count(*), sum(device), round(sum(temp)::numeric, 4) from mergeme; + count | sum | round +--------+---------+--------------- + 518406 | 2854401 | 10373952.7510 +(1 row) + +call merge_chunks(ARRAY['_timescaledb_internal._hyper_1_1_chunk', '_timescaledb_internal._hyper_1_4_chunk','_timescaledb_internal._hyper_1_5_chunk', '_timescaledb_internal._hyper_1_11_chunk']); +select count(*), sum(device), round(sum(temp)::numeric, 4) from mergeme; + count | sum | round +--------+---------+--------------- + 518406 | 2854401 | 10373952.7510 +(1 row) + +select * from partitions; + table_name | column_name | range_start | range_end +-------------------+-------------+----------------------+--------------------- + _hyper_1_2_chunk | time | 1704067200000000 | 1704153600000000 + _hyper_1_3_chunk | time | 1704067200000000 | 1704153600000000 + _hyper_1_1_chunk | time | 1704067200000000 | 1704412800000000 + _hyper_1_10_chunk | time | 1704153600000000 | 1704240000000000 + _hyper_1_9_chunk | time | 1704153600000000 | 1704240000000000 + _hyper_1_12_chunk | time | 1704240000000000 | 1704326400000000 + _hyper_1_13_chunk | time | 1704240000000000 | 1704326400000000 + _hyper_1_15_chunk | time | 1704326400000000 | 1704412800000000 + _hyper_1_14_chunk | time | 1704326400000000 | 1704412800000000 + _hyper_1_1_chunk | device | -9223372036854775808 | 715827882 + _hyper_1_12_chunk | device | 715827882 | 1431655764 + _hyper_1_14_chunk | device | 715827882 | 1431655764 + _hyper_1_9_chunk | device | 715827882 | 1431655764 + _hyper_1_2_chunk | device | 715827882 | 1431655764 + _hyper_1_3_chunk | device | 1431655764 | 9223372036854775807 + _hyper_1_15_chunk | device | 1431655764 | 9223372036854775807 + _hyper_1_10_chunk | device | 1431655764 | 9223372036854775807 + _hyper_1_13_chunk | device | 1431655764 | 9223372036854775807 +(18 rows) + +call merge_chunks(ARRAY['_timescaledb_internal._hyper_1_2_chunk', '_timescaledb_internal._hyper_1_9_chunk','_timescaledb_internal._hyper_1_12_chunk', '_timescaledb_internal._hyper_1_14_chunk']); +select count(*), sum(device), round(sum(temp)::numeric, 4) from mergeme; + count | sum | round +--------+---------+--------------- + 518406 | 2854401 | 10373952.7510 +(1 row) + +select * from partitions; + table_name | column_name | range_start | range_end +-------------------+-------------+----------------------+--------------------- + _hyper_1_3_chunk | time | 1704067200000000 | 1704153600000000 + _hyper_1_1_chunk | time | 1704067200000000 | 1704412800000000 + _hyper_1_2_chunk | time | 1704067200000000 | 1704412800000000 + _hyper_1_10_chunk | time | 1704153600000000 | 1704240000000000 + _hyper_1_13_chunk | time | 1704240000000000 | 1704326400000000 + _hyper_1_15_chunk | time | 1704326400000000 | 1704412800000000 + _hyper_1_1_chunk | device | -9223372036854775808 | 715827882 + _hyper_1_2_chunk | device | 715827882 | 1431655764 + _hyper_1_15_chunk | device | 1431655764 | 9223372036854775807 + _hyper_1_13_chunk | device | 1431655764 | 9223372036854775807 + _hyper_1_10_chunk | device | 1431655764 | 9223372036854775807 + _hyper_1_3_chunk | device | 1431655764 | 9223372036854775807 +(12 rows) + +call merge_chunks(ARRAY['_timescaledb_internal._hyper_1_3_chunk', '_timescaledb_internal._hyper_1_10_chunk','_timescaledb_internal._hyper_1_13_chunk', '_timescaledb_internal._hyper_1_15_chunk']); +select count(*), sum(device), round(sum(temp)::numeric, 4) from mergeme; + count | sum | round +--------+---------+--------------- + 518406 | 2854401 | 10373952.7510 +(1 row) + +select * from partitions; + table_name | column_name | range_start | range_end +------------------+-------------+----------------------+--------------------- + _hyper_1_1_chunk | time | 1704067200000000 | 1704412800000000 + _hyper_1_2_chunk | time | 1704067200000000 | 1704412800000000 + _hyper_1_3_chunk | time | 1704067200000000 | 1704412800000000 + _hyper_1_1_chunk | device | -9223372036854775808 | 715827882 + _hyper_1_2_chunk | device | 715827882 | 1431655764 + _hyper_1_3_chunk | device | 1431655764 | 9223372036854775807 +(6 rows) + +call merge_chunks(ARRAY['_timescaledb_internal._hyper_1_3_chunk', '_timescaledb_internal._hyper_1_1_chunk','_timescaledb_internal._hyper_1_2_chunk']); +select count(*), sum(device), round(sum(temp)::numeric, 4) from mergeme; + count | sum | round +--------+---------+--------------- + 518406 | 2854401 | 10373952.7510 +(1 row) + +select * from partitions; + table_name | column_name | range_start | range_end +------------------+-------------+----------------------+--------------------- + _hyper_1_1_chunk | time | 1704067200000000 | 1704412800000000 + _hyper_1_1_chunk | device | -9223372036854775808 | 9223372036854775807 +(2 rows) + diff --git a/tsl/test/isolation/expected/merge_chunks_concurrent.out b/tsl/test/isolation/expected/merge_chunks_concurrent.out new file mode 100644 index 00000000000..7be59718a0e --- /dev/null +++ b/tsl/test/isolation/expected/merge_chunks_concurrent.out @@ -0,0 +1,611 @@ +Parsed test spec with 4 sessions + +starting permutation: s2_show_chunks s3_show_data s1_begin s3_begin s4_modify s2_merge_chunks s1_show_chunks s3_show_chunks s1_show_data s3_show_data s1_commit s1_show_data s3_commit +step s2_show_chunks: select count(*) from show_chunks('readings'); +count +----- + 2 +(1 row) + +step s3_show_data: + select * from readings order by time desc, device; + select count(*) as num_device_all, count(*) filter (where device=1) as num_device_1, count(*) filter (where device=5) as num_device_5 from readings; + +time |device|temp +----------------------------+------+---- +Mon Jan 01 02:00:00 2024 PST| 3| 3 +Mon Jan 01 02:00:00 2024 PST| 4| 4 +Mon Jan 01 01:01:00 2024 PST| 2| 2 +Mon Jan 01 01:00:00 2024 PST| 1| 1 +(4 rows) + +num_device_all|num_device_1|num_device_5 +--------------+------------+------------ + 4| 1| 0 +(1 row) + +step s1_begin: + start transaction isolation level repeatable read; + select count(*) > 0 from pg_class; + +?column? +-------- +t +(1 row) + +step s3_begin: + start transaction isolation level read committed; + select count(*) > 0 from pg_class; + +?column? +-------- +t +(1 row) + +step s4_modify: + delete from readings where device=1; + insert into readings values ('2024-01-01 01:05', 5, 5.0); + +step s2_merge_chunks: + call merge_all_chunks('readings'); + +step s1_show_chunks: select count(*) from show_chunks('readings'); +count +----- + 1 +(1 row) + +step s3_show_chunks: select count(*) from show_chunks('readings'); +count +----- + 1 +(1 row) + +step s1_show_data: + select * from readings order by time desc, device; + select count(*) as num_device_all, count(*) filter (where device=1) as num_device_1, count(*) filter (where device=5) as num_device_5 from readings; + +time |device|temp +----------------------------+------+---- +Mon Jan 01 02:00:00 2024 PST| 3| 3 +Mon Jan 01 02:00:00 2024 PST| 4| 4 +Mon Jan 01 01:01:00 2024 PST| 2| 2 +Mon Jan 01 01:00:00 2024 PST| 1| 1 +(4 rows) + +num_device_all|num_device_1|num_device_5 +--------------+------------+------------ + 4| 1| 0 +(1 row) + +step s3_show_data: + select * from readings order by time desc, device; + select count(*) as num_device_all, count(*) filter (where device=1) as num_device_1, count(*) filter (where device=5) as num_device_5 from readings; + +time |device|temp +----------------------------+------+---- +Mon Jan 01 02:00:00 2024 PST| 3| 3 +Mon Jan 01 02:00:00 2024 PST| 4| 4 +Mon Jan 01 01:05:00 2024 PST| 5| 5 +Mon Jan 01 01:01:00 2024 PST| 2| 2 +(4 rows) + +num_device_all|num_device_1|num_device_5 +--------------+------------+------------ + 4| 0| 1 +(1 row) + +step s1_commit: commit; +step s1_show_data: + select * from readings order by time desc, device; + select count(*) as num_device_all, count(*) filter (where device=1) as num_device_1, count(*) filter (where device=5) as num_device_5 from readings; + +time |device|temp +----------------------------+------+---- +Mon Jan 01 02:00:00 2024 PST| 3| 3 +Mon Jan 01 02:00:00 2024 PST| 4| 4 +Mon Jan 01 01:05:00 2024 PST| 5| 5 +Mon Jan 01 01:01:00 2024 PST| 2| 2 +(4 rows) + +num_device_all|num_device_1|num_device_5 +--------------+------------+------------ + 4| 0| 1 +(1 row) + +step s3_commit: commit; + +starting permutation: s2_show_chunks s1_begin s1_show_data s2_merge_chunks s1_show_data s1_commit s1_show_data s1_show_chunks +step s2_show_chunks: select count(*) from show_chunks('readings'); +count +----- + 2 +(1 row) + +step s1_begin: + start transaction isolation level repeatable read; + select count(*) > 0 from pg_class; + +?column? +-------- +t +(1 row) + +step s1_show_data: + select * from readings order by time desc, device; + select count(*) as num_device_all, count(*) filter (where device=1) as num_device_1, count(*) filter (where device=5) as num_device_5 from readings; + +time |device|temp +----------------------------+------+---- +Mon Jan 01 02:00:00 2024 PST| 3| 3 +Mon Jan 01 02:00:00 2024 PST| 4| 4 +Mon Jan 01 01:01:00 2024 PST| 2| 2 +Mon Jan 01 01:00:00 2024 PST| 1| 1 +(4 rows) + +num_device_all|num_device_1|num_device_5 +--------------+------------+------------ + 4| 1| 0 +(1 row) + +step s2_merge_chunks: + call merge_all_chunks('readings'); + +step s1_show_data: + select * from readings order by time desc, device; + select count(*) as num_device_all, count(*) filter (where device=1) as num_device_1, count(*) filter (where device=5) as num_device_5 from readings; + +time |device|temp +----------------------------+------+---- +Mon Jan 01 02:00:00 2024 PST| 3| 3 +Mon Jan 01 02:00:00 2024 PST| 4| 4 +Mon Jan 01 01:01:00 2024 PST| 2| 2 +Mon Jan 01 01:00:00 2024 PST| 1| 1 +(4 rows) + +num_device_all|num_device_1|num_device_5 +--------------+------------+------------ + 4| 1| 0 +(1 row) + +step s1_commit: commit; +step s2_merge_chunks: <... completed> +step s1_show_data: + select * from readings order by time desc, device; + select count(*) as num_device_all, count(*) filter (where device=1) as num_device_1, count(*) filter (where device=5) as num_device_5 from readings; + +time |device|temp +----------------------------+------+---- +Mon Jan 01 02:00:00 2024 PST| 3| 3 +Mon Jan 01 02:00:00 2024 PST| 4| 4 +Mon Jan 01 01:01:00 2024 PST| 2| 2 +Mon Jan 01 01:00:00 2024 PST| 1| 1 +(4 rows) + +num_device_all|num_device_1|num_device_5 +--------------+------------+------------ + 4| 1| 0 +(1 row) + +step s1_show_chunks: select count(*) from show_chunks('readings'); +count +----- + 1 +(1 row) + + +starting permutation: s2_set_lock_upgrade s2_show_chunks s1_begin s1_show_data s2_merge_chunks s1_show_data s1_commit s1_show_data s1_show_chunks +step s2_set_lock_upgrade: + set timescaledb.merge_chunks_lock_upgrade_mode='upgrade'; + +step s2_show_chunks: select count(*) from show_chunks('readings'); +count +----- + 2 +(1 row) + +step s1_begin: + start transaction isolation level repeatable read; + select count(*) > 0 from pg_class; + +?column? +-------- +t +(1 row) + +step s1_show_data: + select * from readings order by time desc, device; + select count(*) as num_device_all, count(*) filter (where device=1) as num_device_1, count(*) filter (where device=5) as num_device_5 from readings; + +time |device|temp +----------------------------+------+---- +Mon Jan 01 02:00:00 2024 PST| 3| 3 +Mon Jan 01 02:00:00 2024 PST| 4| 4 +Mon Jan 01 01:01:00 2024 PST| 2| 2 +Mon Jan 01 01:00:00 2024 PST| 1| 1 +(4 rows) + +num_device_all|num_device_1|num_device_5 +--------------+------------+------------ + 4| 1| 0 +(1 row) + +step s2_merge_chunks: + call merge_all_chunks('readings'); + +step s1_show_data: + select * from readings order by time desc, device; + select count(*) as num_device_all, count(*) filter (where device=1) as num_device_1, count(*) filter (where device=5) as num_device_5 from readings; + +time |device|temp +----------------------------+------+---- +Mon Jan 01 02:00:00 2024 PST| 3| 3 +Mon Jan 01 02:00:00 2024 PST| 4| 4 +Mon Jan 01 01:01:00 2024 PST| 2| 2 +Mon Jan 01 01:00:00 2024 PST| 1| 1 +(4 rows) + +num_device_all|num_device_1|num_device_5 +--------------+------------+------------ + 4| 1| 0 +(1 row) + +step s1_commit: commit; +step s2_merge_chunks: <... completed> +step s1_show_data: + select * from readings order by time desc, device; + select count(*) as num_device_all, count(*) filter (where device=1) as num_device_1, count(*) filter (where device=5) as num_device_5 from readings; + +time |device|temp +----------------------------+------+---- +Mon Jan 01 02:00:00 2024 PST| 3| 3 +Mon Jan 01 02:00:00 2024 PST| 4| 4 +Mon Jan 01 01:01:00 2024 PST| 2| 2 +Mon Jan 01 01:00:00 2024 PST| 1| 1 +(4 rows) + +num_device_all|num_device_1|num_device_5 +--------------+------------+------------ + 4| 1| 0 +(1 row) + +step s1_show_chunks: select count(*) from show_chunks('readings'); +count +----- + 1 +(1 row) + + +starting permutation: s2_set_lock_upgrade s4_wp_enable s2_show_chunks s1_begin s1_show_data s2_merge_chunks s1_show_data s1_row_exclusive_lock s4_wp_release s1_commit s1_show_data s1_show_chunks +step s2_set_lock_upgrade: + set timescaledb.merge_chunks_lock_upgrade_mode='upgrade'; + +step s4_wp_enable: SELECT debug_waitpoint_enable('merge_chunks_before_heap_swap'); +debug_waitpoint_enable +---------------------- + +(1 row) + +step s2_show_chunks: select count(*) from show_chunks('readings'); +count +----- + 2 +(1 row) + +step s1_begin: + start transaction isolation level repeatable read; + select count(*) > 0 from pg_class; + +?column? +-------- +t +(1 row) + +step s1_show_data: + select * from readings order by time desc, device; + select count(*) as num_device_all, count(*) filter (where device=1) as num_device_1, count(*) filter (where device=5) as num_device_5 from readings; + +time |device|temp +----------------------------+------+---- +Mon Jan 01 02:00:00 2024 PST| 3| 3 +Mon Jan 01 02:00:00 2024 PST| 4| 4 +Mon Jan 01 01:01:00 2024 PST| 2| 2 +Mon Jan 01 01:00:00 2024 PST| 1| 1 +(4 rows) + +num_device_all|num_device_1|num_device_5 +--------------+------------+------------ + 4| 1| 0 +(1 row) + +step s2_merge_chunks: + call merge_all_chunks('readings'); + +step s1_show_data: + select * from readings order by time desc, device; + select count(*) as num_device_all, count(*) filter (where device=1) as num_device_1, count(*) filter (where device=5) as num_device_5 from readings; + +time |device|temp +----------------------------+------+---- +Mon Jan 01 02:00:00 2024 PST| 3| 3 +Mon Jan 01 02:00:00 2024 PST| 4| 4 +Mon Jan 01 01:01:00 2024 PST| 2| 2 +Mon Jan 01 01:00:00 2024 PST| 1| 1 +(4 rows) + +num_device_all|num_device_1|num_device_5 +--------------+------------+------------ + 4| 1| 0 +(1 row) + +step s1_row_exclusive_lock: call lock_one_chunk('readings'); +step s4_wp_release: SELECT debug_waitpoint_release('merge_chunks_before_heap_swap'); +debug_waitpoint_release +----------------------- + +(1 row) + +step s1_row_exclusive_lock: <... completed> +ERROR: deadlock detected +step s2_merge_chunks: <... completed> +step s1_commit: commit; +step s1_show_data: + select * from readings order by time desc, device; + select count(*) as num_device_all, count(*) filter (where device=1) as num_device_1, count(*) filter (where device=5) as num_device_5 from readings; + +time |device|temp +----------------------------+------+---- +Mon Jan 01 02:00:00 2024 PST| 3| 3 +Mon Jan 01 02:00:00 2024 PST| 4| 4 +Mon Jan 01 01:01:00 2024 PST| 2| 2 +Mon Jan 01 01:00:00 2024 PST| 1| 1 +(4 rows) + +num_device_all|num_device_1|num_device_5 +--------------+------------+------------ + 4| 1| 0 +(1 row) + +step s1_show_chunks: select count(*) from show_chunks('readings'); +count +----- + 1 +(1 row) + + +starting permutation: s2_set_lock_upgrade_conditional s4_wp_enable s2_show_chunks s1_begin s1_show_data s2_merge_chunks s1_show_data s1_row_exclusive_lock s4_wp_release s1_commit s1_show_data s1_show_chunks +step s2_set_lock_upgrade_conditional: + set timescaledb.merge_chunks_lock_upgrade_mode='conditional'; + +step s4_wp_enable: SELECT debug_waitpoint_enable('merge_chunks_before_heap_swap'); +debug_waitpoint_enable +---------------------- + +(1 row) + +step s2_show_chunks: select count(*) from show_chunks('readings'); +count +----- + 2 +(1 row) + +step s1_begin: + start transaction isolation level repeatable read; + select count(*) > 0 from pg_class; + +?column? +-------- +t +(1 row) + +step s1_show_data: + select * from readings order by time desc, device; + select count(*) as num_device_all, count(*) filter (where device=1) as num_device_1, count(*) filter (where device=5) as num_device_5 from readings; + +time |device|temp +----------------------------+------+---- +Mon Jan 01 02:00:00 2024 PST| 3| 3 +Mon Jan 01 02:00:00 2024 PST| 4| 4 +Mon Jan 01 01:01:00 2024 PST| 2| 2 +Mon Jan 01 01:00:00 2024 PST| 1| 1 +(4 rows) + +num_device_all|num_device_1|num_device_5 +--------------+------------+------------ + 4| 1| 0 +(1 row) + +step s2_merge_chunks: + call merge_all_chunks('readings'); + +step s1_show_data: + select * from readings order by time desc, device; + select count(*) as num_device_all, count(*) filter (where device=1) as num_device_1, count(*) filter (where device=5) as num_device_5 from readings; + +time |device|temp +----------------------------+------+---- +Mon Jan 01 02:00:00 2024 PST| 3| 3 +Mon Jan 01 02:00:00 2024 PST| 4| 4 +Mon Jan 01 01:01:00 2024 PST| 2| 2 +Mon Jan 01 01:00:00 2024 PST| 1| 1 +(4 rows) + +num_device_all|num_device_1|num_device_5 +--------------+------------+------------ + 4| 1| 0 +(1 row) + +step s1_row_exclusive_lock: call lock_one_chunk('readings'); +step s4_wp_release: SELECT debug_waitpoint_release('merge_chunks_before_heap_swap'); +debug_waitpoint_release +----------------------- + +(1 row) + +step s2_merge_chunks: <... completed> +ERROR: could not lock relation "_hyper_X_X_chunk" for merge +step s1_row_exclusive_lock: <... completed> +step s1_commit: commit; +step s1_show_data: + select * from readings order by time desc, device; + select count(*) as num_device_all, count(*) filter (where device=1) as num_device_1, count(*) filter (where device=5) as num_device_5 from readings; + +time |device|temp +----------------------------+------+---- +Mon Jan 01 02:00:00 2024 PST| 3| 3 +Mon Jan 01 02:00:00 2024 PST| 4| 4 +Mon Jan 01 01:01:00 2024 PST| 2| 2 +Mon Jan 01 01:00:00 2024 PST| 1| 1 +(4 rows) + +num_device_all|num_device_1|num_device_5 +--------------+------------+------------ + 4| 1| 0 +(1 row) + +step s1_show_chunks: select count(*) from show_chunks('readings'); +count +----- + 2 +(1 row) + + +starting permutation: s4_wp_enable s2_merge_chunks s3_merge_chunks s4_wp_release s1_show_data s1_show_chunks +step s4_wp_enable: SELECT debug_waitpoint_enable('merge_chunks_before_heap_swap'); +debug_waitpoint_enable +---------------------- + +(1 row) + +step s2_merge_chunks: + call merge_all_chunks('readings'); + +step s3_merge_chunks: + call merge_all_chunks('readings'); + +step s4_wp_release: SELECT debug_waitpoint_release('merge_chunks_before_heap_swap'); +debug_waitpoint_release +----------------------- + +(1 row) + +step s2_merge_chunks: <... completed> +step s3_merge_chunks: <... completed> +ERROR: relation does not exist +step s1_show_data: + select * from readings order by time desc, device; + select count(*) as num_device_all, count(*) filter (where device=1) as num_device_1, count(*) filter (where device=5) as num_device_5 from readings; + +time |device|temp +----------------------------+------+---- +Mon Jan 01 02:00:00 2024 PST| 3| 3 +Mon Jan 01 02:00:00 2024 PST| 4| 4 +Mon Jan 01 01:01:00 2024 PST| 2| 2 +Mon Jan 01 01:00:00 2024 PST| 1| 1 +(4 rows) + +num_device_all|num_device_1|num_device_5 +--------------+------------+------------ + 4| 1| 0 +(1 row) + +step s1_show_chunks: select count(*) from show_chunks('readings'); +count +----- + 1 +(1 row) + + +starting permutation: s4_wp_enable s2_merge_chunks s3_compress_chunks s4_wp_release s1_show_data s1_show_chunks +step s4_wp_enable: SELECT debug_waitpoint_enable('merge_chunks_before_heap_swap'); +debug_waitpoint_enable +---------------------- + +(1 row) + +step s2_merge_chunks: + call merge_all_chunks('readings'); + +step s3_compress_chunks: + select compress_chunk(show_chunks('readings')); + +step s4_wp_release: SELECT debug_waitpoint_release('merge_chunks_before_heap_swap'); +debug_waitpoint_release +----------------------- + +(1 row) + +step s1_show_data: + select * from readings order by time desc, device; + select count(*) as num_device_all, count(*) filter (where device=1) as num_device_1, count(*) filter (where device=5) as num_device_5 from readings; + +step s1_show_data: <... completed> +time |device|temp +----------------------------+------+---- +Mon Jan 01 02:00:00 2024 PST| 3| 3 +Mon Jan 01 02:00:00 2024 PST| 4| 4 +Mon Jan 01 01:01:00 2024 PST| 2| 2 +Mon Jan 01 01:00:00 2024 PST| 1| 1 +(4 rows) + +num_device_all|num_device_1|num_device_5 +--------------+------------+------------ + 4| 1| 0 +(1 row) + +step s2_merge_chunks: <... completed> +step s3_compress_chunks: <... completed> +ERROR: deadlock detected +step s1_show_chunks: select count(*) from show_chunks('readings'); +count +----- + 1 +(1 row) + + +starting permutation: s4_wp_enable s2_merge_chunks s3_drop_chunks s4_wp_release s1_show_data s1_show_chunks +step s4_wp_enable: SELECT debug_waitpoint_enable('merge_chunks_before_heap_swap'); +debug_waitpoint_enable +---------------------- + +(1 row) + +step s2_merge_chunks: + call merge_all_chunks('readings'); + +step s3_drop_chunks: + call drop_one_chunk('readings'); + +step s4_wp_release: SELECT debug_waitpoint_release('merge_chunks_before_heap_swap'); +debug_waitpoint_release +----------------------- + +(1 row) + +step s1_show_data: + select * from readings order by time desc, device; + select count(*) as num_device_all, count(*) filter (where device=1) as num_device_1, count(*) filter (where device=5) as num_device_5 from readings; + +step s1_show_data: <... completed> +time |device|temp +----------------------------+------+---- +Mon Jan 01 02:00:00 2024 PST| 3| 3 +Mon Jan 01 02:00:00 2024 PST| 4| 4 +Mon Jan 01 01:01:00 2024 PST| 2| 2 +Mon Jan 01 01:00:00 2024 PST| 1| 1 +(4 rows) + +num_device_all|num_device_1|num_device_5 +--------------+------------+------------ + 4| 1| 0 +(1 row) + +step s2_merge_chunks: <... completed> +step s3_drop_chunks: <... completed> +ERROR: deadlock detected +step s1_show_chunks: select count(*) from show_chunks('readings'); +count +----- + 1 +(1 row) + diff --git a/tsl/test/isolation/specs/CMakeLists.txt b/tsl/test/isolation/specs/CMakeLists.txt index aa20561638e..629f53094fb 100644 --- a/tsl/test/isolation/specs/CMakeLists.txt +++ b/tsl/test/isolation/specs/CMakeLists.txt @@ -42,7 +42,8 @@ if(CMAKE_BUILD_TYPE MATCHES Debug) compression_freeze.spec compression_merge_race.spec compression_recompress.spec - decompression_chunk_and_parallel_query_wo_idx.spec) + decompression_chunk_and_parallel_query_wo_idx.spec + merge_chunks_concurrent.spec) if(PG_VERSION VERSION_GREATER_EQUAL "14.0") list(APPEND TEST_FILES freeze_chunk.spec compression_dml_iso.spec) endif() diff --git a/tsl/test/isolation/specs/merge_chunks_concurrent.spec b/tsl/test/isolation/specs/merge_chunks_concurrent.spec new file mode 100644 index 00000000000..5e02dbade77 --- /dev/null +++ b/tsl/test/isolation/specs/merge_chunks_concurrent.spec @@ -0,0 +1,174 @@ +# This file and its contents are licensed under the Timescale License. +# Please see the included NOTICE for copyright information and +# LICENSE-TIMESCALE for a copy of the license. + +setup +{ + create table readings (time timestamptz, device int, temp float); + select create_hypertable('readings', 'time', chunk_time_interval => interval '1 hour'); + insert into readings values ('2024-01-01 01:00', 1, 1.0), ('2024-01-01 01:01', 2, 2.0), ('2024-01-01 02:00', 3, 3.0), ('2024-01-01 02:00', 4, 4.0); + alter table readings set (timescaledb.compress_orderby='time', timescaledb.compress_segmentby='device'); + + create or replace procedure merge_all_chunks(hypertable regclass) as $$ + declare + chunks_arr regclass[]; + begin + select array_agg(cl.oid) into chunks_arr + from pg_class cl + join pg_inherits inh + on (cl.oid = inh.inhrelid) + where inh.inhparent = hypertable; + + call merge_chunks(variadic chunks_arr); + end; + $$ LANGUAGE plpgsql; + + create or replace procedure drop_one_chunk(hypertable regclass) as $$ + declare + chunk regclass; + begin + select cl.oid into chunk + from pg_class cl + join pg_inherits inh + on (cl.oid = inh.inhrelid) + where inh.inhparent = hypertable + limit 1; + execute format('drop table %s cascade', chunk); + end; + $$ LANGUAGE plpgsql; + + create or replace procedure lock_one_chunk(hypertable regclass) as $$ + declare + chunk regclass; + begin + select ch into chunk from show_chunks(hypertable) ch offset 1 limit 1; + execute format('lock %s in row exclusive mode', chunk); + end; + $$ LANGUAGE plpgsql; + + reset timescaledb.merge_chunks_lock_upgrade_mode; +} + +teardown { + drop table readings; +} + +session "s1" +setup { + set local lock_timeout = '5000ms'; + set local deadlock_timeout = '10ms'; +} + +# The transaction will not "pick" a snapshot until the first query, so +# do a simple select on pg_class to pick one for the transaction. We +# don't want to query any tables involved in the test since that will +# grab locks on them. +step "s1_begin" { + start transaction isolation level repeatable read; + select count(*) > 0 from pg_class; +} + +step "s1_show_chunks" { select count(*) from show_chunks('readings'); } +step "s1_show_data" { + select * from readings order by time desc, device; + select count(*) as num_device_all, count(*) filter (where device=1) as num_device_1, count(*) filter (where device=5) as num_device_5 from readings; +} +step "s1_row_exclusive_lock" { call lock_one_chunk('readings'); } +step "s1_commit" { commit; } + +session "s2" +setup { + set local lock_timeout = '500ms'; + set local deadlock_timeout = '100ms'; + reset timescaledb.merge_chunks_lock_upgrade_mode; +} + +step "s2_show_chunks" { select count(*) from show_chunks('readings'); } +step "s2_merge_chunks" { + call merge_all_chunks('readings'); +} + +step "s2_set_lock_upgrade" { + set timescaledb.merge_chunks_lock_upgrade_mode='upgrade'; +} +step "s2_set_lock_upgrade_conditional" { + set timescaledb.merge_chunks_lock_upgrade_mode='conditional'; +} + +session "s3" +setup { + set local lock_timeout = '500ms'; + set local deadlock_timeout = '100ms'; +} + +step "s3_begin" { + start transaction isolation level read committed; + select count(*) > 0 from pg_class; +} +step "s3_show_data" { + select * from readings order by time desc, device; + select count(*) as num_device_all, count(*) filter (where device=1) as num_device_1, count(*) filter (where device=5) as num_device_5 from readings; +} +step "s3_show_chunks" { select count(*) from show_chunks('readings'); } +step "s3_merge_chunks" { + call merge_all_chunks('readings'); +} +step "s3_compress_chunks" { + select compress_chunk(show_chunks('readings')); +} +step "s3_drop_chunks" { + call drop_one_chunk('readings'); +} +step "s3_commit" { commit; } + +session "s4" +setup { + set local lock_timeout = '500ms'; + set local deadlock_timeout = '100ms'; +} + +step "s4_modify" { + delete from readings where device=1; + insert into readings values ('2024-01-01 01:05', 5, 5.0); +} + +step "s4_wp_enable" { SELECT debug_waitpoint_enable('merge_chunks_before_heap_swap'); } +step "s4_wp_release" { SELECT debug_waitpoint_release('merge_chunks_before_heap_swap'); } + +# Run 4 backends: +# +# s1: will read data in REPEATABLE READ (should not see changes after merge) +# s2: will merge chunks +# s3: will read data in READ COMMITTED (should see changes immediately after merge) +# s4: will modify data during TX s1 and s3 but before merge +# +# Expectation: s1 should see the original data as it was before s4 +# modifications and merge while s3 should see the changes +permutation "s2_show_chunks" "s3_show_data" "s1_begin" "s3_begin" "s4_modify" "s2_merge_chunks" "s1_show_chunks" "s3_show_chunks" "s1_show_data" "s3_show_data" "s1_commit" "s1_show_data" "s3_commit" + +# Merge chunks with AccessExclusiveLock (default). s2_merge_chunks +# need to wait for readers to finish before even starting merge +permutation "s2_show_chunks" "s1_begin" "s1_show_data" "s2_merge_chunks" "s1_show_data" "s1_commit" "s1_show_data" "s1_show_chunks" + +# Merge chunks with lock upgrade. s2_merge_chunks can merge +# concurrently with readers but need to wait for readers to finish +# before doing the heap swap. +permutation "s2_set_lock_upgrade" "s2_show_chunks" "s1_begin" "s1_show_data" "s2_merge_chunks" "s1_show_data" "s1_commit" "s1_show_data" "s1_show_chunks" + +# Same as the above, but it will deadlock because a reader takes a +# heavier lock. +permutation "s2_set_lock_upgrade" "s4_wp_enable" "s2_show_chunks" "s1_begin" "s1_show_data" "s2_merge_chunks" "s1_show_data" "s1_row_exclusive_lock" "s4_wp_release" "s1_commit" "s1_show_data" "s1_show_chunks" + +# Same as above but with a conditional lock. The merge process should +# fail with an error saying it can't take the lock needed for the +# merge. +permutation "s2_set_lock_upgrade_conditional" "s4_wp_enable" "s2_show_chunks" "s1_begin" "s1_show_data" "s2_merge_chunks" "s1_show_data" "s1_row_exclusive_lock" "s4_wp_release" "s1_commit" "s1_show_data" "s1_show_chunks" + +# Test concurrent merges +permutation "s4_wp_enable" "s2_merge_chunks" "s3_merge_chunks" "s4_wp_release" "s1_show_data" "s1_show_chunks" + +# Test concurrent compress_chunk() +permutation "s4_wp_enable" "s2_merge_chunks" "s3_compress_chunks" "s4_wp_release" "s1_show_data" "s1_show_chunks" + +# Test concurrent drop table +permutation "s4_wp_enable" "s2_merge_chunks" "s3_drop_chunks" "s4_wp_release" "s1_show_data" "s1_show_chunks" diff --git a/tsl/test/shared/expected/extension.out b/tsl/test/shared/expected/extension.out index fc8c707689d..310d4ba836c 100644 --- a/tsl/test/shared/expected/extension.out +++ b/tsl/test/shared/expected/extension.out @@ -256,6 +256,8 @@ ORDER BY pronamespace::regnamespace::text COLLATE "C", p.oid::regprocedure::text interpolate(smallint,record,record) last(anyelement,"any") locf(anyelement,anyelement,boolean) + merge_chunks(regclass,regclass) + merge_chunks(regclass[]) move_chunk(regclass,name,name,regclass,boolean) recompress_chunk(regclass,boolean) refresh_continuous_aggregate(regclass,"any","any",boolean) diff --git a/tsl/test/sql/CMakeLists.txt b/tsl/test/sql/CMakeLists.txt index 6401bf70e8f..243aab81a00 100644 --- a/tsl/test/sql/CMakeLists.txt +++ b/tsl/test/sql/CMakeLists.txt @@ -146,6 +146,7 @@ if((${PG_VERSION_MAJOR} GREATER_EQUAL "15")) list( APPEND TEST_FILES + merge_chunks.sql cagg_refresh_using_merge.sql compress_sort_transform.sql hypercore_columnar.sql diff --git a/tsl/test/sql/merge_chunks.sql b/tsl/test/sql/merge_chunks.sql new file mode 100644 index 00000000000..d073c0b0367 --- /dev/null +++ b/tsl/test/sql/merge_chunks.sql @@ -0,0 +1,247 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. + + +\c :TEST_DBNAME :ROLE_SUPERUSER +CREATE ACCESS METHOD testam TYPE TABLE HANDLER heap_tableam_handler; +set role :ROLE_DEFAULT_PERM_USER; + +------------------ +-- Helper views -- +------------------- +create view partitions as +select c.table_name, d.column_name, ds.range_start, ds.range_end +from _timescaledb_catalog.hypertable h +join _timescaledb_catalog.chunk c on (c.hypertable_id = h.id) +join _timescaledb_catalog.dimension d on (d.hypertable_id = h.id) +join _timescaledb_catalog.dimension_slice ds on (d.id = ds.dimension_id) +join _timescaledb_catalog.chunk_constraint cc on (cc.chunk_id = c.id and cc.dimension_slice_id = ds.id) +where h.table_name = 'mergeme' +order by d.id, ds.range_start, ds.range_end; + +create view orphaned_slices as +select ds.id, cc.constraint_name from _timescaledb_catalog.dimension_slice ds +left join _timescaledb_catalog.chunk_constraint cc on (ds.id = cc.dimension_slice_id) +where cc.constraint_name is null; + +----------------- +-- Setup table -- +----------------- +create table mergeme (time timestamptz not null, device int, temp float); +select create_hypertable('mergeme', 'time', 'device', 3, chunk_time_interval => interval '1 day'); + +-- +-- Insert data to create two chunks with same time ranges like this: +-- _______ +-- | | +-- | 1 | +-- |_____| +-- | | +-- | 2 | +-- |_____| +--- +insert into mergeme values ('2024-01-01', 1, 1.0), ('2024-01-01', 2, 2.0); + + +select "Constraint", "Columns", "Expr" from test.show_constraints('_timescaledb_internal._hyper_1_1_chunk'); + +-- Show partition layout +select * from partitions; + +-- Now merge chunk 1 and 2: +begin; +call merge_chunks('_timescaledb_internal._hyper_1_1_chunk', '_timescaledb_internal._hyper_1_2_chunk'); +select * from _timescaledb_internal._hyper_1_1_chunk; +select reltuples from pg_class where oid='_timescaledb_internal._hyper_1_1_chunk'::regclass; +select * from partitions; +select "Constraint", "Columns", "Expr" from test.show_constraints('_timescaledb_internal._hyper_1_1_chunk'); +select count(*) as num_orphaned_slices from orphaned_slices; +select * from show_chunks('mergeme'); +select * from mergeme; +rollback; + + +-- create a new chunk as a third space partition +-- _______ +-- | | +-- | 1 | +-- |_____| +-- | | +-- | 2 | +-- |_____| +-- | | +-- | 3 | +-- |_____| +--- + +insert into mergeme values ('2024-01-01', 3, 3.0); + +-- Test some basic error cases +\set ON_ERROR_STOP 0 +-- Can't merge chunk 1 and 3 +call merge_chunks('_timescaledb_internal._hyper_1_1_chunk', '_timescaledb_internal._hyper_1_3_chunk'); +call merge_chunks(NULL); +call merge_chunks(NULL, NULL); +call merge_chunks(999999,999991); +call merge_chunks(ARRAY['_timescaledb_internal._hyper_1_1_chunk']); +call merge_chunks('_timescaledb_internal._hyper_1_1_chunk', NULL); +call merge_chunks(ARRAY['_timescaledb_internal._hyper_1_1_chunk', NULL]); +call merge_chunks('_timescaledb_internal._hyper_1_1_chunk', '_timescaledb_internal._hyper_1_1_chunk'); + + +-- Check permissions +reset role; +set role :ROLE_1; +call merge_chunks('_timescaledb_internal._hyper_1_1_chunk', '_timescaledb_internal._hyper_1_2_chunk'); +reset role; +set role :ROLE_DEFAULT_PERM_USER; +\set ON_ERROR_STOP 1 + +-- Show new partition +select * from partitions; + +begin; +-- Should be able to merge all three chunks +call merge_chunks(ARRAY['_timescaledb_internal._hyper_1_3_chunk', '_timescaledb_internal._hyper_1_1_chunk', '_timescaledb_internal._hyper_1_2_chunk']); +select * from partitions; +-- Note that no space partition CHECK constraint is added because it +-- now covers the entire range from -inf to +inf. +select "Constraint", "Columns", "Expr" from test.show_constraints('_timescaledb_internal._hyper_1_1_chunk'); +select count(*) as num_orphaned_slices from orphaned_slices; +select * from show_chunks('mergeme'); +select * from mergeme; +rollback; + +-- create two new chunks, 4 and 5, as follows: +-- _____________ _______ +-- | | | | | +-- | 1 | 4 | | 5 | +-- |_____|_____| |_____| +-- | | +-- | 2 | +-- |_____| +-- | | +-- | 3 | +-- |_____| +--- +insert into mergeme values ('2024-01-02', 1, 4.0), ('2024-01-04', 1, 5.0); + +-- Show new partitions +select * from partitions; + +\set ON_ERROR_STOP 0 +-- can't merge 3 and 4 +call merge_chunks('_timescaledb_internal._hyper_1_3_chunk', '_timescaledb_internal._hyper_1_4_chunk'); +-- can't merge 1 and 5 +call merge_chunks('_timescaledb_internal._hyper_1_1_chunk', '_timescaledb_internal._hyper_1_5_chunk'); +-- can't merge 2 and 4 +call merge_chunks('_timescaledb_internal._hyper_1_2_chunk', '_timescaledb_internal._hyper_1_4_chunk'); +-- can't merge 4 and 5 +call merge_chunks('_timescaledb_internal._hyper_1_5_chunk', '_timescaledb_internal._hyper_1_4_chunk'); +-- currently can't merge 1,2,3,4 due to limitation in how we validate the merge +call merge_chunks(ARRAY['_timescaledb_internal._hyper_1_3_chunk', '_timescaledb_internal._hyper_1_2_chunk', '_timescaledb_internal._hyper_1_4_chunk', '_timescaledb_internal._hyper_1_1_chunk']); + +begin; +-- Should be able to merge all three chunks 1,2,3 +call merge_chunks(ARRAY['_timescaledb_internal._hyper_1_1_chunk', '_timescaledb_internal._hyper_1_2_chunk', '_timescaledb_internal._hyper_1_3_chunk']); +-- But merging the merged 1,2,3 chunk with 4 is currently not +-- possible, although we chould do it in theory +call merge_chunks('_timescaledb_internal._hyper_1_1_chunk', '_timescaledb_internal._hyper_1_4_chunk'); +rollback; +\set ON_ERROR_STOP 1 + +alter table mergeme set (timescaledb.compress_orderby='time', timescaledb.compress_segmentby='device'); +select compress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +select compress_chunk('_timescaledb_internal._hyper_1_3_chunk'); + +\set ON_ERROR_STOP 0 +-- Currently cannot merge compressed chunks +call merge_chunks('_timescaledb_internal._hyper_1_1_chunk', '_timescaledb_internal._hyper_1_2_chunk'); +call merge_chunks('_timescaledb_internal._hyper_1_2_chunk', '_timescaledb_internal._hyper_1_3_chunk'); +\set ON_ERROR_STOP 1 + +-- Currently cannot merge chunks using Hypercore TAM +alter table _timescaledb_internal._hyper_1_1_chunk set access method hypercore; +alter table _timescaledb_internal._hyper_1_3_chunk set access method hypercore; + +select relname, amname from pg_class cl +join pg_am am on (cl.relam = am.oid) +where cl.oid in ('_timescaledb_internal._hyper_1_1_chunk'::regclass, '_timescaledb_internal._hyper_1_3_chunk'::regclass); + +\set ON_ERROR_STOP 0 +call merge_chunks('_timescaledb_internal._hyper_1_1_chunk', '_timescaledb_internal._hyper_1_2_chunk'); +call merge_chunks('_timescaledb_internal._hyper_1_2_chunk', '_timescaledb_internal._hyper_1_3_chunk'); +\set ON_ERROR_STOP 1 + +--- +-- Test some error cases when merging chunks with non-chunks or chunks +-- from other hypertables +--- +-- Decompress all chunks to ensure we only have non-compressed chunks +select decompress_chunk(ch) from show_chunks('mergeme') ch; + +-- Create a non-chunk table +create table mergeme_too(time timestamptz not null, device int, temp float); +select create_hypertable('mergeme_too', 'time', 'device', 3, chunk_time_interval => interval '1 day'); +create table mergeme_regular(time timestamptz not null, device int, temp float); + +insert into mergeme_too values ('2024-01-01', 1, 1.0); +insert into mergeme_regular select * from mergeme_too; + +create materialized view mergeme_mat as +select * from mergeme_too where device=1; + +select * from show_chunks('mergeme_too'); + + +\set ON_ERROR_STOP 0 +-- Merge chunk and regular table +call merge_chunks('_timescaledb_internal._hyper_1_1_chunk', 'mergeme_regular'); +call merge_chunks('mergeme_regular', '_timescaledb_internal._hyper_1_1_chunk'); +call merge_chunks('_timescaledb_internal._hyper_1_1_chunk', 'mergeme_mat'); +-- Merge chunks from different hypertables +call merge_chunks('_timescaledb_internal._hyper_1_1_chunk', '_timescaledb_internal._hyper_3_8_chunk'); + +-- Merge with unsupported access method +alter table _timescaledb_internal._hyper_1_1_chunk set access method testam; +call merge_chunks('_timescaledb_internal._hyper_1_1_chunk', '_timescaledb_internal._hyper_1_2_chunk'); +alter table _timescaledb_internal._hyper_1_1_chunk set access method heap; + +-- Merge OSM chunks +reset role; +update _timescaledb_catalog.chunk ch set osm_chunk = true where table_name = '_hyper_1_1_chunk'; +set role :ROLE_DEFAULT_PERM_USER; + +call merge_chunks('_timescaledb_internal._hyper_1_1_chunk', '_timescaledb_internal._hyper_1_2_chunk'); +reset role; +update _timescaledb_catalog.chunk ch set osm_chunk = false where table_name = '_hyper_1_1_chunk'; +set role :ROLE_DEFAULT_PERM_USER; + +\set ON_ERROR_STOP 1 + + +-- Set seed to consistently generate same data and same set of chunks +select setseed(0.2); +-- Test merge with bigger data set and chunks with more blocks +insert into mergeme (time, device, temp) +select t, ceil(random()*10), random()*40 +from generate_series('2024-01-01'::timestamptz, '2024-01-04', '0.5s') t; + +-- Show partitions before merge +select * from partitions; + +-- Merge all chunks until only 1 remains +select count(*), sum(device), round(sum(temp)::numeric, 4) from mergeme; +call merge_chunks(ARRAY['_timescaledb_internal._hyper_1_1_chunk', '_timescaledb_internal._hyper_1_4_chunk','_timescaledb_internal._hyper_1_5_chunk', '_timescaledb_internal._hyper_1_11_chunk']); +select count(*), sum(device), round(sum(temp)::numeric, 4) from mergeme; +select * from partitions; +call merge_chunks(ARRAY['_timescaledb_internal._hyper_1_2_chunk', '_timescaledb_internal._hyper_1_9_chunk','_timescaledb_internal._hyper_1_12_chunk', '_timescaledb_internal._hyper_1_14_chunk']); +select count(*), sum(device), round(sum(temp)::numeric, 4) from mergeme; +select * from partitions; +call merge_chunks(ARRAY['_timescaledb_internal._hyper_1_3_chunk', '_timescaledb_internal._hyper_1_10_chunk','_timescaledb_internal._hyper_1_13_chunk', '_timescaledb_internal._hyper_1_15_chunk']); +select count(*), sum(device), round(sum(temp)::numeric, 4) from mergeme; +select * from partitions; +call merge_chunks(ARRAY['_timescaledb_internal._hyper_1_3_chunk', '_timescaledb_internal._hyper_1_1_chunk','_timescaledb_internal._hyper_1_2_chunk']); +select count(*), sum(device), round(sum(temp)::numeric, 4) from mergeme; +select * from partitions; From 6f2aa305f65d6263cc2547335f0dbe019c31f9fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Nordstr=C3=B6m?= Date: Mon, 20 Jan 2025 09:10:06 +0100 Subject: [PATCH 13/32] Fix potential int overflow in merge chunks When merging chunks, a relation size calculation could potentially overflow an 32-bit integer value. Use a 64-bit integer for the result to prevent overflow. --- tsl/src/chunk.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tsl/src/chunk.c b/tsl/src/chunk.c index 6ad0a406832..9bb68957735 100644 --- a/tsl/src/chunk.c +++ b/tsl/src/chunk.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -661,8 +662,10 @@ static TableAmRoutine routine = {}; static uint64 pq17_workaround_merge_relation_size(Relation rel, ForkNumber forkNumber) { + uint64 nblocks = merge_rel_nblocks; + if (forkNumber == MAIN_FORKNUM) - return merge_rel_nblocks * BLCKSZ; + return nblocks * BLCKSZ; return old_routine->relation_size(rel, forkNumber); } From 954f220079a4de64daf91903d9f86001eaf2a056 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Mon, 20 Jan 2025 16:01:03 +0100 Subject: [PATCH 14/32] Determine backport branch based on version not update_from_version This allows switching backport branch without requiring a published release for the version. This patch also bumps version to 2.19.0-dev so 2.18.x becomes the new backport target. --- scripts/backport.py | 9 +++++---- version.config | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/backport.py b/scripts/backport.py index f5f429e999c..49c8f7613cc 100755 --- a/scripts/backport.py +++ b/scripts/backport.py @@ -196,10 +196,11 @@ def git_returncode(command): ] ) -previous_version = version_config["update_from_version"] -previous_version_parts = previous_version.split(".") -previous_version_parts[-1] = "x" -backport_target = ".".join(previous_version_parts) +version = version_config["version"].split("-") +version_parts = version.split(".") +version_parts[1] = str(int(version_parts[1]) - 1) +version_parts[2] = "x" +backport_target = ".".join(version_parts) backported_label = f"backported-{backport_target}" print(f"Will backport to {backport_target}.") diff --git a/version.config b/version.config index 69be6f54f70..f0cedf97f01 100644 --- a/version.config +++ b/version.config @@ -1,3 +1,3 @@ -version = 2.18.0-dev +version = 2.19.0-dev update_from_version = 2.17.2 downgrade_to_version = 2.17.2 From 7428ef799fe1519ec35b140def68154fecd1d730 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Tue, 21 Jan 2025 11:20:05 +0100 Subject: [PATCH 15/32] Fix the backport script (#7607) --- scripts/backport.py | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/scripts/backport.py b/scripts/backport.py index 49c8f7613cc..80164b910f5 100755 --- a/scripts/backport.py +++ b/scripts/backport.py @@ -18,10 +18,12 @@ def run_query(query): """A simple function to use requests.post to make the GraphQL API call.""" + token = os.environ.get("GITHUB_TOKEN") + request = requests.post( "https://api.github.com/graphql", json={"query": query}, - headers={"Authorization": f'Bearer {os.environ.get("GITHUB_TOKEN")}'}, + headers={"Authorization": f"Bearer {token}"} if token else None, timeout=20, ) response = request.json() @@ -164,21 +166,6 @@ def git_returncode(command): source_repo = github.get_repo(source_repo_name) target_repo = github.get_repo(target_repo_name) -# Set git name and email corresponding to the token user. -token_user = github.get_user() -os.environ["GIT_COMMITTER_NAME"] = token_user.name - -# This is an email that is used by Github when you opt to hide your real email -# address. It is required so that the commits are recognized by Github as made -# by the user. That is, if you use a wrong e-mail, there won't be a clickable -# profile picture next to the commit in the Github interface. -os.environ["GIT_COMMITTER_EMAIL"] = ( - f"{token_user.id}+{token_user.login}@users.noreply.github.com" -) -print( - f"Will commit as {os.environ['GIT_COMMITTER_NAME']} <{os.environ['GIT_COMMITTER_EMAIL']}>" -) - # Fetch the main branch. Apparently the local repo can be shallow in some cases # in Github Actions, so specify the depth. --unshallow will complain on normal # repositories, this is why we don't use it here. @@ -196,8 +183,8 @@ def git_returncode(command): ] ) -version = version_config["version"].split("-") -version_parts = version.split(".") +version = version_config["version"].split("-")[0] # Split off the 'dev' suffix. +version_parts = version.split(".") # Split the three version numbers. version_parts[1] = str(int(version_parts[1]) - 1) version_parts[2] = "x" backport_target = ".".join(version_parts) @@ -205,6 +192,7 @@ def git_returncode(command): print(f"Will backport to {backport_target}.") + # Fetch the target branch. Apparently the local repo can be shallow in some cases # in Github Actions, so specify the depth. --unshallow will complain on normal # repositories, this is why we don't use it here. @@ -390,6 +378,22 @@ def report_backport_not_done(original_pr, reason, details=None): original_pr.add_to_labels("auto-backport-not-done") +# Set git name and email corresponding to the token user. +token_user = github.get_user() +os.environ["GIT_COMMITTER_NAME"] = token_user.name + +# This is an email that is used by Github when you opt to hide your real email +# address. It is required so that the commits are recognized by Github as made +# by the user. That is, if you use a wrong e-mail, there won't be a clickable +# profile picture next to the commit in the Github interface. +os.environ["GIT_COMMITTER_EMAIL"] = ( + f"{token_user.id}+{token_user.login}@users.noreply.github.com" +) +print( + f"Will commit as {os.environ['GIT_COMMITTER_NAME']} <{os.environ['GIT_COMMITTER_EMAIL']}>" +) + + # Now, go over the list of PRs that we have collected, and try to backport # each of them. print(f"Have {len(prs_to_backport)} PRs to backport.") From f0996a46bca26f4914608eac9c25b21ee8b6c764 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Wed, 22 Jan 2025 11:51:32 +0100 Subject: [PATCH 16/32] Match the Postgres NaN behavior in vectorized filters (#7598) It has some nonstandard rules that don't match the IEEE floats. --- .unreleased/nan-vectorized-filters | 2 + .../pred_vector_const_arithmetic_type_pair.c | 15 +- tsl/test/expected/decompress_vector_qual.out | 1331 +++++++++++++++++ tsl/test/sql/decompress_vector_qual.sql | 24 + 4 files changed, 1365 insertions(+), 7 deletions(-) create mode 100644 .unreleased/nan-vectorized-filters diff --git a/.unreleased/nan-vectorized-filters b/.unreleased/nan-vectorized-filters new file mode 100644 index 00000000000..9059ee99758 --- /dev/null +++ b/.unreleased/nan-vectorized-filters @@ -0,0 +1,2 @@ +Fixes: #6884 Match the Postgres NaN comparison behavior in WHERE clause over compressed tables. +Thanks: @jakehedlund for reporting the incompatible NaN behavior in WHERE clause over compressed tables. diff --git a/tsl/src/nodes/decompress_chunk/pred_vector_const_arithmetic_type_pair.c b/tsl/src/nodes/decompress_chunk/pred_vector_const_arithmetic_type_pair.c index 37cfd0ebe22..7f75c0b39c7 100644 --- a/tsl/src/nodes/decompress_chunk/pred_vector_const_arithmetic_type_pair.c +++ b/tsl/src/nodes/decompress_chunk/pred_vector_const_arithmetic_type_pair.c @@ -5,31 +5,32 @@ */ /* - * Vector-const predicates for one pair of arithmetic types. + * Vector-const predicates for one pair of arithmetic types. For NaN comparison, + * Postgres has its own nonstandard rules different from the IEEE floats. */ #define PREDICATE_NAME GE -#define PREDICATE_EXPRESSION(X, Y) ((X) >= (Y)) +#define PREDICATE_EXPRESSION(X, Y) (isnan((double) (X)) || (!isnan((double) (Y)) && (X) >= (Y))) #include "pred_vector_const_arithmetic_single.c" #define PREDICATE_NAME LE -#define PREDICATE_EXPRESSION(X, Y) ((X) <= (Y)) +#define PREDICATE_EXPRESSION(X, Y) (isnan((double) (Y)) || (!isnan((double) (X)) && (X) <= (Y))) #include "pred_vector_const_arithmetic_single.c" #define PREDICATE_NAME LT -#define PREDICATE_EXPRESSION(X, Y) ((X) < (Y)) +#define PREDICATE_EXPRESSION(X, Y) (!isnan((double) (X)) && (isnan((double) (Y)) || (X) < (Y))) #include "pred_vector_const_arithmetic_single.c" #define PREDICATE_NAME GT -#define PREDICATE_EXPRESSION(X, Y) ((X) > (Y)) +#define PREDICATE_EXPRESSION(X, Y) (!isnan((double) (Y)) && (isnan((double) (X)) || (X) > (Y))) #include "pred_vector_const_arithmetic_single.c" #define PREDICATE_NAME EQ -#define PREDICATE_EXPRESSION(X, Y) ((X) == (Y)) +#define PREDICATE_EXPRESSION(X, Y) (isnan((double) (X)) ? isnan((double) (Y)) : ((X) == (Y))) #include "pred_vector_const_arithmetic_single.c" #define PREDICATE_NAME NE -#define PREDICATE_EXPRESSION(X, Y) ((X) != (Y)) +#define PREDICATE_EXPRESSION(X, Y) (isnan((double) (X)) ? !isnan((double) (Y)) : ((X) != (Y))) #include "pred_vector_const_arithmetic_single.c" #undef VECTOR_CTYPE diff --git a/tsl/test/expected/decompress_vector_qual.out b/tsl/test/expected/decompress_vector_qual.out index 595c8e6f994..e4ea46bbb18 100644 --- a/tsl/test/expected/decompress_vector_qual.out +++ b/tsl/test/expected/decompress_vector_qual.out @@ -1585,3 +1585,1334 @@ select count(*), min(ts), max(ts), min(d), max(d) from text_table where a > 'sam reset timescaledb.debug_require_vector_qual; reset timescaledb.enable_bulk_decompression; +-- Test the nonstandard Postgres NaN comparison that doesn't match the IEEE floats. +create table nans(t int, cfloat4 float4, cfloat8 float8); +select create_hypertable('nans', 't', chunk_time_interval => 1024 * 1024 * 1024); +NOTICE: adding not-null constraint to column "t" + create_hypertable +-------------------- + (11,public,nans,t) +(1 row) + +alter table nans set (timescaledb.compress); +WARNING: there was some uncertainty picking the default segment by for the hypertable: You do not have any indexes on columns that can be used for segment_by and thus we are not using segment_by for compression. Please make sure you are not missing any indexes +NOTICE: default segment by for hypertable "nans" is set to "" +NOTICE: default order by for hypertable "nans" is set to "t DESC" +insert into nans select pow(2, n), x, x + from unnest(array[null, 0, 1, 'nan', '-inf', '+inf']::float8[]) + with ordinality as x(x, n) +; +select count(compress_chunk(x)) from show_chunks('nans') x; + count +------- + 1 +(1 row) + +set timescaledb.enable_bulk_decompression to on; +set timescaledb.debug_require_vector_qual to 'require'; +select format('select sum(t) from nans where %s %s %s::%s;', + variable, op, value, type) +from + unnest(array['cfloat4', 'cfloat8']) variable, + unnest(array['=', '!=', '<', '<=', '>', '>=']) op, + unnest(array['null', '0', '1', '''nan''', '''-inf''', '''+inf''']) value, + unnest(array['float4', 'float8', 'numeric']) type +\gexec +select sum(t) from nans where cfloat4 = null::float4; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat4 = null::float8; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat4 = null::numeric; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat8 = null::float4; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat8 = null::float8; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat8 = null::numeric; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat4 != null::float4; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat4 != null::float8; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat4 != null::numeric; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat8 != null::float4; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat8 != null::float8; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat8 != null::numeric; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat4 < null::float4; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat4 < null::float8; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat4 < null::numeric; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat8 < null::float4; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat8 < null::float8; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat8 < null::numeric; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat4 <= null::float4; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat4 <= null::float8; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat4 <= null::numeric; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat8 <= null::float4; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat8 <= null::float8; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat8 <= null::numeric; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat4 > null::float4; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat4 > null::float8; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat4 > null::numeric; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat8 > null::float4; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat8 > null::float8; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat8 > null::numeric; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat4 >= null::float4; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat4 >= null::float8; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat4 >= null::numeric; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat8 >= null::float4; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat8 >= null::float8; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat8 >= null::numeric; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat4 = 0::float4; + sum +----- + 4 +(1 row) + +select sum(t) from nans where cfloat4 = 0::float8; + sum +----- + 4 +(1 row) + +select sum(t) from nans where cfloat4 = 0::numeric; + sum +----- + 4 +(1 row) + +select sum(t) from nans where cfloat8 = 0::float4; + sum +----- + 4 +(1 row) + +select sum(t) from nans where cfloat8 = 0::float8; + sum +----- + 4 +(1 row) + +select sum(t) from nans where cfloat8 = 0::numeric; + sum +----- + 4 +(1 row) + +select sum(t) from nans where cfloat4 != 0::float4; + sum +----- + 120 +(1 row) + +select sum(t) from nans where cfloat4 != 0::float8; + sum +----- + 120 +(1 row) + +select sum(t) from nans where cfloat4 != 0::numeric; + sum +----- + 120 +(1 row) + +select sum(t) from nans where cfloat8 != 0::float4; + sum +----- + 120 +(1 row) + +select sum(t) from nans where cfloat8 != 0::float8; + sum +----- + 120 +(1 row) + +select sum(t) from nans where cfloat8 != 0::numeric; + sum +----- + 120 +(1 row) + +select sum(t) from nans where cfloat4 < 0::float4; + sum +----- + 32 +(1 row) + +select sum(t) from nans where cfloat4 < 0::float8; + sum +----- + 32 +(1 row) + +select sum(t) from nans where cfloat4 < 0::numeric; + sum +----- + 32 +(1 row) + +select sum(t) from nans where cfloat8 < 0::float4; + sum +----- + 32 +(1 row) + +select sum(t) from nans where cfloat8 < 0::float8; + sum +----- + 32 +(1 row) + +select sum(t) from nans where cfloat8 < 0::numeric; + sum +----- + 32 +(1 row) + +select sum(t) from nans where cfloat4 <= 0::float4; + sum +----- + 36 +(1 row) + +select sum(t) from nans where cfloat4 <= 0::float8; + sum +----- + 36 +(1 row) + +select sum(t) from nans where cfloat4 <= 0::numeric; + sum +----- + 36 +(1 row) + +select sum(t) from nans where cfloat8 <= 0::float4; + sum +----- + 36 +(1 row) + +select sum(t) from nans where cfloat8 <= 0::float8; + sum +----- + 36 +(1 row) + +select sum(t) from nans where cfloat8 <= 0::numeric; + sum +----- + 36 +(1 row) + +select sum(t) from nans where cfloat4 > 0::float4; + sum +----- + 88 +(1 row) + +select sum(t) from nans where cfloat4 > 0::float8; + sum +----- + 88 +(1 row) + +select sum(t) from nans where cfloat4 > 0::numeric; + sum +----- + 88 +(1 row) + +select sum(t) from nans where cfloat8 > 0::float4; + sum +----- + 88 +(1 row) + +select sum(t) from nans where cfloat8 > 0::float8; + sum +----- + 88 +(1 row) + +select sum(t) from nans where cfloat8 > 0::numeric; + sum +----- + 88 +(1 row) + +select sum(t) from nans where cfloat4 >= 0::float4; + sum +----- + 92 +(1 row) + +select sum(t) from nans where cfloat4 >= 0::float8; + sum +----- + 92 +(1 row) + +select sum(t) from nans where cfloat4 >= 0::numeric; + sum +----- + 92 +(1 row) + +select sum(t) from nans where cfloat8 >= 0::float4; + sum +----- + 92 +(1 row) + +select sum(t) from nans where cfloat8 >= 0::float8; + sum +----- + 92 +(1 row) + +select sum(t) from nans where cfloat8 >= 0::numeric; + sum +----- + 92 +(1 row) + +select sum(t) from nans where cfloat4 = 1::float4; + sum +----- + 8 +(1 row) + +select sum(t) from nans where cfloat4 = 1::float8; + sum +----- + 8 +(1 row) + +select sum(t) from nans where cfloat4 = 1::numeric; + sum +----- + 8 +(1 row) + +select sum(t) from nans where cfloat8 = 1::float4; + sum +----- + 8 +(1 row) + +select sum(t) from nans where cfloat8 = 1::float8; + sum +----- + 8 +(1 row) + +select sum(t) from nans where cfloat8 = 1::numeric; + sum +----- + 8 +(1 row) + +select sum(t) from nans where cfloat4 != 1::float4; + sum +----- + 116 +(1 row) + +select sum(t) from nans where cfloat4 != 1::float8; + sum +----- + 116 +(1 row) + +select sum(t) from nans where cfloat4 != 1::numeric; + sum +----- + 116 +(1 row) + +select sum(t) from nans where cfloat8 != 1::float4; + sum +----- + 116 +(1 row) + +select sum(t) from nans where cfloat8 != 1::float8; + sum +----- + 116 +(1 row) + +select sum(t) from nans where cfloat8 != 1::numeric; + sum +----- + 116 +(1 row) + +select sum(t) from nans where cfloat4 < 1::float4; + sum +----- + 36 +(1 row) + +select sum(t) from nans where cfloat4 < 1::float8; + sum +----- + 36 +(1 row) + +select sum(t) from nans where cfloat4 < 1::numeric; + sum +----- + 36 +(1 row) + +select sum(t) from nans where cfloat8 < 1::float4; + sum +----- + 36 +(1 row) + +select sum(t) from nans where cfloat8 < 1::float8; + sum +----- + 36 +(1 row) + +select sum(t) from nans where cfloat8 < 1::numeric; + sum +----- + 36 +(1 row) + +select sum(t) from nans where cfloat4 <= 1::float4; + sum +----- + 44 +(1 row) + +select sum(t) from nans where cfloat4 <= 1::float8; + sum +----- + 44 +(1 row) + +select sum(t) from nans where cfloat4 <= 1::numeric; + sum +----- + 44 +(1 row) + +select sum(t) from nans where cfloat8 <= 1::float4; + sum +----- + 44 +(1 row) + +select sum(t) from nans where cfloat8 <= 1::float8; + sum +----- + 44 +(1 row) + +select sum(t) from nans where cfloat8 <= 1::numeric; + sum +----- + 44 +(1 row) + +select sum(t) from nans where cfloat4 > 1::float4; + sum +----- + 80 +(1 row) + +select sum(t) from nans where cfloat4 > 1::float8; + sum +----- + 80 +(1 row) + +select sum(t) from nans where cfloat4 > 1::numeric; + sum +----- + 80 +(1 row) + +select sum(t) from nans where cfloat8 > 1::float4; + sum +----- + 80 +(1 row) + +select sum(t) from nans where cfloat8 > 1::float8; + sum +----- + 80 +(1 row) + +select sum(t) from nans where cfloat8 > 1::numeric; + sum +----- + 80 +(1 row) + +select sum(t) from nans where cfloat4 >= 1::float4; + sum +----- + 88 +(1 row) + +select sum(t) from nans where cfloat4 >= 1::float8; + sum +----- + 88 +(1 row) + +select sum(t) from nans where cfloat4 >= 1::numeric; + sum +----- + 88 +(1 row) + +select sum(t) from nans where cfloat8 >= 1::float4; + sum +----- + 88 +(1 row) + +select sum(t) from nans where cfloat8 >= 1::float8; + sum +----- + 88 +(1 row) + +select sum(t) from nans where cfloat8 >= 1::numeric; + sum +----- + 88 +(1 row) + +select sum(t) from nans where cfloat4 = 'nan'::float4; + sum +----- + 16 +(1 row) + +select sum(t) from nans where cfloat4 = 'nan'::float8; + sum +----- + 16 +(1 row) + +select sum(t) from nans where cfloat4 = 'nan'::numeric; + sum +----- + 16 +(1 row) + +select sum(t) from nans where cfloat8 = 'nan'::float4; + sum +----- + 16 +(1 row) + +select sum(t) from nans where cfloat8 = 'nan'::float8; + sum +----- + 16 +(1 row) + +select sum(t) from nans where cfloat8 = 'nan'::numeric; + sum +----- + 16 +(1 row) + +select sum(t) from nans where cfloat4 != 'nan'::float4; + sum +----- + 108 +(1 row) + +select sum(t) from nans where cfloat4 != 'nan'::float8; + sum +----- + 108 +(1 row) + +select sum(t) from nans where cfloat4 != 'nan'::numeric; + sum +----- + 108 +(1 row) + +select sum(t) from nans where cfloat8 != 'nan'::float4; + sum +----- + 108 +(1 row) + +select sum(t) from nans where cfloat8 != 'nan'::float8; + sum +----- + 108 +(1 row) + +select sum(t) from nans where cfloat8 != 'nan'::numeric; + sum +----- + 108 +(1 row) + +select sum(t) from nans where cfloat4 < 'nan'::float4; + sum +----- + 108 +(1 row) + +select sum(t) from nans where cfloat4 < 'nan'::float8; + sum +----- + 108 +(1 row) + +select sum(t) from nans where cfloat4 < 'nan'::numeric; + sum +----- + 108 +(1 row) + +select sum(t) from nans where cfloat8 < 'nan'::float4; + sum +----- + 108 +(1 row) + +select sum(t) from nans where cfloat8 < 'nan'::float8; + sum +----- + 108 +(1 row) + +select sum(t) from nans where cfloat8 < 'nan'::numeric; + sum +----- + 108 +(1 row) + +select sum(t) from nans where cfloat4 <= 'nan'::float4; + sum +----- + 124 +(1 row) + +select sum(t) from nans where cfloat4 <= 'nan'::float8; + sum +----- + 124 +(1 row) + +select sum(t) from nans where cfloat4 <= 'nan'::numeric; + sum +----- + 124 +(1 row) + +select sum(t) from nans where cfloat8 <= 'nan'::float4; + sum +----- + 124 +(1 row) + +select sum(t) from nans where cfloat8 <= 'nan'::float8; + sum +----- + 124 +(1 row) + +select sum(t) from nans where cfloat8 <= 'nan'::numeric; + sum +----- + 124 +(1 row) + +select sum(t) from nans where cfloat4 > 'nan'::float4; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat4 > 'nan'::float8; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat4 > 'nan'::numeric; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat8 > 'nan'::float4; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat8 > 'nan'::float8; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat8 > 'nan'::numeric; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat4 >= 'nan'::float4; + sum +----- + 16 +(1 row) + +select sum(t) from nans where cfloat4 >= 'nan'::float8; + sum +----- + 16 +(1 row) + +select sum(t) from nans where cfloat4 >= 'nan'::numeric; + sum +----- + 16 +(1 row) + +select sum(t) from nans where cfloat8 >= 'nan'::float4; + sum +----- + 16 +(1 row) + +select sum(t) from nans where cfloat8 >= 'nan'::float8; + sum +----- + 16 +(1 row) + +select sum(t) from nans where cfloat8 >= 'nan'::numeric; + sum +----- + 16 +(1 row) + +select sum(t) from nans where cfloat4 = '-inf'::float4; + sum +----- + 32 +(1 row) + +select sum(t) from nans where cfloat4 = '-inf'::float8; + sum +----- + 32 +(1 row) + +select sum(t) from nans where cfloat4 = '-inf'::numeric; + sum +----- + 32 +(1 row) + +select sum(t) from nans where cfloat8 = '-inf'::float4; + sum +----- + 32 +(1 row) + +select sum(t) from nans where cfloat8 = '-inf'::float8; + sum +----- + 32 +(1 row) + +select sum(t) from nans where cfloat8 = '-inf'::numeric; + sum +----- + 32 +(1 row) + +select sum(t) from nans where cfloat4 != '-inf'::float4; + sum +----- + 92 +(1 row) + +select sum(t) from nans where cfloat4 != '-inf'::float8; + sum +----- + 92 +(1 row) + +select sum(t) from nans where cfloat4 != '-inf'::numeric; + sum +----- + 92 +(1 row) + +select sum(t) from nans where cfloat8 != '-inf'::float4; + sum +----- + 92 +(1 row) + +select sum(t) from nans where cfloat8 != '-inf'::float8; + sum +----- + 92 +(1 row) + +select sum(t) from nans where cfloat8 != '-inf'::numeric; + sum +----- + 92 +(1 row) + +select sum(t) from nans where cfloat4 < '-inf'::float4; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat4 < '-inf'::float8; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat4 < '-inf'::numeric; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat8 < '-inf'::float4; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat8 < '-inf'::float8; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat8 < '-inf'::numeric; + sum +----- + +(1 row) + +select sum(t) from nans where cfloat4 <= '-inf'::float4; + sum +----- + 32 +(1 row) + +select sum(t) from nans where cfloat4 <= '-inf'::float8; + sum +----- + 32 +(1 row) + +select sum(t) from nans where cfloat4 <= '-inf'::numeric; + sum +----- + 32 +(1 row) + +select sum(t) from nans where cfloat8 <= '-inf'::float4; + sum +----- + 32 +(1 row) + +select sum(t) from nans where cfloat8 <= '-inf'::float8; + sum +----- + 32 +(1 row) + +select sum(t) from nans where cfloat8 <= '-inf'::numeric; + sum +----- + 32 +(1 row) + +select sum(t) from nans where cfloat4 > '-inf'::float4; + sum +----- + 92 +(1 row) + +select sum(t) from nans where cfloat4 > '-inf'::float8; + sum +----- + 92 +(1 row) + +select sum(t) from nans where cfloat4 > '-inf'::numeric; + sum +----- + 92 +(1 row) + +select sum(t) from nans where cfloat8 > '-inf'::float4; + sum +----- + 92 +(1 row) + +select sum(t) from nans where cfloat8 > '-inf'::float8; + sum +----- + 92 +(1 row) + +select sum(t) from nans where cfloat8 > '-inf'::numeric; + sum +----- + 92 +(1 row) + +select sum(t) from nans where cfloat4 >= '-inf'::float4; + sum +----- + 124 +(1 row) + +select sum(t) from nans where cfloat4 >= '-inf'::float8; + sum +----- + 124 +(1 row) + +select sum(t) from nans where cfloat4 >= '-inf'::numeric; + sum +----- + 124 +(1 row) + +select sum(t) from nans where cfloat8 >= '-inf'::float4; + sum +----- + 124 +(1 row) + +select sum(t) from nans where cfloat8 >= '-inf'::float8; + sum +----- + 124 +(1 row) + +select sum(t) from nans where cfloat8 >= '-inf'::numeric; + sum +----- + 124 +(1 row) + +select sum(t) from nans where cfloat4 = '+inf'::float4; + sum +----- + 64 +(1 row) + +select sum(t) from nans where cfloat4 = '+inf'::float8; + sum +----- + 64 +(1 row) + +select sum(t) from nans where cfloat4 = '+inf'::numeric; + sum +----- + 64 +(1 row) + +select sum(t) from nans where cfloat8 = '+inf'::float4; + sum +----- + 64 +(1 row) + +select sum(t) from nans where cfloat8 = '+inf'::float8; + sum +----- + 64 +(1 row) + +select sum(t) from nans where cfloat8 = '+inf'::numeric; + sum +----- + 64 +(1 row) + +select sum(t) from nans where cfloat4 != '+inf'::float4; + sum +----- + 60 +(1 row) + +select sum(t) from nans where cfloat4 != '+inf'::float8; + sum +----- + 60 +(1 row) + +select sum(t) from nans where cfloat4 != '+inf'::numeric; + sum +----- + 60 +(1 row) + +select sum(t) from nans where cfloat8 != '+inf'::float4; + sum +----- + 60 +(1 row) + +select sum(t) from nans where cfloat8 != '+inf'::float8; + sum +----- + 60 +(1 row) + +select sum(t) from nans where cfloat8 != '+inf'::numeric; + sum +----- + 60 +(1 row) + +select sum(t) from nans where cfloat4 < '+inf'::float4; + sum +----- + 44 +(1 row) + +select sum(t) from nans where cfloat4 < '+inf'::float8; + sum +----- + 44 +(1 row) + +select sum(t) from nans where cfloat4 < '+inf'::numeric; + sum +----- + 44 +(1 row) + +select sum(t) from nans where cfloat8 < '+inf'::float4; + sum +----- + 44 +(1 row) + +select sum(t) from nans where cfloat8 < '+inf'::float8; + sum +----- + 44 +(1 row) + +select sum(t) from nans where cfloat8 < '+inf'::numeric; + sum +----- + 44 +(1 row) + +select sum(t) from nans where cfloat4 <= '+inf'::float4; + sum +----- + 108 +(1 row) + +select sum(t) from nans where cfloat4 <= '+inf'::float8; + sum +----- + 108 +(1 row) + +select sum(t) from nans where cfloat4 <= '+inf'::numeric; + sum +----- + 108 +(1 row) + +select sum(t) from nans where cfloat8 <= '+inf'::float4; + sum +----- + 108 +(1 row) + +select sum(t) from nans where cfloat8 <= '+inf'::float8; + sum +----- + 108 +(1 row) + +select sum(t) from nans where cfloat8 <= '+inf'::numeric; + sum +----- + 108 +(1 row) + +select sum(t) from nans where cfloat4 > '+inf'::float4; + sum +----- + 16 +(1 row) + +select sum(t) from nans where cfloat4 > '+inf'::float8; + sum +----- + 16 +(1 row) + +select sum(t) from nans where cfloat4 > '+inf'::numeric; + sum +----- + 16 +(1 row) + +select sum(t) from nans where cfloat8 > '+inf'::float4; + sum +----- + 16 +(1 row) + +select sum(t) from nans where cfloat8 > '+inf'::float8; + sum +----- + 16 +(1 row) + +select sum(t) from nans where cfloat8 > '+inf'::numeric; + sum +----- + 16 +(1 row) + +select sum(t) from nans where cfloat4 >= '+inf'::float4; + sum +----- + 80 +(1 row) + +select sum(t) from nans where cfloat4 >= '+inf'::float8; + sum +----- + 80 +(1 row) + +select sum(t) from nans where cfloat4 >= '+inf'::numeric; + sum +----- + 80 +(1 row) + +select sum(t) from nans where cfloat8 >= '+inf'::float4; + sum +----- + 80 +(1 row) + +select sum(t) from nans where cfloat8 >= '+inf'::float8; + sum +----- + 80 +(1 row) + +select sum(t) from nans where cfloat8 >= '+inf'::numeric; + sum +----- + 80 +(1 row) + +reset timescaledb.debug_require_vector_qual; +reset timescaledb.enable_bulk_decompression; diff --git a/tsl/test/sql/decompress_vector_qual.sql b/tsl/test/sql/decompress_vector_qual.sql index f22dacba002..e02b5d98185 100644 --- a/tsl/test/sql/decompress_vector_qual.sql +++ b/tsl/test/sql/decompress_vector_qual.sql @@ -514,6 +514,30 @@ set timescaledb.debug_require_vector_qual to 'forbid'; select count(*), min(ts), max(ts), min(d), max(d) from text_table where a < 'same'; select count(*), min(ts), max(ts), min(d), max(d) from text_table where a > 'same'; +reset timescaledb.debug_require_vector_qual; +reset timescaledb.enable_bulk_decompression; + +-- Test the nonstandard Postgres NaN comparison that doesn't match the IEEE floats. +create table nans(t int, cfloat4 float4, cfloat8 float8); +select create_hypertable('nans', 't', chunk_time_interval => 1024 * 1024 * 1024); +alter table nans set (timescaledb.compress); +insert into nans select pow(2, n), x, x + from unnest(array[null, 0, 1, 'nan', '-inf', '+inf']::float8[]) + with ordinality as x(x, n) +; +select count(compress_chunk(x)) from show_chunks('nans') x; + +set timescaledb.enable_bulk_decompression to on; +set timescaledb.debug_require_vector_qual to 'require'; + +select format('select sum(t) from nans where %s %s %s::%s;', + variable, op, value, type) +from + unnest(array['cfloat4', 'cfloat8']) variable, + unnest(array['=', '!=', '<', '<=', '>', '>=']) op, + unnest(array['null', '0', '1', '''nan''', '''-inf''', '''+inf''']) value, + unnest(array['float4', 'float8', 'numeric']) type +\gexec reset timescaledb.debug_require_vector_qual; reset timescaledb.enable_bulk_decompression; From f3df13ad36cb5eec38d92eff3b2af5a08b68973f Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Wed, 22 Jan 2025 14:30:11 +0100 Subject: [PATCH 17/32] Refactor batch segment metadata builder for bloom filters (#7597) We plan to implement bloom filter sparse indexes for compressed data, so the compressed batch metadata builder requires some cosmetic changes to support other types of metadata. --- tsl/src/compression/CMakeLists.txt | 4 +- tsl/src/compression/batch_metadata_builder.h | 22 +++ .../batch_metadata_builder_minmax.c | 184 ++++++++++++++++++ .../batch_metadata_builder_minmax.h | 44 +++++ tsl/src/compression/compression.c | 74 +++---- tsl/src/compression/compression.h | 6 +- tsl/src/compression/segment_meta.c | 132 ------------- tsl/src/compression/segment_meta.h | 36 ---- tsl/src/init.c | 1 - .../nodes/decompress_chunk/qual_pushdown.c | 1 - tsl/test/src/compression_unit_test.c | 29 +-- 11 files changed, 292 insertions(+), 241 deletions(-) create mode 100644 tsl/src/compression/batch_metadata_builder.h create mode 100644 tsl/src/compression/batch_metadata_builder_minmax.c create mode 100644 tsl/src/compression/batch_metadata_builder_minmax.h delete mode 100644 tsl/src/compression/segment_meta.c delete mode 100644 tsl/src/compression/segment_meta.h diff --git a/tsl/src/compression/CMakeLists.txt b/tsl/src/compression/CMakeLists.txt index d7368c42cc8..f155448b2e7 100644 --- a/tsl/src/compression/CMakeLists.txt +++ b/tsl/src/compression/CMakeLists.txt @@ -1,12 +1,12 @@ set(SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/api.c + ${CMAKE_CURRENT_SOURCE_DIR}/batch_metadata_builder_minmax.c ${CMAKE_CURRENT_SOURCE_DIR}/compression.c ${CMAKE_CURRENT_SOURCE_DIR}/compression_dml.c ${CMAKE_CURRENT_SOURCE_DIR}/compression_scankey.c ${CMAKE_CURRENT_SOURCE_DIR}/compression_storage.c ${CMAKE_CURRENT_SOURCE_DIR}/create.c - ${CMAKE_CURRENT_SOURCE_DIR}/recompress.c - ${CMAKE_CURRENT_SOURCE_DIR}/segment_meta.c) + ${CMAKE_CURRENT_SOURCE_DIR}/recompress.c) target_sources(${TSL_LIBRARY_NAME} PRIVATE ${SOURCES}) add_subdirectory(algorithms) diff --git a/tsl/src/compression/batch_metadata_builder.h b/tsl/src/compression/batch_metadata_builder.h new file mode 100644 index 00000000000..606038e0c2c --- /dev/null +++ b/tsl/src/compression/batch_metadata_builder.h @@ -0,0 +1,22 @@ +/* + * This file and its contents are licensed under the Timescale License. + * Please see the included NOTICE for copyright information and + * LICENSE-TIMESCALE for a copy of the license. + */ +#pragma once + +typedef struct RowCompressor RowCompressor; + +typedef struct BatchMetadataBuilder +{ + void (*update_val)(void *builder, Datum val); + void (*update_null)(void *builder); + + void (*insert_to_compressed_row)(void *builder, RowCompressor *compressor); + + void (*reset)(void *builder, RowCompressor *compressor); +} BatchMetadataBuilder; + +BatchMetadataBuilder *batch_metadata_builder_minmax_create(Oid type, Oid collation, + int min_attr_offset, + int max_attr_offset); diff --git a/tsl/src/compression/batch_metadata_builder_minmax.c b/tsl/src/compression/batch_metadata_builder_minmax.c new file mode 100644 index 00000000000..899487396c5 --- /dev/null +++ b/tsl/src/compression/batch_metadata_builder_minmax.c @@ -0,0 +1,184 @@ +/* + * This file and its contents are licensed under the Timescale License. + * Please see the included NOTICE for copyright information and + * LICENSE-TIMESCALE for a copy of the license. + */ +#include +#include +#include +#include +#include +#include + +#include "batch_metadata_builder_minmax.h" + +#include "compression.h" + +static void minmax_update_val(void *builder_, Datum val); +static void minmax_update_null(void *builder_); +static void minmax_insert_to_compressed_row(void *builder_, RowCompressor *compressor); +static void minmax_reset(void *builder_, RowCompressor *compressor); + +BatchMetadataBuilder * +batch_metadata_builder_minmax_create(Oid type_oid, Oid collation, int min_attr_offset, + int max_attr_offset) +{ + BatchMetadataBuilderMinMax *builder = palloc(sizeof(*builder)); + TypeCacheEntry *type = lookup_type_cache(type_oid, TYPECACHE_LT_OPR); + + if (!OidIsValid(type->lt_opr)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FUNCTION), + errmsg("could not identify an less-than operator for type %s", + format_type_be(type_oid)))); + + *builder = (BatchMetadataBuilderMinMax){ + .functions = + (BatchMetadataBuilder){ + .update_val = minmax_update_val, + .update_null = minmax_update_null, + .insert_to_compressed_row = minmax_insert_to_compressed_row, + .reset = minmax_reset, + }, + .type_oid = type_oid, + .empty = true, + .has_null = false, + .type_by_val = type->typbyval, + .type_len = type->typlen, + .min_metadata_attr_offset = min_attr_offset, + .max_metadata_attr_offset = max_attr_offset, + }; + + builder->ssup.ssup_cxt = CurrentMemoryContext; + builder->ssup.ssup_collation = collation; + builder->ssup.ssup_nulls_first = false; + + PrepareSortSupportFromOrderingOp(type->lt_opr, &builder->ssup); + + return &builder->functions; +} + +void +minmax_update_val(void *builder_, Datum val) +{ + BatchMetadataBuilderMinMax *builder = (BatchMetadataBuilderMinMax *) builder_; + + int cmp; + + if (builder->empty) + { + builder->min = datumCopy(val, builder->type_by_val, builder->type_len); + builder->max = datumCopy(val, builder->type_by_val, builder->type_len); + builder->empty = false; + return; + } + + cmp = ApplySortComparator(builder->min, false, val, false, &builder->ssup); + if (cmp > 0) + { + if (!builder->type_by_val) + pfree(DatumGetPointer(builder->min)); + builder->min = datumCopy(val, builder->type_by_val, builder->type_len); + } + + cmp = ApplySortComparator(builder->max, false, val, false, &builder->ssup); + if (cmp < 0) + { + if (!builder->type_by_val) + pfree(DatumGetPointer(builder->max)); + builder->max = datumCopy(val, builder->type_by_val, builder->type_len); + } +} + +void +minmax_update_null(void *builder_) +{ + BatchMetadataBuilderMinMax *builder = (BatchMetadataBuilderMinMax *) builder_; + builder->has_null = true; +} + +static void +minmax_reset(void *builder_, RowCompressor *compressor) +{ + BatchMetadataBuilderMinMax *builder = (BatchMetadataBuilderMinMax *) builder_; + if (!builder->empty) + { + if (!builder->type_by_val) + { + pfree(DatumGetPointer(builder->min)); + pfree(DatumGetPointer(builder->max)); + } + builder->min = 0; + builder->max = 0; + } + builder->empty = true; + builder->has_null = false; + + compressor->compressed_is_null[builder->max_metadata_attr_offset] = true; + compressor->compressed_is_null[builder->min_metadata_attr_offset] = true; + compressor->compressed_values[builder->min_metadata_attr_offset] = 0; + compressor->compressed_values[builder->max_metadata_attr_offset] = 0; +} + +Datum +batch_metadata_builder_minmax_min(void *builder_) +{ + BatchMetadataBuilderMinMax *builder = (BatchMetadataBuilderMinMax *) builder_; + if (builder->empty) + elog(ERROR, "trying to get min from an empty builder"); + if (builder->type_len == -1) + { + Datum unpacked = PointerGetDatum(PG_DETOAST_DATUM_PACKED(builder->min)); + if (builder->min != unpacked) + pfree(DatumGetPointer(builder->min)); + builder->min = unpacked; + } + return builder->min; +} + +Datum +batch_metadata_builder_minmax_max(void *builder_) +{ + BatchMetadataBuilderMinMax *builder = (BatchMetadataBuilderMinMax *) builder_; + if (builder->empty) + elog(ERROR, "trying to get max from an empty builder"); + if (builder->type_len == -1) + { + Datum unpacked = PointerGetDatum(PG_DETOAST_DATUM_PACKED(builder->max)); + if (builder->max != unpacked) + pfree(DatumGetPointer(builder->max)); + builder->max = unpacked; + } + return builder->max; +} + +bool +batch_metadata_builder_minmax_empty(void *builder_) +{ + BatchMetadataBuilderMinMax *builder = (BatchMetadataBuilderMinMax *) builder_; + return builder->empty; +} + +static void +minmax_insert_to_compressed_row(void *builder_, RowCompressor *compressor) +{ + BatchMetadataBuilderMinMax *builder = (BatchMetadataBuilderMinMax *) builder_; + Assert(builder->min_metadata_attr_offset >= 0); + Assert(builder->max_metadata_attr_offset >= 0); + + if (!batch_metadata_builder_minmax_empty(builder)) + { + compressor->compressed_is_null[builder->min_metadata_attr_offset] = false; + compressor->compressed_is_null[builder->max_metadata_attr_offset] = false; + + compressor->compressed_values[builder->min_metadata_attr_offset] = + batch_metadata_builder_minmax_min(builder); + compressor->compressed_values[builder->max_metadata_attr_offset] = + batch_metadata_builder_minmax_max(builder); + } + else + { + compressor->compressed_is_null[builder->min_metadata_attr_offset] = true; + compressor->compressed_is_null[builder->max_metadata_attr_offset] = true; + } +} diff --git a/tsl/src/compression/batch_metadata_builder_minmax.h b/tsl/src/compression/batch_metadata_builder_minmax.h new file mode 100644 index 00000000000..5d0645b8a2d --- /dev/null +++ b/tsl/src/compression/batch_metadata_builder_minmax.h @@ -0,0 +1,44 @@ +/* + * This file and its contents are licensed under the Timescale License. + * Please see the included NOTICE for copyright information and + * LICENSE-TIMESCALE for a copy of the license. + */ +#pragma once + +#include +#include +#include +#include + +#include "batch_metadata_builder.h" + +typedef struct BatchMetadataBuilderMinMax +{ + BatchMetadataBuilder functions; + + Oid type_oid; + bool empty; + bool has_null; + + SortSupportData ssup; + bool type_by_val; + int16 type_len; + Datum min; + Datum max; + + int16 min_metadata_attr_offset; + int16 max_metadata_attr_offset; +} BatchMetadataBuilderMinMax; + +typedef struct BatchMetadataBuilderMinMax BatchMetadataBuilderMinMax; + +typedef struct RowCompressor RowCompressor; + +/* + * This is exposed only for the old unit tests. Ideally they should be replaced + * with functional tests inspecting the compressed chunk table, and this + * test-only interface should be removed. + */ +Datum batch_metadata_builder_minmax_min(void *builder_); +Datum batch_metadata_builder_minmax_max(void *builder_); +bool batch_metadata_builder_minmax_empty(void *builder_); diff --git a/tsl/src/compression/compression.c b/tsl/src/compression/compression.c index c8a239f9ae8..94259fce222 100644 --- a/tsl/src/compression/compression.c +++ b/tsl/src/compression/compression.c @@ -22,6 +22,7 @@ #include "algorithms/deltadelta.h" #include "algorithms/dictionary.h" #include "algorithms/gorilla.h" +#include "batch_metadata_builder.h" #include "chunk.h" #include "compression.h" #include "create.h" @@ -31,7 +32,6 @@ #include "guc.h" #include "hypercore/hypercore_handler.h" #include "nodes/chunk_dispatch/chunk_insert_state.h" -#include "segment_meta.h" #include "ts_catalog/array_utils.h" #include "ts_catalog/catalog.h" #include "ts_catalog/compression_chunk_size.h" @@ -742,7 +742,7 @@ build_column_map(CompressionSettings *settings, Relation uncompressed_table, int16 segment_min_attr_offset = segment_min_attr_number - 1; int16 segment_max_attr_offset = segment_max_attr_number - 1; - SegmentMetaMinMaxBuilder *segment_min_max_builder = NULL; + BatchMetadataBuilder *batch_minmax_builder = NULL; if (segment_min_attr_number != InvalidAttrNumber || segment_max_attr_number != InvalidAttrNumber) { @@ -750,18 +750,19 @@ build_column_map(CompressionSettings *settings, Relation uncompressed_table, "could not find the min metadata column"); Ensure(segment_max_attr_number != InvalidAttrNumber, "could not find the min metadata column"); - segment_min_max_builder = - segment_meta_min_max_builder_create(attr->atttypid, attr->attcollation); + batch_minmax_builder = + batch_metadata_builder_minmax_create(attr->atttypid, + attr->attcollation, + segment_min_attr_offset, + segment_max_attr_offset); } - Ensure(!is_orderby || segment_min_max_builder != NULL, + Ensure(!is_orderby || batch_minmax_builder != NULL, "orderby columns must have minmax metadata"); *column = (PerColumn){ .compressor = compressor_for_type(attr->atttypid), - .min_metadata_attr_offset = segment_min_attr_offset, - .max_metadata_attr_offset = segment_max_attr_offset, - .min_max_metadata_builder = segment_min_max_builder, + .metadata_builder = batch_minmax_builder, .segmentby_column_index = -1, }; } @@ -775,8 +776,6 @@ build_column_map(CompressionSettings *settings, Relation uncompressed_table, *column = (PerColumn){ .segment_info = segment_info_new(attr), .segmentby_column_index = index, - .min_metadata_attr_offset = -1, - .max_metadata_attr_offset = -1, }; } } @@ -972,21 +971,23 @@ row_compressor_append_row(RowCompressor *row_compressor, TupleTableSlot *row) /* Performance Improvement: Since we call getallatts at the beginning, slot_getattr is * useless overhead here, and we should just access the array directly. */ + BatchMetadataBuilder *builder = row_compressor->per_column[col].metadata_builder; val = slot_getattr(row, AttrOffsetGetAttrNumber(col), &is_null); if (is_null) { compressor->append_null(compressor); - if (row_compressor->per_column[col].min_max_metadata_builder != NULL) - segment_meta_min_max_builder_update_null( - row_compressor->per_column[col].min_max_metadata_builder); + if (builder != NULL) + { + builder->update_null(builder); + } } else { compressor->append_val(compressor, val); - if (row_compressor->per_column[col].min_max_metadata_builder != NULL) - segment_meta_min_max_builder_update_val(row_compressor->per_column[col] - .min_max_metadata_builder, - val); + if (builder != NULL) + { + builder->update_val(builder, val); + } } } @@ -1024,28 +1025,10 @@ row_compressor_flush(RowCompressor *row_compressor, CommandId mycid, bool change row_compressor->compressed_values[compressed_col] = PointerGetDatum(compressed_data); - if (column->min_max_metadata_builder != NULL) + if (column->metadata_builder != NULL) { - Assert(column->min_metadata_attr_offset >= 0); - Assert(column->max_metadata_attr_offset >= 0); - - if (!segment_meta_min_max_builder_empty(column->min_max_metadata_builder)) - { - Assert(compressed_data != NULL); - row_compressor->compressed_is_null[column->min_metadata_attr_offset] = false; - row_compressor->compressed_is_null[column->max_metadata_attr_offset] = false; - - row_compressor->compressed_values[column->min_metadata_attr_offset] = - segment_meta_min_max_builder_min(column->min_max_metadata_builder); - row_compressor->compressed_values[column->max_metadata_attr_offset] = - segment_meta_min_max_builder_max(column->min_max_metadata_builder); - } - else - { - Assert(compressed_data == NULL); - row_compressor->compressed_is_null[column->min_metadata_attr_offset] = true; - row_compressor->compressed_is_null[column->max_metadata_attr_offset] = true; - } + column->metadata_builder->insert_to_compressed_row(column->metadata_builder, + row_compressor); } } else if (column->segment_info != NULL) @@ -1096,20 +1079,9 @@ row_compressor_flush(RowCompressor *row_compressor, CommandId mycid, bool change if (column->compressor != NULL || !column->segment_info->typ_by_val) pfree(DatumGetPointer(row_compressor->compressed_values[compressed_col])); - if (column->min_max_metadata_builder != NULL) + if (column->metadata_builder != NULL) { - /* segment_meta_min_max_builder_reset will free the values, so clear here */ - if (!row_compressor->compressed_is_null[column->min_metadata_attr_offset]) - { - row_compressor->compressed_values[column->min_metadata_attr_offset] = 0; - row_compressor->compressed_is_null[column->min_metadata_attr_offset] = true; - } - if (!row_compressor->compressed_is_null[column->max_metadata_attr_offset]) - { - row_compressor->compressed_values[column->max_metadata_attr_offset] = 0; - row_compressor->compressed_is_null[column->max_metadata_attr_offset] = true; - } - segment_meta_min_max_builder_reset(column->min_max_metadata_builder); + column->metadata_builder->reset(column->metadata_builder, row_compressor); } row_compressor->compressed_values[compressed_col] = 0; diff --git a/tsl/src/compression/compression.h b/tsl/src/compression/compression.h index 0ede01bdf0e..25d2c9c5fd1 100644 --- a/tsl/src/compression/compression.h +++ b/tsl/src/compression/compression.h @@ -16,9 +16,9 @@ typedef struct BulkInsertStateData *BulkInsertState; #include "compat/compat.h" +#include "batch_metadata_builder_minmax.h" #include "hypertable.h" #include "nodes/decompress_chunk/detoaster.h" -#include "segment_meta.h" #include "ts_catalog/compression_settings.h" /* @@ -215,9 +215,7 @@ typedef struct PerColumn * Information on the metadata we'll store for this column (currently only min/max). * Only used for order-by columns right now, will be {-1, NULL} for others. */ - int16 min_metadata_attr_offset; - int16 max_metadata_attr_offset; - SegmentMetaMinMaxBuilder *min_max_metadata_builder; + BatchMetadataBuilder *metadata_builder; /* segment info; only used if compressor is NULL */ SegmentInfo *segment_info; diff --git a/tsl/src/compression/segment_meta.c b/tsl/src/compression/segment_meta.c deleted file mode 100644 index f51fceb3145..00000000000 --- a/tsl/src/compression/segment_meta.c +++ /dev/null @@ -1,132 +0,0 @@ - -/* - * This file and its contents are licensed under the Timescale License. - * Please see the included NOTICE for copyright information and - * LICENSE-TIMESCALE for a copy of the license. - */ -#include -#include -#include -#include -#include -#include - -#include "segment_meta.h" - -SegmentMetaMinMaxBuilder * -segment_meta_min_max_builder_create(Oid type_oid, Oid collation) -{ - SegmentMetaMinMaxBuilder *builder = palloc(sizeof(*builder)); - TypeCacheEntry *type = lookup_type_cache(type_oid, TYPECACHE_LT_OPR); - - if (!OidIsValid(type->lt_opr)) - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_FUNCTION), - errmsg("could not identify an less-than operator for type %s", - format_type_be(type_oid)))); - - *builder = (SegmentMetaMinMaxBuilder){ - .type_oid = type_oid, - .empty = true, - .has_null = false, - .type_by_val = type->typbyval, - .type_len = type->typlen, - }; - - builder->ssup.ssup_cxt = CurrentMemoryContext; - builder->ssup.ssup_collation = collation; - builder->ssup.ssup_nulls_first = false; - - PrepareSortSupportFromOrderingOp(type->lt_opr, &builder->ssup); - - return builder; -} - -void -segment_meta_min_max_builder_update_val(SegmentMetaMinMaxBuilder *builder, Datum val) -{ - int cmp; - - if (builder->empty) - { - builder->min = datumCopy(val, builder->type_by_val, builder->type_len); - builder->max = datumCopy(val, builder->type_by_val, builder->type_len); - builder->empty = false; - return; - } - - cmp = ApplySortComparator(builder->min, false, val, false, &builder->ssup); - if (cmp > 0) - { - if (!builder->type_by_val) - pfree(DatumGetPointer(builder->min)); - builder->min = datumCopy(val, builder->type_by_val, builder->type_len); - } - - cmp = ApplySortComparator(builder->max, false, val, false, &builder->ssup); - if (cmp < 0) - { - if (!builder->type_by_val) - pfree(DatumGetPointer(builder->max)); - builder->max = datumCopy(val, builder->type_by_val, builder->type_len); - } -} - -void -segment_meta_min_max_builder_update_null(SegmentMetaMinMaxBuilder *builder) -{ - builder->has_null = true; -} - -void -segment_meta_min_max_builder_reset(SegmentMetaMinMaxBuilder *builder) -{ - if (!builder->empty) - { - if (!builder->type_by_val) - { - pfree(DatumGetPointer(builder->min)); - pfree(DatumGetPointer(builder->max)); - } - builder->min = 0; - builder->max = 0; - } - builder->empty = true; - builder->has_null = false; -} - -Datum -segment_meta_min_max_builder_min(SegmentMetaMinMaxBuilder *builder) -{ - if (builder->empty) - elog(ERROR, "trying to get min from an empty builder"); - if (builder->type_len == -1) - { - Datum unpacked = PointerGetDatum(PG_DETOAST_DATUM_PACKED(builder->min)); - if (builder->min != unpacked) - pfree(DatumGetPointer(builder->min)); - builder->min = unpacked; - } - return builder->min; -} - -Datum -segment_meta_min_max_builder_max(SegmentMetaMinMaxBuilder *builder) -{ - if (builder->empty) - elog(ERROR, "trying to get max from an empty builder"); - if (builder->type_len == -1) - { - Datum unpacked = PointerGetDatum(PG_DETOAST_DATUM_PACKED(builder->max)); - if (builder->max != unpacked) - pfree(DatumGetPointer(builder->max)); - builder->max = unpacked; - } - return builder->max; -} - -bool -segment_meta_min_max_builder_empty(SegmentMetaMinMaxBuilder *builder) -{ - return builder->empty; -} diff --git a/tsl/src/compression/segment_meta.h b/tsl/src/compression/segment_meta.h deleted file mode 100644 index 2b8876dd4ed..00000000000 --- a/tsl/src/compression/segment_meta.h +++ /dev/null @@ -1,36 +0,0 @@ -/* - * This file and its contents are licensed under the Timescale License. - * Please see the included NOTICE for copyright information and - * LICENSE-TIMESCALE for a copy of the license. - */ -#pragma once - -#include -#include -#include -#include - -typedef struct SegmentMetaMinMaxBuilder -{ - Oid type_oid; - bool empty; - bool has_null; - - SortSupportData ssup; - bool type_by_val; - int16 type_len; - Datum min; - Datum max; -} SegmentMetaMinMaxBuilder; - -typedef struct SegmentMetaMinMaxBuilder SegmentMetaMinMaxBuilder; - -SegmentMetaMinMaxBuilder *segment_meta_min_max_builder_create(Oid type, Oid collation); -void segment_meta_min_max_builder_update_val(SegmentMetaMinMaxBuilder *builder, Datum val); -void segment_meta_min_max_builder_update_null(SegmentMetaMinMaxBuilder *builder); - -Datum segment_meta_min_max_builder_min(SegmentMetaMinMaxBuilder *builder); -Datum segment_meta_min_max_builder_max(SegmentMetaMinMaxBuilder *builder); -bool segment_meta_min_max_builder_empty(SegmentMetaMinMaxBuilder *builder); - -void segment_meta_min_max_builder_reset(SegmentMetaMinMaxBuilder *builder); diff --git a/tsl/src/init.c b/tsl/src/init.c index 594d2277987..550fcb0d14e 100644 --- a/tsl/src/init.c +++ b/tsl/src/init.c @@ -24,7 +24,6 @@ #include "compression/compression.h" #include "compression/create.h" #include "compression/recompress.h" -#include "compression/segment_meta.h" #include "config.h" #include "continuous_aggs/create.h" #include "continuous_aggs/insert.h" diff --git a/tsl/src/nodes/decompress_chunk/qual_pushdown.c b/tsl/src/nodes/decompress_chunk/qual_pushdown.c index 514202793e9..dfc666d6668 100644 --- a/tsl/src/nodes/decompress_chunk/qual_pushdown.c +++ b/tsl/src/nodes/decompress_chunk/qual_pushdown.c @@ -14,7 +14,6 @@ #include #include "compression/create.h" -#include "compression/segment_meta.h" #include "custom_type_cache.h" #include "decompress_chunk.h" #include "qual_pushdown.h" diff --git a/tsl/test/src/compression_unit_test.c b/tsl/test/src/compression_unit_test.c index 0276af438b0..d49b4997272 100644 --- a/tsl/test/src/compression_unit_test.c +++ b/tsl/test/src/compression_unit_test.c @@ -29,7 +29,7 @@ #include "compression/algorithms/float_utils.h" #include "compression/algorithms/gorilla.h" #include "compression/arrow_c_data_interface.h" -#include "compression/segment_meta.h" +#include "compression/batch_metadata_builder_minmax.h" #define TEST_ELEMENTS 1015 @@ -649,8 +649,8 @@ TS_FUNCTION_INFO_V1(ts_segment_meta_min_max_append); Datum ts_segment_meta_min_max_append(PG_FUNCTION_ARGS) { - SegmentMetaMinMaxBuilder *builder = - (SegmentMetaMinMaxBuilder *) (PG_ARGISNULL(0) ? NULL : PG_GETARG_POINTER(0)); + BatchMetadataBuilder *builder = + (BatchMetadataBuilder *) (PG_ARGISNULL(0) ? NULL : PG_GETARG_POINTER(0)); MemoryContext agg_context; MemoryContext old_context; @@ -665,12 +665,13 @@ ts_segment_meta_min_max_append(PG_FUNCTION_ARGS) if (builder == NULL) { Oid type_to_compress = get_fn_expr_argtype(fcinfo->flinfo, 1); - builder = segment_meta_min_max_builder_create(type_to_compress, fcinfo->fncollation); + builder = + batch_metadata_builder_minmax_create(type_to_compress, fcinfo->fncollation, -1, -1); } if (PG_ARGISNULL(1)) - segment_meta_min_max_builder_update_null(builder); + builder->update_null(builder); else - segment_meta_min_max_builder_update_val(builder, PG_GETARG_DATUM(1)); + builder->update_val(builder, PG_GETARG_DATUM(1)); MemoryContextSwitchTo(old_context); PG_RETURN_POINTER(builder); @@ -680,26 +681,26 @@ TS_FUNCTION_INFO_V1(ts_segment_meta_min_max_finish_max); Datum ts_segment_meta_min_max_finish_max(PG_FUNCTION_ARGS) { - SegmentMetaMinMaxBuilder *builder = - (SegmentMetaMinMaxBuilder *) (PG_ARGISNULL(0) ? NULL : PG_GETARG_POINTER(0)); + BatchMetadataBuilderMinMax *builder = + (BatchMetadataBuilderMinMax *) (PG_ARGISNULL(0) ? NULL : PG_GETARG_POINTER(0)); - if (builder == NULL || segment_meta_min_max_builder_empty(builder)) + if (builder == NULL || batch_metadata_builder_minmax_empty(builder)) PG_RETURN_NULL(); - PG_RETURN_DATUM(segment_meta_min_max_builder_max(builder)); + PG_RETURN_DATUM(batch_metadata_builder_minmax_max(builder)); } TS_FUNCTION_INFO_V1(ts_segment_meta_min_max_finish_min); Datum ts_segment_meta_min_max_finish_min(PG_FUNCTION_ARGS) { - SegmentMetaMinMaxBuilder *builder = - (SegmentMetaMinMaxBuilder *) (PG_ARGISNULL(0) ? NULL : PG_GETARG_POINTER(0)); + BatchMetadataBuilderMinMax *builder = + (BatchMetadataBuilderMinMax *) (PG_ARGISNULL(0) ? NULL : PG_GETARG_POINTER(0)); - if (builder == NULL || segment_meta_min_max_builder_empty(builder)) + if (builder == NULL || batch_metadata_builder_minmax_empty(builder)) PG_RETURN_NULL(); - PG_RETURN_DATUM(segment_meta_min_max_builder_min(builder)); + PG_RETURN_DATUM(batch_metadata_builder_minmax_min(builder)); } TS_FUNCTION_INFO_V1(ts_compression_custom_type_in); From cb5ff43560a216e12364b06c8004c11dcfbd0547 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Wed, 22 Jan 2025 19:34:11 +0100 Subject: [PATCH 18/32] Upload CMake logs to the action artifacts (#7611) For debugging. --- .github/workflows/linux-32bit-build-and-test.yaml | 14 +++++++++++++- .github/workflows/linux-build-and-test.yaml | 12 ++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/.github/workflows/linux-32bit-build-and-test.yaml b/.github/workflows/linux-32bit-build-and-test.yaml index b159f83ec7e..51fcc458cf5 100644 --- a/.github/workflows/linux-32bit-build-and-test.yaml +++ b/.github/workflows/linux-32bit-build-and-test.yaml @@ -128,7 +128,7 @@ jobs: if: always() && steps.cache-postgresql.outputs.cache-hit != 'true' uses: actions/upload-artifact@v4 with: - name: config.log for i386 PostgreSQL ${{ matrix.pg }} + name: config.log linux-i386 PG${{ matrix.pg }} path: ~/${{ env.PG_SRC_DIR }}/config.log - name: Build TimescaleDB @@ -145,6 +145,18 @@ jobs: make -C build install chown -R postgres:postgres . + - name: Upload CMake Logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: CMake Logs linux-i386 PG${{ matrix.pg }} + path: | + build/CMakeCache.txt + build/CMakeFiles/CMakeConfigureLog.yaml + build/CMakeFiles/CMakeError.log + build/CMakeFiles/CMakeOutput.log + build/compile_commands.json + - name: make installcheck id: installcheck shell: bash diff --git a/.github/workflows/linux-build-and-test.yaml b/.github/workflows/linux-build-and-test.yaml index 34933021b1f..00544c3987e 100644 --- a/.github/workflows/linux-build-and-test.yaml +++ b/.github/workflows/linux-build-and-test.yaml @@ -165,6 +165,18 @@ jobs: make -j $(nproc) -C build make -C build install + - name: Upload CMake Logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: CMake Logs ${{ matrix.os }} ${{ matrix.name }} ${{ matrix.pg }} + path: | + build/CMakeCache.txt + build/CMakeFiles/CMakeConfigureLog.yaml + build/CMakeFiles/CMakeError.log + build/CMakeFiles/CMakeOutput.log + build/compile_commands.json + - name: Check exported symbols run: ./build/scripts/export_prefix_check.sh From bbf183c23b19c300cf749b9695281c12c97f2ccb Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Thu, 23 Jan 2025 09:16:44 +0100 Subject: [PATCH 19/32] Remove bitnami from docker test and release workflow --- .github/workflows/docker-images.yaml | 2 -- .github/workflows/release_build_packages.yml | 1 - 2 files changed, 3 deletions(-) diff --git a/.github/workflows/docker-images.yaml b/.github/workflows/docker-images.yaml index e7e6b4ff30f..63c523222e0 100644 --- a/.github/workflows/docker-images.yaml +++ b/.github/workflows/docker-images.yaml @@ -45,7 +45,6 @@ jobs: "timescaledb-ha:pg15", "timescaledb-ha:pg16", "timescaledb-ha:pg17", - "timescaledb:latest-pg17-bitnami", ] steps: @@ -65,7 +64,6 @@ jobs: echo "version=${version}" >>$GITHUB_OUTPUT - name: Wait for services to start - # bitnami images have trouble using the docker builtin healthcheck so we are doing it here run: | sleep 10 pg_isready -t 30 diff --git a/.github/workflows/release_build_packages.yml b/.github/workflows/release_build_packages.yml index 7635c5592bc..98c41bf714a 100644 --- a/.github/workflows/release_build_packages.yml +++ b/.github/workflows/release_build_packages.yml @@ -14,7 +14,6 @@ jobs: GH_TOKEN: ${{ secrets.ORG_AUTOMATION_TOKEN }} run: | gh workflow run docker-image.yml -R timescale/timescaledb-docker -f version=${{ github.event.release.tag_name }} - gh workflow run bitnami.yml -R timescale/timescaledb-docker -f version=${{ github.event.release.tag_name }} gh workflow run timescaledb-debian.yml -R timescale/release-build-scripts -f version=${{ github.event.release.tag_name }} -f upload-artifacts=true gh workflow run timescaledb-ubuntu.yml -R timescale/release-build-scripts -f version=${{ github.event.release.tag_name }} -f upload-artifacts=true gh workflow run timescaledb-apt-arm64.yml -R timescale/release-build-scripts -f version=${{ github.event.release.tag_name }} -f upload-artifacts=true From b2d6612e81d95190621e12fb18f0878a374a686b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Nordstr=C3=B6m?= Date: Thu, 23 Jan 2025 09:05:18 +0100 Subject: [PATCH 20/32] Refactor vector qual handling for arrow slots Move the code for vector qual execution to its own module. The vector qual execution will produce a result in the form of a bitmap filter for the arrow array. Add functions to the arrow slot to carry the result bitmap in the arrow tuple table slot. This allows passing the filter result to nodes above the node that computed the vector qual result. This is necessary to, e.g., run vectorized aggregation above a columnar scan. --- tsl/src/hypercore/CMakeLists.txt | 3 +- tsl/src/hypercore/arrow_tts.c | 9 ++ tsl/src/hypercore/arrow_tts.h | 38 ++++- tsl/src/hypercore/vector_quals.c | 138 ++++++++++++++++++ tsl/src/hypercore/vector_quals.h | 17 +++ tsl/src/nodes/columnar_scan/columnar_scan.c | 150 +++++--------------- tsl/src/nodes/columnar_scan/columnar_scan.h | 1 + 7 files changed, 236 insertions(+), 120 deletions(-) create mode 100644 tsl/src/hypercore/vector_quals.c create mode 100644 tsl/src/hypercore/vector_quals.h diff --git a/tsl/src/hypercore/CMakeLists.txt b/tsl/src/hypercore/CMakeLists.txt index 8719d603956..06d0efee6f3 100644 --- a/tsl/src/hypercore/CMakeLists.txt +++ b/tsl/src/hypercore/CMakeLists.txt @@ -7,7 +7,8 @@ set(SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/hypercore_handler.c ${CMAKE_CURRENT_SOURCE_DIR}/hypercore_proxy.c ${CMAKE_CURRENT_SOURCE_DIR}/relstats.c - ${CMAKE_CURRENT_SOURCE_DIR}/utils.c) + ${CMAKE_CURRENT_SOURCE_DIR}/utils.c + ${CMAKE_CURRENT_SOURCE_DIR}/vector_quals.c) if(PG_VERSION VERSION_GREATER_EQUAL "17.0") list(APPEND SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/import/analyze.c) endif() diff --git a/tsl/src/hypercore/arrow_tts.c b/tsl/src/hypercore/arrow_tts.c index b6e91e874e9..c666434cdfc 100644 --- a/tsl/src/hypercore/arrow_tts.c +++ b/tsl/src/hypercore/arrow_tts.c @@ -93,6 +93,7 @@ tts_arrow_init(TupleTableSlot *slot) aslot->tuple_index = InvalidTupleIndex; aslot->total_row_count = 0; aslot->referenced_attrs = NULL; + aslot->arrow_qual_result = NULL; /* * Set up child slots, one for the non-compressed relation and one for the @@ -120,6 +121,11 @@ tts_arrow_init(TupleTableSlot *slot) Assert(TTS_EMPTY(slot)); Assert(TTS_EMPTY(aslot->noncompressed_slot)); + + /* Memory context reset every new segment. Used to store, e.g., vectorized + * filters */ + aslot->per_segment_mcxt = + GenerationContextCreateCompat(slot->tts_mcxt, "Per-segment memory context", 64 * 1024); } /* @@ -262,6 +268,8 @@ tts_arrow_clear(TupleTableSlot *slot) /* Clear arrow slot fields */ memset(aslot->valid_attrs, 0, sizeof(bool) * slot->tts_tupleDescriptor->natts); aslot->arrow_cache_entry = NULL; + aslot->arrow_qual_result = NULL; + MemoryContextReset(aslot->per_segment_mcxt); } static inline void @@ -333,6 +341,7 @@ tts_arrow_store_tuple(TupleTableSlot *slot, TupleTableSlot *child_slot, uint16 t aslot->arrow_cache_entry = NULL; /* Clear valid attributes */ memset(aslot->valid_attrs, 0, sizeof(bool) * slot->tts_tupleDescriptor->natts); + MemoryContextReset(aslot->per_segment_mcxt); } /* diff --git a/tsl/src/hypercore/arrow_tts.h b/tsl/src/hypercore/arrow_tts.h index 48c9f9c8303..2af5306d96f 100644 --- a/tsl/src/hypercore/arrow_tts.h +++ b/tsl/src/hypercore/arrow_tts.h @@ -79,6 +79,15 @@ typedef struct ArrowTupleTableSlot int16 *attrs_offset_map; /* Offset number mappings between the * non-compressed and compressed * relation */ + + /* Per-segment data. The following data is allocated on the per-segment + * memory context which is reset for every new segment stored and + * processed in the slot. */ + MemoryContext per_segment_mcxt; + + const uint64 *arrow_qual_result; /* Bitmap with result of qual + * filtering over arrow_array. NULL if + * no filtering has been applied. */ } ArrowTupleTableSlot; extern const TupleTableSlotOps TTSOpsArrowTuple; @@ -197,9 +206,9 @@ arrow_slot_get_noncompressed_slot(TupleTableSlot *slot) } static inline uint16 -arrow_slot_total_row_count(TupleTableSlot *slot) +arrow_slot_total_row_count(const TupleTableSlot *slot) { - ArrowTupleTableSlot *aslot = (ArrowTupleTableSlot *) slot; + const ArrowTupleTableSlot *aslot = (const ArrowTupleTableSlot *) slot; Assert(TTS_IS_ARROWTUPLE(slot)); Assert(aslot->total_row_count > 0); @@ -271,6 +280,23 @@ arrow_slot_is_last(const TupleTableSlot *slot) return aslot->tuple_index == InvalidTupleIndex || aslot->tuple_index == aslot->total_row_count; } +static inline void +arrow_slot_set_qual_result(TupleTableSlot *slot, const uint64 *qual_result) +{ + ArrowTupleTableSlot *aslot = (ArrowTupleTableSlot *) slot; + + Assert(TTS_IS_ARROWTUPLE(slot)); + aslot->arrow_qual_result = qual_result; +} + +static inline const uint64 * +arrow_slot_get_qual_result(const TupleTableSlot *slot) +{ + const ArrowTupleTableSlot *aslot = (const ArrowTupleTableSlot *) slot; + + return aslot->arrow_qual_result; +} + /* * Increment or decrement an arrow slot to point to a subsequent row. * @@ -368,6 +394,14 @@ arrow_slot_try_getnext(TupleTableSlot *slot, ScanDirection direction) return false; } +static inline MemoryContext +arrow_slot_per_segment_memory_context(const TupleTableSlot *slot) +{ + const ArrowTupleTableSlot *aslot = (const ArrowTupleTableSlot *) slot; + Assert(TTS_IS_ARROWTUPLE(slot)); + return aslot->per_segment_mcxt; +} + extern bool is_compressed_col(const TupleDesc tupdesc, AttrNumber attno); extern const ArrowArray *arrow_slot_get_array(TupleTableSlot *slot, AttrNumber attno); extern void arrow_slot_set_referenced_attrs(TupleTableSlot *slot, Bitmapset *attrs); diff --git a/tsl/src/hypercore/vector_quals.c b/tsl/src/hypercore/vector_quals.c new file mode 100644 index 00000000000..3635a5247c3 --- /dev/null +++ b/tsl/src/hypercore/vector_quals.c @@ -0,0 +1,138 @@ +/* + * This file and its contents are licensed under the Timescale License. + * Please see the included NOTICE for copyright information and + * LICENSE-TIMESCALE for a copy of the license. + */ +#include +#include "nodes/decompress_chunk/vector_quals.h" +#include + +#include "arrow_tts.h" +#include "vector_quals.h" + +/* + * Support functions to execute vectorized quals over arrow tuple table slots. + */ + +/* + * Initialize the vector qual state. + */ +void +vector_qual_state_init(VectorQualState *vqstate, List *quals, TupleTableSlot *slot) +{ + MemSet(vqstate, 0, sizeof(VectorQualState)); + vqstate->vectorized_quals_constified = quals; + vqstate->per_vector_mcxt = arrow_slot_per_segment_memory_context(slot); + vqstate->get_arrow_array = vector_qual_state_get_arrow_array; + vqstate->num_results = TupIsNull(slot) ? 0 : arrow_slot_total_row_count(slot); + vqstate->slot = slot; +} + +/* + * Reset the vector qual state. + * + * The function should be called when all values in the arrow array have been + * processed. + */ +void +vector_qual_state_reset(VectorQualState *vqstate) +{ + MemoryContextReset(vqstate->per_vector_mcxt); + vqstate->vector_qual_result = NULL; + vqstate->num_results = arrow_slot_total_row_count(vqstate->slot); + arrow_slot_set_qual_result(vqstate->slot, NULL); +} + +/* + * Implementation of VectorQualState->get_arrow_array() for arrow tuple table + * slots. + * + * Given a VectorQualState return the ArrowArray in the contained slot. + */ +const ArrowArray * +vector_qual_state_get_arrow_array(VectorQualState *vqstate, Expr *expr, bool *is_default_value) +{ + TupleTableSlot *slot = vqstate->slot; + const Var *var = castNode(Var, expr); + const int attoff = AttrNumberGetAttrOffset(var->varattno); + const ArrowArray *array = arrow_slot_get_array(slot, var->varattno); + + if (array == NULL) + { + Form_pg_attribute attr = &slot->tts_tupleDescriptor->attrs[attoff]; + /* + * If getting here, this is a non-compressed value or a compressed + * column with a default value. We can treat non-compressed values the + * same as default ones. It is not possible to fall back to the + * non-vectorized quals now, so build a single-value ArrowArray with + * this (default) value, check if it passes the predicate, and apply + * it to the entire batch. + */ + array = make_single_value_arrow(attr->atttypid, + slot->tts_values[attoff], + slot->tts_isnull[attoff]); + *is_default_value = true; + } + else + *is_default_value = false; + + return array; +} + +/* + * Execute vectorized filter over a vector/array of values. + * + * Returns the number of values filtered until the first valid value. + */ +uint16 +ExecVectorQual(VectorQualState *vqstate, ExprContext *econtext) +{ + TupleTableSlot *slot = econtext->ecxt_scantuple; + const uint16 rowindex = arrow_slot_row_index(slot); + + /* Compute the vector quals over both compressed and non-compressed + * tuples. In case a non-compressed tuple is filtered, return SomeRowsPass + * although only one row will pass. */ + if (rowindex <= 1) + { + vector_qual_state_reset(vqstate); + VectorQualSummary vector_qual_summary = vqstate->vectorized_quals_constified != NIL ? + vector_qual_compute(vqstate) : + AllRowsPass; + + switch (vector_qual_summary) + { + case NoRowsPass: + return arrow_slot_total_row_count(slot); + case AllRowsPass: + /* + * If all rows pass, no need to test the vector qual for each row. This + * is a common case for time range conditions. + */ + vector_qual_state_reset(vqstate); + return 0; + case SomeRowsPass: + break; + } + } + + /* Fast path when all rows have passed (i.e., no rows filtered). No need + * to check qual result and it should be NULL. */ + if (vqstate->vector_qual_result == NULL) + return 0; + + const uint16 nrows = arrow_slot_total_row_count(slot); + const uint16 off = arrow_slot_arrow_offset(slot); + uint16 nfiltered = 0; + + for (uint16 i = off; i < nrows; i++) + { + if (arrow_row_is_valid(vqstate->vector_qual_result, i)) + break; + nfiltered++; + } + + arrow_slot_set_qual_result(slot, vqstate->vector_qual_result); + + return nfiltered; +} diff --git a/tsl/src/hypercore/vector_quals.h b/tsl/src/hypercore/vector_quals.h new file mode 100644 index 00000000000..5563f0676a1 --- /dev/null +++ b/tsl/src/hypercore/vector_quals.h @@ -0,0 +1,17 @@ +/* + * This file and its contents are licensed under the Timescale License. + * Please see the included NOTICE for copyright information and + * LICENSE-TIMESCALE for a copy of the license. + */ +#pragma once + +#include +#include + +#include "nodes/decompress_chunk/vector_quals.h" + +extern void vector_qual_state_init(VectorQualState *vqstate, List *quals, TupleTableSlot *slot); +extern void vector_qual_state_reset(VectorQualState *vqstate); +extern const ArrowArray *vector_qual_state_get_arrow_array(VectorQualState *vqstate, Expr *expr, + bool *is_default_value); +extern uint16 ExecVectorQual(VectorQualState *vqstate, ExprContext *econtext); diff --git a/tsl/src/nodes/columnar_scan/columnar_scan.c b/tsl/src/nodes/columnar_scan/columnar_scan.c index 1231eeb85d3..db48f3ea492 100644 --- a/tsl/src/nodes/columnar_scan/columnar_scan.c +++ b/tsl/src/nodes/columnar_scan/columnar_scan.c @@ -27,12 +27,11 @@ #include #include "columnar_scan.h" -#include "compression/arrow_c_data_interface.h" #include "compression/compression.h" #include "hypercore/arrow_tts.h" #include "hypercore/hypercore_handler.h" +#include "hypercore/vector_quals.h" #include "import/ts_explain.h" -#include "nodes/decompress_chunk/vector_quals.h" typedef struct SimpleProjInfo { @@ -67,60 +66,6 @@ match_relvar(Expr *expr, Index relid) return false; } -/* - * ColumnarScan implementation of VectorQualState->get_arrow_array(). - * - * Given a VectorQualState return the ArrowArray in the contained slot. - */ -static const ArrowArray * -vector_qual_state_get_arrow_array(VectorQualState *vqstate, Expr *expr, bool *is_default_value) -{ - TupleTableSlot *slot = vqstate->slot; - const Var *var = castNode(Var, expr); - const int attoff = AttrNumberGetAttrOffset(var->varattno); - const ArrowArray *array = arrow_slot_get_array(slot, var->varattno); - - if (array == NULL) - { - Form_pg_attribute attr = &slot->tts_tupleDescriptor->attrs[attoff]; - /* - * If getting here, this is a non-compressed value or a compressed - * column with a default value. We can treat non-compressed values the - * same as default ones. It is not possible to fall back to the - * non-vectorized quals now, so build a single-value ArrowArray with - * this (default) value, check if it passes the predicate, and apply - * it to the entire batch. - */ - array = make_single_value_arrow(attr->atttypid, - slot->tts_values[attoff], - slot->tts_isnull[attoff]); - *is_default_value = true; - } - else - *is_default_value = false; - - return array; -} - -static void -vector_qual_state_reset(VectorQualState *vqstate, ExprContext *econtext) -{ - MemoryContextReset(vqstate->per_vector_mcxt); - vqstate->vector_qual_result = NULL; - vqstate->slot = econtext->ecxt_scantuple; - vqstate->num_results = arrow_slot_total_row_count(vqstate->slot); -} - -static void -vector_qual_state_init(VectorQualState *vqstate, ExprContext *econtext) -{ - vqstate->per_vector_mcxt = GenerationContextCreateCompat(econtext->ecxt_per_query_memory, - "Per-vector memory context", - 64 * 1024); - vqstate->get_arrow_array = vector_qual_state_get_arrow_array; - vqstate->slot = econtext->ecxt_scantuple; -} - /* * Utility function to extract quals that can be used as scankeys. The * remaining "normal" quals are optionally collected in the corresponding @@ -284,62 +229,6 @@ create_scankeys_from_quals(const HypercoreInfo *hsinfo, Index relid, const List return scankeys; } -/* - * Execute vectorized filter over a vector/array of values. - * - * Returns the number of values filtered until the first valid value. - */ -static inline uint16 -ExecVectorQual(VectorQualState *vqstate, ExprContext *econtext) -{ - TupleTableSlot *slot = econtext->ecxt_scantuple; - const uint16 rowindex = arrow_slot_row_index(slot); - - /* Compute the vector quals over both compressed and non-compressed - * tuples. In case a non-compressed tuple is filtered, return SomeRowsPass - * although only one row will pass. */ - if (rowindex <= 1) - { - vector_qual_state_reset(vqstate, econtext); - VectorQualSummary vector_qual_summary = vqstate->vectorized_quals_constified != NIL ? - vector_qual_compute(vqstate) : - AllRowsPass; - - switch (vector_qual_summary) - { - case NoRowsPass: - return arrow_slot_total_row_count(slot); - case AllRowsPass: - /* - * If all rows pass, no need to test the vector qual for each row. This - * is a common case for time range conditions. - */ - vector_qual_state_reset(vqstate, econtext); - return 0; - case SomeRowsPass: - break; - } - } - - /* Fast path when all rows have passed (i.e., no rows filtered). No need - * to check qual result and it should be NULL. */ - if (vqstate->vector_qual_result == NULL) - return 0; - - const uint16 nrows = arrow_slot_total_row_count(slot); - const uint16 off = arrow_slot_arrow_offset(slot); - uint16 nfiltered = 0; - - for (uint16 i = off; i < nrows; i++) - { - if (arrow_row_is_valid(vqstate->vector_qual_result, i)) - break; - nfiltered++; - } - - return nfiltered; -} - static pg_attribute_always_inline TupleTableSlot * exec_projection(SimpleProjInfo *spi) { @@ -391,6 +280,17 @@ getnextslot(TableScanDesc scandesc, ScanDirection direction, TupleTableSlot *slo return table_scan_getnextslot(scandesc, direction, slot); } +static bool +should_project(const CustomScanState *state) +{ +#if PG15_GE + const CustomScan *scan = castNode(CustomScan, state->ss.ps.plan); + return scan->flags & CUSTOMPATH_SUPPORT_PROJECTION; +#else + return false; +#endif +} + static TupleTableSlot * columnar_scan_exec(CustomScanState *state) { @@ -399,16 +299,19 @@ columnar_scan_exec(CustomScanState *state) EState *estate; ExprContext *econtext; ExprState *qual; - ProjectionInfo *projinfo; ScanDirection direction; TupleTableSlot *slot; bool has_vecquals = cstate->vqstate.vectorized_quals_constified != NIL; + /* + * The VectorAgg node could have requested no projection by unsetting the + * "projection support flag", so only project if the flag is still set. + */ + ProjectionInfo *projinfo = should_project(state) ? state->ss.ps.ps_ProjInfo : NULL; scandesc = state->ss.ss_currentScanDesc; estate = state->ss.ps.state; econtext = state->ss.ps.ps_ExprContext; qual = state->ss.ps.qual; - projinfo = state->ss.ps.ps_ProjInfo; direction = estate->es_direction; slot = state->ss.ss_ScanTupleSlot; @@ -627,7 +530,7 @@ columnar_scan_begin(CustomScanState *state, EState *estate, int eflags) ExecAssignScanProjectionInfo(&state->ss); state->ss.ps.qual = ExecInitQual(state->ss.ps.plan->qual, (PlanState *) state); #endif - vector_qual_state_init(&cstate->vqstate, state->ss.ps.ps_ExprContext); + List *vectorized_quals_constified = NIL; if (cstate->nscankeys > 0) { @@ -647,10 +550,16 @@ columnar_scan_begin(CustomScanState *state, EState *estate, int eflags) foreach (lc, cstate->vectorized_quals_orig) { Node *constified = estimate_expression_value(&root, (Node *) lfirst(lc)); - cstate->vqstate.vectorized_quals_constified = - lappend(cstate->vqstate.vectorized_quals_constified, constified); + vectorized_quals_constified = lappend(vectorized_quals_constified, constified); } + /* + * Initialize the state to compute vectorized quals. + */ + vector_qual_state_init(&cstate->vqstate, + vectorized_quals_constified, + state->ss.ss_ScanTupleSlot); + /* If the node is supposed to project, then try to make it a simple * projection. If not possible, it will fall back to standard PostgreSQL * projection. */ @@ -811,6 +720,7 @@ columnar_scan_initialize_worker(CustomScanState *node, shm_toc *toc, void *arg) } static CustomExecMethods columnar_scan_state_methods = { + .CustomName = "ColumnarScan", .BeginCustomScan = columnar_scan_begin, .ExecCustomScan = columnar_scan_exec, .EndCustomScan = columnar_scan_end, @@ -845,6 +755,12 @@ static CustomScanMethods columnar_scan_plan_methods = { .CreateCustomScanState = columnar_scan_state_create, }; +bool +is_columnar_scan(const CustomScan *scan) +{ + return scan->methods == &columnar_scan_plan_methods; +} + typedef struct VectorQualInfoHypercore { VectorQualInfo vqinfo; diff --git a/tsl/src/nodes/columnar_scan/columnar_scan.h b/tsl/src/nodes/columnar_scan/columnar_scan.h index 9350e8fe2c5..1dc9c94f436 100644 --- a/tsl/src/nodes/columnar_scan/columnar_scan.h +++ b/tsl/src/nodes/columnar_scan/columnar_scan.h @@ -20,6 +20,7 @@ typedef struct ColumnarScanPath extern ColumnarScanPath *columnar_scan_path_create(PlannerInfo *root, RelOptInfo *rel, Relids required_outer, int parallel_workers); extern void columnar_scan_set_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, Hypertable *ht); +extern bool is_columnar_scan(const CustomScan *scan); extern void _columnar_scan_init(void); #endif /* TIMESCALEDB_COLUMNAR_SCAN_H */ From 79418787b87b74b90323a8fc18704053f2f31524 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Thu, 23 Jan 2025 23:49:26 +0100 Subject: [PATCH 21/32] Allow table_am_handler and index_am_handler C function references We need to allow these in the update script as we try to create an operator class later that is gonna call those functions during upgrade process. --- scripts/check_updates_ast.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/check_updates_ast.py b/scripts/check_updates_ast.py index 2ea4a05edf9..b6b7a73ac69 100644 --- a/scripts/check_updates_ast.py +++ b/scripts/check_updates_ast.py @@ -161,6 +161,8 @@ def visit_CreateFunctionStmt( lang and lang[0].arg.sval == "c" and code[-1].sval != "ts_update_placeholder" + and node.returnType.names[0].sval + not in ["table_am_handler", "index_am_handler"] ): self.errors += 1 functype = "procedure" if node.is_procedure else "function" From dc67130185f7b66d70e9ec15b23741a78dd36d43 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Thu, 23 Jan 2025 20:59:49 +0100 Subject: [PATCH 22/32] Fix TAM handling in update script The TAM SQL code was not written with update and downgrade scripts in mind prevents further releases past 2.18.0 due to not splitting up the parts that need to be part of every update script and those that can only run once during initial installation. --- cmake/ScriptFiles.cmake | 4 +++- sql/pre_install/tam.functions.sql | 10 ++++++++++ sql/{hypercore.sql => pre_install/tam.sql} | 10 ++-------- sql/updates/latest-dev.sql | 17 +++++++++++++++++ sql/updates/reverse-dev.sql | 4 ++-- tsl/test/shared/expected/extension.out | 4 ++-- 6 files changed, 36 insertions(+), 13 deletions(-) create mode 100644 sql/pre_install/tam.functions.sql rename sql/{hypercore.sql => pre_install/tam.sql} (66%) diff --git a/cmake/ScriptFiles.cmake b/cmake/ScriptFiles.cmake index e0cff8d8f93..b5c19b023af 100644 --- a/cmake/ScriptFiles.cmake +++ b/cmake/ScriptFiles.cmake @@ -16,18 +16,20 @@ set(PRE_INSTALL_SOURCE_FILES pre_install/types.post.sql # Must be before tables.sql pre_install/tables.sql pre_install/cache.sql + pre_install/tam.functions.sql + pre_install/tam.sql pre_install/insert_data.sql) # Source files that define functions and need to be rerun in update set(PRE_INSTALL_FUNCTION_FILES pre_install/types.functions.sql + pre_install/tam.functions.sql ) # The rest of the source files defining mostly functions set(SOURCE_FILES hypertable.sql chunk.sql - hypercore.sql ddl_internal.sql util_time.sql util_internal_table_ddl.sql diff --git a/sql/pre_install/tam.functions.sql b/sql/pre_install/tam.functions.sql new file mode 100644 index 00000000000..bbbb6d6b79f --- /dev/null +++ b/sql/pre_install/tam.functions.sql @@ -0,0 +1,10 @@ +-- This file and its contents are licensed under the Apache License 2.0. +-- Please see the included NOTICE for copyright information and +-- LICENSE-APACHE for a copy of the license. + +CREATE OR REPLACE FUNCTION @extschema@.ts_hypercore_handler(internal) RETURNS table_am_handler +AS '@MODULE_PATHNAME@', 'ts_hypercore_handler' LANGUAGE C; + +CREATE OR REPLACE FUNCTION @extschema@.ts_hypercore_proxy_handler(internal) RETURNS index_am_handler +AS '@MODULE_PATHNAME@', 'ts_hypercore_proxy_handler' LANGUAGE C; + diff --git a/sql/hypercore.sql b/sql/pre_install/tam.sql similarity index 66% rename from sql/hypercore.sql rename to sql/pre_install/tam.sql index 5fe85bfd785..9241e29bbd0 100644 --- a/sql/hypercore.sql +++ b/sql/pre_install/tam.sql @@ -2,16 +2,10 @@ -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -CREATE FUNCTION ts_hypercore_handler(internal) RETURNS table_am_handler -AS '@MODULE_PATHNAME@', 'ts_hypercore_handler' LANGUAGE C; - -CREATE ACCESS METHOD hypercore TYPE TABLE HANDLER ts_hypercore_handler; +CREATE ACCESS METHOD hypercore TYPE TABLE HANDLER @extschema@.ts_hypercore_handler; COMMENT ON ACCESS METHOD hypercore IS 'Storage engine using hybrid row/columnar compression'; -CREATE FUNCTION ts_hypercore_proxy_handler(internal) RETURNS index_am_handler -AS '@MODULE_PATHNAME@', 'ts_hypercore_proxy_handler' LANGUAGE C; - -CREATE ACCESS METHOD hypercore_proxy TYPE INDEX HANDLER ts_hypercore_proxy_handler; +CREATE ACCESS METHOD hypercore_proxy TYPE INDEX HANDLER @extschema@.ts_hypercore_proxy_handler; COMMENT ON ACCESS METHOD hypercore_proxy IS 'Hypercore proxy index access method'; -- An index AM needs at least one operator class for the column type diff --git a/sql/updates/latest-dev.sql b/sql/updates/latest-dev.sql index 5863834971c..49788b504e8 100644 --- a/sql/updates/latest-dev.sql +++ b/sql/updates/latest-dev.sql @@ -153,3 +153,20 @@ CREATE PROCEDURE @extschema@.merge_chunks( CREATE PROCEDURE @extschema@.merge_chunks( chunks REGCLASS[] ) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_update_placeholder'; + +CREATE FUNCTION @extschema@.ts_hypercore_handler(internal) RETURNS table_am_handler +AS '@MODULE_PATHNAME@', 'ts_hypercore_handler' LANGUAGE C; + +CREATE FUNCTION @extschema@.ts_hypercore_proxy_handler(internal) RETURNS index_am_handler +AS '@MODULE_PATHNAME@', 'ts_hypercore_proxy_handler' LANGUAGE C; + +CREATE ACCESS METHOD hypercore TYPE TABLE HANDLER @extschema@.ts_hypercore_handler; +COMMENT ON ACCESS METHOD hypercore IS 'Storage engine using hybrid row/columnar compression'; + +CREATE ACCESS METHOD hypercore_proxy TYPE INDEX HANDLER @extschema@.ts_hypercore_proxy_handler; +COMMENT ON ACCESS METHOD hypercore_proxy IS 'Hypercore proxy index access method'; + +CREATE OPERATOR CLASS int4_ops +DEFAULT FOR TYPE int4 USING hypercore_proxy AS + OPERATOR 1 = (int4, int4), + FUNCTION 1 hashint4(int4); diff --git a/sql/updates/reverse-dev.sql b/sql/updates/reverse-dev.sql index 89c7935ebbf..bc918202f24 100644 --- a/sql/updates/reverse-dev.sql +++ b/sql/updates/reverse-dev.sql @@ -1,8 +1,8 @@ -- Hypercore AM DROP ACCESS METHOD IF EXISTS hypercore_proxy; -DROP FUNCTION IF EXISTS ts_hypercore_proxy_handler; +DROP FUNCTION IF EXISTS @extschema@.ts_hypercore_proxy_handler; DROP ACCESS METHOD IF EXISTS hypercore; -DROP FUNCTION IF EXISTS ts_hypercore_handler; +DROP FUNCTION IF EXISTS @extschema@.ts_hypercore_handler; DROP FUNCTION IF EXISTS _timescaledb_debug.is_compressed_tid; DROP FUNCTION IF EXISTS @extschema@.compress_chunk(uncompressed_chunk REGCLASS, if_not_compressed BOOLEAN, recompress BOOLEAN, hypercore_use_access_method BOOL); diff --git a/tsl/test/shared/expected/extension.out b/tsl/test/shared/expected/extension.out index 310d4ba836c..f3fd30c1c84 100644 --- a/tsl/test/shared/expected/extension.out +++ b/tsl/test/shared/expected/extension.out @@ -208,8 +208,6 @@ ORDER BY pronamespace::regnamespace::text COLLATE "C", p.oid::regprocedure::text debug_waitpoint_enable(text) debug_waitpoint_id(text) debug_waitpoint_release(text) - ts_hypercore_handler(internal) - ts_hypercore_proxy_handler(internal) ts_now_mock() add_columnstore_policy(regclass,"any",boolean,interval,timestamp with time zone,text,interval,boolean) add_compression_policy(regclass,"any",boolean,interval,timestamp with time zone,text,interval,boolean) @@ -300,6 +298,8 @@ ORDER BY pronamespace::regnamespace::text COLLATE "C", p.oid::regprocedure::text time_bucket_gapfill(smallint,smallint,smallint,smallint) timescaledb_post_restore() timescaledb_pre_restore() + ts_hypercore_handler(internal) + ts_hypercore_proxy_handler(internal) timescaledb_experimental.add_policies(regclass,boolean,"any","any","any","any",boolean) timescaledb_experimental.alter_policies(regclass,boolean,"any","any","any","any") timescaledb_experimental.remove_all_policies(regclass,boolean) From 2b2d02fa27b60bbf42da0cd1bc33d90af8d6a7fa Mon Sep 17 00:00:00 2001 From: Pallavi Sontakke Date: Thu, 16 Jan 2025 17:49:29 +0530 Subject: [PATCH 23/32] Release 2.18.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This release introduces the ability to add secondary indexes to the columnstore, improves group by and filtering performance through columnstore vectorization, and contains the highly upvoted community request of transition table support. We recommend that you upgrade at the next available opportunity. **Highlighted features in TimescaleDB v2.18.0** * The ability to add secondary indexes to the columnstore through the new hypercore table access method. * Significant performance improvements through vectorization (`SIMD`) for aggregations using a group by with one column and/or using a filter clause when querying the columnstore. * Hypertables support triggers for transition tables, which is one of the most upvoted community feature requests. * Updated methods to manage Timescale's hybrid row-columnar store (hypercore) that highlight the usage of the columnstore which includes both an optimized columnar format as well as compression. **Dropping support for Bitnami images** After the recent change in Bitnami’s [LTS support policy](https://github.com/bitnami/containers/issues/75671), we are no longer building Bitnami images for TimescaleDB. We recommend using the [official TimescaleDB Docker image](https://hub.docker.com/r/timescale/timescaledb-ha) **Deprecation Notice** We are deprecating the following parameters, functions, procedures and views. They will be removed with the next major release of TimescaleDB. Please find the replacements in the table below: | Deprecated | Replacement | Type | | --- | --- | --- | | decompress_chunk | convert_to_rowstore | Procedure | | compress_chunk | convert_to_columnstore | Procedure | | add_compression_policy | add_columnstore_policy | Function | | remove_compression_policy | remove_columnstore_policy | Function | | hypertable_compression_stats | hypertable_columnstore_stats | Function | | chunk_compression_stats | chunk_columnstore_stats | Function | | hypertable_compression_settings | hypertable_columnstore_settings | View | | chunk_compression_settings | chunk_columnstore_settings | View | | compression_settings | columnstore_settings | View | | timescaledb.compress | timescaledb.enable_columnstore | Parameter | | timescaledb.compress_segmentby | timescaledb.segmentby | Parameter | | timescaledb.compress_orderby | timescaledb.orderby | Parameter | **Features** * #7341: Vectorized aggregation with grouping by one fixed-size by-value compressed column (such as arithmetic types). * #7104: Hypercore table access method. * #6901: Add hypertable support for transition tables. * #7482: Optimize recompression of partially compressed chunks. * #7458: Support vectorized aggregation with aggregate `filter` clauses that are also vectorizable. * #7433: Add support for merging chunks. * #7271: Push down `order by` in real-time continuous aggregate queries. * #7455: Support `drop not null` on compressed hypertables. * #7295: Support `alter table set access method` on hypertable. * #7411: Change parameter name to enable hypercore table access method. * #7436: Add index creation on `order by` columns. * #7443: Add hypercore function and view aliases. * #7521: Add optional `force` argument to `refresh_continuous_aggregate`. * #7528: Transform sorting on `time_bucket` to sorting on time for compressed chunks in some cases. * #7565: Add hint when hypertable creation fails. * #7390: Disable custom `hashagg` planner code. * #7587: Add `include_tiered_data` parameter to `add_continuous_aggregate_policy` API. * #7486: Prevent building against PostgreSQL versions with broken ABI. * #7412: Add [GUC](https://www.postgresql.org/docs/current/acronyms.html#:~:text=GUC) for the `hypercore_use_access_method` default. * #7413: Add GUC for segmentwise recompression. **Bugfixes** * #7378: Remove obsolete job referencing `policy_job_error_retention`. * #7409: Update `bgw_job` table when altering procedure. * #7410: Fix the `aggregated compressed column not found` error on aggregation query. * #7426: Fix `datetime` parsing error in chunk constraint creation. * #7432: Verify that the heap tuple is valid before using. * #7434: Fix the segfault when internally setting the replica identity for a given chunk. * #7488: Emit error for transition table trigger on chunks. * #7514: Fix the error: `invalid child of chunk append`. * #7517: Fix the performance regression on the `cagg_migrate` procedure. * #7527: Restart scheduler on error. * #7557: Fix null handling for in-memory tuple filtering. * #7566: Improve transaction check in CAGG refresh. * #7584: Fix NaN-handling for vectorized aggregation. * #7598: Match the Postgres NaN comparison behavior in WHERE clause over compressed tables. **Thanks** * @bharrisau for reporting the segfault when creating chunks. * @jakehedlund for reporting the incompatible NaN behavior in WHERE clause over compressed tables. * @k-rus for suggesting that we add a hint when hypertable creation fails. * @staticlibs for sending the pull request that improves the transaction check in CAGG refresh. * @uasiddiqi for reporting the `aggregated compressed column not found` error. --- .unreleased/compressed-sort-transform | 1 - .unreleased/nan-vectorized-filters | 2 - .unreleased/pr_6901 | 1 - .unreleased/pr_7104 | 1 - .unreleased/pr_7271 | 1 - .unreleased/pr_7295 | 1 - .unreleased/pr_7378 | 2 - .unreleased/pr_7390 | 1 - .unreleased/pr_7409 | 1 - .unreleased/pr_7411 | 1 - .unreleased/pr_7412 | 1 - .unreleased/pr_7413 | 1 - .unreleased/pr_7426 | 1 - .unreleased/pr_7432 | 1 - .unreleased/pr_7433 | 1 - .unreleased/pr_7434 | 2 - .unreleased/pr_7436 | 1 - .unreleased/pr_7443 | 1 - .unreleased/pr_7455 | 1 - .unreleased/pr_7482 | 1 - .unreleased/pr_7486 | 1 - .unreleased/pr_7488 | 1 - .unreleased/pr_7514 | 1 - .unreleased/pr_7517 | 1 - .unreleased/pr_7521 | 1 - .unreleased/pr_7527 | 1 - .unreleased/pr_7557 | 1 - .unreleased/pr_7565 | 2 - .unreleased/pr_7566 | 2 - .unreleased/pr_7584 | 1 - .unreleased/pr_7587 | 1 - .unreleased/resolve-vars | 2 - .unreleased/vectorized-agg-filter | 1 - .unreleased/vectorized-grouping-one-fixed | 1 - CHANGELOG.md | 81 ++++++++++ sql/CMakeLists.txt | 6 +- sql/pre_install/tam.functions.sql | 4 +- sql/pre_install/tam.sql | 4 +- sql/updates/2.17.2--2.18.0.sql | 172 ++++++++++++++++++++++ sql/updates/2.18.0--2.17.2.sql | 93 ++++++++++++ sql/updates/latest-dev.sql | 171 --------------------- sql/updates/reverse-dev.sql | 93 ------------ tsl/test/shared/expected/extension.out | 4 +- version.config | 4 +- 44 files changed, 358 insertions(+), 314 deletions(-) delete mode 100644 .unreleased/compressed-sort-transform delete mode 100644 .unreleased/nan-vectorized-filters delete mode 100644 .unreleased/pr_6901 delete mode 100644 .unreleased/pr_7104 delete mode 100644 .unreleased/pr_7271 delete mode 100644 .unreleased/pr_7295 delete mode 100644 .unreleased/pr_7378 delete mode 100644 .unreleased/pr_7390 delete mode 100644 .unreleased/pr_7409 delete mode 100644 .unreleased/pr_7411 delete mode 100644 .unreleased/pr_7412 delete mode 100644 .unreleased/pr_7413 delete mode 100644 .unreleased/pr_7426 delete mode 100644 .unreleased/pr_7432 delete mode 100644 .unreleased/pr_7433 delete mode 100644 .unreleased/pr_7434 delete mode 100644 .unreleased/pr_7436 delete mode 100644 .unreleased/pr_7443 delete mode 100644 .unreleased/pr_7455 delete mode 100644 .unreleased/pr_7482 delete mode 100644 .unreleased/pr_7486 delete mode 100644 .unreleased/pr_7488 delete mode 100644 .unreleased/pr_7514 delete mode 100644 .unreleased/pr_7517 delete mode 100644 .unreleased/pr_7521 delete mode 100644 .unreleased/pr_7527 delete mode 100644 .unreleased/pr_7557 delete mode 100644 .unreleased/pr_7565 delete mode 100644 .unreleased/pr_7566 delete mode 100644 .unreleased/pr_7584 delete mode 100644 .unreleased/pr_7587 delete mode 100644 .unreleased/resolve-vars delete mode 100644 .unreleased/vectorized-agg-filter delete mode 100644 .unreleased/vectorized-grouping-one-fixed create mode 100644 sql/updates/2.17.2--2.18.0.sql create mode 100644 sql/updates/2.18.0--2.17.2.sql diff --git a/.unreleased/compressed-sort-transform b/.unreleased/compressed-sort-transform deleted file mode 100644 index 5e5e91ed0ff..00000000000 --- a/.unreleased/compressed-sort-transform +++ /dev/null @@ -1 +0,0 @@ -Implements: #7528 Transform sorting on `time_bucket` to sorting on time for compressed chunks in some cases. diff --git a/.unreleased/nan-vectorized-filters b/.unreleased/nan-vectorized-filters deleted file mode 100644 index 9059ee99758..00000000000 --- a/.unreleased/nan-vectorized-filters +++ /dev/null @@ -1,2 +0,0 @@ -Fixes: #6884 Match the Postgres NaN comparison behavior in WHERE clause over compressed tables. -Thanks: @jakehedlund for reporting the incompatible NaN behavior in WHERE clause over compressed tables. diff --git a/.unreleased/pr_6901 b/.unreleased/pr_6901 deleted file mode 100644 index 70af3960865..00000000000 --- a/.unreleased/pr_6901 +++ /dev/null @@ -1 +0,0 @@ -Implements: #6901: Add hypertable support for transition tables. diff --git a/.unreleased/pr_7104 b/.unreleased/pr_7104 deleted file mode 100644 index 143808aa3cf..00000000000 --- a/.unreleased/pr_7104 +++ /dev/null @@ -1 +0,0 @@ -Implements: #7104: Hypercore table access method. diff --git a/.unreleased/pr_7271 b/.unreleased/pr_7271 deleted file mode 100644 index 3f002c8995f..00000000000 --- a/.unreleased/pr_7271 +++ /dev/null @@ -1 +0,0 @@ -Implements: #7271: Push down `order by` in real-time continuous aggregate queries. diff --git a/.unreleased/pr_7295 b/.unreleased/pr_7295 deleted file mode 100644 index 4ccbebe088e..00000000000 --- a/.unreleased/pr_7295 +++ /dev/null @@ -1 +0,0 @@ -Implements: #7295: Support `alter table set access method` on hypertable. diff --git a/.unreleased/pr_7378 b/.unreleased/pr_7378 deleted file mode 100644 index b917991e10d..00000000000 --- a/.unreleased/pr_7378 +++ /dev/null @@ -1,2 +0,0 @@ -Fixes: #7378: Remove obsolete job referencing `policy_job_error_retention`. -Thanks: @pgloader for reporting the issue in an internal background job. diff --git a/.unreleased/pr_7390 b/.unreleased/pr_7390 deleted file mode 100644 index 74c1e4d0947..00000000000 --- a/.unreleased/pr_7390 +++ /dev/null @@ -1 +0,0 @@ -Implements: #7390: Disable custom `hashagg` planner code. diff --git a/.unreleased/pr_7409 b/.unreleased/pr_7409 deleted file mode 100644 index ac9cfb22d4f..00000000000 --- a/.unreleased/pr_7409 +++ /dev/null @@ -1 +0,0 @@ -Fixes: #7409: Update `bgw_job` table when altering procedure. diff --git a/.unreleased/pr_7411 b/.unreleased/pr_7411 deleted file mode 100644 index 537885a7dd2..00000000000 --- a/.unreleased/pr_7411 +++ /dev/null @@ -1 +0,0 @@ -Implements: #7411: Change parameter name to enable hypercore table access method. diff --git a/.unreleased/pr_7412 b/.unreleased/pr_7412 deleted file mode 100644 index 86f2368b534..00000000000 --- a/.unreleased/pr_7412 +++ /dev/null @@ -1 +0,0 @@ -Implements: #7412: Add GUC for `hypercore_use_access_method` default. diff --git a/.unreleased/pr_7413 b/.unreleased/pr_7413 deleted file mode 100644 index 53f2a223bae..00000000000 --- a/.unreleased/pr_7413 +++ /dev/null @@ -1 +0,0 @@ -Implements: #7413: Add GUC for segmentwise recompression. diff --git a/.unreleased/pr_7426 b/.unreleased/pr_7426 deleted file mode 100644 index d2eb7768462..00000000000 --- a/.unreleased/pr_7426 +++ /dev/null @@ -1 +0,0 @@ -Fixes: #7426: Fix `datetime` parsing error in chunk constraint creation. diff --git a/.unreleased/pr_7432 b/.unreleased/pr_7432 deleted file mode 100644 index 30ff2ac3c5e..00000000000 --- a/.unreleased/pr_7432 +++ /dev/null @@ -1 +0,0 @@ -Fixes: #7432: Verify that the heap tuple is valid before using. diff --git a/.unreleased/pr_7433 b/.unreleased/pr_7433 deleted file mode 100644 index c733d50d429..00000000000 --- a/.unreleased/pr_7433 +++ /dev/null @@ -1 +0,0 @@ -Implements: #7433 Add support for merging chunks diff --git a/.unreleased/pr_7434 b/.unreleased/pr_7434 deleted file mode 100644 index dfc43438871..00000000000 --- a/.unreleased/pr_7434 +++ /dev/null @@ -1,2 +0,0 @@ -Fixes: #7434: Fixes the segfault when internally setting the replica identity for a given chunk. -Thanks: @bharrisau for reporting the segfault when creating chunks. diff --git a/.unreleased/pr_7436 b/.unreleased/pr_7436 deleted file mode 100644 index 8f7b34d56bb..00000000000 --- a/.unreleased/pr_7436 +++ /dev/null @@ -1 +0,0 @@ -Implements: #7436 Add index creation on orderby columns diff --git a/.unreleased/pr_7443 b/.unreleased/pr_7443 deleted file mode 100644 index 17ae70064cb..00000000000 --- a/.unreleased/pr_7443 +++ /dev/null @@ -1 +0,0 @@ -Implements: #7443: Add hypercore function and view aliases. diff --git a/.unreleased/pr_7455 b/.unreleased/pr_7455 deleted file mode 100644 index 27624c0c2dd..00000000000 --- a/.unreleased/pr_7455 +++ /dev/null @@ -1 +0,0 @@ -Implements: #7455: Support `drop not null` on compressed hypertables. diff --git a/.unreleased/pr_7482 b/.unreleased/pr_7482 deleted file mode 100644 index ea8c07aa726..00000000000 --- a/.unreleased/pr_7482 +++ /dev/null @@ -1 +0,0 @@ -Implements: #7482: Optimize recompression of partially compressed chunks. diff --git a/.unreleased/pr_7486 b/.unreleased/pr_7486 deleted file mode 100644 index a4d1a58498e..00000000000 --- a/.unreleased/pr_7486 +++ /dev/null @@ -1 +0,0 @@ -Implements: #7486: Prevent building against postgres versions with broken ABI. diff --git a/.unreleased/pr_7488 b/.unreleased/pr_7488 deleted file mode 100644 index a1507ecc518..00000000000 --- a/.unreleased/pr_7488 +++ /dev/null @@ -1 +0,0 @@ -Fixes: #7488: Emit error for transition table trigger on chunks. diff --git a/.unreleased/pr_7514 b/.unreleased/pr_7514 deleted file mode 100644 index fe62a0f9d14..00000000000 --- a/.unreleased/pr_7514 +++ /dev/null @@ -1 +0,0 @@ -Fixes: #7514: Fix the error: `invalid child of chunk append`. diff --git a/.unreleased/pr_7517 b/.unreleased/pr_7517 deleted file mode 100644 index d664cdfd347..00000000000 --- a/.unreleased/pr_7517 +++ /dev/null @@ -1 +0,0 @@ -Fixes: #7517 Fixes performance regression on `cagg_migrate` procedure diff --git a/.unreleased/pr_7521 b/.unreleased/pr_7521 deleted file mode 100644 index a230550a94c..00000000000 --- a/.unreleased/pr_7521 +++ /dev/null @@ -1 +0,0 @@ -Implements: #7521 Add optional `force` argument to `refresh_continuous_aggregate` diff --git a/.unreleased/pr_7527 b/.unreleased/pr_7527 deleted file mode 100644 index 534b8bec629..00000000000 --- a/.unreleased/pr_7527 +++ /dev/null @@ -1 +0,0 @@ -Fixes: #7527 Restart scheduler on error diff --git a/.unreleased/pr_7557 b/.unreleased/pr_7557 deleted file mode 100644 index 2fcd79a6dbe..00000000000 --- a/.unreleased/pr_7557 +++ /dev/null @@ -1 +0,0 @@ -Fixes: #7557: Fix null handling for in-memory tuple filtering. diff --git a/.unreleased/pr_7565 b/.unreleased/pr_7565 deleted file mode 100644 index 1dde2c8222b..00000000000 --- a/.unreleased/pr_7565 +++ /dev/null @@ -1,2 +0,0 @@ -Implements: #7565 Add hint when hypertable creation fails -Thanks: @k-rus for suggesting the improvement diff --git a/.unreleased/pr_7566 b/.unreleased/pr_7566 deleted file mode 100644 index ca3c51d8392..00000000000 --- a/.unreleased/pr_7566 +++ /dev/null @@ -1,2 +0,0 @@ -Fixes: #7566 Improve transaction check in CAgg refresh -Thanks: @staticlibs for sending PR to improve transaction check in CAgg refresh diff --git a/.unreleased/pr_7584 b/.unreleased/pr_7584 deleted file mode 100644 index dec2e4e1912..00000000000 --- a/.unreleased/pr_7584 +++ /dev/null @@ -1 +0,0 @@ -Fixes: #7584 Fix NaN-handling for vectorized aggregation diff --git a/.unreleased/pr_7587 b/.unreleased/pr_7587 deleted file mode 100644 index bd14aaf1c90..00000000000 --- a/.unreleased/pr_7587 +++ /dev/null @@ -1 +0,0 @@ -Implements: #7587 Add `include_tiered_data` parameter to `add_continuous_aggregate_policy` API diff --git a/.unreleased/resolve-vars b/.unreleased/resolve-vars deleted file mode 100644 index aa006e923a4..00000000000 --- a/.unreleased/resolve-vars +++ /dev/null @@ -1,2 +0,0 @@ -Fixes: #7410: Fix the `aggregated compressed column not found` error on aggregation query. -Thanks: @uasiddiqi for reporting the `aggregated compressed column not found` error. diff --git a/.unreleased/vectorized-agg-filter b/.unreleased/vectorized-agg-filter deleted file mode 100644 index b68087e0355..00000000000 --- a/.unreleased/vectorized-agg-filter +++ /dev/null @@ -1 +0,0 @@ -Implements: #7458: Support vecorized aggregation with aggregate `filter` clauses that are also vectorizable. diff --git a/.unreleased/vectorized-grouping-one-fixed b/.unreleased/vectorized-grouping-one-fixed deleted file mode 100644 index c2ad06aa2b9..00000000000 --- a/.unreleased/vectorized-grouping-one-fixed +++ /dev/null @@ -1 +0,0 @@ -Implements: #7341: Vectorized aggregation with grouping by one fixed-size by-value compressed column (such as arithmetic types). diff --git a/CHANGELOG.md b/CHANGELOG.md index d745eb8a037..9555057b9e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,87 @@ `psql` with the `-X` flag to prevent any `.psqlrc` commands from accidentally triggering the load of a previous DB version.** + +## 2.18.0 (2025-01-23) + +This release introduces the ability to add secondary indexes to the columnstore, improves group by and filtering performance through columnstore vectorization, and contains the highly upvoted community request of transition table support. We recommend that you upgrade at the next available opportunity. + +**Highlighted features in TimescaleDB v2.18.0** + +* The ability to add secondary indexes to the columnstore through the new hypercore table access method. +* Significant performance improvements through vectorization (`SIMD`) for aggregations using a group by with one column and/or using a filter clause when querying the columnstore. +* Hypertables support triggers for transition tables, which is one of the most upvoted community feature requests. +* Updated methods to manage Timescale's hybrid row-columnar store (hypercore) that highlight the usage of the columnstore which includes both an optimized columnar format as well as compression. + +**Dropping support for Bitnami images** + +After the recent change in Bitnami’s [LTS support policy](https://github.com/bitnami/containers/issues/75671), we are no longer building Bitnami images for TimescaleDB. We recommend using the [official TimescaleDB Docker image](https://hub.docker.com/r/timescale/timescaledb-ha) + +**Deprecation Notice** + +We are deprecating the following parameters, functions, procedures and views. They will be removed with the next major release of TimescaleDB. Please find the replacements in the table below: + +| Deprecated | Replacement | Type | +| --- | --- | --- | +| decompress_chunk | convert_to_rowstore | Procedure | +| compress_chunk | convert_to_columnstore | Procedure | +| add_compression_policy | add_columnstore_policy | Function | +| remove_compression_policy | remove_columnstore_policy | Function | +| hypertable_compression_stats | hypertable_columnstore_stats | Function | +| chunk_compression_stats | chunk_columnstore_stats | Function | +| hypertable_compression_settings | hypertable_columnstore_settings | View | +| chunk_compression_settings | chunk_columnstore_settings | View | +| compression_settings | columnstore_settings | View | +| timescaledb.compress | timescaledb.enable_columnstore | Parameter | +| timescaledb.compress_segmentby | timescaledb.segmentby | Parameter | +| timescaledb.compress_orderby | timescaledb.orderby | Parameter | + +**Features** +* #7341: Vectorized aggregation with grouping by one fixed-size by-value compressed column (such as arithmetic types). +* #7104: Hypercore table access method. +* #6901: Add hypertable support for transition tables. +* #7482: Optimize recompression of partially compressed chunks. +* #7458: Support vectorized aggregation with aggregate `filter` clauses that are also vectorizable. +* #7433: Add support for merging chunks. +* #7271: Push down `order by` in real-time continuous aggregate queries. +* #7455: Support `drop not null` on compressed hypertables. +* #7295: Support `alter table set access method` on hypertable. +* #7411: Change parameter name to enable hypercore table access method. +* #7436: Add index creation on `order by` columns. +* #7443: Add hypercore function and view aliases. +* #7521: Add optional `force` argument to `refresh_continuous_aggregate`. +* #7528: Transform sorting on `time_bucket` to sorting on time for compressed chunks in some cases. +* #7565: Add hint when hypertable creation fails. +* #7390: Disable custom `hashagg` planner code. +* #7587: Add `include_tiered_data` parameter to `add_continuous_aggregate_policy` API. +* #7486: Prevent building against PostgreSQL versions with broken ABI. +* #7412: Add [GUC](https://www.postgresql.org/docs/current/acronyms.html#:~:text=GUC) for the `hypercore_use_access_method` default. +* #7413: Add GUC for segmentwise recompression. + +**Bugfixes** +* #7378: Remove obsolete job referencing `policy_job_error_retention`. +* #7409: Update `bgw_job` table when altering procedure. +* #7410: Fix the `aggregated compressed column not found` error on aggregation query. +* #7426: Fix `datetime` parsing error in chunk constraint creation. +* #7432: Verify that the heap tuple is valid before using. +* #7434: Fix the segfault when internally setting the replica identity for a given chunk. +* #7488: Emit error for transition table trigger on chunks. +* #7514: Fix the error: `invalid child of chunk append`. +* #7517: Fix the performance regression on the `cagg_migrate` procedure. +* #7527: Restart scheduler on error. +* #7557: Fix null handling for in-memory tuple filtering. +* #7566: Improve transaction check in CAGG refresh. +* #7584: Fix NaN-handling for vectorized aggregation. +* #7598: Match the Postgres NaN comparison behavior in WHERE clause over compressed tables. + +**Thanks** +* @bharrisau for reporting the segfault when creating chunks. +* @jakehedlund for reporting the incompatible NaN behavior in WHERE clause over compressed tables. +* @k-rus for suggesting that we add a hint when hypertable creation fails. +* @staticlibs for sending the pull request that improves the transaction check in CAGG refresh. +* @uasiddiqi for reporting the `aggregated compressed column not found` error. + + ## 2.17.2 (2024-11-06) This release contains bug fixes since the 2.17.1 release. We recommend that you diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index b3d7f3d39b5..d01adbb17aa 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -45,7 +45,8 @@ set(MOD_FILES updates/2.16.0--2.16.1.sql updates/2.16.1--2.17.0.sql updates/2.17.0--2.17.1.sql - updates/2.17.1--2.17.2.sql) + updates/2.17.1--2.17.2.sql + updates/2.17.2--2.18.0.sql) # The downgrade file to generate a downgrade script for the current version, as # specified in version.config @@ -90,7 +91,8 @@ set(OLD_REV_FILES 2.16.1--2.16.0.sql 2.17.0--2.16.1.sql 2.17.1--2.17.0.sql - 2.17.2--2.17.1.sql) + 2.17.2--2.17.1.sql + 2.18.0--2.17.2.sql) set(MODULE_PATHNAME "$libdir/timescaledb-${PROJECT_VERSION_MOD}") set(LOADER_PATHNAME "$libdir/timescaledb") diff --git a/sql/pre_install/tam.functions.sql b/sql/pre_install/tam.functions.sql index bbbb6d6b79f..b1405a09142 100644 --- a/sql/pre_install/tam.functions.sql +++ b/sql/pre_install/tam.functions.sql @@ -2,9 +2,9 @@ -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -CREATE OR REPLACE FUNCTION @extschema@.ts_hypercore_handler(internal) RETURNS table_am_handler +CREATE OR REPLACE FUNCTION ts_hypercore_handler(internal) RETURNS table_am_handler AS '@MODULE_PATHNAME@', 'ts_hypercore_handler' LANGUAGE C; -CREATE OR REPLACE FUNCTION @extschema@.ts_hypercore_proxy_handler(internal) RETURNS index_am_handler +CREATE OR REPLACE FUNCTION ts_hypercore_proxy_handler(internal) RETURNS index_am_handler AS '@MODULE_PATHNAME@', 'ts_hypercore_proxy_handler' LANGUAGE C; diff --git a/sql/pre_install/tam.sql b/sql/pre_install/tam.sql index 9241e29bbd0..684155cdc03 100644 --- a/sql/pre_install/tam.sql +++ b/sql/pre_install/tam.sql @@ -2,10 +2,10 @@ -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -CREATE ACCESS METHOD hypercore TYPE TABLE HANDLER @extschema@.ts_hypercore_handler; +CREATE ACCESS METHOD hypercore TYPE TABLE HANDLER ts_hypercore_handler; COMMENT ON ACCESS METHOD hypercore IS 'Storage engine using hybrid row/columnar compression'; -CREATE ACCESS METHOD hypercore_proxy TYPE INDEX HANDLER @extschema@.ts_hypercore_proxy_handler; +CREATE ACCESS METHOD hypercore_proxy TYPE INDEX HANDLER ts_hypercore_proxy_handler; COMMENT ON ACCESS METHOD hypercore_proxy IS 'Hypercore proxy index access method'; -- An index AM needs at least one operator class for the column type diff --git a/sql/updates/2.17.2--2.18.0.sql b/sql/updates/2.17.2--2.18.0.sql new file mode 100644 index 00000000000..db6c4c056a5 --- /dev/null +++ b/sql/updates/2.17.2--2.18.0.sql @@ -0,0 +1,172 @@ +-- remove obsolete job +DELETE FROM _timescaledb_config.bgw_job WHERE id = 2; + +-- Hypercore updates +CREATE FUNCTION _timescaledb_debug.is_compressed_tid(tid) RETURNS BOOL +AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C STRICT; + +DROP FUNCTION IF EXISTS @extschema@.compress_chunk(uncompressed_chunk REGCLASS, if_not_compressed BOOLEAN, recompress BOOLEAN); + +CREATE FUNCTION @extschema@.compress_chunk( + uncompressed_chunk REGCLASS, + if_not_compressed BOOLEAN = true, + recompress BOOLEAN = false, + hypercore_use_access_method BOOL = NULL +) RETURNS REGCLASS AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C VOLATILE; + +DROP FUNCTION IF EXISTS @extschema@.add_compression_policy(hypertable REGCLASS, compress_after "any", if_not_exists BOOL, schedule_interval INTERVAL, initial_start TIMESTAMPTZ, timezone TEXT, compress_created_before INTERVAL); + +CREATE FUNCTION @extschema@.add_compression_policy( + hypertable REGCLASS, + compress_after "any" = NULL, + if_not_exists BOOL = false, + schedule_interval INTERVAL = NULL, + initial_start TIMESTAMPTZ = NULL, + timezone TEXT = NULL, + compress_created_before INTERVAL = NULL, + hypercore_use_access_method BOOL = NULL +) +RETURNS INTEGER +AS '@MODULE_PATHNAME@', 'ts_update_placeholder' +LANGUAGE C VOLATILE; + +DROP FUNCTION IF EXISTS timescaledb_experimental.add_policies(relation REGCLASS, if_not_exists BOOL, refresh_start_offset "any", refresh_end_offset "any", compress_after "any", drop_after "any"); + +CREATE FUNCTION timescaledb_experimental.add_policies( + relation REGCLASS, + if_not_exists BOOL = false, + refresh_start_offset "any" = NULL, + refresh_end_offset "any" = NULL, + compress_after "any" = NULL, + drop_after "any" = NULL, + hypercore_use_access_method BOOL = NULL) +RETURNS BOOL +AS '@MODULE_PATHNAME@', 'ts_update_placeholder' +LANGUAGE C VOLATILE; + +DROP PROCEDURE IF EXISTS _timescaledb_functions.policy_compression_execute(job_id INTEGER, htid INTEGER, lag ANYELEMENT, maxchunks INTEGER, verbose_log BOOLEAN, recompress_enabled BOOLEAN, use_creation_time BOOLEAN); + +DROP PROCEDURE IF EXISTS _timescaledb_functions.policy_compression(job_id INTEGER, config JSONB); + +CREATE PROCEDURE @extschema@.convert_to_columnstore( + chunk REGCLASS, + if_not_columnstore BOOLEAN = true, + recompress BOOLEAN = false, + hypercore_use_access_method BOOL = NULL) +AS '@MODULE_PATHNAME@', 'ts_update_placeholder' +LANGUAGE C; + +CREATE PROCEDURE @extschema@.convert_to_rowstore( + chunk REGCLASS, + if_columnstore BOOLEAN = true) +AS '@MODULE_PATHNAME@', 'ts_update_placeholder' +LANGUAGE C; + +CREATE PROCEDURE @extschema@.add_columnstore_policy( + hypertable REGCLASS, + after "any" = NULL, + if_not_exists BOOL = false, + schedule_interval INTERVAL = NULL, + initial_start TIMESTAMPTZ = NULL, + timezone TEXT = NULL, + created_before INTERVAL = NULL, + hypercore_use_access_method BOOL = NULL +) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_update_placeholder'; + +CREATE PROCEDURE @extschema@.remove_columnstore_policy( + hypertable REGCLASS, + if_exists BOOL = false +) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_update_placeholder'; + +CREATE FUNCTION @extschema@.chunk_columnstore_stats (hypertable REGCLASS) + RETURNS TABLE ( + chunk_schema name, + chunk_name name, + compression_status text, + before_compression_table_bytes bigint, + before_compression_index_bytes bigint, + before_compression_toast_bytes bigint, + before_compression_total_bytes bigint, + after_compression_table_bytes bigint, + after_compression_index_bytes bigint, + after_compression_toast_bytes bigint, + after_compression_total_bytes bigint, + node_name name) + LANGUAGE SQL + STABLE STRICT + AS 'SELECT * FROM @extschema@.chunk_compression_stats($1)' + SET search_path TO pg_catalog, pg_temp; + +CREATE FUNCTION @extschema@.hypertable_columnstore_stats (hypertable REGCLASS) + RETURNS TABLE ( + total_chunks bigint, + number_compressed_chunks bigint, + before_compression_table_bytes bigint, + before_compression_index_bytes bigint, + before_compression_toast_bytes bigint, + before_compression_total_bytes bigint, + after_compression_table_bytes bigint, + after_compression_index_bytes bigint, + after_compression_toast_bytes bigint, + after_compression_total_bytes bigint, + node_name name) + LANGUAGE SQL + STABLE STRICT + AS 'SELECT * FROM @extschema@.hypertable_compression_stats($1)' + SET search_path TO pg_catalog, pg_temp; + +-- Recreate `refresh_continuous_aggregate` procedure to add `force` argument +DROP PROCEDURE IF EXISTS @extschema@.refresh_continuous_aggregate (continuous_aggregate REGCLASS, window_start "any", window_end "any"); + +CREATE PROCEDURE @extschema@.refresh_continuous_aggregate( + continuous_aggregate REGCLASS, + window_start "any", + window_end "any", + force BOOLEAN = FALSE +) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_update_placeholder'; + +-- Add `include_tiered_data` argument to `add_continuous_aggregate_policy` +DROP FUNCTION @extschema@.add_continuous_aggregate_policy( + continuous_aggregate REGCLASS, start_offset "any", + end_offset "any", schedule_interval INTERVAL, + if_not_exists BOOL, + initial_start TIMESTAMPTZ, + timezone TEXT +); +CREATE FUNCTION @extschema@.add_continuous_aggregate_policy( + continuous_aggregate REGCLASS, start_offset "any", + end_offset "any", schedule_interval INTERVAL, + if_not_exists BOOL = false, + initial_start TIMESTAMPTZ = NULL, + timezone TEXT = NULL, + include_tiered_data BOOL = NULL +) +RETURNS INTEGER +AS '@MODULE_PATHNAME@', 'ts_update_placeholder' +LANGUAGE C VOLATILE; + +-- Merge chunks +CREATE PROCEDURE @extschema@.merge_chunks( + chunk1 REGCLASS, chunk2 REGCLASS +) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_update_placeholder'; + +CREATE PROCEDURE @extschema@.merge_chunks( + chunks REGCLASS[] +) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_update_placeholder'; + +CREATE FUNCTION ts_hypercore_handler(internal) RETURNS table_am_handler +AS '@MODULE_PATHNAME@', 'ts_hypercore_handler' LANGUAGE C; + +CREATE FUNCTION ts_hypercore_proxy_handler(internal) RETURNS index_am_handler +AS '@MODULE_PATHNAME@', 'ts_hypercore_proxy_handler' LANGUAGE C; + +CREATE ACCESS METHOD hypercore TYPE TABLE HANDLER ts_hypercore_handler; +COMMENT ON ACCESS METHOD hypercore IS 'Storage engine using hybrid row/columnar compression'; + +CREATE ACCESS METHOD hypercore_proxy TYPE INDEX HANDLER ts_hypercore_proxy_handler; +COMMENT ON ACCESS METHOD hypercore_proxy IS 'Hypercore proxy index access method'; + +CREATE OPERATOR CLASS int4_ops +DEFAULT FOR TYPE int4 USING hypercore_proxy AS + OPERATOR 1 = (int4, int4), + FUNCTION 1 hashint4(int4); diff --git a/sql/updates/2.18.0--2.17.2.sql b/sql/updates/2.18.0--2.17.2.sql new file mode 100644 index 00000000000..bc918202f24 --- /dev/null +++ b/sql/updates/2.18.0--2.17.2.sql @@ -0,0 +1,93 @@ +-- Hypercore AM +DROP ACCESS METHOD IF EXISTS hypercore_proxy; +DROP FUNCTION IF EXISTS @extschema@.ts_hypercore_proxy_handler; +DROP ACCESS METHOD IF EXISTS hypercore; +DROP FUNCTION IF EXISTS @extschema@.ts_hypercore_handler; +DROP FUNCTION IF EXISTS _timescaledb_debug.is_compressed_tid; + +DROP FUNCTION IF EXISTS @extschema@.compress_chunk(uncompressed_chunk REGCLASS, if_not_compressed BOOLEAN, recompress BOOLEAN, hypercore_use_access_method BOOL); + +CREATE FUNCTION @extschema@.compress_chunk( + uncompressed_chunk REGCLASS, + if_not_compressed BOOLEAN = true, + recompress BOOLEAN = false +) RETURNS REGCLASS AS '@MODULE_PATHNAME@', 'ts_compress_chunk' LANGUAGE C STRICT VOLATILE; + +DROP FUNCTION IF EXISTS @extschema@.add_compression_policy(hypertable REGCLASS, compress_after "any", if_not_exists BOOL, schedule_interval INTERVAL, initial_start TIMESTAMPTZ, timezone TEXT, compress_created_before INTERVAL, hypercore_use_access_method BOOL); + +CREATE FUNCTION @extschema@.add_compression_policy( + hypertable REGCLASS, + compress_after "any" = NULL, + if_not_exists BOOL = false, + schedule_interval INTERVAL = NULL, + initial_start TIMESTAMPTZ = NULL, + timezone TEXT = NULL, + compress_created_before INTERVAL = NULL +) +RETURNS INTEGER +AS '@MODULE_PATHNAME@', 'ts_policy_compression_add' +LANGUAGE C VOLATILE; + +DROP FUNCTION IF EXISTS timescaledb_experimental.add_policies(relation REGCLASS, if_not_exists BOOL, refresh_start_offset "any", refresh_end_offset "any", compress_after "any", drop_after "any", hypercore_use_access_method BOOL); + +CREATE FUNCTION timescaledb_experimental.add_policies( + relation REGCLASS, + if_not_exists BOOL = false, + refresh_start_offset "any" = NULL, + refresh_end_offset "any" = NULL, + compress_after "any" = NULL, + drop_after "any" = NULL) +RETURNS BOOL +AS '@MODULE_PATHNAME@', 'ts_policies_add' +LANGUAGE C VOLATILE; + +DROP PROCEDURE IF EXISTS _timescaledb_functions.policy_compression_execute(job_id INTEGER, htid INTEGER, lag ANYELEMENT, maxchunks INTEGER, verbose_log BOOLEAN, recompress_enabled BOOLEAN, use_creation_time BOOLEAN, useam BOOLEAN); + +DROP PROCEDURE IF EXISTS _timescaledb_functions.policy_compression(job_id INTEGER, config JSONB); +DROP PROCEDURE IF EXISTS @extschema@.convert_to_columnstore(REGCLASS, BOOLEAN, BOOLEAN, BOOLEAN); +DROP PROCEDURE IF EXISTS @extschema@.convert_to_rowstore(REGCLASS, BOOLEAN); +DROP PROCEDURE IF EXISTS @extschema@.add_columnstore_policy(REGCLASS, "any", BOOL, INTERVAL, TIMESTAMPTZ, TEXT, INTERVAL, BOOL); +DROP PROCEDURE IF EXISTS @extschema@.remove_columnstore_policy(REGCLASS, BOOL); +DROP FUNCTION IF EXISTS @extschema@.hypertable_columnstore_stats(REGCLASS); +DROP FUNCTION IF EXISTS @extschema@.chunk_columnstore_stats(REGCLASS); + +ALTER EXTENSION timescaledb DROP VIEW timescaledb_information.hypertable_columnstore_settings; +ALTER EXTENSION timescaledb DROP VIEW timescaledb_information.chunk_columnstore_settings; + +DROP VIEW timescaledb_information.hypertable_columnstore_settings; +DROP VIEW timescaledb_information.chunk_columnstore_settings; + +DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_update_watermark(INTEGER); + +-- Recreate `refresh_continuous_aggregate` procedure to remove the `force` argument +DROP PROCEDURE IF EXISTS @extschema@.refresh_continuous_aggregate (continuous_aggregate REGCLASS, window_start "any", window_end "any", force BOOLEAN); + +CREATE PROCEDURE @extschema@.refresh_continuous_aggregate( + continuous_aggregate REGCLASS, + window_start "any", + window_end "any" +) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_continuous_agg_refresh'; + +-- Remove `include_tiered_data` argument from `add_continuous_aggregate_policy` +DROP FUNCTION @extschema@.add_continuous_aggregate_policy( + continuous_aggregate REGCLASS, start_offset "any", + end_offset "any", schedule_interval INTERVAL, + if_not_exists BOOL, + initial_start TIMESTAMPTZ, + timezone TEXT, + include_tiered_data BOOL +); +CREATE FUNCTION @extschema@.add_continuous_aggregate_policy( + continuous_aggregate REGCLASS, start_offset "any", + end_offset "any", schedule_interval INTERVAL, + if_not_exists BOOL = false, + initial_start TIMESTAMPTZ = NULL, + timezone TEXT = NULL +) +RETURNS INTEGER +AS '@MODULE_PATHNAME@', 'ts_policy_refresh_cagg_add' +LANGUAGE C VOLATILE; + +-- Merge chunks +DROP PROCEDURE IF EXISTS @extschema@.merge_chunks(chunk1 REGCLASS, chunk2 REGCLASS); +DROP PROCEDURE IF EXISTS @extschema@.merge_chunks(chunks REGCLASS[]); diff --git a/sql/updates/latest-dev.sql b/sql/updates/latest-dev.sql index 49788b504e8..8b137891791 100644 --- a/sql/updates/latest-dev.sql +++ b/sql/updates/latest-dev.sql @@ -1,172 +1 @@ --- remove obsolete job -DELETE FROM _timescaledb_config.bgw_job WHERE id = 2; --- Hypercore updates -CREATE FUNCTION _timescaledb_debug.is_compressed_tid(tid) RETURNS BOOL -AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C STRICT; - -DROP FUNCTION IF EXISTS @extschema@.compress_chunk(uncompressed_chunk REGCLASS, if_not_compressed BOOLEAN, recompress BOOLEAN); - -CREATE FUNCTION @extschema@.compress_chunk( - uncompressed_chunk REGCLASS, - if_not_compressed BOOLEAN = true, - recompress BOOLEAN = false, - hypercore_use_access_method BOOL = NULL -) RETURNS REGCLASS AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C VOLATILE; - -DROP FUNCTION IF EXISTS @extschema@.add_compression_policy(hypertable REGCLASS, compress_after "any", if_not_exists BOOL, schedule_interval INTERVAL, initial_start TIMESTAMPTZ, timezone TEXT, compress_created_before INTERVAL); - -CREATE FUNCTION @extschema@.add_compression_policy( - hypertable REGCLASS, - compress_after "any" = NULL, - if_not_exists BOOL = false, - schedule_interval INTERVAL = NULL, - initial_start TIMESTAMPTZ = NULL, - timezone TEXT = NULL, - compress_created_before INTERVAL = NULL, - hypercore_use_access_method BOOL = NULL -) -RETURNS INTEGER -AS '@MODULE_PATHNAME@', 'ts_update_placeholder' -LANGUAGE C VOLATILE; - -DROP FUNCTION IF EXISTS timescaledb_experimental.add_policies(relation REGCLASS, if_not_exists BOOL, refresh_start_offset "any", refresh_end_offset "any", compress_after "any", drop_after "any"); - -CREATE FUNCTION timescaledb_experimental.add_policies( - relation REGCLASS, - if_not_exists BOOL = false, - refresh_start_offset "any" = NULL, - refresh_end_offset "any" = NULL, - compress_after "any" = NULL, - drop_after "any" = NULL, - hypercore_use_access_method BOOL = NULL) -RETURNS BOOL -AS '@MODULE_PATHNAME@', 'ts_update_placeholder' -LANGUAGE C VOLATILE; - -DROP PROCEDURE IF EXISTS _timescaledb_functions.policy_compression_execute(job_id INTEGER, htid INTEGER, lag ANYELEMENT, maxchunks INTEGER, verbose_log BOOLEAN, recompress_enabled BOOLEAN, use_creation_time BOOLEAN); - -DROP PROCEDURE IF EXISTS _timescaledb_functions.policy_compression(job_id INTEGER, config JSONB); - -CREATE PROCEDURE @extschema@.convert_to_columnstore( - chunk REGCLASS, - if_not_columnstore BOOLEAN = true, - recompress BOOLEAN = false, - hypercore_use_access_method BOOL = NULL) -AS '@MODULE_PATHNAME@', 'ts_update_placeholder' -LANGUAGE C; - -CREATE PROCEDURE @extschema@.convert_to_rowstore( - chunk REGCLASS, - if_columnstore BOOLEAN = true) -AS '@MODULE_PATHNAME@', 'ts_update_placeholder' -LANGUAGE C; - -CREATE PROCEDURE @extschema@.add_columnstore_policy( - hypertable REGCLASS, - after "any" = NULL, - if_not_exists BOOL = false, - schedule_interval INTERVAL = NULL, - initial_start TIMESTAMPTZ = NULL, - timezone TEXT = NULL, - created_before INTERVAL = NULL, - hypercore_use_access_method BOOL = NULL -) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_update_placeholder'; - -CREATE PROCEDURE @extschema@.remove_columnstore_policy( - hypertable REGCLASS, - if_exists BOOL = false -) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_update_placeholder'; - -CREATE FUNCTION @extschema@.chunk_columnstore_stats (hypertable REGCLASS) - RETURNS TABLE ( - chunk_schema name, - chunk_name name, - compression_status text, - before_compression_table_bytes bigint, - before_compression_index_bytes bigint, - before_compression_toast_bytes bigint, - before_compression_total_bytes bigint, - after_compression_table_bytes bigint, - after_compression_index_bytes bigint, - after_compression_toast_bytes bigint, - after_compression_total_bytes bigint, - node_name name) - LANGUAGE SQL - STABLE STRICT - AS 'SELECT * FROM @extschema@.chunk_compression_stats($1)' - SET search_path TO pg_catalog, pg_temp; - -CREATE FUNCTION @extschema@.hypertable_columnstore_stats (hypertable REGCLASS) - RETURNS TABLE ( - total_chunks bigint, - number_compressed_chunks bigint, - before_compression_table_bytes bigint, - before_compression_index_bytes bigint, - before_compression_toast_bytes bigint, - before_compression_total_bytes bigint, - after_compression_table_bytes bigint, - after_compression_index_bytes bigint, - after_compression_toast_bytes bigint, - after_compression_total_bytes bigint, - node_name name) - LANGUAGE SQL - STABLE STRICT - AS 'SELECT * FROM @extschema@.hypertable_compression_stats($1)' - SET search_path TO pg_catalog, pg_temp; - --- Recreate `refresh_continuous_aggregate` procedure to add `force` argument -DROP PROCEDURE IF EXISTS @extschema@.refresh_continuous_aggregate (continuous_aggregate REGCLASS, window_start "any", window_end "any"); - -CREATE PROCEDURE @extschema@.refresh_continuous_aggregate( - continuous_aggregate REGCLASS, - window_start "any", - window_end "any", - force BOOLEAN = FALSE -) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_update_placeholder'; - --- Add `include_tiered_data` argument to `add_continuous_aggregate_policy` -DROP FUNCTION @extschema@.add_continuous_aggregate_policy( - continuous_aggregate REGCLASS, start_offset "any", - end_offset "any", schedule_interval INTERVAL, - if_not_exists BOOL, - initial_start TIMESTAMPTZ, - timezone TEXT -); -CREATE FUNCTION @extschema@.add_continuous_aggregate_policy( - continuous_aggregate REGCLASS, start_offset "any", - end_offset "any", schedule_interval INTERVAL, - if_not_exists BOOL = false, - initial_start TIMESTAMPTZ = NULL, - timezone TEXT = NULL, - include_tiered_data BOOL = NULL -) -RETURNS INTEGER -AS '@MODULE_PATHNAME@', 'ts_update_placeholder' -LANGUAGE C VOLATILE; - --- Merge chunks -CREATE PROCEDURE @extschema@.merge_chunks( - chunk1 REGCLASS, chunk2 REGCLASS -) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_update_placeholder'; - -CREATE PROCEDURE @extschema@.merge_chunks( - chunks REGCLASS[] -) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_update_placeholder'; - -CREATE FUNCTION @extschema@.ts_hypercore_handler(internal) RETURNS table_am_handler -AS '@MODULE_PATHNAME@', 'ts_hypercore_handler' LANGUAGE C; - -CREATE FUNCTION @extschema@.ts_hypercore_proxy_handler(internal) RETURNS index_am_handler -AS '@MODULE_PATHNAME@', 'ts_hypercore_proxy_handler' LANGUAGE C; - -CREATE ACCESS METHOD hypercore TYPE TABLE HANDLER @extschema@.ts_hypercore_handler; -COMMENT ON ACCESS METHOD hypercore IS 'Storage engine using hybrid row/columnar compression'; - -CREATE ACCESS METHOD hypercore_proxy TYPE INDEX HANDLER @extschema@.ts_hypercore_proxy_handler; -COMMENT ON ACCESS METHOD hypercore_proxy IS 'Hypercore proxy index access method'; - -CREATE OPERATOR CLASS int4_ops -DEFAULT FOR TYPE int4 USING hypercore_proxy AS - OPERATOR 1 = (int4, int4), - FUNCTION 1 hashint4(int4); diff --git a/sql/updates/reverse-dev.sql b/sql/updates/reverse-dev.sql index bc918202f24..e69de29bb2d 100644 --- a/sql/updates/reverse-dev.sql +++ b/sql/updates/reverse-dev.sql @@ -1,93 +0,0 @@ --- Hypercore AM -DROP ACCESS METHOD IF EXISTS hypercore_proxy; -DROP FUNCTION IF EXISTS @extschema@.ts_hypercore_proxy_handler; -DROP ACCESS METHOD IF EXISTS hypercore; -DROP FUNCTION IF EXISTS @extschema@.ts_hypercore_handler; -DROP FUNCTION IF EXISTS _timescaledb_debug.is_compressed_tid; - -DROP FUNCTION IF EXISTS @extschema@.compress_chunk(uncompressed_chunk REGCLASS, if_not_compressed BOOLEAN, recompress BOOLEAN, hypercore_use_access_method BOOL); - -CREATE FUNCTION @extschema@.compress_chunk( - uncompressed_chunk REGCLASS, - if_not_compressed BOOLEAN = true, - recompress BOOLEAN = false -) RETURNS REGCLASS AS '@MODULE_PATHNAME@', 'ts_compress_chunk' LANGUAGE C STRICT VOLATILE; - -DROP FUNCTION IF EXISTS @extschema@.add_compression_policy(hypertable REGCLASS, compress_after "any", if_not_exists BOOL, schedule_interval INTERVAL, initial_start TIMESTAMPTZ, timezone TEXT, compress_created_before INTERVAL, hypercore_use_access_method BOOL); - -CREATE FUNCTION @extschema@.add_compression_policy( - hypertable REGCLASS, - compress_after "any" = NULL, - if_not_exists BOOL = false, - schedule_interval INTERVAL = NULL, - initial_start TIMESTAMPTZ = NULL, - timezone TEXT = NULL, - compress_created_before INTERVAL = NULL -) -RETURNS INTEGER -AS '@MODULE_PATHNAME@', 'ts_policy_compression_add' -LANGUAGE C VOLATILE; - -DROP FUNCTION IF EXISTS timescaledb_experimental.add_policies(relation REGCLASS, if_not_exists BOOL, refresh_start_offset "any", refresh_end_offset "any", compress_after "any", drop_after "any", hypercore_use_access_method BOOL); - -CREATE FUNCTION timescaledb_experimental.add_policies( - relation REGCLASS, - if_not_exists BOOL = false, - refresh_start_offset "any" = NULL, - refresh_end_offset "any" = NULL, - compress_after "any" = NULL, - drop_after "any" = NULL) -RETURNS BOOL -AS '@MODULE_PATHNAME@', 'ts_policies_add' -LANGUAGE C VOLATILE; - -DROP PROCEDURE IF EXISTS _timescaledb_functions.policy_compression_execute(job_id INTEGER, htid INTEGER, lag ANYELEMENT, maxchunks INTEGER, verbose_log BOOLEAN, recompress_enabled BOOLEAN, use_creation_time BOOLEAN, useam BOOLEAN); - -DROP PROCEDURE IF EXISTS _timescaledb_functions.policy_compression(job_id INTEGER, config JSONB); -DROP PROCEDURE IF EXISTS @extschema@.convert_to_columnstore(REGCLASS, BOOLEAN, BOOLEAN, BOOLEAN); -DROP PROCEDURE IF EXISTS @extschema@.convert_to_rowstore(REGCLASS, BOOLEAN); -DROP PROCEDURE IF EXISTS @extschema@.add_columnstore_policy(REGCLASS, "any", BOOL, INTERVAL, TIMESTAMPTZ, TEXT, INTERVAL, BOOL); -DROP PROCEDURE IF EXISTS @extschema@.remove_columnstore_policy(REGCLASS, BOOL); -DROP FUNCTION IF EXISTS @extschema@.hypertable_columnstore_stats(REGCLASS); -DROP FUNCTION IF EXISTS @extschema@.chunk_columnstore_stats(REGCLASS); - -ALTER EXTENSION timescaledb DROP VIEW timescaledb_information.hypertable_columnstore_settings; -ALTER EXTENSION timescaledb DROP VIEW timescaledb_information.chunk_columnstore_settings; - -DROP VIEW timescaledb_information.hypertable_columnstore_settings; -DROP VIEW timescaledb_information.chunk_columnstore_settings; - -DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_update_watermark(INTEGER); - --- Recreate `refresh_continuous_aggregate` procedure to remove the `force` argument -DROP PROCEDURE IF EXISTS @extschema@.refresh_continuous_aggregate (continuous_aggregate REGCLASS, window_start "any", window_end "any", force BOOLEAN); - -CREATE PROCEDURE @extschema@.refresh_continuous_aggregate( - continuous_aggregate REGCLASS, - window_start "any", - window_end "any" -) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_continuous_agg_refresh'; - --- Remove `include_tiered_data` argument from `add_continuous_aggregate_policy` -DROP FUNCTION @extschema@.add_continuous_aggregate_policy( - continuous_aggregate REGCLASS, start_offset "any", - end_offset "any", schedule_interval INTERVAL, - if_not_exists BOOL, - initial_start TIMESTAMPTZ, - timezone TEXT, - include_tiered_data BOOL -); -CREATE FUNCTION @extschema@.add_continuous_aggregate_policy( - continuous_aggregate REGCLASS, start_offset "any", - end_offset "any", schedule_interval INTERVAL, - if_not_exists BOOL = false, - initial_start TIMESTAMPTZ = NULL, - timezone TEXT = NULL -) -RETURNS INTEGER -AS '@MODULE_PATHNAME@', 'ts_policy_refresh_cagg_add' -LANGUAGE C VOLATILE; - --- Merge chunks -DROP PROCEDURE IF EXISTS @extschema@.merge_chunks(chunk1 REGCLASS, chunk2 REGCLASS); -DROP PROCEDURE IF EXISTS @extschema@.merge_chunks(chunks REGCLASS[]); diff --git a/tsl/test/shared/expected/extension.out b/tsl/test/shared/expected/extension.out index f3fd30c1c84..310d4ba836c 100644 --- a/tsl/test/shared/expected/extension.out +++ b/tsl/test/shared/expected/extension.out @@ -208,6 +208,8 @@ ORDER BY pronamespace::regnamespace::text COLLATE "C", p.oid::regprocedure::text debug_waitpoint_enable(text) debug_waitpoint_id(text) debug_waitpoint_release(text) + ts_hypercore_handler(internal) + ts_hypercore_proxy_handler(internal) ts_now_mock() add_columnstore_policy(regclass,"any",boolean,interval,timestamp with time zone,text,interval,boolean) add_compression_policy(regclass,"any",boolean,interval,timestamp with time zone,text,interval,boolean) @@ -298,8 +300,6 @@ ORDER BY pronamespace::regnamespace::text COLLATE "C", p.oid::regprocedure::text time_bucket_gapfill(smallint,smallint,smallint,smallint) timescaledb_post_restore() timescaledb_pre_restore() - ts_hypercore_handler(internal) - ts_hypercore_proxy_handler(internal) timescaledb_experimental.add_policies(regclass,boolean,"any","any","any","any",boolean) timescaledb_experimental.alter_policies(regclass,boolean,"any","any","any","any") timescaledb_experimental.remove_all_policies(regclass,boolean) diff --git a/version.config b/version.config index f0cedf97f01..6cb6958dff3 100644 --- a/version.config +++ b/version.config @@ -1,3 +1,3 @@ version = 2.19.0-dev -update_from_version = 2.17.2 -downgrade_to_version = 2.17.2 +update_from_version = 2.18.0 +downgrade_to_version = 2.18.0 From 5c2f6b5bc5374d13a8e93b5da36f354fbe5af0f7 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Fri, 24 Jan 2025 12:02:26 +0100 Subject: [PATCH 24/32] Temporarily disable downgrade test The 2.18.0 sql files for building downgrade scripts have some incompatible changes in them that prevent downgrade script generation. This patch disable downgrade test until the necessary adjustments in downgrade script generation are made. --- .github/workflows/update-test.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/update-test.yaml b/.github/workflows/update-test.yaml index bb51021ba7c..be4138a8c92 100644 --- a/.github/workflows/update-test.yaml +++ b/.github/workflows/update-test.yaml @@ -37,11 +37,12 @@ jobs: PATH="/usr/lib/postgresql/${{ matrix.pg }}/bin:$PATH" ./scripts/test_updates.sh - - name: Downgrade tests PG${{ matrix.pg }} - if: always() - run: | - PATH="/usr/lib/postgresql/${{ matrix.pg }}/bin:$PATH" - ./scripts/test_downgrade.sh +# Temporary disabled downgrade for 2.18.0 +# - name: Downgrade tests PG${{ matrix.pg }} +# if: always() +# run: | +# PATH="/usr/lib/postgresql/${{ matrix.pg }}/bin:$PATH" +# ./scripts/test_downgrade.sh - name: Update diff if: failure() From 2b0011e7e8810538ea79c1184141cb27ce8e6708 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Fri, 24 Jan 2025 15:45:05 +0100 Subject: [PATCH 25/32] Import the UMASH hashing library (#7616) We use it for vectorized hash grouping. For now, add the library separately to figure out the required CMake and CI changes. --- .github/workflows/windows-build-and-test.yaml | 17 + scripts/clang_format_all.sh | 2 +- tsl/CMakeLists.txt | 58 + tsl/src/import/CMakeLists.txt | 17 +- tsl/src/import/umash.c | 1568 +++++++++++++++++ tsl/src/import/umash.h | 333 ++++ 6 files changed, 1992 insertions(+), 3 deletions(-) create mode 100644 tsl/src/import/umash.c create mode 100644 tsl/src/import/umash.h diff --git a/.github/workflows/windows-build-and-test.yaml b/.github/workflows/windows-build-and-test.yaml index 85b123c3f09..fab95ef71ed 100644 --- a/.github/workflows/windows-build-and-test.yaml +++ b/.github/workflows/windows-build-and-test.yaml @@ -245,6 +245,23 @@ jobs: name: PostgreSQL ${{ matrix.pg }} log ${{ matrix.os }} ${{ matrix.build_type }} Build path: ${{ env.PGDATA }}\log\postmaster.log + - name: Upload CMake Logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: CMake Logs ${{ matrix.pg }} ${{ matrix.os }} ${{ matrix.build_type }} + path: | + build_win/CMakeCache.txt + build_win/CMakeFiles/CMakeConfigureLog.yaml + build_win/CMakeFiles/CMakeError.log + build_win/CMakeFiles/CMakeOutput.log + build_win/compile_commands.json + build_wsl/CMakeCache.txt + build_wsl/CMakeFiles/CMakeConfigureLog.yaml + build_wsl/CMakeFiles/CMakeError.log + build_wsl/CMakeFiles/CMakeOutput.log + build_wsl/compile_commands.json + - name: Upload test results to the database if: always() shell: wsl-bash {0} diff --git a/scripts/clang_format_all.sh b/scripts/clang_format_all.sh index ea1bb03e8b8..5828d39b855 100755 --- a/scripts/clang_format_all.sh +++ b/scripts/clang_format_all.sh @@ -5,5 +5,5 @@ SCRIPT_DIR=$(cd "$(dirname $0)" || exit; pwd) BASE_DIR=$(dirname $SCRIPT_DIR) find ${BASE_DIR} \( -path "${BASE_DIR}/src/*" -or -path "${BASE_DIR}/test/*" -or -path "${BASE_DIR}/tsl/*" \) \ - -and -not \( -path "*/.*" -or -path "*CMake*" \) \ + -and -not \( -path "*/.*" -or -path "*CMake*" -or -path "${BASE_DIR}/tsl/src/import/*" \) \ -and \( -name '*.c' -or -name '*.h' \) -print0 | xargs -0 ${SCRIPT_DIR}/clang_format_wrapper.sh -style=file -i diff --git a/tsl/CMakeLists.txt b/tsl/CMakeLists.txt index 9058a14625a..4dde58522d1 100644 --- a/tsl/CMakeLists.txt +++ b/tsl/CMakeLists.txt @@ -4,5 +4,63 @@ if(COMPRESSION_FUZZING) add_compile_definitions(TS_COMPRESSION_FUZZING=1) endif() +# We use the UMASH library for hashing in vectorized grouping. If it was not +# explicitly disabled already, detect if we can compile it on this platform. +if((NOT DEFINED USE_UMASH) OR USE_UMASH) + # Check whether we can enable the pclmul instruction required for the UMASH + # hashing on amd64. Shouldn't be done if the user has manually specified the + # target architecture, no idea how to detect this, but at least we shouldn't + # do this when cross-compiling. + if(NOT CMAKE_CROSSCOMPILING) + check_c_compiler_flag(-mpclmul CC_PCLMUL) + if(CC_PCLMUL) + add_compile_options(-mpclmul) + # The "C source compiles" check below doesn't use the global compilation + # flags, so we have to modify its flags separately. + set(CMAKE_REQUIRED_FLAGS -mpclmul) + endif() + endif() + + set(CMAKE_REQUIRED_FLAGS + "${CMAKE_REQUIRED_FLAGS} -Werror=implicit-function-declaration") + check_c_source_compiles( + " +#if defined(__PCLMUL__) +#include +#include +/* + * For some reason, this doesn't compile on our i386 CI, but I also can't detect + * it using the standard condition of defined(__x86_64__) && !defined(__ILP32__), + * as described at https://wiki.debian.org/X32Port . + */ +static void test() { (void) _mm_cvtsi64_si128((uint64_t) 0); } +#elif defined(__ARM_FEATURE_CRYPTO) +/* OK */ +#else +#error Unsupported platform for UMASH +#endif +void main(void) {}; +" + UMASH_SUPPORTED) + unset(CMAKE_REQUIRED_FLAGS) +else() + set(UMASH_SUPPORTED OFF) +endif() + +option(USE_UMASH + "Use the UMASH hash for string and multi-column vectorized grouping" + ${UMASH_SUPPORTED}) + +if(USE_UMASH) + if(NOT UMASH_SUPPORTED) + message( + FATAL_ERROR + "UMASH use is requested, but it is not supported in the current configuration" + ) + endif() + add_compile_definitions(TS_USE_UMASH) +endif() + +# Add the subdirectories add_subdirectory(test) add_subdirectory(src) diff --git a/tsl/src/import/CMakeLists.txt b/tsl/src/import/CMakeLists.txt index ccac900c3ee..b938decf792 100644 --- a/tsl/src/import/CMakeLists.txt +++ b/tsl/src/import/CMakeLists.txt @@ -1,2 +1,15 @@ -set(SOURCES "") -target_sources(${PROJECT_NAME} PRIVATE ${SOURCES}) +set(SOURCES) + +if(USE_UMASH) + list(APPEND SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/umash.c) +endif() + +if(SOURCES) + # Disable clang-tidy for imported code + add_library(target_no_static_code_analysis OBJECT ${SOURCES}) + set_target_properties(target_no_static_code_analysis PROPERTIES C_CLANG_TIDY + "") + + target_link_libraries(${TSL_LIBRARY_NAME} + $) +endif() diff --git a/tsl/src/import/umash.c b/tsl/src/import/umash.c new file mode 100644 index 00000000000..eebcf8e2ced --- /dev/null +++ b/tsl/src/import/umash.c @@ -0,0 +1,1568 @@ +/* + * This file and its contents are licensed under the Timescale License. + * Please see the included NOTICE for copyright information and + * LICENSE-TIMESCALE for a copy of the license. + */ + +/* + * This file contains source code that was copied and/or modified from + * the UMASH hash implementation at https://github.com/backtrace-labs/umash. + * + * This is a copy of umash.c, git commit sha + * fc4c5b6ca1f06c308e96c43aa080bd766238e092. + */ + +#include "umash.h" + +/* + * UMASH is distributed under the MIT license. + * + * SPDX-License-Identifier: MIT + * + * Copyright 2020-2022 Backtrace I/O, Inc. + * Copyright 2022 Paul Khuong + * Copyright 2022 Dougall Johnson + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if !defined(UMASH_TEST_ONLY) && !defined(NDEBUG) +#define NDEBUG +#endif + +/** + * -DUMASH_LONG_INPUTS=0 to disable the routine specialised for long + * inputs, and -DUMASH_LONG_INPUTS=1 to enable it. If the variable + * isn't defined, we try to probe for `umash_long.inc`: that's where + * the long input routines are defined. + */ +#ifndef UMASH_LONG_INPUTS +#ifdef __has_include +#if __has_include("umash_long.inc") +#define UMASH_LONG_INPUTS 1 +#endif /* __has_include() */ +#endif /* __has_include */ + +#ifndef UMASH_LONG_INPUTS +#define UMASH_LONG_INPUTS 0 +#endif /* !UMASH_LONG_INPUTS */ +#endif /* !UMASH_LONG_INPUTS */ + +/* + * Default to dynamically dispatching implementations on x86-64 + * (there's nothing to dispatch on aarch64). + */ +#ifndef UMASH_DYNAMIC_DISPATCH +#ifdef __x86_64__ +#define UMASH_DYNAMIC_DISPATCH 1 +#else +#define UMASH_DYNAMIC_DISPATCH 0 +#endif +#endif + +/* + * Enable inline assembly by default when building with recent GCC or + * compatible compilers. It should always be safe to disable this + * option, although there may be a performance cost. + */ +#ifndef UMASH_INLINE_ASM + +#if defined(__clang__) +/* + * We need clang 8+ for output flags, and 10+ for relaxed vector + * constraints. + */ +#if __clang_major__ >= 10 +#define UMASH_INLINE_ASM 1 +#else +#define UMASH_INLINE_ASM 0 +#endif /* __clang_major__ */ + +#elif defined(__GNUC__) +#if __GNUC__ >= 6 +#define UMASH_INLINE_ASM 1 +#else +#define UMASH_INLINE_ASM 0 +#endif /* __GNUC__ */ + +#else +#define UMASH_INLINE_ASM 0 +#endif + +#endif + +#include +#include + +#ifdef __PCLMUL__ +/* If we have access to x86 PCLMUL (and some basic SSE). */ +#include + +/* We only use 128-bit vector, as pairs of 64-bit integers. */ +typedef __m128i v128; + +#define V128_ZERO { 0 }; + +static inline v128 +v128_create(uint64_t lo, uint64_t hi) +{ + return _mm_set_epi64x(hi, lo); +} + +/* Shift each 64-bit lane left by one bit. */ +static inline v128 +v128_shift(v128 x) +{ + return _mm_add_epi64(x, x); +} + +/* Computes the 128-bit carryless product of x and y. */ +static inline v128 +v128_clmul(uint64_t x, uint64_t y) +{ + return _mm_clmulepi64_si128(_mm_cvtsi64_si128(x), _mm_cvtsi64_si128(y), 0); +} + +/* Computes the 128-bit carryless product of the high and low halves of x. */ +static inline v128 +v128_clmul_cross(v128 x) +{ + return _mm_clmulepi64_si128(x, x, 1); +} + +#elif defined(__ARM_FEATURE_CRYPTO) + +#include + +typedef uint64x2_t v128; + +#define V128_ZERO { 0 }; + +static inline v128 +v128_create(uint64_t lo, uint64_t hi) +{ + return vcombine_u64(vcreate_u64(lo), vcreate_u64(hi)); +} + +static inline v128 +v128_shift(v128 x) +{ + return vshlq_n_u64(x, 1); +} + +static inline v128 +v128_clmul(uint64_t x, uint64_t y) +{ + return vreinterpretq_u64_p128(vmull_p64(x, y)); +} + +static inline v128 +v128_clmul_cross(v128 x) +{ + v128 swapped = vextq_u64(x, x, 1); +#if UMASH_INLINE_ASM + /* Keep the result out of GPRs. */ + __asm__("" : "+w"(swapped)); +#endif + + return v128_clmul(vgetq_lane_u64(x, 0), vgetq_lane_u64(swapped, 0)); +} + +#else + +#error \ + "Unsupported platform: umash requires CLMUL (-mpclmul) on x86-64, or crypto (-march=...+crypto) extensions on aarch64." +#endif + +/* + * #define UMASH_STAP_PROBE=1 to insert probe points in public UMASH + * functions. + * + * This functionality depends on Systemtap's SDT header file. + */ +#if defined(UMASH_STAP_PROBE) && UMASH_STAP_PROBE +#include +#else +#define DTRACE_PROBE1(lib, name, a0) +#define DTRACE_PROBE2(lib, name, a0, a1) +#define DTRACE_PROBE3(lib, name, a0, a1, a2) +#define DTRACE_PROBE4(lib, name, a0, a1, a2, a3) +#endif + +/* + * #define UMASH_SECTION="special_section" to emit all UMASH symbols + * in the `special_section` ELF section. + */ +#if defined(UMASH_SECTION) && defined(__GNUC__) +#define FN __attribute__((__section__(UMASH_SECTION))) +#else +#define FN +#endif + +/* + * Defining UMASH_TEST_ONLY switches to a debug build with internal + * symbols exposed. + */ +#ifdef UMASH_TEST_ONLY +#define TEST_DEF FN +#include "t/umash_test_only.h" +#else +#define TEST_DEF static FN +#endif + +#ifdef __GNUC__ +#define LIKELY(X) __builtin_expect(!!(X), 1) +#define UNLIKELY(X) __builtin_expect(!!(X), 0) +#define HOT __attribute__((__hot__)) +#define COLD __attribute__((__cold__)) +#else +#define LIKELY(X) X +#define UNLIKELY(X) X +#define HOT +#define COLD +#endif + +#define ARRAY_SIZE(ARR) (sizeof(ARR) / sizeof(ARR[0])) + +#define BLOCK_SIZE (sizeof(uint64_t) * UMASH_OH_PARAM_COUNT) + +/* + * We derive independent short hashes by offsetting the constant array + * by four u64s. In theory, any positive even number works, but this + * is the constant we used in an earlier incarnation, and it works. + */ +#define OH_SHORT_HASH_SHIFT 4 + +/* Incremental UMASH consumes 16 bytes at a time. */ +#define INCREMENTAL_GRANULARITY 16 + +/** + * Modular arithmetic utilities. + * + * The code below uses GCC extensions. It should be possible to add + * support for other compilers. + */ + +#if !defined(__x86_64__) || !UMASH_INLINE_ASM +static inline void +mul128(uint64_t x, uint64_t y, uint64_t *hi, uint64_t *lo) +{ + __uint128_t product = x; + + product *= y; + *hi = product >> 64; + *lo = product; + return; +} +#else +static inline void +mul128(uint64_t x, uint64_t y, uint64_t *hi, uint64_t *lo) +{ + uint64_t mulhi, mullo; + + __asm__("mul %3" : "=a"(mullo), "=d"(mulhi) : "%a"(x), "r"(y) : "cc"); + *hi = mulhi; + *lo = mullo; + return; +} +#endif + +TEST_DEF inline uint64_t +add_mod_fast(uint64_t x, uint64_t y) +{ + unsigned long long sum; + + /* If `sum` overflows, `sum + 8` does not. */ + return (__builtin_uaddll_overflow(x, y, &sum) ? sum + 8 : sum); +} + +static FN COLD uint64_t +add_mod_slow_slow_path(uint64_t sum, uint64_t fixup) +{ + /* Reduce sum, mod 2**64 - 8. */ + sum = (sum >= (uint64_t)-8) ? sum + 8 : sum; + /* sum < 2**64 - 8, so this doesn't overflow. */ + sum += fixup; + /* Reduce again. */ + sum = (sum >= (uint64_t)-8) ? sum + 8 : sum; + return sum; +} + +TEST_DEF inline uint64_t +add_mod_slow(uint64_t x, uint64_t y) +{ + unsigned long long sum; + uint64_t fixup = 0; + + /* x + y \equiv sum + fixup */ + if (__builtin_uaddll_overflow(x, y, &sum)) + fixup = 8; + + /* + * We must ensure `sum + fixup < 2**64 - 8`. + * + * We want a conditional branch here, but not in the + * overflowing add: overflows happen roughly half the time on + * pseudorandom inputs, but `sum < 2**64 - 16` is almost + * always true, for pseudorandom `sum`. + */ + if (LIKELY(sum < (uint64_t)-16)) + return sum + fixup; + +#ifdef UMASH_INLINE_ASM + /* + * Some compilers like to compile the likely branch above with + * conditional moves or predication. Insert a compiler barrier + * in the slow path here to force a branch. + */ + __asm__("" : "+r"(sum)); +#endif + return add_mod_slow_slow_path(sum, fixup); +} + +TEST_DEF inline uint64_t +mul_mod_fast(uint64_t m, uint64_t x) +{ + uint64_t hi, lo; + + mul128(m, x, &hi, &lo); + return add_mod_fast(lo, 8 * hi); +} + +TEST_DEF inline uint64_t +horner_double_update(uint64_t acc, uint64_t m0, uint64_t m1, uint64_t x, uint64_t y) +{ + + acc = add_mod_fast(acc, x); + return add_mod_slow(mul_mod_fast(m0, acc), mul_mod_fast(m1, y)); +} + +/** + * Salsa20 stream generator, used to derive struct umash_param. + * + * Slightly prettified version of D. J. Bernstein's public domain NaCL + * (version 20110121), without paying any attention to constant time + * execution or any other side-channel. + */ +static inline uint32_t +rotate(uint32_t u, int c) +{ + + return (u << c) | (u >> (32 - c)); +} + +static inline uint32_t +load_littleendian(const void *buf) +{ + uint32_t ret = 0; + uint8_t x[4]; + + memcpy(x, buf, sizeof(x)); + for (size_t i = 0; i < 4; i++) + ret |= (uint32_t)x[i] << (8 * i); + + return ret; +} + +static inline void +store_littleendian(void *dst, uint32_t u) +{ + + for (size_t i = 0; i < 4; i++) { + uint8_t lo = u; + + memcpy(dst, &lo, 1); + u >>= 8; + dst = (char *)dst + 1; + } + + return; +} + +static FN void +core_salsa20(char *out, const uint8_t in[static 16], const uint8_t key[static 32], + const uint8_t constant[16]) +{ + enum { ROUNDS = 20 }; + uint32_t x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15; + uint32_t j0, j1, j2, j3, j4, j5, j6, j7, j8, j9, j10, j11, j12, j13, j14, j15; + + j0 = x0 = load_littleendian(constant + 0); + j1 = x1 = load_littleendian(key + 0); + j2 = x2 = load_littleendian(key + 4); + j3 = x3 = load_littleendian(key + 8); + j4 = x4 = load_littleendian(key + 12); + j5 = x5 = load_littleendian(constant + 4); + j6 = x6 = load_littleendian(in + 0); + j7 = x7 = load_littleendian(in + 4); + j8 = x8 = load_littleendian(in + 8); + j9 = x9 = load_littleendian(in + 12); + j10 = x10 = load_littleendian(constant + 8); + j11 = x11 = load_littleendian(key + 16); + j12 = x12 = load_littleendian(key + 20); + j13 = x13 = load_littleendian(key + 24); + j14 = x14 = load_littleendian(key + 28); + j15 = x15 = load_littleendian(constant + 12); + + for (size_t i = 0; i < ROUNDS; i += 2) { + x4 ^= rotate(x0 + x12, 7); + x8 ^= rotate(x4 + x0, 9); + x12 ^= rotate(x8 + x4, 13); + x0 ^= rotate(x12 + x8, 18); + x9 ^= rotate(x5 + x1, 7); + x13 ^= rotate(x9 + x5, 9); + x1 ^= rotate(x13 + x9, 13); + x5 ^= rotate(x1 + x13, 18); + x14 ^= rotate(x10 + x6, 7); + x2 ^= rotate(x14 + x10, 9); + x6 ^= rotate(x2 + x14, 13); + x10 ^= rotate(x6 + x2, 18); + x3 ^= rotate(x15 + x11, 7); + x7 ^= rotate(x3 + x15, 9); + x11 ^= rotate(x7 + x3, 13); + x15 ^= rotate(x11 + x7, 18); + x1 ^= rotate(x0 + x3, 7); + x2 ^= rotate(x1 + x0, 9); + x3 ^= rotate(x2 + x1, 13); + x0 ^= rotate(x3 + x2, 18); + x6 ^= rotate(x5 + x4, 7); + x7 ^= rotate(x6 + x5, 9); + x4 ^= rotate(x7 + x6, 13); + x5 ^= rotate(x4 + x7, 18); + x11 ^= rotate(x10 + x9, 7); + x8 ^= rotate(x11 + x10, 9); + x9 ^= rotate(x8 + x11, 13); + x10 ^= rotate(x9 + x8, 18); + x12 ^= rotate(x15 + x14, 7); + x13 ^= rotate(x12 + x15, 9); + x14 ^= rotate(x13 + x12, 13); + x15 ^= rotate(x14 + x13, 18); + } + + x0 += j0; + x1 += j1; + x2 += j2; + x3 += j3; + x4 += j4; + x5 += j5; + x6 += j6; + x7 += j7; + x8 += j8; + x9 += j9; + x10 += j10; + x11 += j11; + x12 += j12; + x13 += j13; + x14 += j14; + x15 += j15; + + store_littleendian(out + 0, x0); + store_littleendian(out + 4, x1); + store_littleendian(out + 8, x2); + store_littleendian(out + 12, x3); + store_littleendian(out + 16, x4); + store_littleendian(out + 20, x5); + store_littleendian(out + 24, x6); + store_littleendian(out + 28, x7); + store_littleendian(out + 32, x8); + store_littleendian(out + 36, x9); + store_littleendian(out + 40, x10); + store_littleendian(out + 44, x11); + store_littleendian(out + 48, x12); + store_littleendian(out + 52, x13); + store_littleendian(out + 56, x14); + store_littleendian(out + 60, x15); + return; +} + +TEST_DEF void +salsa20_stream( + void *dst, size_t len, const uint8_t nonce[static 8], const uint8_t key[static 32]) +{ + static const uint8_t sigma[16] = "expand 32-byte k"; + uint8_t in[16]; + + if (len == 0) + return; + + memcpy(in, nonce, 8); + memset(in + 8, 0, 8); + + while (len >= 64) { + unsigned int u; + + core_salsa20(dst, in, key, sigma); + u = 1; + for (size_t i = 8; i < 16; i++) { + u += in[i]; + in[i] = u; + u >>= 8; + } + + dst = (char *)dst + 64; + len -= 64; + } + + if (len > 0) { + char block[64]; + + core_salsa20(block, in, key, sigma); + memcpy(dst, block, len); + } + + return; +} + +#if defined(UMASH_TEST_ONLY) || UMASH_LONG_INPUTS +#include "umash_long.inc" +#endif + +/** + * OH block compression. + */ +TEST_DEF struct umash_oh +oh_varblock(const uint64_t *params, uint64_t tag, const void *block, size_t n_bytes) +{ + struct umash_oh ret; + v128 acc = V128_ZERO; + + /* The final block processes `remaining > 0` bytes. */ + size_t remaining = 1 + ((n_bytes - 1) % sizeof(v128)); + size_t end_full_pairs = (n_bytes - remaining) / sizeof(uint64_t); + const void *last_ptr = (const char *)block + n_bytes - sizeof(v128); + size_t i; + + for (i = 0; i < end_full_pairs; i += 2) { + v128 x, k; + + memcpy(&x, block, sizeof(x)); + block = (const char *)block + sizeof(x); + + memcpy(&k, ¶ms[i], sizeof(k)); + x ^= k; + acc ^= v128_clmul_cross(x); + } + + memcpy(&ret, &acc, sizeof(ret)); + + /* Compress the final (potentially partial) pair. */ + { + uint64_t x, y, enh_hi, enh_lo; + + memcpy(&x, last_ptr, sizeof(x)); + last_ptr = (const char *)last_ptr + sizeof(x); + memcpy(&y, last_ptr, sizeof(y)); + + x += params[i]; + y += params[i + 1]; + mul128(x, y, &enh_hi, &enh_lo); + enh_hi += tag; + + ret.bits[0] ^= enh_lo; + ret.bits[1] ^= enh_hi ^ enh_lo; + } + + return ret; +} + +TEST_DEF void +oh_varblock_fprint(struct umash_oh dst[static restrict 2], + const uint64_t *restrict params, uint64_t tag, const void *restrict block, + size_t n_bytes) +{ + v128 acc = V128_ZERO; /* Base umash */ + v128 acc_shifted = V128_ZERO; /* Accumulates shifted values */ + v128 lrc; + /* The final block processes `remaining > 0` bytes. */ + size_t remaining = 1 + ((n_bytes - 1) % sizeof(v128)); + size_t end_full_pairs = (n_bytes - remaining) / sizeof(uint64_t); + const void *last_ptr = (const char *)block + n_bytes - sizeof(v128); + size_t i; + + lrc = v128_create(params[UMASH_OH_PARAM_COUNT], params[UMASH_OH_PARAM_COUNT + 1]); + for (i = 0; i < end_full_pairs; i += 2) { + v128 x, k; + + memcpy(&x, block, sizeof(x)); + block = (const char *)block + sizeof(x); + + memcpy(&k, ¶ms[i], sizeof(k)); + + x ^= k; + lrc ^= x; + + x = v128_clmul_cross(x); + + acc ^= x; + if (i + 2 >= end_full_pairs) + break; + + acc_shifted ^= x; + acc_shifted = v128_shift(acc_shifted); + } + + /* + * Update the LRC for the last chunk before treating it + * specially. + */ + { + v128 x, k; + + memcpy(&x, last_ptr, sizeof(x)); + memcpy(&k, ¶ms[end_full_pairs], sizeof(k)); + + lrc ^= x ^ k; + } + + acc_shifted ^= acc; + acc_shifted = v128_shift(acc_shifted); + + acc_shifted ^= v128_clmul_cross(lrc); + + memcpy(&dst[0], &acc, sizeof(dst[0])); + memcpy(&dst[1], &acc_shifted, sizeof(dst[1])); + + { + uint64_t x, y, kx, ky, enh_hi, enh_lo; + + memcpy(&x, last_ptr, sizeof(x)); + last_ptr = (const char *)last_ptr + sizeof(x); + memcpy(&y, last_ptr, sizeof(y)); + + kx = x + params[end_full_pairs]; + ky = y + params[end_full_pairs + 1]; + + mul128(kx, ky, &enh_hi, &enh_lo); + enh_hi += tag; + + enh_hi ^= enh_lo; + dst[0].bits[0] ^= enh_lo; + dst[0].bits[1] ^= enh_hi; + + dst[1].bits[0] ^= enh_lo; + dst[1].bits[1] ^= enh_hi; + } + + return; +} + +/** + * Returns `then` if `cond` is true, `otherwise` if false. + * + * This noise helps compiler emit conditional moves. + */ +static inline const void * +select_ptr(bool cond, const void *then, const void *otherwise) +{ + const char *ret; + +#if UMASH_INLINE_ASM + /* Force strict evaluation of both arguments. */ + __asm__("" ::"r"(then), "r"(otherwise)); +#endif + + ret = (cond) ? then : otherwise; + +#if UMASH_INLINE_ASM + /* And also force the result to be materialised with a blackhole. */ + __asm__("" : "+r"(ret)); +#endif + return ret; +} + +/** + * Short UMASH (<= 8 bytes). + */ +TEST_DEF inline uint64_t +vec_to_u64(const void *data, size_t n_bytes) +{ + const char zeros[2] = { 0 }; + uint32_t hi, lo; + + /* + * If there are at least 4 bytes to read, read the first 4 in + * `lo`, and the last 4 in `hi`. This covers the whole range, + * since `n_bytes` is at most 8. + */ + if (LIKELY(n_bytes >= sizeof(lo))) { + memcpy(&lo, data, sizeof(lo)); + memcpy(&hi, (const char *)data + n_bytes - sizeof(hi), sizeof(hi)); + } else { + /* 0 <= n_bytes < 4. Decode the size in binary. */ + uint16_t word; + uint8_t byte; + + /* + * If the size is odd, load the first byte in `byte`; + * otherwise, load in a zero. + */ + memcpy(&byte, select_ptr(n_bytes & 1, data, zeros), 1); + lo = byte; + + /* + * If the size is 2 or 3, load the last two bytes in `word`; + * otherwise, load in a zero. + */ + memcpy(&word, + select_ptr(n_bytes & 2, (const char *)data + n_bytes - 2, zeros), 2); + /* + * We have now read `bytes[0 ... n_bytes - 1]` + * exactly once without overwriting any data. + */ + hi = word; + } + + /* + * Mix `hi` with the `lo` bits: SplitMix64 seems to have + * trouble with the top 4 bits. + */ + return ((uint64_t)hi << 32) | (lo + hi); +} + +TEST_DEF uint64_t +umash_short(const uint64_t *params, uint64_t seed, const void *data, size_t n_bytes) +{ + uint64_t h; + + seed += params[n_bytes]; + h = vec_to_u64(data, n_bytes); + h ^= h >> 30; + h *= 0xbf58476d1ce4e5b9ULL; + h = (h ^ seed) ^ (h >> 27); + h *= 0x94d049bb133111ebULL; + h ^= h >> 31; + return h; +} + +static FN struct umash_fp +umash_fp_short(const uint64_t *params, uint64_t seed, const void *data, size_t n_bytes) +{ + struct umash_fp ret; + uint64_t h; + + ret.hash[0] = seed + params[n_bytes]; + ret.hash[1] = seed + params[n_bytes + OH_SHORT_HASH_SHIFT]; + + h = vec_to_u64(data, n_bytes); + h ^= h >> 30; + h *= 0xbf58476d1ce4e5b9ULL; + h ^= h >> 27; + +#define TAIL(i) \ + do { \ + ret.hash[i] ^= h; \ + ret.hash[i] *= 0x94d049bb133111ebULL; \ + ret.hash[i] ^= ret.hash[i] >> 31; \ + } while (0) + + TAIL(0); + TAIL(1); +#undef TAIL + + return ret; +} + +/** + * Rotates `x` left by `n` bits. + */ +static inline uint64_t +rotl64(uint64_t x, int n) +{ + + return (x << n) | (x >> (64 - n)); +} + +TEST_DEF inline uint64_t +finalize(uint64_t x) +{ + + return (x ^ rotl64(x, 8)) ^ rotl64(x, 33); +} + +TEST_DEF uint64_t +umash_medium(const uint64_t multipliers[static 2], const uint64_t *oh, uint64_t seed, + const void *data, size_t n_bytes) +{ + uint64_t enh_hi, enh_lo; + + { + uint64_t x, y; + + memcpy(&x, data, sizeof(x)); + memcpy(&y, (const char *)data + n_bytes - sizeof(y), sizeof(y)); + x += oh[0]; + y += oh[1]; + + mul128(x, y, &enh_hi, &enh_lo); + enh_hi += seed ^ n_bytes; + } + + enh_hi ^= enh_lo; + return finalize(horner_double_update( + /*acc=*/0, multipliers[0], multipliers[1], enh_lo, enh_hi)); +} + +static FN struct umash_fp +umash_fp_medium(const uint64_t multipliers[static 2][2], const uint64_t *oh, + uint64_t seed, const void *data, size_t n_bytes) +{ + struct umash_fp ret; + const uint64_t offset = seed ^ n_bytes; + uint64_t enh_hi, enh_lo; + union { + v128 v; + uint64_t u64[2]; + } mixed_lrc; + uint64_t lrc[2] = { oh[UMASH_OH_PARAM_COUNT], oh[UMASH_OH_PARAM_COUNT + 1] }; + uint64_t x, y; + uint64_t a, b; + + /* Expand the 9-16 bytes to 16. */ + memcpy(&x, data, sizeof(x)); + memcpy(&y, (const char *)data + n_bytes - sizeof(y), sizeof(y)); + + a = oh[0]; + b = oh[1]; + + lrc[0] ^= x ^ a; + lrc[1] ^= y ^ b; + mixed_lrc.v = v128_clmul(lrc[0], lrc[1]); + + a += x; + b += y; + + mul128(a, b, &enh_hi, &enh_lo); + enh_hi += offset; + enh_hi ^= enh_lo; + + ret.hash[0] = finalize(horner_double_update( + /*acc=*/0, multipliers[0][0], multipliers[0][1], enh_lo, enh_hi)); + + ret.hash[1] = finalize(horner_double_update(/*acc=*/0, multipliers[1][0], + multipliers[1][1], enh_lo ^ mixed_lrc.u64[0], enh_hi ^ mixed_lrc.u64[1])); + + return ret; +} + +TEST_DEF uint64_t +umash_long(const uint64_t multipliers[static 2], const uint64_t *oh, uint64_t seed, + const void *data, size_t n_bytes) +{ + uint64_t acc = 0; + + /* + * umash_long.inc defines this variable when the long input + * routine is enabled. + */ +#ifdef UMASH_MULTIPLE_BLOCKS_THRESHOLD + if (UNLIKELY(n_bytes >= UMASH_MULTIPLE_BLOCKS_THRESHOLD)) { + size_t n_block = n_bytes / BLOCK_SIZE; + const void *remaining; + + n_bytes %= BLOCK_SIZE; + remaining = (const char *)data + (n_block * BLOCK_SIZE); + acc = umash_multiple_blocks(acc, multipliers, oh, seed, data, n_block); + + data = remaining; + if (n_bytes == 0) + goto finalize; + + goto last_block; + } +#else + /* Avoid warnings about the unused labels. */ + if (0) { + goto last_block; + goto finalize; + } +#endif + + while (n_bytes > BLOCK_SIZE) { + struct umash_oh compressed; + + compressed = oh_varblock(oh, seed, data, BLOCK_SIZE); + data = (const char *)data + BLOCK_SIZE; + n_bytes -= BLOCK_SIZE; + + acc = horner_double_update(acc, multipliers[0], multipliers[1], + compressed.bits[0], compressed.bits[1]); + } + +last_block: + /* Do the final block. */ + { + struct umash_oh compressed; + + seed ^= (uint8_t)n_bytes; + compressed = oh_varblock(oh, seed, data, n_bytes); + acc = horner_double_update(acc, multipliers[0], multipliers[1], + compressed.bits[0], compressed.bits[1]); + } + +finalize: + return finalize(acc); +} + +TEST_DEF struct umash_fp +umash_fp_long(const uint64_t multipliers[static 2][2], const uint64_t *oh, uint64_t seed, + const void *data, size_t n_bytes) +{ + struct umash_oh compressed[2]; + struct umash_fp ret; + uint64_t acc[2] = { 0, 0 }; + +#ifdef UMASH_MULTIPLE_BLOCKS_THRESHOLD + if (UNLIKELY(n_bytes >= UMASH_MULTIPLE_BLOCKS_THRESHOLD)) { + struct umash_fp poly = { .hash = { 0, 0 } }; + size_t n_block = n_bytes / BLOCK_SIZE; + const void *remaining; + + n_bytes %= BLOCK_SIZE; + remaining = (const char *)data + (n_block * BLOCK_SIZE); + poly = umash_fprint_multiple_blocks( + poly, multipliers, oh, seed, data, n_block); + + acc[0] = poly.hash[0]; + acc[1] = poly.hash[1]; + + data = remaining; + if (n_bytes == 0) + goto finalize; + + goto last_block; + } +#else + /* Avoid warnings about the unused labels. */ + if (0) { + goto last_block; + goto finalize; + } +#endif + + while (n_bytes > BLOCK_SIZE) { + oh_varblock_fprint(compressed, oh, seed, data, BLOCK_SIZE); + +#define UPDATE(i) \ + acc[i] = horner_double_update(acc[i], multipliers[i][0], multipliers[i][1], \ + compressed[i].bits[0], compressed[i].bits[1]) + + UPDATE(0); + UPDATE(1); +#undef UPDATE + + data = (const char *)data + BLOCK_SIZE; + n_bytes -= BLOCK_SIZE; + } + +last_block: + oh_varblock_fprint(compressed, oh, seed ^ (uint8_t)n_bytes, data, n_bytes); + +#define FINAL(i) \ + do { \ + acc[i] = horner_double_update(acc[i], multipliers[i][0], \ + multipliers[i][1], compressed[i].bits[0], compressed[i].bits[1]); \ + } while (0) + + FINAL(0); + FINAL(1); +#undef FINAL + +finalize: + ret.hash[0] = finalize(acc[0]); + ret.hash[1] = finalize(acc[1]); + return ret; +} + +static FN bool +value_is_repeated(const uint64_t *values, size_t n, uint64_t needle) +{ + + for (size_t i = 0; i < n; i++) { + if (values[i] == needle) + return true; + } + + return false; +} + +FN bool +umash_params_prepare(struct umash_params *params) +{ + static const uint64_t modulo = (1UL << 61) - 1; + /* + * The polynomial parameters have two redundant fields (for + * the pre-squared multipliers). Use them as our source of + * extra entropy if needed. + */ + uint64_t buf[] = { params->poly[0][0], params->poly[1][0] }; + size_t buf_idx = 0; + +#define GET_RANDOM(DST) \ + do { \ + if (buf_idx >= ARRAY_SIZE(buf)) \ + return false; \ + \ + (DST) = buf[buf_idx++]; \ + } while (0) + + /* Check the polynomial multipliers: we don't want 0s. */ + for (size_t i = 0; i < ARRAY_SIZE(params->poly); i++) { + uint64_t f = params->poly[i][1]; + + while (true) { + /* + * Zero out bits and use rejection sampling to + * guarantee uniformity. + */ + f &= (1UL << 61) - 1; + if (f != 0 && f < modulo) + break; + + GET_RANDOM(f); + } + + /* We can work in 2**64 - 8 and reduce after the fact. */ + params->poly[i][0] = mul_mod_fast(f, f) % modulo; + params->poly[i][1] = f; + } + + /* Avoid repeated OH noise values. */ + for (size_t i = 0; i < ARRAY_SIZE(params->oh); i++) { + while (value_is_repeated(params->oh, i, params->oh[i])) + GET_RANDOM(params->oh[i]); + } + + return true; +} + +FN void +umash_params_derive(struct umash_params *params, uint64_t bits, const void *key) +{ + uint8_t umash_key[32] = "Do not use UMASH VS adversaries."; + + if (key != NULL) + memcpy(umash_key, key, sizeof(umash_key)); + + while (true) { + uint8_t nonce[8]; + + for (size_t i = 0; i < 8; i++) + nonce[i] = bits >> (8 * i); + + salsa20_stream(params, sizeof(*params), nonce, umash_key); + if (umash_params_prepare(params)) + return; + + /* + * This should practically never fail, so really + * shouldn't happen multiple times. If it does, an + * infinite loop is as good as anything else. + */ + bits++; + } +} + +/* + * Updates the polynomial state at the end of a block. + */ +static FN void +sink_update_poly(struct umash_sink *sink) +{ + uint64_t oh0, oh1; + + oh0 = sink->oh_acc.bits[0]; + oh1 = sink->oh_acc.bits[1]; + sink->poly_state[0].acc = horner_double_update(sink->poly_state[0].acc, + sink->poly_state[0].mul[0], sink->poly_state[0].mul[1], oh0, oh1); + + sink->oh_acc = (struct umash_oh) { .bits = { 0 } }; + if (sink->hash_wanted == 0) + return; + + oh0 = sink->oh_twisted.acc.bits[0]; + oh1 = sink->oh_twisted.acc.bits[1]; + sink->poly_state[1].acc = horner_double_update(sink->poly_state[1].acc, + sink->poly_state[1].mul[0], sink->poly_state[1].mul[1], oh0, oh1); + + sink->oh_twisted = + (struct umash_twisted_oh) { .lrc = { sink->oh[UMASH_OH_PARAM_COUNT], + sink->oh[UMASH_OH_PARAM_COUNT + 1] } }; + return; +} + +/* + * Updates the OH state with 16 bytes of data. If `final` is true, we + * are definitely consuming the last chunk in the input. + */ +static FN void +sink_consume_buf( + struct umash_sink *sink, const char buf[static INCREMENTAL_GRANULARITY], bool final) +{ + const size_t buf_begin = sizeof(sink->buf) - INCREMENTAL_GRANULARITY; + const size_t param = sink->oh_iter; + const uint64_t k0 = sink->oh[param]; + const uint64_t k1 = sink->oh[param + 1]; + uint64_t x, y; + + /* Use GPR loads to avoid forwarding stalls. */ + memcpy(&x, buf, sizeof(x)); + memcpy(&y, buf + sizeof(x), sizeof(y)); + + /* All but the last 16-byte chunk of each block goes through PH. */ + if (sink->oh_iter < UMASH_OH_PARAM_COUNT - 2 && !final) { + v128 acc, h, twisted_acc, prev; + uint64_t m0, m1; + + m0 = x ^ k0; + m1 = y ^ k1; + + memcpy(&acc, &sink->oh_acc, sizeof(acc)); + h = v128_clmul(m0, m1); + acc ^= h; + memcpy(&sink->oh_acc, &acc, sizeof(acc)); + + if (sink->hash_wanted == 0) + goto next; + + sink->oh_twisted.lrc[0] ^= m0; + sink->oh_twisted.lrc[1] ^= m1; + + memcpy(&twisted_acc, &sink->oh_twisted.acc, sizeof(twisted_acc)); + memcpy(&prev, sink->oh_twisted.prev, sizeof(prev)); + + twisted_acc ^= prev; + twisted_acc = v128_shift(twisted_acc); + memcpy(&sink->oh_twisted.acc, &twisted_acc, sizeof(twisted_acc)); + memcpy(&sink->oh_twisted.prev, &h, sizeof(h)); + } else { + /* The last chunk is combined with the size tag with ENH. */ + uint64_t tag = sink->seed ^ (uint8_t)(sink->block_size + sink->bufsz); + uint64_t enh_hi, enh_lo; + + mul128(x + k0, y + k1, &enh_hi, &enh_lo); + enh_hi += tag; + enh_hi ^= enh_lo; + + if (sink->hash_wanted != 0) { + union { + v128 vec; + uint64_t h[2]; + } lrc_hash; + uint64_t lrc0, lrc1; + uint64_t oh0, oh1; + uint64_t oh_twisted0, oh_twisted1; + + lrc0 = sink->oh_twisted.lrc[0] ^ x ^ k0; + lrc1 = sink->oh_twisted.lrc[1] ^ y ^ k1; + lrc_hash.vec = v128_clmul(lrc0, lrc1); + + oh_twisted0 = sink->oh_twisted.acc.bits[0]; + oh_twisted1 = sink->oh_twisted.acc.bits[1]; + + oh0 = sink->oh_acc.bits[0]; + oh1 = sink->oh_acc.bits[1]; + oh0 ^= oh_twisted0; + oh0 <<= 1; + oh1 ^= oh_twisted1; + oh1 <<= 1; + + oh0 ^= lrc_hash.h[0]; + oh1 ^= lrc_hash.h[1]; + sink->oh_twisted.acc.bits[0] = oh0 ^ enh_lo; + sink->oh_twisted.acc.bits[1] = oh1 ^ enh_hi; + } + + sink->oh_acc.bits[0] ^= enh_lo; + sink->oh_acc.bits[1] ^= enh_hi; + } + +next: + memmove(&sink->buf, buf, buf_begin); + sink->block_size += sink->bufsz; + sink->bufsz = 0; + sink->oh_iter += 2; + + if (sink->oh_iter == UMASH_OH_PARAM_COUNT || final) { + sink_update_poly(sink); + sink->block_size = 0; + sink->oh_iter = 0; + } + + return; +} + +/** + * Hashes full 256-byte blocks into a sink that just dumped its OH + * state in the toplevel polynomial hash and reset the block state. + */ +static FN size_t +block_sink_update(struct umash_sink *sink, const void *data, size_t n_bytes) +{ + size_t consumed = 0; + + assert(n_bytes >= BLOCK_SIZE); + assert(sink->bufsz == 0); + assert(sink->block_size == 0); + assert(sink->oh_iter == 0); + +#ifdef UMASH_MULTIPLE_BLOCKS_THRESHOLD + if (UNLIKELY(n_bytes > UMASH_MULTIPLE_BLOCKS_THRESHOLD)) { + /* + * We leave the last block (partial or not) for the + * caller: incremental hashing must save some state + * at the end of a block. + */ + size_t n_blocks = (n_bytes - 1) / BLOCK_SIZE; + + if (sink->hash_wanted != 0) { + const uint64_t multipliers[2][2] = { + [0][0] = sink->poly_state[0].mul[0], + [0][1] = sink->poly_state[0].mul[1], + [1][0] = sink->poly_state[1].mul[0], + [1][1] = sink->poly_state[1].mul[1], + }; + struct umash_fp poly = { + .hash[0] = sink->poly_state[0].acc, + .hash[1] = sink->poly_state[1].acc, + }; + + poly = umash_fprint_multiple_blocks( + poly, multipliers, sink->oh, sink->seed, data, n_blocks); + + sink->poly_state[0].acc = poly.hash[0]; + sink->poly_state[1].acc = poly.hash[1]; + } else { + sink->poly_state[0].acc = umash_multiple_blocks( + sink->poly_state[0].acc, sink->poly_state[0].mul, sink->oh, + sink->seed, data, n_blocks); + } + + return n_blocks * BLOCK_SIZE; + } +#endif + + while (n_bytes > BLOCK_SIZE) { + /* + * Is this worth unswitching? Not obviously, given + * the amount of work in one OH block. + */ + if (sink->hash_wanted != 0) { + struct umash_oh hashes[2]; + + oh_varblock_fprint( + hashes, sink->oh, sink->seed, data, BLOCK_SIZE); + sink->oh_acc = hashes[0]; + sink->oh_twisted.acc = hashes[1]; + } else { + sink->oh_acc = + oh_varblock(sink->oh, sink->seed, data, BLOCK_SIZE); + } + + sink_update_poly(sink); + consumed += BLOCK_SIZE; + data = (const char *)data + BLOCK_SIZE; + n_bytes -= BLOCK_SIZE; + } + + return consumed; +} + +FN void +umash_sink_update(struct umash_sink *sink, const void *data, size_t n_bytes) +{ + const size_t buf_begin = sizeof(sink->buf) - INCREMENTAL_GRANULARITY; + size_t remaining = INCREMENTAL_GRANULARITY - sink->bufsz; + + DTRACE_PROBE4(libumash, umash_sink_update, sink, remaining, data, n_bytes); + + if (n_bytes < remaining) { + memcpy(&sink->buf[buf_begin + sink->bufsz], data, n_bytes); + sink->bufsz += n_bytes; + return; + } + + memcpy(&sink->buf[buf_begin + sink->bufsz], data, remaining); + data = (const char *)data + remaining; + n_bytes -= remaining; + /* We know we're hashing at least 16 bytes. */ + sink->large_umash = true; + sink->bufsz = INCREMENTAL_GRANULARITY; + + /* + * We can't compress a 16-byte buffer until we know whether + * data is coming: the last 16-byte chunk goes to `NH` instead + * of `PH`. We could try to detect when the buffer is the + * last chunk in a block and immediately go to `NH`, but it + * seems more robust to always let the stores settle before we + * read them, just in case the combination is bad for forwarding. + */ + if (n_bytes == 0) + return; + + sink_consume_buf(sink, sink->buf + buf_begin, /*final=*/false); + + while (n_bytes > INCREMENTAL_GRANULARITY) { + size_t consumed; + + if (sink->oh_iter == 0 && n_bytes > BLOCK_SIZE) { + consumed = block_sink_update(sink, data, n_bytes); + assert(consumed >= BLOCK_SIZE); + + /* + * Save the tail of the data we just consumed + * in `sink->buf[0 ... buf_begin - 1]`: the + * final digest may need those bytes for its + * redundant read. + */ + memcpy(sink->buf, + (const char *)data + (consumed - INCREMENTAL_GRANULARITY), + buf_begin); + } else { + consumed = INCREMENTAL_GRANULARITY; + sink->bufsz = INCREMENTAL_GRANULARITY; + sink_consume_buf(sink, data, /*final=*/false); + } + + n_bytes -= consumed; + data = (const char *)data + consumed; + } + + memcpy(&sink->buf[buf_begin], data, n_bytes); + sink->bufsz = n_bytes; + return; +} + +FN uint64_t +umash_full(const struct umash_params *params, uint64_t seed, int which, const void *data, + size_t n_bytes) +{ + + DTRACE_PROBE4(libumash, umash_full, params, which, data, n_bytes); + + /* + * We don't (yet) implement code that only evaluates the + * second hash. We don't currently use that logic, and it's + * about to become a bit more complex, so let's just go for a + * full fingerprint and take what we need. + * + * umash_full is also rarely used that way: usually we want + * either the main hash, or the full fingerprint. + */ + if (UNLIKELY(which != 0)) { + struct umash_fp fp; + + fp = umash_fprint(params, seed, data, n_bytes); + return fp.hash[1]; + } + + /* + * It's not that short inputs are necessarily more likely, but + * we want to make sure they fall through correctly to + * minimise latency. + */ + if (LIKELY(n_bytes <= sizeof(v128))) { + if (LIKELY(n_bytes <= sizeof(uint64_t))) + return umash_short(params->oh, seed, data, n_bytes); + + return umash_medium(params->poly[0], params->oh, seed, data, n_bytes); + } + + return umash_long(params->poly[0], params->oh, seed, data, n_bytes); +} + +FN struct umash_fp +umash_fprint( + const struct umash_params *params, uint64_t seed, const void *data, size_t n_bytes) +{ + + DTRACE_PROBE3(libumash, umash_fprint, params, data, n_bytes); + if (LIKELY(n_bytes <= sizeof(v128))) { + if (LIKELY(n_bytes <= sizeof(uint64_t))) + return umash_fp_short(params->oh, seed, data, n_bytes); + + return umash_fp_medium(params->poly, params->oh, seed, data, n_bytes); + } + + return umash_fp_long(params->poly, params->oh, seed, data, n_bytes); +} + +FN void +umash_init(struct umash_state *state, const struct umash_params *params, uint64_t seed, + int which) +{ + + which = (which == 0) ? 0 : 1; + DTRACE_PROBE3(libumash, umash_init, state, params, which); + + state->sink = (struct umash_sink) { + .poly_state[0] = { + .mul = { + params->poly[0][0], + params->poly[0][1], + }, + }, + .poly_state[1]= { + .mul = { + params->poly[1][0], + params->poly[1][1], + }, + }, + .oh = params->oh, + .hash_wanted = which, + .oh_twisted.lrc = { params->oh[UMASH_OH_PARAM_COUNT], + params->oh[UMASH_OH_PARAM_COUNT + 1] }, + .seed = seed, + }; + + return; +} + +FN void +umash_fp_init( + struct umash_fp_state *state, const struct umash_params *params, uint64_t seed) +{ + + DTRACE_PROBE2(libumash, umash_fp_init, state, params); + + state->sink = (struct umash_sink) { + .poly_state[0] = { + .mul = { + params->poly[0][0], + params->poly[0][1], + }, + }, + .poly_state[1]= { + .mul = { + params->poly[1][0], + params->poly[1][1], + }, + }, + .oh = params->oh, + .hash_wanted = 2, + .oh_twisted.lrc = { params->oh[UMASH_OH_PARAM_COUNT], + params->oh[UMASH_OH_PARAM_COUNT + 1] }, + .seed = seed, + }; + + return; +} + +/** + * Pumps any last block out of the incremental state. + */ +static FN void +digest_flush(struct umash_sink *sink) +{ + + if (sink->bufsz > 0) + sink_consume_buf(sink, &sink->buf[sink->bufsz], /*final=*/true); + return; +} + +/** + * Finalizes a digest out of `sink`'s current state. + * + * The `sink` must be `digest_flush`ed if it is a `large_umash`. + * + * @param index 0 to return the first (only, if hashing) value, 1 for the + * second independent value for fingerprinting. + */ +static FN uint64_t +digest(const struct umash_sink *sink, int index) +{ + const size_t buf_begin = sizeof(sink->buf) - INCREMENTAL_GRANULARITY; + const size_t shift = (index == 0) ? 0 : OH_SHORT_HASH_SHIFT; + + if (sink->large_umash) + return finalize(sink->poly_state[index].acc); + + if (sink->bufsz <= sizeof(uint64_t)) + return umash_short( + &sink->oh[shift], sink->seed, &sink->buf[buf_begin], sink->bufsz); + + return umash_medium(sink->poly_state[index].mul, sink->oh, sink->seed, + &sink->buf[buf_begin], sink->bufsz); +} + +static FN struct umash_fp +fp_digest_sink(const struct umash_sink *sink) +{ + struct umash_sink copy; + struct umash_fp ret; + const size_t buf_begin = sizeof(sink->buf) - INCREMENTAL_GRANULARITY; + + if (sink->large_umash) { + copy = *sink; + digest_flush(©); + sink = © + } else if (sink->bufsz <= sizeof(uint64_t)) { + return umash_fp_short( + sink->oh, sink->seed, &sink->buf[buf_begin], sink->bufsz); + } else { + const struct umash_params *params; + + /* + * Back out the params struct from our pointer to its + * `oh` member. + */ + params = (const void *)((const char *)sink->oh - + __builtin_offsetof(struct umash_params, oh)); + return umash_fp_medium(params->poly, sink->oh, sink->seed, + &sink->buf[buf_begin], sink->bufsz); + } + + for (size_t i = 0; i < ARRAY_SIZE(ret.hash); i++) + ret.hash[i] = digest(sink, i); + + return ret; +} + +FN uint64_t +umash_digest(const struct umash_state *state) +{ + struct umash_sink copy; + const struct umash_sink *sink = &state->sink; + + DTRACE_PROBE1(libumash, umash_digest, state); + + if (sink->hash_wanted == 1) { + struct umash_fp fp; + + fp = fp_digest_sink(sink); + return fp.hash[1]; + } + + if (sink->large_umash) { + copy = *sink; + digest_flush(©); + sink = © + } + + return digest(sink, 0); +} + +FN struct umash_fp +umash_fp_digest(const struct umash_fp_state *state) +{ + + DTRACE_PROBE1(libumash, umash_fp_digest, state); + return fp_digest_sink(&state->sink); +} diff --git a/tsl/src/import/umash.h b/tsl/src/import/umash.h new file mode 100644 index 00000000000..3ad9460b46a --- /dev/null +++ b/tsl/src/import/umash.h @@ -0,0 +1,333 @@ +/* + * This file and its contents are licensed under the Timescale License. + * Please see the included NOTICE for copyright information and + * LICENSE-TIMESCALE for a copy of the license. + */ + +/* + * This file contains source code that was copied and/or modified from + * the UMASH hash implementation at https://github.com/backtrace-labs/umash. + * + * This is a copy of umash.h, git commit sha + * fc4c5b6ca1f06c308e96c43aa080bd766238e092. + */ + +/* + * UMASH is distributed under the MIT license. + * + * SPDX-License-Identifier: MIT + * + * Copyright 2020-2022 Backtrace I/O, Inc. + * Copyright 2022 Paul Khuong + * Copyright 2022 Dougall Johnson + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef UMASH_H +#define UMASH_H +#include +#include +#include + +#ifndef TS_USE_UMASH +#error "UMASH usage is disabled, but the header is included" +#endif + +/** + * # UMASH: a non-cryptographic hash function with collision bounds + * + * SPDX-License-Identifier: MIT + * Copyright 2020-2022 Backtrace I/O, Inc. + * Copyright 2022 Paul Khuong + * + * UMASH is a fast (9-22 ns latency for inputs of 1-64 bytes and 22 + * GB/s peak throughput, on a 2.5 GHz Intel 8175M) 64-bit hash + * function with mathematically proven collision bounds: it is + * [ceil(s / 4096) * 2^{-55}]-almost-universal for inputs of s or + * fewer bytes. + * + * When that's not enough, UMASH can also generate a pair of 64-bit + * hashes in a single traversal. The resulting fingerprint reduces + * the collision probability to less than [ceil(s / 2^{26})^2 * 2^{-83}]; + * the probability that two distinct inputs receive the same + * fingerprint is less 2^{-83} for inputs up to 64 MB, and less than + * 2^{-70} as long as the inputs are shorter than 5 GB each. This + * expectation is taken over the randomly generated `umash_params`. + * If an attacker can infer the contents of these parameters, the + * bounds do not apply. + * + * ## Initialisation + * + * In order to use `UMASH`, one must first generate a `struct + * umash_params`; each such param defines a distinct `UMASH` function + * (a pair of such functions, in fact). Ideally, one would fill + * a struct with random bytes and call`umash_params_prepare`. + * + * - `umash_params_prepare`: attempts to convert the contents of + * randomly filled `struct umash_params` into a valid UMASH + * parameter struct (key). When the input consists of uniformly + * generated random bytes, the probability of failure is + * astronomically small. + * + * - `umash_params_derive`: deterministically constructs a `struct + * umash_params` from a 64-bit seed and an optional 32-byte secret. + * The seed and secret are expanded into random bytes with Salsa20; + * the resulting `umash_params` should be practically random, as + * long the seed or secret are unknown. + * + * ## Batch hashing and fingerprinting + * + * Once we have a `struct umash_params`, we can use `umash_full` or + * `umash_fprint` like regular hash functions. + * + * - `umash_full` can compute either of the two UMASH functions + * described by a `struct umash_params`. Its `seed` argument will + * change the output, but is not associated with any collision + * bound. + * + * - `umash_fprint` computes both `UMASH` functions described by a + * `struct umash_params`. `umash_fp::hash[0]` corresponds to + * calling `umash_full` with the same arguments and `which = 0`; + * `umash_fp::hash[1]` corresponds to `which = 1`. + * + * ## Incremental hashing and fingerprinting + * + * We can also compute UMASH values by feeding bytes incrementally. + * The result is guaranteed to the same as if we had buffered all the + * bytes and called `umash_full` or `umash_fprint`. + * + * - `umash_init` initialises a `struct umash_state` with the same + * parameters one would pass to `umash_full`. + * + * - `umash_digest` computes the value `umash_full` would return + * were it passed the arguments that were given to `umash_init`, + * and the bytes "fed" into the `umash_state`. + * + * - `umash_fp_init` initialises a `struct umash_fp_state` with the + * same parameters one would pass to `umash_fprint`. + * + * - `umash_fp_digest` computes the value `umash_fprint` would return + * for the bytes "fed" into the `umash_fp_state`. + * + * In both cases, one passes a pointer to `struct umash_state::sink` + * or `struct umash_fp_state::sink` to callees that wish to feed bytes + * into the `umash_state` or `umash_fp_state`. + * + * - `umash_sink_update` feeds a byte range to the `umash_sink` + * initialised by calling `umash_init` or `umash_fp_init`. The sink + * does not take ownership of anything and the input bytes may be + * overwritten or freed as soon as `umash_sink_update` returns. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +enum { UMASH_OH_PARAM_COUNT = 32, UMASH_OH_TWISTING_COUNT = 2 }; + +/** + * A single UMASH params struct stores the parameters for a pair of + * independent `UMASH` functions. + */ +struct umash_params { + /* + * Each uint64_t[2] array consists of {f^2, f}, where f is a + * random multiplier in mod 2**61 - 1. + */ + uint64_t poly[2][2]; + /* + * The second (twisted) OH function uses an additional + * 128-bit constant stored in the last two elements. + */ + uint64_t oh[UMASH_OH_PARAM_COUNT + UMASH_OH_TWISTING_COUNT]; +}; + +/** + * A fingerprint consists of two independent `UMASH` hash values. + */ +struct umash_fp { + uint64_t hash[2]; +}; + +/** + * This struct holds the state for incremental UMASH hashing or + * fingerprinting. + * + * A sink owns no allocation, and simply borrows a pointer to its + * `umash_params`. It can be byte-copied to snapshot its state. + * + * The layout works best with alignment to 64 bytes, but does not + * require it. + */ +struct umash_sink { + /* + * We incrementally maintain two states when fingerprinting. + * When hashing, only the first `poly_state` and `oh_acc` + * entries are active. + */ + struct { + uint64_t mul[2]; /* Multiplier, and multiplier^2. */ + uint64_t acc; /* Current Horner accumulator. */ + } poly_state[2]; + + /* + * We write new bytes to the second half, and keep the previous + * 16 byte chunk in the first half. + * + * We may temporarily have a full 16-byte buffer in the second half: + * we must know if the first 16 byte chunk is the first of many, or + * the whole input. + */ + char buf[2 * 16]; + + /* The next 64 bytes are accessed in the `OH` inner loop. */ + + /* key->oh. */ + const uint64_t *oh; + + /* oh_iter tracks where we are in the inner loop, times 2. */ + uint32_t oh_iter; + uint8_t bufsz; /* Write pointer in `buf + 16`. */ + uint8_t block_size; /* Current OH block size, excluding `bufsz`. */ + bool large_umash; /* True once we definitely have >= 16 bytes. */ + /* + * 0 if we're computing the first umash, 1 for the second, and + * 2 for a fingerprint. + * + * In practice, we treat 1 and 2 the same (always compute a + * full fingerprint), and return only the second half if we + * only want that half. + */ + uint8_t hash_wanted; + + /* Accumulators for the current OH value. */ + struct umash_oh { + uint64_t bits[2]; + } oh_acc; + struct umash_twisted_oh { + uint64_t lrc[2]; + uint64_t prev[2]; + struct umash_oh acc; + } oh_twisted; + + uint64_t seed; +}; + +/** + * The `umash_state` struct wraps a sink in a type-safe interface: we + * don't want to try and extract a fingerprint from a sink configured + * for hashing. + */ +struct umash_state { + struct umash_sink sink; +}; + +/** + * Similarly, the `umash_fp_state` struct wraps a sink from which we + * should extract a fingerprint. + */ +struct umash_fp_state { + struct umash_sink sink; +}; + +/** + * Converts a `umash_params` struct filled with random values into + * something usable by the UMASH functions below. + * + * When it succeeds, this function is idempotent. Failure happens + * with probability < 2**-110 is `params` is filled with uniformly + * distributed random bits. That's an astronomically unlikely event, + * and most likely signals an issue with the caller's (pseudo-)random + * number generator. + * + * @return false on failure, probably because the input was not random. + */ +bool umash_params_prepare(struct umash_params *params); + +/** + * Deterministically derives a `umash_params` struct from `bits` and + * `key`. The `bits` values do not have to be particularly well + * distributed, and can be generated sequentially. + * + * @param key a pointer to exactly 32 secret bytes. NULL will be + * replaced with "Do not use UMASH VS adversaries.", the default + * UMASH secret. + */ +void umash_params_derive(struct umash_params *, uint64_t bits, const void *key); + +/** + * Updates a `umash_sink` to take into account `data[0 ... n_bytes)`. + */ +void umash_sink_update(struct umash_sink *, const void *data, size_t n_bytes); + +/** + * Computes the UMASH hash of `data[0 ... n_bytes)`. + * + * Randomly generated `param` lead to independent UMASH values and + * associated worst-case collision bounds; changing the `seed` comes + * with no guarantee. + * + * @param which 0 to compute the first UMASH defined by `params`, 1 + * for the second. + */ +uint64_t umash_full(const struct umash_params *params, uint64_t seed, int which, + const void *data, size_t n_bytes); + +/** + * Computes the UMASH fingerprint of `data[0 ... n_bytes)`. + * + * Randomly generated `param` lead to independent UMASH values and + * associated worst-case collision bounds; changing the `seed` comes + * with no guarantee. + */ +struct umash_fp umash_fprint( + const struct umash_params *params, uint64_t seed, const void *data, size_t n_bytes); + +/** + * Prepares a `umash_state` for computing the `which`th UMASH function in + * `params`. + */ +void umash_init( + struct umash_state *, const struct umash_params *params, uint64_t seed, int which); + +/** + * Returns the UMASH value for the bytes that have been + * `umash_sink_update`d into the state. + */ +uint64_t umash_digest(const struct umash_state *); + +/** + * Prepares a `umash_fp_state` for computing the UMASH fingerprint in + * `params`. + */ +void umash_fp_init( + struct umash_fp_state *, const struct umash_params *params, uint64_t seed); + +/** + * Returns the UMASH fingerprint for the bytes that have been + * `umash_sink_update`d into the state. + */ +struct umash_fp umash_fp_digest(const struct umash_fp_state *); + +#ifdef __cplusplus +} +#endif +#endif /* !UMASH_H */ From 2c52b54724e919b2f29a674b08acc73a963b6526 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Fri, 24 Jan 2025 18:56:54 +0100 Subject: [PATCH 26/32] Fix downgrade path for 2.18.0 This patch adjusts the downgrade script generation to not include incompatible files from the 2.18.0 release that would break script generation and replaces them with a working version. This adjustment can be removed after we release of 2.18.1. This patch also reenables the downgrade test. --- .github/workflows/update-test.yaml | 11 +++++------ cmake/GenerateScripts.cmake | 4 ++++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/update-test.yaml b/.github/workflows/update-test.yaml index be4138a8c92..bb51021ba7c 100644 --- a/.github/workflows/update-test.yaml +++ b/.github/workflows/update-test.yaml @@ -37,12 +37,11 @@ jobs: PATH="/usr/lib/postgresql/${{ matrix.pg }}/bin:$PATH" ./scripts/test_updates.sh -# Temporary disabled downgrade for 2.18.0 -# - name: Downgrade tests PG${{ matrix.pg }} -# if: always() -# run: | -# PATH="/usr/lib/postgresql/${{ matrix.pg }}/bin:$PATH" -# ./scripts/test_downgrade.sh + - name: Downgrade tests PG${{ matrix.pg }} + if: always() + run: | + PATH="/usr/lib/postgresql/${{ matrix.pg }}/bin:$PATH" + ./scripts/test_downgrade.sh - name: Update diff if: failure() diff --git a/cmake/GenerateScripts.cmake b/cmake/GenerateScripts.cmake index c45706015f3..3664dd6a8d6 100644 --- a/cmake/GenerateScripts.cmake +++ b/cmake/GenerateScripts.cmake @@ -172,6 +172,10 @@ function(generate_downgrade_script) _epilog_files IGNORE_ERRORS) + if(_downgrade_TARGET_VERSION VERSION_EQUAL 2.18.0) + list(TRANSFORM _epilog_files REPLACE "^.*/hypercore.sql" "${CMAKE_CURRENT_SOURCE_DIR}/pre_install/tam.functions.sql") + endif() + foreach(_downgrade_file ${_downgrade_PRE_FILES}) get_filename_component(_downgrade_filename ${_downgrade_file} NAME) configure_file(${_downgrade_file} ${_downgrade_INPUT_DIRECTORY}/${_downgrade_filename} COPYONLY) From 2194a6109f4410a31dacddf083759dd953546cf5 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Fri, 24 Jan 2025 21:37:52 +0100 Subject: [PATCH 27/32] Allow more workflows to be triggered by github ui This is mostly to allow triggering package tests directly from github UI but no harm enabling it for a couple more. --- .github/workflows/abi.yaml | 1 + .github/workflows/apt-arm-packages.yaml | 1 + .github/workflows/apt-packages.yaml | 1 + .github/workflows/coverity.yaml | 1 + .github/workflows/docker-images.yaml | 1 + .github/workflows/homebrew.yaml | 1 + .github/workflows/libfuzzer.yaml | 1 + .github/workflows/rpm-packages.yaml | 1 + .github/workflows/sanitizer-build-and-test.yaml | 1 + .github/workflows/snapshot-abi.yaml | 2 ++ .github/workflows/update-test.yaml | 2 ++ .github/workflows/windows-build-and-test.yaml | 1 + .github/workflows/windows-packages.yaml | 1 + 13 files changed, 15 insertions(+) diff --git a/.github/workflows/abi.yaml b/.github/workflows/abi.yaml index b4240ab1afb..c99594ada4b 100644 --- a/.github/workflows/abi.yaml +++ b/.github/workflows/abi.yaml @@ -17,6 +17,7 @@ name: ABI Test - trigger/abi pull_request: paths: .github/workflows/abi.yaml + workflow_dispatch: jobs: config: runs-on: ubuntu-latest diff --git a/.github/workflows/apt-arm-packages.yaml b/.github/workflows/apt-arm-packages.yaml index fea4881b064..e9f6c6fbdbc 100644 --- a/.github/workflows/apt-arm-packages.yaml +++ b/.github/workflows/apt-arm-packages.yaml @@ -12,6 +12,7 @@ name: APT ARM64 packages branches: - release_test - trigger/package_test + workflow_dispatch: jobs: apt_tests: name: APT ARM64 ${{ matrix.image }} PG${{ matrix.pg }} diff --git a/.github/workflows/apt-packages.yaml b/.github/workflows/apt-packages.yaml index eb118f791b2..ac9919f7456 100644 --- a/.github/workflows/apt-packages.yaml +++ b/.github/workflows/apt-packages.yaml @@ -12,6 +12,7 @@ name: APT packages branches: - release_test - trigger/package_test + workflow_dispatch: jobs: apt_tests: name: APT ${{ matrix.image }} PG${{ matrix.pg }} ${{ matrix.license }} diff --git a/.github/workflows/coverity.yaml b/.github/workflows/coverity.yaml index f22c1415c57..b62914e3ce8 100644 --- a/.github/workflows/coverity.yaml +++ b/.github/workflows/coverity.yaml @@ -6,6 +6,7 @@ name: Coverity push: branches: - coverity_scan + workflow_dispatch: jobs: coverity: diff --git a/.github/workflows/docker-images.yaml b/.github/workflows/docker-images.yaml index 63c523222e0..0c54fd76349 100644 --- a/.github/workflows/docker-images.yaml +++ b/.github/workflows/docker-images.yaml @@ -15,6 +15,7 @@ name: Test Docker images branches: - release_test - trigger/package_test + workflow_dispatch: jobs: docker_tests: name: ${{ matrix.image }} diff --git a/.github/workflows/homebrew.yaml b/.github/workflows/homebrew.yaml index 8c20c2f5b21..72b725224e0 100644 --- a/.github/workflows/homebrew.yaml +++ b/.github/workflows/homebrew.yaml @@ -11,6 +11,7 @@ name: Homebrew - release_test - trigger/package_test - trigger/homebrew_test + workflow_dispatch: jobs: homebrew: diff --git a/.github/workflows/libfuzzer.yaml b/.github/workflows/libfuzzer.yaml index d4a164ef36e..119cca1931f 100644 --- a/.github/workflows/libfuzzer.yaml +++ b/.github/workflows/libfuzzer.yaml @@ -12,6 +12,7 @@ name: Libfuzzer paths: - .github/workflows/libfuzzer.yaml - 'tsl/test/fuzzing/compression/**' + workflow_dispatch: jobs: build: diff --git a/.github/workflows/rpm-packages.yaml b/.github/workflows/rpm-packages.yaml index 560719b2fe5..c978f5d285b 100644 --- a/.github/workflows/rpm-packages.yaml +++ b/.github/workflows/rpm-packages.yaml @@ -12,6 +12,7 @@ name: RPM packages branches: - release_test - trigger/package_test + workflow_dispatch: jobs: rpm_tests: diff --git a/.github/workflows/sanitizer-build-and-test.yaml b/.github/workflows/sanitizer-build-and-test.yaml index 703593d3beb..0e8e4e9f812 100644 --- a/.github/workflows/sanitizer-build-and-test.yaml +++ b/.github/workflows/sanitizer-build-and-test.yaml @@ -11,6 +11,7 @@ name: Sanitizer test - trigger/sanitizer pull_request: paths: .github/workflows/sanitizer-build-and-test.yaml + workflow_dispatch: env: name: "Sanitizer" diff --git a/.github/workflows/snapshot-abi.yaml b/.github/workflows/snapshot-abi.yaml index b0f0cb4dfa0..a11676986db 100644 --- a/.github/workflows/snapshot-abi.yaml +++ b/.github/workflows/snapshot-abi.yaml @@ -13,6 +13,8 @@ name: ABI Test Against Snapshot - trigger/snapshot-abi pull_request: paths: .github/workflows/snapshot-abi.yaml + workflow_dispatch: + jobs: config: runs-on: ubuntu-latest diff --git a/.github/workflows/update-test.yaml b/.github/workflows/update-test.yaml index bb51021ba7c..561631d7cf7 100644 --- a/.github/workflows/update-test.yaml +++ b/.github/workflows/update-test.yaml @@ -5,6 +5,8 @@ name: Test Update and Downgrade - main - prerelease_test pull_request: + workflow_dispatch: + jobs: update_test: name: Update test PG${{ matrix.pg }} diff --git a/.github/workflows/windows-build-and-test.yaml b/.github/workflows/windows-build-and-test.yaml index fab95ef71ed..fe5b40ea05b 100644 --- a/.github/workflows/windows-build-and-test.yaml +++ b/.github/workflows/windows-build-and-test.yaml @@ -20,6 +20,7 @@ name: Regression Windows - 'LICENSE*' - NOTICE - 'bootstrap*' + workflow_dispatch: jobs: config: runs-on: ubuntu-latest diff --git a/.github/workflows/windows-packages.yaml b/.github/workflows/windows-packages.yaml index 62fa7b97a1f..c44c83bbd4b 100644 --- a/.github/workflows/windows-packages.yaml +++ b/.github/workflows/windows-packages.yaml @@ -12,6 +12,7 @@ name: Windows Packages branches: - release_test - trigger/windows_packages + workflow_dispatch: jobs: config: From f35930b82291267293f07ec42af9c9c50a762ad5 Mon Sep 17 00:00:00 2001 From: Ante Kresic Date: Mon, 27 Jan 2025 10:47:51 +0100 Subject: [PATCH 28/32] Add Windows package test for latest pg17 --- .github/workflows/windows-packages.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/windows-packages.yaml b/.github/workflows/windows-packages.yaml index c44c83bbd4b..852c635bb99 100644 --- a/.github/workflows/windows-packages.yaml +++ b/.github/workflows/windows-packages.yaml @@ -25,6 +25,7 @@ jobs: pg14_latest: ${{ steps.config.outputs.pg14_latest }} pg15_latest: ${{ steps.config.outputs.pg15_latest }} pg16_latest: ${{ steps.config.outputs.pg16_latest }} + pg17_latest: ${{ steps.config.outputs.pg17_latest }} steps: - name: Checkout source code @@ -41,7 +42,7 @@ jobs: fail-fast: false matrix: os: [ windows-2019 ] - test: [ "14min", "14max", "15min", "15max", "16min", "16max", "17min" ] + test: [ "14min", "14max", "15min", "15max", "16min", "16max", "17min", "17max" ] include: - test: 14min pg: 14 @@ -64,6 +65,9 @@ jobs: - test: 17min pg: 17 pkg_version: ${{ fromJson(needs.config.outputs.pg17_earliest) }}.0 + - test: 17max + pg: 17 + pkg_version: ${{ fromJson(needs.config.outputs.pg17_latest) }}.0 env: # PostgreSQL configuration PGPORT: 6543 From e9da072f87134e4ef02495cc9f39fac267ee8ff0 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 26 Jan 2025 18:30:27 +0100 Subject: [PATCH 29/32] Add ci check for idempotency in sql scripts To detect the problematic patterns that were part of the 2.18 release we can check the sql scripts against a list of allowed statements. Any non idempotent operation should be in the pre_install scripts and not the scripts that get appended for the update scripts. --- .github/workflows/catalog-updates-check.yaml | 6 + scripts/check_sql_script.py | 138 +++++++++++++++++++ scripts/check_updates_ast.py | 6 + sql/metadata.sql | 4 +- 4 files changed, 151 insertions(+), 3 deletions(-) create mode 100644 scripts/check_sql_script.py diff --git a/.github/workflows/catalog-updates-check.yaml b/.github/workflows/catalog-updates-check.yaml index 893a7a59447..e1f234093da 100644 --- a/.github/workflows/catalog-updates-check.yaml +++ b/.github/workflows/catalog-updates-check.yaml @@ -22,9 +22,15 @@ jobs: run: | python scripts/check_updates_ast.py --latest "sql/updates/latest-dev.sql" + - name: Check for idempotency in SQL scripts + if: always() + run: | + python scripts/check_sql_script.py sql/*.sql + # To allow fixing previous mistakes we run the check against reverse-dev but don't # fail it on errors. - name: Check reverse-dev contents if: always() run: | python scripts/check_updates_ast.py "sql/updates/reverse-dev.sql" || true + diff --git a/scripts/check_sql_script.py b/scripts/check_sql_script.py new file mode 100644 index 00000000000..983c4c5693f --- /dev/null +++ b/scripts/check_sql_script.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python + +# Check SQL script components for problematic patterns. This script is +# intended to be run on the scripts that are added to every update script, +# but not the compiled update script or the pre_install scripts. +# +# This script will find patterns that are not idempotent and therefore +# should be moved to the pre_install part. + +from pglast import parse_sql +from pglast.visitors import Visitor, Skip, Continue +from pglast.stream import RawStream +import sys +import re +import argparse + +parser = argparse.ArgumentParser() +parser.add_argument("filename", type=argparse.FileType("r"), nargs="+") +args = parser.parse_args() + + +class SQLVisitor(Visitor): + def __init__(self, file): + self.errors = 0 + self.file = file + super().__init__() + + def error(self, node, hint): + self.errors += 1 + print( + f"Invalid statement found in sql script({self.file}):\n", + RawStream()(node), + ) + print(hint, "\n") + + def visit_RawStmt(self, _ancestors, _node): + # Statements are nested in RawStmt so we need to let the visitor descend + return Continue + + def visit(self, _ancestors, node): + self.error(node, "Consider moving the statement into a pre_install script") + + # We are only interested in checking top-level statements + return Skip + + def visit_CommentStmt(self, _ancestors, _node): + return Skip + + def visit_GrantStmt(self, _ancestors, _node): + return Skip + + def visit_SelectStmt(self, _ancestors, _node): + return Skip + + def visit_InsertStmt(self, _ancestors, _node): + return Skip + + def visit_DeleteStmt(self, _ancestors, _node): + return Skip + + def visit_DoStmt(self, _ancestors, _node): + return Skip + + def visit_CreateEventTrigStmt(self, _ancestors, _node): + return Skip + + def visit_CreateTrigStmt(self, _ancestors, node): + if not node.replace: + self.error(node, "Consider using CREATE OR REPLACE TRIGGER") + + return Skip + + def visit_DefineStmt(self, _ancestors, node): + if not node.replace: + self.error(node, "Consider using CREATE OR REPLACE") + + return Skip + + def visit_DropStmt(self, _ancestors, node): + if not node.missing_ok: + self.error(node, "Consider using DROP IF EXISTS") + + return Skip + + def visit_ViewStmt(self, _ancestors, node): + if not node.replace: + self.error(node, "Consider using CREATE OR REPLACE VIEW") + + return Skip + + def visit_CreateFunctionStmt(self, _ancestors, node): + if not node.replace: + self.error(node, "Consider using CREATE OR REPLACE FUNCTION") + + return Skip + + +# copied from pgspot +def visit_sql(sql, file): + # @extschema@ is placeholder in extension scripts for + # the schema the extension gets installed in + sql = sql.replace("@extschema@", "extschema") + sql = sql.replace("@extowner@", "extowner") + sql = sql.replace("@database_owner@", "database_owner") + # postgres contrib modules are protected by psql meta commands to + # prevent running extension files in psql. + # The SQL parser will error on those since they are not valid + # SQL, so we comment out all psql meta commands before parsing. + sql = re.sub(r"^\\", "-- \\\\", sql, flags=re.MULTILINE) + + visitor = SQLVisitor(file) + for stmt in parse_sql(sql): + visitor(stmt) + return visitor.errors + + +def main(args): + errors = 0 + error_files = [] + for file in args.filename: + sql = file.read() + result = visit_sql(sql, file.name) + if result > 0: + errors += result + error_files.append(file.name) + + if errors > 0: + numbering = "errors" if errors > 1 else "error" + print( + f"{errors} {numbering} detected in {len(error_files)} files({', '.join(error_files)})" + ) + sys.exit(1) + sys.exit(0) + + +if __name__ == "__main__": + main(args) + sys.exit(0) diff --git a/scripts/check_updates_ast.py b/scripts/check_updates_ast.py index b6b7a73ac69..3c505de68aa 100644 --- a/scripts/check_updates_ast.py +++ b/scripts/check_updates_ast.py @@ -1,3 +1,9 @@ +#!/usr/bin/env python + +# Check SQL update script for undesirable patterns. This script is +# intended to be run on the compiled update script or subsets of +# the update script (e.g. latest-dev.sql and reverse-dev.sql) + from pglast import parse_sql from pglast.ast import ColumnDef from pglast.visitors import Visitor diff --git a/sql/metadata.sql b/sql/metadata.sql index eabb4f2702c..39a0b77a71c 100644 --- a/sql/metadata.sql +++ b/sql/metadata.sql @@ -21,9 +21,7 @@ BEGIN END $$ SET search_path TO pg_catalog, pg_temp; --- CREATE OR REPLACE TRIGGER is PG14+ only -DROP TRIGGER IF EXISTS metadata_insert_trigger ON _timescaledb_catalog.metadata; -CREATE TRIGGER metadata_insert_trigger BEFORE INSERT ON _timescaledb_catalog.metadata FOR EACH ROW EXECUTE PROCEDURE _timescaledb_functions.metadata_insert_trigger(); +CREATE OR REPLACE TRIGGER metadata_insert_trigger BEFORE INSERT ON _timescaledb_catalog.metadata FOR EACH ROW EXECUTE PROCEDURE _timescaledb_functions.metadata_insert_trigger(); -- Insert uuid and install_timestamp on database creation since the trigger -- will turn these into UPDATEs on conflicts we can't use ON CONFLICT DO NOTHING. From 8479ba738aa1d4848c3efef06c4cfc8d7a95802c Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 26 Jan 2025 10:33:05 +0100 Subject: [PATCH 30/32] Remove some dead code from downgrade script generation Since we only support generating downgrade script for the previous version anything targeting versions before 2.18 will never be executed in current context. So we can safely remove the code that deals with version before 2.3. --- cmake/GenerateScripts.cmake | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/cmake/GenerateScripts.cmake b/cmake/GenerateScripts.cmake index 3664dd6a8d6..1ea65dfc9da 100644 --- a/cmake/GenerateScripts.cmake +++ b/cmake/GenerateScripts.cmake @@ -137,21 +137,9 @@ function(generate_downgrade_script) endforeach() # Fetch manifest with list of files for the prolog and epilog from the target - # version, if we are in a version that supports downgrades. Otherwise, take - # the one in the current version. - # - # We have a specific exception where we allow a missing manifest for the first - # version that supports downgrades and assume that the files to include are - # the same in the target version as the current one. - if(_downgrade_TARGET_VERSION VERSION_GREATER 2.3) - git_versioned_get(VERSION ${_downgrade_TARGET_VERSION} FILES - ${CMAKE_SOURCE_DIR}/cmake/ScriptFiles.cmake) - else() - file(MAKE_DIRECTORY - "${CMAKE_BINARY_DIR}/v${_downgrade_TARGET_VERSION}/cmake") - file(COPY "${CMAKE_SOURCE_DIR}/cmake/ScriptFiles.cmake" - DESTINATION "${CMAKE_BINARY_DIR}/v${_downgrade_TARGET_VERSION}/cmake") - endif() + # version. + git_versioned_get(VERSION ${_downgrade_TARGET_VERSION} FILES + ${CMAKE_SOURCE_DIR}/cmake/ScriptFiles.cmake) # This will include the variables in this scope, but not in the parent scope # so we can read them locally without affecting the parent scope. From 06ccc50674f8483a81b4a9222ec6e86493f21135 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabr=C3=ADzio=20de=20Royes=20Mello?= Date: Tue, 28 Jan 2025 19:49:57 -0300 Subject: [PATCH 31/32] Update gitignore Leftover from dc671301 --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 5716b04e335..b426d831392 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ **/CMakeCache.txt /sql/tests/unit/testoutputs.tmp /sql/timescaledb--*.sql +/sql/pre_install/*.gen /sql/updates/*.gen /data/ /src/*.o From 538c4430e73a51dd3ace5d9031c6ac43df8fca0c Mon Sep 17 00:00:00 2001 From: Mats Kindahl Date: Fri, 17 Jan 2025 10:58:37 +0100 Subject: [PATCH 32/32] Fix lock order when dropping index If an index is dropped, it is necessary to lock the heap table (of the index) before the index since all normal operations do it in this order. When dropping an index, we did not take all the necessary locks in the right order before calling `performMultipleDeletions`, which can cause deadlocks when dropping an index on a hypertable at the same time as running a utility statement that takes heavy locks, e.g., VACUUM or ANALYZE. Adding a isolation test as well that will generate a deadlock if the index and table locks are not taken in the correct order. --- .unreleased/pr_7600 | 1 + src/chunk_index.c | 106 +++++++++++--- .../expected/deadlock_drop_index_vacuum.out | 24 ++++ tsl/test/isolation/specs/CMakeLists.txt | 1 + .../specs/deadlock_drop_index_vacuum.spec | 130 ++++++++++++++++++ 5 files changed, 240 insertions(+), 22 deletions(-) create mode 100644 .unreleased/pr_7600 create mode 100644 tsl/test/isolation/expected/deadlock_drop_index_vacuum.out create mode 100644 tsl/test/isolation/specs/deadlock_drop_index_vacuum.spec diff --git a/.unreleased/pr_7600 b/.unreleased/pr_7600 new file mode 100644 index 00000000000..bb3a4008f81 --- /dev/null +++ b/.unreleased/pr_7600 @@ -0,0 +1 @@ +Fixes: #7600 Fix lock order when dropping index diff --git a/src/chunk_index.c b/src/chunk_index.c index da11d4a5658..5e85282f2fc 100644 --- a/src/chunk_index.c +++ b/src/chunk_index.c @@ -573,17 +573,70 @@ typedef struct ChunkIndexDeleteData bool drop_index; } ChunkIndexDeleteData; -/* Find all internal dependencies to be able to delete all the objects in one +/* + * Lock object. + * + * In particular, we need to ensure that we lock the table of an index before + * locking the index, or run the risk of ending up in a deadlock since the + * normal locking order is table first, index second. Since we're not a + * concurrent delete, we take a strong lock for this. + * + * It is also necessary that the parent table is locked first, but we have + * already done that at this stage, so it does not need to be done explicitly. + */ +static bool +chunk_lock_object_for_deletion(const ObjectAddress *obj) +{ + /* + * If we're locking an index, we need to lock the table first. See + * RangeVarCallbackForDropRelation() in tablecmds.c. We can ignore + * partition indexes since we're not using that. + */ + char relkind = get_rel_relkind(obj->objectId); + + /* + * If we cannot find the object, it might have been concurrently deleted + * (we do not have locks on objects yet). + */ + if (relkind == '\0') + return false; + if (relkind == RELKIND_INDEX) + { + Oid heapOid = IndexGetRelation(obj->objectId, true); + if (OidIsValid(heapOid)) + LockRelationOid(heapOid, AccessExclusiveLock); + } + + LockRelationOid(obj->objectId, AccessExclusiveLock); + return true; +} + +/* + * Find all internal dependencies to be able to delete all the objects in one * go. We do this by scanning the dependency table and keeping all the tables - * in our internal schema. */ -static void -chunk_collect_objects_for_deletion(const ObjectAddress *relobj, ObjectAddresses *objects) + * in our internal schema. + * + * We also lock the objects in the correct order (meaning table first, index + * second) here to make sure that we do not end up with deadlocks. + * + * We return 'true' if we added any objects, and 'false' otherwise. + */ +static bool +chunk_collect_and_lock_objects_for_deletion(const ObjectAddress *relobj, ObjectAddresses *objects) { Relation deprel = table_open(DependRelationId, RowExclusiveLock); ScanKeyData scankey[2]; SysScanDesc scan; HeapTuple tup; + /* + * If the object disappeared before we managed to get a lock on it, there + * is nothing more to do so just return early and indicate that there are + * no objects to delete. + */ + if (!chunk_lock_object_for_deletion(relobj)) + return false; + add_exact_object_address(relobj, objects); ScanKeyInit(&scankey[0], @@ -608,18 +661,13 @@ chunk_collect_objects_for_deletion(const ObjectAddress *relobj, ObjectAddresses { Form_pg_depend record = (Form_pg_depend) GETSTRUCT(tup); ObjectAddress refobj = { .classId = record->refclassid, .objectId = record->refobjid }; - - switch (record->deptype) - { - case DEPENDENCY_INTERNAL: - add_exact_object_address(&refobj, objects); - break; - default: - continue; /* Do nothing */ - } + if (record->deptype == DEPENDENCY_INTERNAL && chunk_lock_object_for_deletion(&refobj)) + add_exact_object_address(&refobj, objects); } + systable_endscan(scan); table_close(deprel, RowExclusiveLock); + return true; } static ScanTupleResult @@ -642,7 +690,8 @@ chunk_index_tuple_delete(TupleInfo *ti, void *data) if (OidIsValid(idxobj.objectId)) { - /* If we use performDeletion here it will fail if there are + /* + * If we use performDeletion() here it will fail if there are * internal dependencies on the object since we are restricting * the cascade. * @@ -651,15 +700,28 @@ chunk_index_tuple_delete(TupleInfo *ti, void *data) * internal dependencies and use the function * performMultipleDeletions. * - * The function performMultipleDeletions accept a list of objects - * and if there are dependencies between any of the objects given - * to the function, it will not generate an error for that but - * rather proceed with the deletion. If there are any dependencies - * (internal or not) outside this set of objects, it will still - * abort the deletion and print an error. */ + * We lock the objects to delete first to make sure that the lock + * order is correct. This is done inside RemoveRelations and + * performMultipleDeletions() expect these locks to be taken + * first. If not, it will take very rudimentary locks, which will + * cause deadlocks in some cases because the lock order is not + * correct. + * + * Since we do not have any locks on any objects at this point, + * the relations might have disappeared before we had a chance to + * lock them. In this case it is not necessary to do an explicit + * call to performMultipleDeletions(). + * + * The function performMultipleDeletions() accept a list of + * objects and if there are dependencies between any of the + * objects given to the function, it will not generate an error + * for that but rather proceed with the deletion. If there are any + * dependencies (internal or not) outside this set of objects, it + * will still abort the deletion and print an error. + */ ObjectAddresses *objects = new_object_addresses(); - chunk_collect_objects_for_deletion(&idxobj, objects); - performMultipleDeletions(objects, DROP_RESTRICT, 0); + if (chunk_collect_and_lock_objects_for_deletion(&idxobj, objects)) + performMultipleDeletions(objects, DROP_RESTRICT, 0); free_object_addresses(objects); } } diff --git a/tsl/test/isolation/expected/deadlock_drop_index_vacuum.out b/tsl/test/isolation/expected/deadlock_drop_index_vacuum.out new file mode 100644 index 00000000000..03fa5f15125 --- /dev/null +++ b/tsl/test/isolation/expected/deadlock_drop_index_vacuum.out @@ -0,0 +1,24 @@ +Parsed test spec with 4 sessions + +starting permutation: S1_lock S3_vacuum S2_lock S1_commit S4_drop_index S2_commit +step S1_lock: + LOCK TABLE _timescaledb_internal.metrics_chunk_2 IN ACCESS EXCLUSIVE MODE; + +step S3_vacuum: + VACUUM ANALYZE _timescaledb_internal.metrics_chunk_2; + +step S2_lock: + LOCK TABLE _timescaledb_internal.metrics_chunk_2 IN ACCESS EXCLUSIVE MODE; + +step S1_commit: + COMMIT; + +step S2_lock: <... completed> +step S4_drop_index: + DROP INDEX metrics_device_time_idx; + +step S2_commit: + COMMIT; + +step S3_vacuum: <... completed> +step S4_drop_index: <... completed> diff --git a/tsl/test/isolation/specs/CMakeLists.txt b/tsl/test/isolation/specs/CMakeLists.txt index 629f53094fb..960abb4b0d4 100644 --- a/tsl/test/isolation/specs/CMakeLists.txt +++ b/tsl/test/isolation/specs/CMakeLists.txt @@ -20,6 +20,7 @@ list( cagg_multi_iso.spec cagg_concurrent_refresh.spec deadlock_drop_chunks_compress.spec + deadlock_drop_index_vacuum.spec parallel_compression.spec osm_range_updates_iso.spec) diff --git a/tsl/test/isolation/specs/deadlock_drop_index_vacuum.spec b/tsl/test/isolation/specs/deadlock_drop_index_vacuum.spec new file mode 100644 index 00000000000..be11e373984 --- /dev/null +++ b/tsl/test/isolation/specs/deadlock_drop_index_vacuum.spec @@ -0,0 +1,130 @@ +# This file and its contents are licensed under the Timescale License. +# Please see the included NOTICE for copyright information and +# LICENSE-TIMESCALE for a copy of the license. + +# Order of locks in drop index and vacuum analyze was wrong, and DROP +# INDEX took the locks in order index-table, while VACUUM ANALYZE took +# them in order table-index. +# +# Create deadlock if the locking order is wrong. Problem here is that +# vacuum takes two locks. First one to read a table and then one to do +# the actual vacuum. For this reason we need two processes that end up +# in the deadlock (VACUUM ANALYZE and DROP INDEX respectively) and +# then two extra sessions working in lock-step to create the deadlock. +# +# It can be illustrated with this sequence, where we have a chunk +# table and a chunk index. The sessions between the brackets ([]) is +# the lock queue and session holding the lock is in angle brackets +# (<>) is the session that holds the lock on the object in question: +# +# S1: Lock chunk from hypertable +# index: [] +# table: [] +# S3: Start VACUUM ANALYZE, will attempt to lock chunk table, but get queued. +# index: [] +# table: [S3] +# S2: Lock chunk table from hypertable, which will be queued +# index: [] +# table: [S3 S2] +# S1: Unlock chunk table +# index: [] +# table: [S3 S2] +# S3: VACUUM ANALYZE continues and takes the lock on the chunk table. +# index: [] +# table: [S2] +# S3: VACUUM ANALYZE will release the lock on the chunk table. +# index: [] +# table: [S2] +# S3: VACUUM ANALYZE will attempt to lock the chunk table again +# index: [] +# table: [S2 S3] +# S2: The LOCK statement gets the lock and VACUUM will wait +# index: [] +# table: [S3] +# S4: DROP INDEX starts and takes lock in index first and then is +# queued for the chunk table +# index: [] +# table: [S3 S4] +# S2: Release lock on chunk table +# index: [] +# table: [S3 S4] +# S3: VACUUM continues and takes table lock and then tries index lock +# index: [S3] +# table: [S4] +# Deadlock + +setup { + CREATE TABLE metrics (time timestamptz, device_id integer, temp float); + SELECT create_hypertable('metrics', 'time', chunk_time_interval => interval '1 day'); + INSERT INTO metrics + SELECT generate_series('2018-12-01 00:00'::timestamp, + '2018-12-03 00:00', + '1 hour'), + (random()*30)::int, + random()*80 - 40; + + CREATE INDEX metrics_device_time_idx ON metrics(device_id, time NULLS FIRST); + + -- Rename chunks so that we have known names. We cannot execute + -- VACUUM in a function block. + DO $$ + DECLARE + chunk regclass; + count int = 1; + BEGIN + FOR chunk IN SELECT ch FROM show_chunks('metrics') ch + LOOP + EXECUTE format('ALTER TABLE %s RENAME TO metrics_chunk_%s', chunk, count); + count = count + 1; + END LOOP; + END + $$; +} + +teardown { + DROP TABLE metrics; +} + +session "S1" +setup { + START TRANSACTION; + SET TRANSACTION ISOLATION LEVEL READ COMMITTED; + SET LOCAL lock_timeout = '500ms'; + SET LOCAL deadlock_timeout = '300ms'; +} + +step "S1_lock" { + LOCK TABLE _timescaledb_internal.metrics_chunk_2 IN ACCESS EXCLUSIVE MODE; +} + +step "S1_commit" { + COMMIT; +} + +session "S2" +setup { + START TRANSACTION; + SET TRANSACTION ISOLATION LEVEL READ COMMITTED; + SET LOCAL lock_timeout = '500ms'; + SET LOCAL deadlock_timeout = '300ms'; +} + +step "S2_lock" { + LOCK TABLE _timescaledb_internal.metrics_chunk_2 IN ACCESS EXCLUSIVE MODE; +} + +step "S2_commit" { + COMMIT; +} + +session "S3" +step "S3_vacuum" { + VACUUM ANALYZE _timescaledb_internal.metrics_chunk_2; +} + +session "S4" +step "S4_drop_index" { + DROP INDEX metrics_device_time_idx; +} + +permutation S1_lock S3_vacuum S2_lock S1_commit S4_drop_index S2_commit