From 252fb327643fe74e25bc467fe4ed237f80f0befb Mon Sep 17 00:00:00 2001 From: Dmitry Sherstobitov Date: Wed, 8 Nov 2023 10:28:45 +0400 Subject: [PATCH 1/3] Added flags to control test features --- yugabyte/src/yugabyte/auto.clj | 10 ++++++++++ yugabyte/src/yugabyte/core.clj | 9 +++++++++ yugabyte/src/yugabyte/runner.clj | 6 ++++++ 3 files changed, 25 insertions(+) diff --git a/yugabyte/src/yugabyte/auto.clj b/yugabyte/src/yugabyte/auto.clj index ac0cd6993..82a79449e 100644 --- a/yugabyte/src/yugabyte/auto.clj +++ b/yugabyte/src/yugabyte/auto.clj @@ -421,6 +421,14 @@ :--rpc_connection_timeout_ms 1500] [])) +(defn master-tserver-trace-sql-transactions + "Speed up recovery from partitions and crashes. Right now it looks like + these actually make the cluster slower to, or unable to, recover." + [test] + (if (:trace-sql-tx test) + [:--vmodule "transaction=2,transaction_coordinator=4,conflict_resolution=4"] + [])) + (def limits-conf "Ulimits, in the format for /etc/security/limits.conf." " @@ -476,6 +484,7 @@ :--replication_factor (:replication-factor test) ;:--auto_create_local_transaction_tables=false (master-tserver-experimental-tuning-flags test) + (master-tserver-trace-sql-transactions test) (master-tserver-random-clock-skew test node) (master-tserver-wait-on-conflict-flags test) (master-tserver-packed-columns (:yb-packed-columns-enabled test)) @@ -497,6 +506,7 @@ :--enable_tracing :--rpc_slow_query_threshold_ms 1000 (master-tserver-experimental-tuning-flags test) + (master-tserver-trace-sql-transactions test) (master-tserver-random-clock-skew test node) (master-tserver-wait-on-conflict-flags test) (master-tserver-packed-columns (:yb-packed-columns-enabled test)) diff --git a/yugabyte/src/yugabyte/core.clj b/yugabyte/src/yugabyte/core.clj index 5ff84a28d..921192563 100644 --- a/yugabyte/src/yugabyte/core.clj +++ b/yugabyte/src/yugabyte/core.clj @@ -317,6 +317,15 @@ colocated (and (not (utils/is-test-geo-partitioned? opts)) (> (rand) 0.5))] (assoc opts :yb-packed-columns-enabled packed-columns-enabled :yb-colocated colocated))) +(defn test-3 + "Final phase where we define global cluster configuration parameters" + [opts] + (let [packed-columns-enabled (> (rand) 0.5) + colocated (and (not (utils/is-test-geo-partitioned? opts)) (> (rand) 0.5))] + (if (:additional-features test) + (assoc opts :yb-packed-columns-enabled packed-columns-enabled + :yb-colocated colocated)))) + (defn yb-test "Constructs a yugabyte test from CLI options." [opts] diff --git a/yugabyte/src/yugabyte/runner.clj b/yugabyte/src/yugabyte/runner.clj index 35e809b9c..d3882903f 100644 --- a/yugabyte/src/yugabyte/runner.clj +++ b/yugabyte/src/yugabyte/runner.clj @@ -56,6 +56,9 @@ [nil "--heartbeat-flags" "Enable heartbeat tserver tracing flags on YB" :default false] + [nil "--additional-features" "Enable additional features like packed columns, colocated, etc." + :default true] + [nil "--clock-skew-flags" "Enable soft clock skew flags on YB" :default true] @@ -112,6 +115,9 @@ [nil "--url URL" "URL to Yugabyte tarball to install, has precedence over --version" :default nil] + [nil "--trace-sql-tx" "Trace SQL transactions using vmodule gflag, needed for fast debug." + :default false] + [nil "--trace-cql" "If provided, logs CQL queries" :default false]]) From 9efd6a777913e7bafcb5e0d90a3b1b5e53643798 Mon Sep 17 00:00:00 2001 From: Dmitry Sherstobitov Date: Tue, 9 Jan 2024 15:12:48 +0400 Subject: [PATCH 2/3] Fixed duplicated code --- yugabyte/src/yugabyte/core.clj | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/yugabyte/src/yugabyte/core.clj b/yugabyte/src/yugabyte/core.clj index 921192563..8cdc5c6af 100644 --- a/yugabyte/src/yugabyte/core.clj +++ b/yugabyte/src/yugabyte/core.clj @@ -315,16 +315,8 @@ [opts] (let [packed-columns-enabled (> (rand) 0.5) colocated (and (not (utils/is-test-geo-partitioned? opts)) (> (rand) 0.5))] - (assoc opts :yb-packed-columns-enabled packed-columns-enabled :yb-colocated colocated))) - -(defn test-3 - "Final phase where we define global cluster configuration parameters" - [opts] - (let [packed-columns-enabled (> (rand) 0.5) - colocated (and (not (utils/is-test-geo-partitioned? opts)) (> (rand) 0.5))] - (if (:additional-features test) - (assoc opts :yb-packed-columns-enabled packed-columns-enabled - :yb-colocated colocated)))) + (assoc opts :yb-packed-columns-enabled packed-columns-enabled + :yb-colocated colocated))) (defn yb-test "Constructs a yugabyte test from CLI options." From d9947307b1dde0b41f90449d8eccb5a9699be5c4 Mon Sep 17 00:00:00 2001 From: Dmitry Sherstobitov Date: Tue, 19 Mar 2024 10:37:29 +0400 Subject: [PATCH 3/3] Added log all statements --- yugabyte/src/yugabyte/auto.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yugabyte/src/yugabyte/auto.clj b/yugabyte/src/yugabyte/auto.clj index 82a79449e..0b9d6854c 100644 --- a/yugabyte/src/yugabyte/auto.clj +++ b/yugabyte/src/yugabyte/auto.clj @@ -426,7 +426,8 @@ these actually make the cluster slower to, or unable to, recover." [test] (if (:trace-sql-tx test) - [:--vmodule "transaction=2,transaction_coordinator=4,conflict_resolution=4"] + [:--vmodule "transaction=2,transaction_coordinator=4,conflict_resolution=4" + :--ysql-log-statement "all"] [])) (def limits-conf