From 6eed90532d88093bdebd90cc492b00c0c7b024e0 Mon Sep 17 00:00:00 2001 From: chiragsalian Date: Wed, 6 Nov 2024 14:41:13 -0800 Subject: [PATCH 1/5] outputting crash command list --- BedrockServer.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/BedrockServer.cpp b/BedrockServer.cpp index 8fbd245a9..747655ad9 100644 --- a/BedrockServer.cpp +++ b/BedrockServer.cpp @@ -1671,11 +1671,14 @@ void BedrockServer::_status(unique_ptr& command) { { // Make it known if anything is known to cause crashes. shared_lock lock(_crashCommandMutex); + vector crashCommandList; size_t totalCount = 0; for (const auto& s : _crashCommands) { + crashCommandList.push_back(s.first); totalCount += s.second.size(); } content["crashCommands"] = totalCount; + content["crashCommandList"] = SComposeList(crashCommandList); } // On leader, return the current multi-write blacklists. From 8c8bea16255a0da87fc1c9f1890cb795eff2e342 Mon Sep 17 00:00:00 2001 From: chiragsalian Date: Fri, 8 Nov 2024 10:35:16 -0800 Subject: [PATCH 2/5] working params --- BedrockServer.cpp | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/BedrockServer.cpp b/BedrockServer.cpp index 747655ad9..c20efe629 100644 --- a/BedrockServer.cpp +++ b/BedrockServer.cpp @@ -1671,14 +1671,26 @@ void BedrockServer::_status(unique_ptr& command) { { // Make it known if anything is known to cause crashes. shared_lock lock(_crashCommandMutex); - vector crashCommandList; + map> crashCommandData; + size_t totalCount = 0; for (const auto& s : _crashCommands) { - crashCommandList.push_back(s.first); totalCount += s.second.size(); + for (const STable& params : s.second) { + crashCommandData[s.first].push_back(SComposeJSONObject(params)); + } + } + + // Create a final JSON-like structure + STable crashCommandList; + vector crashCommandListArray; + for (const auto& [command, data] : crashCommandData) { + STable commandObject; + commandObject[command] = SComposeJSONArray(data); + crashCommandListArray.push_back(SComposeJSONObject(commandObject)); } content["crashCommands"] = totalCount; - content["crashCommandList"] = SComposeList(crashCommandList); + content["crashCommandList"] = SComposeJSONArray(crashCommandListArray); } // On leader, return the current multi-write blacklists. From 05b5902ff39445990fff9d5ac14b9e00ab239d9b Mon Sep 17 00:00:00 2001 From: chiragsalian Date: Fri, 8 Nov 2024 10:38:39 -0800 Subject: [PATCH 3/5] consolidating the blocks into one --- BedrockServer.cpp | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/BedrockServer.cpp b/BedrockServer.cpp index c20efe629..e45ed0ff1 100644 --- a/BedrockServer.cpp +++ b/BedrockServer.cpp @@ -1671,22 +1671,19 @@ void BedrockServer::_status(unique_ptr& command) { { // Make it known if anything is known to cause crashes. shared_lock lock(_crashCommandMutex); - map> crashCommandData; + vector crashCommandListArray; size_t totalCount = 0; for (const auto& s : _crashCommands) { totalCount += s.second.size(); + + vector paramsArray; for (const STable& params : s.second) { - crashCommandData[s.first].push_back(SComposeJSONObject(params)); + paramsArray.push_back(SComposeJSONObject(params)); } - } - - // Create a final JSON-like structure - STable crashCommandList; - vector crashCommandListArray; - for (const auto& [command, data] : crashCommandData) { + STable commandObject; - commandObject[command] = SComposeJSONArray(data); + commandObject[s.first] = SComposeJSONArray(paramsArray); crashCommandListArray.push_back(SComposeJSONObject(commandObject)); } content["crashCommands"] = totalCount; From 87f8dbd19524408602710c7532eccc03f49a3c91 Mon Sep 17 00:00:00 2001 From: chiragsalian Date: Fri, 8 Nov 2024 11:02:07 -0800 Subject: [PATCH 4/5] empty param cleanup --- BedrockServer.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/BedrockServer.cpp b/BedrockServer.cpp index e45ed0ff1..33005c2b4 100644 --- a/BedrockServer.cpp +++ b/BedrockServer.cpp @@ -1679,7 +1679,9 @@ void BedrockServer::_status(unique_ptr& command) { vector paramsArray; for (const STable& params : s.second) { - paramsArray.push_back(SComposeJSONObject(params)); + if (!params.empty()) { + paramsArray.push_back(SComposeJSONObject(params)); + } } STable commandObject; From e00e96fc5eb8d3d9e5247f6efc60b53878698350 Mon Sep 17 00:00:00 2001 From: Florent De Neve Date: Wed, 13 Nov 2024 13:58:48 -0400 Subject: [PATCH 5/5] Cleanup: Always set "PRAGMA synchronous = OFF;" --- BedrockServer.cpp | 4 ++-- docker/README.md | 5 ++--- main.cpp | 3 --- sqlitecluster/SQLite.cpp | 12 +++--------- sqlitecluster/SQLite.h | 13 ++++--------- sqlitecluster/SQLitePool.cpp | 3 +-- sqlitecluster/SQLitePool.h | 2 +- test/lib/BedrockTester.cpp | 2 +- 8 files changed, 14 insertions(+), 30 deletions(-) diff --git a/BedrockServer.cpp b/BedrockServer.cpp index 8fbd245a9..cff74bd47 100644 --- a/BedrockServer.cpp +++ b/BedrockServer.cpp @@ -97,7 +97,7 @@ void BedrockServer::sync() // We use fewer FDs on test machines that have other resource restrictions in place. SINFO("Setting dbPool size to: " << _dbPoolSize); - _dbPool = make_shared(_dbPoolSize, args["-db"], args.calc("-cacheSize"), args.calc("-maxJournalSize"), journalTables, args["-synchronous"], mmapSizeGB, args.isSet("-hctree")); + _dbPool = make_shared(_dbPoolSize, args["-db"], args.calc("-cacheSize"), args.calc("-maxJournalSize"), journalTables, mmapSizeGB, args.isSet("-hctree")); SQLite& db = _dbPool->getBase(); // Initialize the command processor. @@ -1026,7 +1026,7 @@ void BedrockServer::runCommand(unique_ptr&& _command, bool isBlo if (_enableConflictPageLocks) { lastConflictTable = db.getLastConflictTable(); - // Journals are always chosen at the time of commit. So in case there was a conflict on the journal in + // Journals are always chosen at the time of commit. So in case there was a conflict on the journal in // the previous commit, the chances are very low (1/192) that we'll choose the same journal, thus, we // don't need to lock our next commit on this page conflict. if (!SStartsWith(lastConflictTable, "journal")) { diff --git a/docker/README.md b/docker/README.md index 878a16eeb..e00cb61d1 100644 --- a/docker/README.md +++ b/docker/README.md @@ -10,7 +10,7 @@ This folder contains the necessary files to build and run Bedrock in a Docker co ## Dockerfile -The Dockerfile sets up a container based on [`phusion/baseimage:noble-1.0.0`](https://github.com/phusion/baseimage-docker) +The Dockerfile sets up a container based on [`phusion/baseimage:noble-1.0.0`](https://github.com/phusion/baseimage-docker) and includes the following key steps: 1. Installs necessary packages @@ -90,7 +90,6 @@ The following environment variables can be used to configure Bedrock: - `WORKER_THREADS`: Number of worker threads - `QUERY_LOG`: Query log configuration - `MAX_JOURNAL_SIZE`: Maximum journal size -- `SYNCHRONOUS`: Synchronous mode setting Additional flags can be set using the `VERBOSE`, `QUIET`, and `CLEAN` environment variables. @@ -118,4 +117,4 @@ bedrock-node2 | Oct 17 16:14:52 da62315a18ec bedrock: xxxxxx (SQLiteNode.cpp:12 ## Network -The Bedrock nodes are connected to a custom bridge network `bedrock-cluster`, allowing for easy communication between nodes. \ No newline at end of file +The Bedrock nodes are connected to a custom bridge network `bedrock-cluster`, allowing for easy communication between nodes. diff --git a/main.cpp b/main.cpp index f14683522..240a1da88 100644 --- a/main.cpp +++ b/main.cpp @@ -236,9 +236,6 @@ int main(int argc, char* argv[]) { << endl; cout << "-maxJournalSize <#commits> Number of commits to retain in the historical journal (default 1000000)" << endl; - cout << "-synchronous Set the PRAGMA schema.synchronous " - "(defaults see https://sqlite.org/pragma.html#pragma_synchronous)" - << endl; cout << endl; cout << "Quick Start Tips:" << endl; cout << "-----------------" << endl; diff --git a/sqlitecluster/SQLite.cpp b/sqlitecluster/SQLite.cpp index d13d04f90..5ea59ced0 100644 --- a/sqlitecluster/SQLite.cpp +++ b/sqlitecluster/SQLite.cpp @@ -218,16 +218,12 @@ void SQLite::commonConstructorInitialization(bool hctree) { // Setting a wal hook prevents auto-checkpointing. sqlite3_wal_hook(_db, _walHookCallback, this); - // Check if synchronous has been set and run query to use a custom synchronous setting - if (!_synchronous.empty()) { - SASSERT(!SQuery(_db, "setting custom synchronous commits", "PRAGMA synchronous = " + SQ(_synchronous) + ";")); - } else { - DBINFO("Using SQLite default PRAGMA synchronous"); - } + // Always set synchronous commits to off for best commit performance in WAL mode. + SASSERT(!SQuery(_db, "setting synchronous commits to off", "PRAGMA synchronous = OFF;")); } SQLite::SQLite(const string& filename, int cacheSize, int maxJournalSize, - int minJournalTables, const string& synchronous, int64_t mmapSizeGB, bool hctree) : + int minJournalTables, int64_t mmapSizeGB, bool hctree) : _filename(initializeFilename(filename)), _maxJournalSize(maxJournalSize), _db(initializeDB(_filename, mmapSizeGB, hctree)), @@ -235,7 +231,6 @@ SQLite::SQLite(const string& filename, int cacheSize, int maxJournalSize, _sharedData(initializeSharedData(_db, _filename, _journalNames, hctree)), _journalSize(initializeJournalSize(_db, _journalNames)), _cacheSize(cacheSize), - _synchronous(synchronous), _mmapSizeGB(mmapSizeGB) { commonConstructorInitialization(hctree); @@ -249,7 +244,6 @@ SQLite::SQLite(const SQLite& from) : _sharedData(from._sharedData), _journalSize(from._journalSize), _cacheSize(from._cacheSize), - _synchronous(from._synchronous), _mmapSizeGB(from._mmapSizeGB) { // This can always pass "true" because the copy constructor does not need to set the DB to WAL2 mode, it would have been set in the object being copied. diff --git a/sqlitecluster/SQLite.h b/sqlitecluster/SQLite.h index 246f4dd66..842b51430 100644 --- a/sqlitecluster/SQLite.h +++ b/sqlitecluster/SQLite.h @@ -56,12 +56,8 @@ class SQLite { // // mmapSizeGB: address space to use for memory-mapped IO, in GB. SQLite(const string& filename, int cacheSize, int maxJournalSize, int minJournalTables, - const string& synchronous = "", int64_t mmapSizeGB = 0, bool hctree = false); + int64_t mmapSizeGB = 0, bool hctree = false); - // Compatibility constructor. Remove when AuthTester::getStripeSQLiteDB no longer uses this outdated version. - SQLite(const string& filename, int cacheSize, int maxJournalSize, int minJournalTables, int synchronous) : - SQLite(filename, cacheSize, maxJournalSize, minJournalTables, "") {} - // This constructor is not exactly a copy constructor. It creates an other SQLite object based on the first except // with a *different* journal table. This avoids a lot of locking around creating structures that we know already // exist because we already have a SQLite object for this file. @@ -154,7 +150,7 @@ class SQLite { void setRewriteHandler(bool (*handler)(int, const char*, string&)); // Enables the on prepare handler. - // The on commit handler allows a plugin to be notified when a transaction is prepared but not yet committed. + // The on commit handler allows a plugin to be notified when a transaction is prepared but not yet committed. // This allows the plugin to take arbitrary actions prior to committing to the database. Bedrock does not // pass up any information in this case, it simply notifies the plugin that a transaction was prepared. void enablePrepareNotifications(bool enable); @@ -162,10 +158,10 @@ class SQLite { // Update the on prepare handler. // The on prepare handler accepts a reference to this SQLiteDB object and an int tableID. The tableID is the // same ID that is used for the journal number in the current running thread. This allows the handler to utilize - // SQLite.cpp's method for avoiding conflicts on tables written on every command. + // SQLite.cpp's method for avoiding conflicts on tables written on every command. // IMPORTANT: The on prepare handler allows a plugin to run code inside the commit lock. This code should be time sensitive // as increases to the amount of time this lock is held increase conflict chances and decreases the parallelness - // of bedrock commands. + // of bedrock commands. // IMPORTANT: there can be only one on-prepare handler for a given DB at once. void setOnPrepareHandler(void (*handler)(SQLite& _db, int64_t tableID)); @@ -505,7 +501,6 @@ class SQLite { // Copies of parameters used to initialize the DB that we store if we make child objects based on this one. int _cacheSize; - const string _synchronous; int64_t _mmapSizeGB; // This is a string (which may be empty) containing the most recent logged error by SQLite in this thread. diff --git a/sqlitecluster/SQLitePool.cpp b/sqlitecluster/SQLitePool.cpp index 7c0e0cec6..ca3d7a4ad 100644 --- a/sqlitecluster/SQLitePool.cpp +++ b/sqlitecluster/SQLitePool.cpp @@ -7,11 +7,10 @@ SQLitePool::SQLitePool(size_t maxDBs, int cacheSize, int maxJournalSize, int minJournalTables, - const string& synchronous, int64_t mmapSizeGB, bool hctree) : _maxDBs(max(maxDBs, 1ul)), - _baseDB(filename, cacheSize, maxJournalSize, minJournalTables, synchronous, mmapSizeGB, hctree), + _baseDB(filename, cacheSize, maxJournalSize, minJournalTables, mmapSizeGB, hctree), _objects(_maxDBs, nullptr) { } diff --git a/sqlitecluster/SQLitePool.h b/sqlitecluster/SQLitePool.h index 24637893b..8cbc6c92e 100644 --- a/sqlitecluster/SQLitePool.h +++ b/sqlitecluster/SQLitePool.h @@ -7,7 +7,7 @@ class SQLitePool { public: // Create a pool of DB handles. SQLitePool(size_t maxDBs, const string& filename, int cacheSize, int maxJournalSize, int minJournalTables, - const string& synchronous = "", int64_t mmapSizeGB = 0, bool hctree = false); + int64_t mmapSizeGB = 0, bool hctree = false); ~SQLitePool(); // Get the base object (the first one created, which uses the `journal` table). Note that if called by multiple diff --git a/test/lib/BedrockTester.cpp b/test/lib/BedrockTester.cpp index 53f0afc9e..6b6f66a25 100644 --- a/test/lib/BedrockTester.cpp +++ b/test/lib/BedrockTester.cpp @@ -536,7 +536,7 @@ SQLite& BedrockTester::getSQLiteDB() { if (!_db) { // Assumes wal2 mode. - _db = new SQLite(_args["-db"], 1000000, 3000000, -1, "", 0, ENABLE_HCTREE); + _db = new SQLite(_args["-db"], 1000000, 3000000, -1, 0, ENABLE_HCTREE); } return *_db; }