From b55bf86e0741c6d6f19d7e0b243fbd649b4be5a3 Mon Sep 17 00:00:00 2001 From: Moray Grieve Date: Wed, 18 Dec 2024 10:32:45 +0000 Subject: [PATCH 01/22] New props --- .default.properties | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.default.properties b/.default.properties index 9b67f013..77fcb0ff 100644 --- a/.default.properties +++ b/.default.properties @@ -20,6 +20,12 @@ node_path = /usr/lib/node_modules:/usr/local/lib/node_modules npm=/usr/bin/npm npx = /usr/bin/npx +[persistence.db] +host = ten-test.mysql.database.azure.com +user = tentestadmin +database = tentest +password = t3n_T3st + # Constants relating to the different environments the tests can run against namely Ten, Sepolia, Arbitrum and Ganache. # The account PKs used are randomly generated and do not relate to any real values. To use real values from a metamask # wallet, override in your ~/.tentest/user.properties file. From 46816a3af54b3ec7f345265de2b971aacf41469a Mon Sep 17 00:00:00 2001 From: Moray Grieve Date: Thu, 19 Dec 2024 16:49:21 +0000 Subject: [PATCH 02/22] Get azure metadata --- src/python/ten/test/baserunner.py | 8 +++++++- src/python/ten/test/utils/cloud.py | 14 ++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 src/python/ten/test/utils/cloud.py diff --git a/src/python/ten/test/baserunner.py b/src/python/ten/test/baserunner.py index aeb7ac2a..c98d70ea 100644 --- a/src/python/ten/test/baserunner.py +++ b/src/python/ten/test/baserunner.py @@ -8,6 +8,7 @@ from pysys.exceptions import AbortExecution from pysys.constants import LOG_TRACEBACK from pysys.utils.logutils import BaseLogFormatter +from ten.test.utils.cloud import is_azure_vm from ten.test.persistence.rates import RatesPersistence from ten.test.persistence.nonce import NoncePersistence from ten.test.persistence.funds import FundsPersistence @@ -58,7 +59,12 @@ def setup(self, runner): if os.path.exists(runner.output): shutil.rmtree(runner.output) os.makedirs(runner.output) - # create the nonce db if it does not already exist, clean it out if using ganache + # set up the persistence layer based on if we are running in the cloud, or locally + metadata = is_azure_vm() + if metadata is not None: + runner.log.info('Running on an azure VM') + runner.log.info(metadata) + db_dir = os.path.join(str(Path.home()), '.tentest') if not os.path.exists(db_dir): os.makedirs(db_dir) rates_db = RatesPersistence(db_dir) diff --git a/src/python/ten/test/utils/cloud.py b/src/python/ten/test/utils/cloud.py new file mode 100644 index 00000000..73615f81 --- /dev/null +++ b/src/python/ten/test/utils/cloud.py @@ -0,0 +1,14 @@ +import requests + +def is_azure_vm(): + metadata_url = "http://169.254.169.254/metadata/instance?api-version=2021-02-01" + headers = {"Metadata": "true"} + + try: + response = requests.get(metadata_url, headers=headers, timeout=5) + if response.status_code == 200: + return response.json() + else: + return None + except requests.exceptions.RequestException as e: + return None From 4fde5a70d4bc70d389f885a034051a962fd176a6 Mon Sep 17 00:00:00 2001 From: Moray Grieve Date: Mon, 30 Dec 2024 09:03:29 +0000 Subject: [PATCH 03/22] Naming --- .github/workflows/health_check_sepolia.yml | 2 +- .github/workflows/health_check_sepolia_funds.yml | 2 +- .github/workflows/health_check_uat.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/health_check_sepolia.yml b/.github/workflows/health_check_sepolia.yml index e870771b..2024fc4a 100644 --- a/.github/workflows/health_check_sepolia.yml +++ b/.github/workflows/health_check_sepolia.yml @@ -1,6 +1,6 @@ # Run a set of sanity tests on sepolia-testnet to ensure it is up and running # -name: '[health] Check sepolia health' +name: '[liveness] Check sepolia health' run-name: Check sepolia-testnet health on: schedule: diff --git a/.github/workflows/health_check_sepolia_funds.yml b/.github/workflows/health_check_sepolia_funds.yml index 67dbc22d..4cc197d1 100644 --- a/.github/workflows/health_check_sepolia_funds.yml +++ b/.github/workflows/health_check_sepolia_funds.yml @@ -1,6 +1,6 @@ # Check key balances on sepolia-testnet # -name: '[health] Check sepolia funds' +name: '[liveness] Check sepolia funds' run-name: Check sepolia-testnet funds on: schedule: diff --git a/.github/workflows/health_check_uat.yml b/.github/workflows/health_check_uat.yml index f3dae920..acfe3dfc 100644 --- a/.github/workflows/health_check_uat.yml +++ b/.github/workflows/health_check_uat.yml @@ -1,6 +1,6 @@ # Run a set of sanity tests on uat-testnet to ensure it is up and running # -name: '[health] Check uat health' +name: '[liveness] Check uat health' run-name: Check uat-testnet health on: schedule: From 5fc602969200825a394734b0c509e92dc7d0360d Mon Sep 17 00:00:00 2001 From: Moray Grieve Date: Mon, 6 Jan 2025 11:28:40 +0000 Subject: [PATCH 04/22] dumps --- src/python/ten/test/baserunner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/ten/test/baserunner.py b/src/python/ten/test/baserunner.py index c98d70ea..fe7bcf45 100644 --- a/src/python/ten/test/baserunner.py +++ b/src/python/ten/test/baserunner.py @@ -63,7 +63,7 @@ def setup(self, runner): metadata = is_azure_vm() if metadata is not None: runner.log.info('Running on an azure VM') - runner.log.info(metadata) + runner.log.info(json.dumps(metadata, indent=4)) db_dir = os.path.join(str(Path.home()), '.tentest') if not os.path.exists(db_dir): os.makedirs(db_dir) From 2a592c9d2e5dfa36b03961b25cc3ef02d672feb7 Mon Sep 17 00:00:00 2001 From: Moray Grieve Date: Mon, 6 Jan 2025 11:35:15 +0000 Subject: [PATCH 05/22] Log name and location --- src/python/ten/test/baserunner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/ten/test/baserunner.py b/src/python/ten/test/baserunner.py index fe7bcf45..52abe24d 100644 --- a/src/python/ten/test/baserunner.py +++ b/src/python/ten/test/baserunner.py @@ -63,7 +63,7 @@ def setup(self, runner): metadata = is_azure_vm() if metadata is not None: runner.log.info('Running on an azure VM') - runner.log.info(json.dumps(metadata, indent=4)) + runner.log.info('Machine %s, location %s' % (metadata['compute']['location'], metadata['compute']['name'])) db_dir = os.path.join(str(Path.home()), '.tentest') if not os.path.exists(db_dir): os.makedirs(db_dir) From 93f991a07e9a55c9cac67ebd1cf48cebfb92a358 Mon Sep 17 00:00:00 2001 From: Moray Grieve Date: Mon, 6 Jan 2025 11:35:29 +0000 Subject: [PATCH 06/22] 3 secs timeout --- src/python/ten/test/utils/cloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/ten/test/utils/cloud.py b/src/python/ten/test/utils/cloud.py index 73615f81..ae1ecb32 100644 --- a/src/python/ten/test/utils/cloud.py +++ b/src/python/ten/test/utils/cloud.py @@ -5,7 +5,7 @@ def is_azure_vm(): headers = {"Metadata": "true"} try: - response = requests.get(metadata_url, headers=headers, timeout=5) + response = requests.get(metadata_url, headers=headers, timeout=3) if response.status_code == 200: return response.json() else: From 34715bd12b60ab94e4e659a1482afdd4d7a669c0 Mon Sep 17 00:00:00 2001 From: Moray Grieve Date: Mon, 6 Jan 2025 11:37:58 +0000 Subject: [PATCH 07/22] 3 secs timeout --- src/python/ten/test/utils/cloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/ten/test/utils/cloud.py b/src/python/ten/test/utils/cloud.py index ae1ecb32..e6be61ac 100644 --- a/src/python/ten/test/utils/cloud.py +++ b/src/python/ten/test/utils/cloud.py @@ -5,7 +5,7 @@ def is_azure_vm(): headers = {"Metadata": "true"} try: - response = requests.get(metadata_url, headers=headers, timeout=3) + response = requests.get(metadata_url, headers=headers, timeout=10) if response.status_code == 200: return response.json() else: From c933e1e575aaea1a84477136e1fe4f1f91e6f4b9 Mon Sep 17 00:00:00 2001 From: Moray Grieve Date: Mon, 6 Jan 2025 11:39:55 +0000 Subject: [PATCH 08/22] 3 secs timeout --- src/python/ten/test/utils/cloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/ten/test/utils/cloud.py b/src/python/ten/test/utils/cloud.py index e6be61ac..ae1ecb32 100644 --- a/src/python/ten/test/utils/cloud.py +++ b/src/python/ten/test/utils/cloud.py @@ -5,7 +5,7 @@ def is_azure_vm(): headers = {"Metadata": "true"} try: - response = requests.get(metadata_url, headers=headers, timeout=10) + response = requests.get(metadata_url, headers=headers, timeout=3) if response.status_code == 200: return response.json() else: From 40314d8a7b8d781114e4e7e93c821a601362bbee Mon Sep 17 00:00:00 2001 From: Moray Grieve Date: Mon, 6 Jan 2025 11:42:52 +0000 Subject: [PATCH 09/22] Updated --- src/python/ten/test/baserunner.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/python/ten/test/baserunner.py b/src/python/ten/test/baserunner.py index 52abe24d..51b43e76 100644 --- a/src/python/ten/test/baserunner.py +++ b/src/python/ten/test/baserunner.py @@ -62,8 +62,7 @@ def setup(self, runner): # set up the persistence layer based on if we are running in the cloud, or locally metadata = is_azure_vm() if metadata is not None: - runner.log.info('Running on an azure VM') - runner.log.info('Machine %s, location %s' % (metadata['compute']['location'], metadata['compute']['name'])) + runner.log.info('Running on azure (%s, %s)' % (metadata['compute']['name'], metadata['compute']['location'])) db_dir = os.path.join(str(Path.home()), '.tentest') if not os.path.exists(db_dir): os.makedirs(db_dir) From 74812e0ffee9d8862ded68b1e2892083b23db065 Mon Sep 17 00:00:00 2001 From: Moray Grieve Date: Mon, 6 Jan 2025 11:47:37 +0000 Subject: [PATCH 10/22] Set in user properties --- .default.properties | 8 +++---- sql.py | 52 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 4 deletions(-) create mode 100644 sql.py diff --git a/.default.properties b/.default.properties index 77fcb0ff..526aa05d 100644 --- a/.default.properties +++ b/.default.properties @@ -21,10 +21,10 @@ npm=/usr/bin/npm npx = /usr/bin/npx [persistence.db] -host = ten-test.mysql.database.azure.com -user = tentestadmin -database = tentest -password = t3n_T3st +host = +user = +database = +password = # Constants relating to the different environments the tests can run against namely Ten, Sepolia, Arbitrum and Ganache. # The account PKs used are randomly generated and do not relate to any real values. To use real values from a metamask diff --git a/sql.py b/sql.py new file mode 100644 index 00000000..7420e94c --- /dev/null +++ b/sql.py @@ -0,0 +1,52 @@ +import mysql.connector +from mysql.connector import errorcode + +# Obtain connection string information from the portal + +config = { + 'host':'ten-test.mysql.database.azure.com', + 'user':'tentestadmin', + 'password':'t3n_T3st', + 'database':'tentest' +} + +# Construct connection string + +try: + conn = mysql.connector.connect(**config) + print("Connection established") +except mysql.connector.Error as err: + if err.errno == errorcode.ER_ACCESS_DENIED_ERROR: + print("Something is wrong with the user name or password") + elif err.errno == errorcode.ER_BAD_DB_ERROR: + print("Database does not exist") + else: + print(err) +else: + cursor = conn.cursor() + +# Drop previous table of same name if one exists +cursor.execute("DROP TABLE IF EXISTS inventory;") +print("Finished dropping table (if existed).") + +# Create table +cursor.execute("CREATE TABLE inventory (id serial PRIMARY KEY, name VARCHAR(50), quantity INTEGER);") +print("Finished creating table.") + +# Insert some data into table +cursor.execute("INSERT INTO inventory (name, quantity) VALUES (%s, %s);", ("banana", 150)) +print("Inserted",cursor.rowcount,"row(s) of data.") +cursor.execute("INSERT INTO inventory (name, quantity) VALUES (%s, %s);", ("orange", 154)) +print("Inserted",cursor.rowcount,"row(s) of data.") +cursor.execute("INSERT INTO inventory (name, quantity) VALUES (%s, %s);", ("apple", 100)) +print("Inserted",cursor.rowcount,"row(s) of data.") + +# Cleanup +conn.commit() +cursor.close() +conn.close() +print("Done.") + + + + From 93569cb277bae85df2b407de8f38eb56186c22ac Mon Sep 17 00:00:00 2001 From: Moray Grieve Date: Mon, 6 Jan 2025 14:21:27 +0000 Subject: [PATCH 11/22] Refactor --- sql.py | 52 --------------------- src/python/ten/test/baserunner.py | 36 ++++++++------ src/python/ten/test/basetest.py | 25 ++++++---- src/python/ten/test/persistence/__init__.py | 18 +++++++ src/python/ten/test/persistence/contract.py | 10 ++-- src/python/ten/test/persistence/counts.py | 10 ++-- src/python/ten/test/persistence/funds.py | 10 ++-- src/python/ten/test/persistence/nonce.py | 12 ++--- src/python/ten/test/persistence/rates.py | 10 ++-- src/python/ten/test/persistence/results.py | 10 ++-- src/python/ten/test/utils/cloud.py | 2 +- src/python/ten/test/utils/properties.py | 6 +++ 12 files changed, 82 insertions(+), 119 deletions(-) delete mode 100644 sql.py diff --git a/sql.py b/sql.py deleted file mode 100644 index 7420e94c..00000000 --- a/sql.py +++ /dev/null @@ -1,52 +0,0 @@ -import mysql.connector -from mysql.connector import errorcode - -# Obtain connection string information from the portal - -config = { - 'host':'ten-test.mysql.database.azure.com', - 'user':'tentestadmin', - 'password':'t3n_T3st', - 'database':'tentest' -} - -# Construct connection string - -try: - conn = mysql.connector.connect(**config) - print("Connection established") -except mysql.connector.Error as err: - if err.errno == errorcode.ER_ACCESS_DENIED_ERROR: - print("Something is wrong with the user name or password") - elif err.errno == errorcode.ER_BAD_DB_ERROR: - print("Database does not exist") - else: - print(err) -else: - cursor = conn.cursor() - -# Drop previous table of same name if one exists -cursor.execute("DROP TABLE IF EXISTS inventory;") -print("Finished dropping table (if existed).") - -# Create table -cursor.execute("CREATE TABLE inventory (id serial PRIMARY KEY, name VARCHAR(50), quantity INTEGER);") -print("Finished creating table.") - -# Insert some data into table -cursor.execute("INSERT INTO inventory (name, quantity) VALUES (%s, %s);", ("banana", 150)) -print("Inserted",cursor.rowcount,"row(s) of data.") -cursor.execute("INSERT INTO inventory (name, quantity) VALUES (%s, %s);", ("orange", 154)) -print("Inserted",cursor.rowcount,"row(s) of data.") -cursor.execute("INSERT INTO inventory (name, quantity) VALUES (%s, %s);", ("apple", 100)) -print("Inserted",cursor.rowcount,"row(s) of data.") - -# Cleanup -conn.commit() -cursor.close() -conn.close() -print("Done.") - - - - diff --git a/src/python/ten/test/baserunner.py b/src/python/ten/test/baserunner.py index 51b43e76..c55748a9 100644 --- a/src/python/ten/test/baserunner.py +++ b/src/python/ten/test/baserunner.py @@ -1,4 +1,4 @@ -import os, shutil, sys, json, requests, time +import os, shutil, sys, json, requests, time, socket from collections import OrderedDict from web3 import Web3 from pathlib import Path @@ -8,7 +8,6 @@ from pysys.exceptions import AbortExecution from pysys.constants import LOG_TRACEBACK from pysys.utils.logutils import BaseLogFormatter -from ten.test.utils.cloud import is_azure_vm from ten.test.persistence.rates import RatesPersistence from ten.test.persistence.nonce import NoncePersistence from ten.test.persistence.funds import FundsPersistence @@ -16,6 +15,8 @@ from ten.test.persistence.results import ResultsPersistence from ten.test.persistence.contract import ContractPersistence from ten.test.utils.properties import Properties +from ten.test.persistence import get_connection +from ten.test.utils.cloud import is_cloud_vm class TenRunnerPlugin(): @@ -33,6 +34,9 @@ def __init__(self): self.env = None self.NODE_HOST = None self.balances = OrderedDict() + self.cloud_metadata = is_cloud_vm() + self.user_dir = os.path.join(str(Path.home()), '.tentest') + if not os.path.exists(self.user_dir): os.makedirs(self.user_dir) def setup(self, runner): """Set up a runner plugin to start any processes required to execute the tests. """ @@ -60,23 +64,25 @@ def setup(self, runner): os.makedirs(runner.output) # set up the persistence layer based on if we are running in the cloud, or locally - metadata = is_azure_vm() - if metadata is not None: - runner.log.info('Running on azure (%s, %s)' % (metadata['compute']['name'], metadata['compute']['location'])) + if self.cloud_metadata is None: + machine_name = socket.gethostname() + runner.log.info('Running on local (%s)' % machine_name) + else: + machine_name = self.cloud_metadata['compute']['name'] + runner.log.info('Running on azure (%s, %s)' % (machine_name, self.cloud_metadata['compute']['location'])) + connection = get_connection(self.cloud_metadata is None, self.user_dir) - db_dir = os.path.join(str(Path.home()), '.tentest') - if not os.path.exists(db_dir): os.makedirs(db_dir) - rates_db = RatesPersistence(db_dir) + rates_db = RatesPersistence(connection) rates_db.create() - nonce_db = NoncePersistence(db_dir) + nonce_db = NoncePersistence(connection) nonce_db.create() - contracts_db = ContractPersistence(db_dir) + contracts_db = ContractPersistence(connection) contracts_db.create() - funds_db = FundsPersistence(db_dir) + funds_db = FundsPersistence(connection) funds_db.create() - counts_db = CountsPersistence(db_dir) + counts_db = CountsPersistence(connection) counts_db.create() - results_db = ResultsPersistence(db_dir) + results_db = ResultsPersistence(connection) results_db.create() eth_price = self.get_eth_price() @@ -165,9 +171,13 @@ def setup(self, runner): runner.cleanup() sys.exit(1) + rates_db.close() nonce_db.close() contracts_db.close() funds_db.close() + counts_db.close() + results_db.close() + connection.close() def run_ganache(self, runner): """Run ganache for use by the tests. """ diff --git a/src/python/ten/test/basetest.py b/src/python/ten/test/basetest.py index e7e96316..1b67585c 100644 --- a/src/python/ten/test/basetest.py +++ b/src/python/ten/test/basetest.py @@ -1,7 +1,6 @@ import os, copy, sys, json, secrets, re import threading, requests from web3 import Web3 -from pathlib import Path from pysys.basetest import BaseTest from pysys.constants import PROJECT, BACKGROUND, FAILED from pysys.constants import LOG_TRACEBACK @@ -19,6 +18,7 @@ from ten.test.networks.arbitrum import ArbitrumSepolia from ten.test.networks.sepolia import Sepolia from ten.test.networks.ten import Ten, TenL1Geth, TenL1Sepolia +from ten.test.persistence import get_connection class GenericNetworkTest(BaseTest): @@ -33,14 +33,14 @@ def __init__(self, descriptor, outsubdir, runner): self.block_time = Properties().block_time_secs(self.env) self.log.info('Running test in thread %s', threading.currentThread().getName()) - # every test has its own connection to the nonce and contract db - db_dir = os.path.join(str(Path.home()), '.tentest') - self.rates_db = RatesPersistence(db_dir) - self.nonce_db = NoncePersistence(db_dir) - self.contract_db = ContractPersistence(db_dir) - self.funds_db = FundsPersistence(db_dir) - self.counts_db = CountsPersistence(db_dir) - self.results_db = ResultsPersistence(db_dir) + # every test has its own connection to the dbs + self.connection = get_connection(is_local=runner.ten_runner.cloud_metadata is None, db_dir=runner.ten_runner.user_dir) + self.rates_db = RatesPersistence(self.connection) + self.nonce_db = NoncePersistence(self.connection) + self.contract_db = ContractPersistence(self.connection) + self.funds_db = FundsPersistence(self.connection) + self.counts_db = CountsPersistence(self.connection) + self.results_db = ResultsPersistence(self.connection) self.addCleanupFunction(self.close_db) # every test has a unique connection for the funded account @@ -72,9 +72,14 @@ def __test_cost(self): self.log.info(" %s: %s%.3f USD", 'Test cost', sign, self.eth_price*change, extra=BaseLogFormatter.tag(LOG_TRACEBACK, 0)) def close_db(self): - """Close the connection to the nonce database on completion. """ + """Close the connection to the databases on completion. """ + self.rates_db.close() self.nonce_db.close() self.contract_db.close() + self.funds_db.close() + self.counts_db.close() + self.results_db.close() + self.connection.close() def drain_ephemeral_pks(self): """Drain any ephemeral accounts of their funds. """ diff --git a/src/python/ten/test/persistence/__init__.py b/src/python/ten/test/persistence/__init__.py index 77f8f005..8dd0d77e 100644 --- a/src/python/ten/test/persistence/__init__.py +++ b/src/python/ten/test/persistence/__init__.py @@ -1,2 +1,20 @@ """Package of persistence for transaction counts. """ +import os, sqlite3, mysql.connector +from ten.test.utils.properties import Properties + + +def get_connection(is_local, db_dir): + if is_local: + db = os.path.join(db_dir, 'ten-test.db') + connection = sqlite3.connect(db) + else: + props = Properties() + config = { + 'host': props.persistence_host(), + 'user': props.persistence_user(), + 'password': props.persistence_password(), + 'database': props.persistence_database() + } + connection = mysql.connector.connect(**config) + return connection \ No newline at end of file diff --git a/src/python/ten/test/persistence/contract.py b/src/python/ten/test/persistence/contract.py index 53783274..4bb3c1bc 100644 --- a/src/python/ten/test/persistence/contract.py +++ b/src/python/ten/test/persistence/contract.py @@ -1,6 +1,3 @@ -import sqlite3, os - - class ContractPersistence: """Abstracts the persistence of contract addresses into a local database. """ @@ -19,10 +16,9 @@ class ContractPersistence: SQL_SEL_PARAMS = "SELECT value FROM params WHERE address=? AND environment=? AND key=? " \ "ORDER BY address DESC LIMIT 1" - def __init__(self, db_dir): + def __init__(self, connection): """Instantiate an instance.""" - self.db = os.path.join(db_dir, 'contracts.db') - self.connection = sqlite3.connect(self.db) + self.connection = connection self.cursor = self.connection.cursor() def create(self): @@ -32,7 +28,7 @@ def create(self): def close(self): """Close the connection to the underlying persistence.""" - self.connection.close() + self.cursor.close() def delete_environment(self, environment): """Delete all stored contract details for a particular environment.""" diff --git a/src/python/ten/test/persistence/counts.py b/src/python/ten/test/persistence/counts.py index 3e749d74..7296865b 100644 --- a/src/python/ten/test/persistence/counts.py +++ b/src/python/ten/test/persistence/counts.py @@ -1,6 +1,3 @@ -import sqlite3, os - - class CountsPersistence: """Abstracts the persistence of transaction counts across accounts into a local database. """ @@ -12,10 +9,9 @@ class CountsPersistence: SQL_SELECT_THREE = "SELECT time, count FROM counts WHERE name=? and environment=? ORDER BY time DESC LIMIT 3" SQL_SELECT_HOUR = "SELECT time, count FROM counts WHERE name=? and environment=? and time >= ? ORDER BY time DESC" - def __init__(self, db_dir): + def __init__(self, connection): """Instantiate an instance.""" - self.db = os.path.join(db_dir, 'counts.db') - self.connection = sqlite3.connect(self.db) + self.connection = connection self.cursor = self.connection.cursor() def create(self): @@ -24,7 +20,7 @@ def create(self): def close(self): """Close the connection to the underlying persistence.""" - self.connection.close() + self.cursor.close() def delete_environment(self, environment): """Delete all stored contract details for a particular environment.""" diff --git a/src/python/ten/test/persistence/funds.py b/src/python/ten/test/persistence/funds.py index 80f28133..8a987d7c 100644 --- a/src/python/ten/test/persistence/funds.py +++ b/src/python/ten/test/persistence/funds.py @@ -1,6 +1,3 @@ -import sqlite3, os - - class FundsPersistence: """Abstracts the persistence of funds across accounts into a local database. """ @@ -11,10 +8,9 @@ class FundsPersistence: SQL_DELETE = "DELETE from funds WHERE environment=?" SQL_SELECT = "SELECT time, balance FROM funds WHERE name=? and environment=? ORDER BY time DESC" - def __init__(self, db_dir): + def __init__(self, connection): """Instantiate an instance.""" - self.db = os.path.join(db_dir, 'funds.db') - self.connection = sqlite3.connect(self.db) + self.connection = connection self.cursor = self.connection.cursor() def create(self): @@ -23,7 +19,7 @@ def create(self): def close(self): """Close the connection to the underlying persistence.""" - self.connection.close() + self.cursor.close() def delete_environment(self, environment): """Delete all stored details for a particular environment.""" diff --git a/src/python/ten/test/persistence/nonce.py b/src/python/ten/test/persistence/nonce.py index c65fdc5a..664ee4ad 100644 --- a/src/python/ten/test/persistence/nonce.py +++ b/src/python/ten/test/persistence/nonce.py @@ -1,6 +1,3 @@ -import sqlite3, os - - class NoncePersistence: """Abstracts the persistence of nonces into a local database. """ @@ -14,10 +11,9 @@ class NoncePersistence: SQL_ACCNTS = "SELECT DISTINCT account from nonce_db where environment=?" SQL_DELENT = "DELETE from nonce_db WHERE account=? AND environment=? AND nonce=?" - def __init__(self, db_dir): - """Instantiate an instance. """ - self.db = os.path.join(db_dir, 'nonce.db') - self.connection = sqlite3.connect(self.db) + def __init__(self, connection): + """Instantiate an instance.""" + self.connection = connection self.cursor = self.connection.cursor() def create(self): @@ -26,7 +22,7 @@ def create(self): def close(self): """Close the connection to the underlying persistence. """ - self.connection.close() + self.cursor.close() def get_next_nonce(self, test, web3, account, environment, persist_nonce=True, log=True): """Get the next nonce to use in a transaction. diff --git a/src/python/ten/test/persistence/rates.py b/src/python/ten/test/persistence/rates.py index 4cc7f826..2829b38e 100644 --- a/src/python/ten/test/persistence/rates.py +++ b/src/python/ten/test/persistence/rates.py @@ -1,6 +1,3 @@ -import sqlite3, os - - class RatesPersistence: """Abstracts the persistence of rates across cryptos into a local database. """ @@ -11,10 +8,9 @@ class RatesPersistence: SQL_DELETE = "DELETE from rates WHERE crypto=?" SQL_SELECT = "SELECT time, rate FROM rates WHERE crypto=? and currency=? ORDER BY time DESC LIMIT 1" - def __init__(self, db_dir): + def __init__(self, connection): """Instantiate an instance.""" - self.db = os.path.join(db_dir, 'rates.db') - self.connection = sqlite3.connect(self.db) + self.connection = connection self.cursor = self.connection.cursor() def create(self): @@ -23,7 +19,7 @@ def create(self): def close(self): """Close the connection to the underlying persistence.""" - self.connection.close() + self.cursor.close() def delete_crypto(self, crypto): """Delete all stored rates for a particular crypto.""" diff --git a/src/python/ten/test/persistence/results.py b/src/python/ten/test/persistence/results.py index 5c0f680e..6dfa1443 100644 --- a/src/python/ten/test/persistence/results.py +++ b/src/python/ten/test/persistence/results.py @@ -1,6 +1,3 @@ -import sqlite3, os - - class ResultsPersistence: """Abstracts the persistence of performance results into a local database. """ @@ -11,10 +8,9 @@ class ResultsPersistence: SQL_DELETE = "DELETE from results WHERE environment=?" SQL_SELECT = "SELECT time, result FROM results WHERE test=? AND environment=? ORDER BY time ASC" - def __init__(self, db_dir): + def __init__(self, connection): """Instantiate an instance.""" - self.db = os.path.join(db_dir, 'results.db') - self.connection = sqlite3.connect(self.db) + self.connection = connection self.cursor = self.connection.cursor() def create(self): @@ -23,7 +19,7 @@ def create(self): def close(self): """Close the connection to the underlying persistence.""" - self.connection.close() + self.cursor.close() def delete_environment(self, environment): """Delete all stored performance results for a particular environment.""" diff --git a/src/python/ten/test/utils/cloud.py b/src/python/ten/test/utils/cloud.py index ae1ecb32..b8c9e98e 100644 --- a/src/python/ten/test/utils/cloud.py +++ b/src/python/ten/test/utils/cloud.py @@ -1,6 +1,6 @@ import requests -def is_azure_vm(): +def is_cloud_vm(): metadata_url = "http://169.254.169.254/metadata/instance?api-version=2021-02-01" headers = {"Metadata": "true"} diff --git a/src/python/ten/test/utils/properties.py b/src/python/ten/test/utils/properties.py index 6b20673e..cc50efcd 100644 --- a/src/python/ten/test/utils/properties.py +++ b/src/python/ten/test/utils/properties.py @@ -77,6 +77,12 @@ def npx_binary(self): raise FileNotFoundException('npx binary not found at default location %s' % path) return path + # persistence + def persistence_host(self): return self.get('persistence.db', 'host') + def persistence_user(self): return self.get('persistence.db', 'user') + def persistence_database(self): return self.get('persistence.db', 'database') + def persistence_password(self): return self.get('persistence.db', 'password') + # common to all environments def block_time_secs(self, key): return self.get('env.'+key, 'BlockTimeSecs') From 8c226511fd5266b21cf2c375b8233cde37279173 Mon Sep 17 00:00:00 2001 From: Moray Grieve Date: Mon, 6 Jan 2025 14:33:55 +0000 Subject: [PATCH 12/22] Timeout --- README.md | 1 + src/python/ten/test/persistence/__init__.py | 3 ++- utils/docker/image.Dockerfile | 1 + utils/github/install.sh | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 81d43f8e..8b27b0c4 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,7 @@ pip3 install pysys==1.6.1 pip3 install solc-select==1.0.4 pip3 install py-solc-x==2.0.2 pip3 install numpy==1.26.4 +pip3 install mysql-connector-python==9.0.0 solc-select install 0.8.15 solc-select use 0.8.15 diff --git a/src/python/ten/test/persistence/__init__.py b/src/python/ten/test/persistence/__init__.py index 8dd0d77e..f6c62a1a 100644 --- a/src/python/ten/test/persistence/__init__.py +++ b/src/python/ten/test/persistence/__init__.py @@ -14,7 +14,8 @@ def get_connection(is_local, db_dir): 'host': props.persistence_host(), 'user': props.persistence_user(), 'password': props.persistence_password(), - 'database': props.persistence_database() + 'database': props.persistence_database(), + 'connection_timeout': 10 } connection = mysql.connector.connect(**config) return connection \ No newline at end of file diff --git a/utils/docker/image.Dockerfile b/utils/docker/image.Dockerfile index 80d7c7fa..8df09c81 100644 --- a/utils/docker/image.Dockerfile +++ b/utils/docker/image.Dockerfile @@ -27,6 +27,7 @@ RUN python3 -m pip install web3==6.13.0 RUN python3 -m pip install pysys==1.6.1 RUN python3 -m pip install py-solc-x RUN python3 -m pip install numpy==1.24.4 +RUN python3 -m pip install mysql-connector-python==9.0.0 RUN mkdir /home/ten-test RUN mkdir /home/go-ten diff --git a/utils/github/install.sh b/utils/github/install.sh index ca19b489..01e88734 100755 --- a/utils/github/install.sh +++ b/utils/github/install.sh @@ -24,6 +24,7 @@ python3 -m pip install pysys==1.6.1 python3 -m pip install solc-select python3 -m pip install py-solc-x python3 -m pip install numpy==1.24.4 +python3 -m pip install mysql-connector-python==9.0.0 snap install go --classic curl -fsSL https://get.docker.com -o get-docker.sh sh ./get-docker.sh From c39d0256d06d196200cf83ab25058405ec7e5619 Mon Sep 17 00:00:00 2001 From: Moray Grieve Date: Mon, 6 Jan 2025 15:00:43 +0000 Subject: [PATCH 13/22] Rates --- src/python/ten/test/persistence/rates.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/python/ten/test/persistence/rates.py b/src/python/ten/test/persistence/rates.py index 2829b38e..adeff8ec 100644 --- a/src/python/ten/test/persistence/rates.py +++ b/src/python/ten/test/persistence/rates.py @@ -2,7 +2,10 @@ class RatesPersistence: """Abstracts the persistence of rates across cryptos into a local database. """ SQL_CREATE = "CREATE TABLE IF NOT EXISTS rates " \ - "(crypto TEXT, currency INTEGER, time INTEGER, rate TEXT, " \ + "(crypto VARCHAR(3), " \ + "currency INTEGER, " \ + "time INTEGER, " \ + "rate VARCHAR(255), " \ "PRIMARY KEY (crypto, currency, time))" SQL_INSERT = "INSERT INTO rates VALUES (?, ?, ?, ?)" SQL_DELETE = "DELETE from rates WHERE crypto=?" From 05e900ac337e320435184965863b5ec9cfaeed5a Mon Sep 17 00:00:00 2001 From: Moray Grieve Date: Mon, 6 Jan 2025 16:41:12 +0000 Subject: [PATCH 14/22] Interim update --- src/python/ten/test/baserunner.py | 17 +++++++++-------- src/python/ten/test/basetest.py | 17 +++++++++-------- src/python/ten/test/persistence/__init__.py | 21 ++++++++++++--------- src/python/ten/test/persistence/contract.py | 12 ++++++------ src/python/ten/test/persistence/counts.py | 10 +++++----- src/python/ten/test/persistence/funds.py | 10 +++++----- src/python/ten/test/persistence/nonce.py | 18 +++++++++--------- src/python/ten/test/persistence/rates.py | 18 +++++++++--------- src/python/ten/test/persistence/results.py | 10 +++++----- 9 files changed, 69 insertions(+), 64 deletions(-) diff --git a/src/python/ten/test/baserunner.py b/src/python/ten/test/baserunner.py index c55748a9..508bfc69 100644 --- a/src/python/ten/test/baserunner.py +++ b/src/python/ten/test/baserunner.py @@ -70,19 +70,19 @@ def setup(self, runner): else: machine_name = self.cloud_metadata['compute']['name'] runner.log.info('Running on azure (%s, %s)' % (machine_name, self.cloud_metadata['compute']['location'])) - connection = get_connection(self.cloud_metadata is None, self.user_dir) + dbconnection1, dbconnection2 = get_connection(self.cloud_metadata is None, self.user_dir) - rates_db = RatesPersistence(connection) + rates_db = RatesPersistence(dbconnection1) rates_db.create() - nonce_db = NoncePersistence(connection) + nonce_db = NoncePersistence(dbconnection1) nonce_db.create() - contracts_db = ContractPersistence(connection) + contracts_db = ContractPersistence(dbconnection1) contracts_db.create() - funds_db = FundsPersistence(connection) + funds_db = FundsPersistence(dbconnection1) funds_db.create() - counts_db = CountsPersistence(connection) + counts_db = CountsPersistence(dbconnection1) counts_db.create() - results_db = ResultsPersistence(connection) + results_db = ResultsPersistence(dbconnection1) results_db.create() eth_price = self.get_eth_price() @@ -177,7 +177,8 @@ def setup(self, runner): funds_db.close() counts_db.close() results_db.close() - connection.close() + dbconnection1.connection.close() + dbconnection2.connection.close() def run_ganache(self, runner): """Run ganache for use by the tests. """ diff --git a/src/python/ten/test/basetest.py b/src/python/ten/test/basetest.py index 1b67585c..ae7bd2e5 100644 --- a/src/python/ten/test/basetest.py +++ b/src/python/ten/test/basetest.py @@ -34,13 +34,13 @@ def __init__(self, descriptor, outsubdir, runner): self.log.info('Running test in thread %s', threading.currentThread().getName()) # every test has its own connection to the dbs - self.connection = get_connection(is_local=runner.ten_runner.cloud_metadata is None, db_dir=runner.ten_runner.user_dir) - self.rates_db = RatesPersistence(self.connection) - self.nonce_db = NoncePersistence(self.connection) - self.contract_db = ContractPersistence(self.connection) - self.funds_db = FundsPersistence(self.connection) - self.counts_db = CountsPersistence(self.connection) - self.results_db = ResultsPersistence(self.connection) + self.dbconnection1, self.dbconnection2 = get_connection(is_local=runner.ten_runner.cloud_metadata is None, db_dir=runner.ten_runner.user_dir) + self.rates_db = RatesPersistence(self.dbconnection1) + self.nonce_db = NoncePersistence(self.dbconnection1) + self.contract_db = ContractPersistence(self.dbconnection1) + self.funds_db = FundsPersistence(self.dbconnection1) + self.counts_db = CountsPersistence(self.dbconnection1) + self.results_db = ResultsPersistence(self.dbconnection1) self.addCleanupFunction(self.close_db) # every test has a unique connection for the funded account @@ -79,7 +79,8 @@ def close_db(self): self.funds_db.close() self.counts_db.close() self.results_db.close() - self.connection.close() + self.dbconnection1.connection.close() + self.dbconnection2.connection.close() def drain_ephemeral_pks(self): """Drain any ephemeral accounts of their funds. """ diff --git a/src/python/ten/test/persistence/__init__.py b/src/python/ten/test/persistence/__init__.py index f6c62a1a..64d1086d 100644 --- a/src/python/ten/test/persistence/__init__.py +++ b/src/python/ten/test/persistence/__init__.py @@ -1,21 +1,24 @@ """Package of persistence for transaction counts. """ import os, sqlite3, mysql.connector +from collections import namedtuple from ten.test.utils.properties import Properties +DBConnection = namedtuple('DBConnection', 'connection type') + def get_connection(is_local, db_dir): - if is_local: - db = os.path.join(db_dir, 'ten-test.db') - connection = sqlite3.connect(db) - else: - props = Properties() - config = { + #if is_local: + db = os.path.join(db_dir, 'ten-test.db') + connection1 = sqlite3.connect(db) + #else: + props = Properties() + config = { 'host': props.persistence_host(), 'user': props.persistence_user(), 'password': props.persistence_password(), 'database': props.persistence_database(), 'connection_timeout': 10 - } - connection = mysql.connector.connect(**config) - return connection \ No newline at end of file + } + connection2 = mysql.connector.connect(**config) + return DBConnection(connection1, 'sqlite3'), DBConnection(connection2, 'mysql') \ No newline at end of file diff --git a/src/python/ten/test/persistence/contract.py b/src/python/ten/test/persistence/contract.py index 4bb3c1bc..8d412ce7 100644 --- a/src/python/ten/test/persistence/contract.py +++ b/src/python/ten/test/persistence/contract.py @@ -16,10 +16,10 @@ class ContractPersistence: SQL_SEL_PARAMS = "SELECT value FROM params WHERE address=? AND environment=? AND key=? " \ "ORDER BY address DESC LIMIT 1" - def __init__(self, connection): + def __init__(self, dbconnection): """Instantiate an instance.""" - self.connection = connection - self.cursor = self.connection.cursor() + self.dbconnection = dbconnection + self.cursor = self.dbconnection.connection.cursor() def create(self): """Create the cursor to the underlying persistence.""" @@ -34,12 +34,12 @@ def delete_environment(self, environment): """Delete all stored contract details for a particular environment.""" self.cursor.execute(self.SQL_DELETE, (environment, )) self.cursor.execute(self.SQL_DEL_PARAMS, (environment, )) - self.connection.commit() + self.dbconnection.connection.commit() def insert_contract(self, name, environment, address, abi): """Insert a new contract into the persistence. """ self.cursor.execute(self.SQL_INSERT, (name, environment, address, abi)) - self.connection.commit() + self.dbconnection.connection.commit() def get_contract(self, name, environment): """Return the address and abi for a particular deployed contract. """ @@ -51,7 +51,7 @@ def get_contract(self, name, environment): def insert_param(self, address, environment, key, value): """Insert a parameter for a named contract. """ self.cursor.execute(self.SQL_INS_PARAMS, (address, environment, key, value)) - self.connection.commit() + self.dbconnection.connection.commit() def get_param(self, address, environment, key): """Return the address and abi for a particular deployed contract. """ diff --git a/src/python/ten/test/persistence/counts.py b/src/python/ten/test/persistence/counts.py index 7296865b..eb39d0f2 100644 --- a/src/python/ten/test/persistence/counts.py +++ b/src/python/ten/test/persistence/counts.py @@ -9,10 +9,10 @@ class CountsPersistence: SQL_SELECT_THREE = "SELECT time, count FROM counts WHERE name=? and environment=? ORDER BY time DESC LIMIT 3" SQL_SELECT_HOUR = "SELECT time, count FROM counts WHERE name=? and environment=? and time >= ? ORDER BY time DESC" - def __init__(self, connection): + def __init__(self, dbconnection): """Instantiate an instance.""" - self.connection = connection - self.cursor = self.connection.cursor() + self.dbconnection = dbconnection + self.cursor = self.dbconnection.connection.cursor() def create(self): """Create the cursor to the underlying persistence.""" @@ -25,12 +25,12 @@ def close(self): def delete_environment(self, environment): """Delete all stored contract details for a particular environment.""" self.cursor.execute(self.SQL_DELETE, (environment, )) - self.connection.commit() + self.dbconnection.connection.commit() def insert_count(self, name, address, environment, time, count): """Insert a new counts entry for a particular logical account.""" self.cursor.execute(self.SQL_INSERT, (name, address, environment, time, str(count))) - self.connection.commit() + self.dbconnection.connection.commit() def get_last_three_counts(self, name, environment): """Return the transaction count with time for a particular logical account.""" diff --git a/src/python/ten/test/persistence/funds.py b/src/python/ten/test/persistence/funds.py index 8a987d7c..f65cb542 100644 --- a/src/python/ten/test/persistence/funds.py +++ b/src/python/ten/test/persistence/funds.py @@ -8,10 +8,10 @@ class FundsPersistence: SQL_DELETE = "DELETE from funds WHERE environment=?" SQL_SELECT = "SELECT time, balance FROM funds WHERE name=? and environment=? ORDER BY time DESC" - def __init__(self, connection): + def __init__(self, dbconnection): """Instantiate an instance.""" - self.connection = connection - self.cursor = self.connection.cursor() + self.dbconnection = dbconnection + self.cursor = self.dbconnection.connection.cursor() def create(self): """Create the cursor to the underlying persistence.""" @@ -24,12 +24,12 @@ def close(self): def delete_environment(self, environment): """Delete all stored details for a particular environment.""" self.cursor.execute(self.SQL_DELETE, (environment, )) - self.connection.commit() + self.dbconnection.connection.commit() def insert_funds(self, name, address, environment, time, balance): """Insert a new funds entry for a particular logical account.""" self.cursor.execute(self.SQL_INSERT, (name, address, environment, time, str(balance))) - self.connection.commit() + self.dbconnection.connection.commit() def get_funds(self, name, environment): """Return the funds with time for a particular logical account.""" diff --git a/src/python/ten/test/persistence/nonce.py b/src/python/ten/test/persistence/nonce.py index 664ee4ad..cb0e06aa 100644 --- a/src/python/ten/test/persistence/nonce.py +++ b/src/python/ten/test/persistence/nonce.py @@ -11,10 +11,10 @@ class NoncePersistence: SQL_ACCNTS = "SELECT DISTINCT account from nonce_db where environment=?" SQL_DELENT = "DELETE from nonce_db WHERE account=? AND environment=? AND nonce=?" - def __init__(self, connection): + def __init__(self, dbconnection): """Instantiate an instance.""" - self.connection = connection - self.cursor = self.connection.cursor() + self.dbconnection = dbconnection + self.cursor = self.dbconnection.connection.cursor() def create(self): """Create the cursor to the underlying persistence. """ @@ -47,32 +47,32 @@ def get_next_nonce(self, test, web3, account, environment, persist_nonce=True, l def insert(self, account, environment, nonce, status='PENDING'): """Insert a new nonce into the persistence. """ self.cursor.execute(self.SQL_INSERT, (account, environment, nonce, status)) - self.connection.commit() + self.dbconnection.connection.commit() def update(self, account, environment, nonce, status): """Update the status of a transaction for a given nonce into the persistence. """ self.cursor.execute(self.SQL_UPDATE, (status, account, environment, nonce)) - self.connection.commit() + self.dbconnection.connection.commit() def delete(self, account, environment): """Delete all nonce entries in the persistence for a given account and environment. """ self.cursor.execute(self.SQL_DELETE, (account, environment)) - self.connection.commit() + self.dbconnection.connection.commit() def delete_from(self, account, environment, nonce): """Delete all nonce entries in the persistence for a given account and environment. """ self.cursor.execute(self.SQL_DELFRO, (account, environment, nonce)) - self.connection.commit() + self.dbconnection.connection.commit() def delete_environment(self, environment): """Delete all nonce entries for all accounts for a given environment. """ self.cursor.execute(self.SQL_DELENV, (environment, )) - self.connection.commit() + self.dbconnection.connection.commit() def delete_entries(self, account, environment, nonce): """Delete all nonce entries in the persistence for a given account and environment and nonce. """ self.cursor.execute(self.SQL_DELENT, (account, environment, nonce)) - self.connection.commit() + self.dbconnection.connection.commit() def get_accounts(self, environment): """Return a list of all accounts with persisted values for a given environment. """ diff --git a/src/python/ten/test/persistence/rates.py b/src/python/ten/test/persistence/rates.py index adeff8ec..8096635e 100644 --- a/src/python/ten/test/persistence/rates.py +++ b/src/python/ten/test/persistence/rates.py @@ -3,18 +3,18 @@ class RatesPersistence: SQL_CREATE = "CREATE TABLE IF NOT EXISTS rates " \ "(crypto VARCHAR(3), " \ - "currency INTEGER, " \ + "currency VARCHAR(3), " \ "time INTEGER, " \ "rate VARCHAR(255), " \ "PRIMARY KEY (crypto, currency, time))" - SQL_INSERT = "INSERT INTO rates VALUES (?, ?, ?, ?)" - SQL_DELETE = "DELETE from rates WHERE crypto=?" - SQL_SELECT = "SELECT time, rate FROM rates WHERE crypto=? and currency=? ORDER BY time DESC LIMIT 1" + SQL_INSERT = "INSERT INTO rates VALUES (?, ?, ?, ?);" + SQL_DELETE = "DELETE from rates WHERE crypto=?;" + SQL_SELECT = "SELECT time, rate FROM rates WHERE crypto=? and currency=? ORDER BY time DESC LIMIT 1;" - def __init__(self, connection): + def __init__(self, dbconnection): """Instantiate an instance.""" - self.connection = connection - self.cursor = self.connection.cursor() + self.dbconnection = dbconnection + self.cursor = self.dbconnection.connection.cursor() def create(self): """Create the cursor to the underlying persistence.""" @@ -27,12 +27,12 @@ def close(self): def delete_crypto(self, crypto): """Delete all stored rates for a particular crypto.""" self.cursor.execute(self.SQL_DELETE, (crypto, )) - self.connection.commit() + self.dbconnection.connection.commit() def insert_rates(self, crypto, currency, time, rate): """Insert a new rate for a particular crypto and currency.""" self.cursor.execute(self.SQL_INSERT, (crypto, currency, time, str(rate))) - self.connection.commit() + self.dbconnection.connection.commit() def get_latest_rate(self, crypto, currency): """Return the latest rate for the crypto and currency.""" diff --git a/src/python/ten/test/persistence/results.py b/src/python/ten/test/persistence/results.py index 6dfa1443..9e2f2a34 100644 --- a/src/python/ten/test/persistence/results.py +++ b/src/python/ten/test/persistence/results.py @@ -8,10 +8,10 @@ class ResultsPersistence: SQL_DELETE = "DELETE from results WHERE environment=?" SQL_SELECT = "SELECT time, result FROM results WHERE test=? AND environment=? ORDER BY time ASC" - def __init__(self, connection): + def __init__(self, dbconnection): """Instantiate an instance.""" - self.connection = connection - self.cursor = self.connection.cursor() + self.dbconnection = dbconnection + self.cursor = self.dbconnection.connection.cursor() def create(self): """Create the cursor to the underlying persistence.""" @@ -24,12 +24,12 @@ def close(self): def delete_environment(self, environment): """Delete all stored performance results for a particular environment.""" self.cursor.execute(self.SQL_DELETE, (environment, )) - self.connection.commit() + self.dbconnection.connection.commit() def insert_result(self, test, environment, time, result): """Insert a new performance result into the persistence. """ self.cursor.execute(self.SQL_INSERT, (test, environment, time, result)) - self.connection.commit() + self.dbconnection.connection.commit() def get_results(self, test, environment): """Return the performance results for a particular test and environment. """ From 257111aced62b5fc5bbe7a75d49ac4ae4d9e8577 Mon Sep 17 00:00:00 2001 From: Moray Grieve Date: Mon, 6 Jan 2025 16:50:45 +0000 Subject: [PATCH 15/22] Interim update --- src/python/ten/test/baserunner.py | 2 +- src/python/ten/test/basetest.py | 2 +- src/python/ten/test/persistence/__init__.py | 4 ++++ src/python/ten/test/persistence/rates.py | 14 +++++++++++--- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/python/ten/test/baserunner.py b/src/python/ten/test/baserunner.py index 508bfc69..fbf2b74c 100644 --- a/src/python/ten/test/baserunner.py +++ b/src/python/ten/test/baserunner.py @@ -72,7 +72,7 @@ def setup(self, runner): runner.log.info('Running on azure (%s, %s)' % (machine_name, self.cloud_metadata['compute']['location'])) dbconnection1, dbconnection2 = get_connection(self.cloud_metadata is None, self.user_dir) - rates_db = RatesPersistence(dbconnection1) + rates_db = RatesPersistence(dbconnection2) rates_db.create() nonce_db = NoncePersistence(dbconnection1) nonce_db.create() diff --git a/src/python/ten/test/basetest.py b/src/python/ten/test/basetest.py index ae7bd2e5..4dfde434 100644 --- a/src/python/ten/test/basetest.py +++ b/src/python/ten/test/basetest.py @@ -35,7 +35,7 @@ def __init__(self, descriptor, outsubdir, runner): # every test has its own connection to the dbs self.dbconnection1, self.dbconnection2 = get_connection(is_local=runner.ten_runner.cloud_metadata is None, db_dir=runner.ten_runner.user_dir) - self.rates_db = RatesPersistence(self.dbconnection1) + self.rates_db = RatesPersistence(self.dbconnection2) self.nonce_db = NoncePersistence(self.dbconnection1) self.contract_db = ContractPersistence(self.dbconnection1) self.funds_db = FundsPersistence(self.dbconnection1) diff --git a/src/python/ten/test/persistence/__init__.py b/src/python/ten/test/persistence/__init__.py index 64d1086d..f632fe71 100644 --- a/src/python/ten/test/persistence/__init__.py +++ b/src/python/ten/test/persistence/__init__.py @@ -7,6 +7,10 @@ DBConnection = namedtuple('DBConnection', 'connection type') +def normalise(statement, _type): + return statement if _type != 'mysql' else statement.replace('?', '%s') + + def get_connection(is_local, db_dir): #if is_local: db = os.path.join(db_dir, 'ten-test.db') diff --git a/src/python/ten/test/persistence/rates.py b/src/python/ten/test/persistence/rates.py index 8096635e..ecfb8250 100644 --- a/src/python/ten/test/persistence/rates.py +++ b/src/python/ten/test/persistence/rates.py @@ -1,3 +1,6 @@ +from ten.test.persistence import normalise + + class RatesPersistence: """Abstracts the persistence of rates across cryptos into a local database. """ @@ -14,6 +17,9 @@ class RatesPersistence: def __init__(self, dbconnection): """Instantiate an instance.""" self.dbconnection = dbconnection + self.insert = normalise(self.SQL_INSERT, dbconnection.type) + self.delete = normalise(self.SQL_DELETE, dbconnection.type) + self.select = normalise(self.SQL_SELECT, dbconnection.type) self.cursor = self.dbconnection.connection.cursor() def create(self): @@ -26,17 +32,17 @@ def close(self): def delete_crypto(self, crypto): """Delete all stored rates for a particular crypto.""" - self.cursor.execute(self.SQL_DELETE, (crypto, )) + self.cursor.execute(self.delete, (crypto, )) self.dbconnection.connection.commit() def insert_rates(self, crypto, currency, time, rate): """Insert a new rate for a particular crypto and currency.""" - self.cursor.execute(self.SQL_INSERT, (crypto, currency, time, str(rate))) + self.cursor.execute(self.insert, (crypto, currency, time, str(rate))) self.dbconnection.connection.commit() def get_latest_rate(self, crypto, currency): """Return the latest rate for the crypto and currency.""" - self.cursor.execute(self.SQL_SELECT, (crypto, currency)) + self.cursor.execute(self.select, (crypto, currency)) try: result = self.cursor.fetchone()[1] return float(result) @@ -44,3 +50,5 @@ def get_latest_rate(self, crypto, currency): return None + + From d7ad8d2a4ca9b6066b9046e4b3f163047e5387ae Mon Sep 17 00:00:00 2001 From: Moray Grieve Date: Mon, 6 Jan 2025 20:02:01 +0000 Subject: [PATCH 16/22] Initial pass through --- src/python/ten/test/baserunner.py | 23 ++++++++------- src/python/ten/test/basetest.py | 14 ++++----- src/python/ten/test/persistence/__init__.py | 24 ++++++++-------- src/python/ten/test/persistence/contract.py | 26 ++++++++++++----- src/python/ten/test/persistence/counts.py | 20 +++++++++---- src/python/ten/test/persistence/funds.py | 17 ++++++++--- src/python/ten/test/persistence/nonce.py | 32 +++++++++++++++------ src/python/ten/test/persistence/rates.py | 4 +-- src/python/ten/test/persistence/results.py | 17 ++++++++--- 9 files changed, 116 insertions(+), 61 deletions(-) diff --git a/src/python/ten/test/baserunner.py b/src/python/ten/test/baserunner.py index fbf2b74c..c27c4fec 100644 --- a/src/python/ten/test/baserunner.py +++ b/src/python/ten/test/baserunner.py @@ -35,6 +35,7 @@ def __init__(self): self.NODE_HOST = None self.balances = OrderedDict() self.cloud_metadata = is_cloud_vm() + self.is_cloud_vm = self.cloud_metadata is not None self.user_dir = os.path.join(str(Path.home()), '.tentest') if not os.path.exists(self.user_dir): os.makedirs(self.user_dir) @@ -64,25 +65,25 @@ def setup(self, runner): os.makedirs(runner.output) # set up the persistence layer based on if we are running in the cloud, or locally - if self.cloud_metadata is None: - machine_name = socket.gethostname() - runner.log.info('Running on local (%s)' % machine_name) - else: + if self.is_cloud_vm: machine_name = self.cloud_metadata['compute']['name'] runner.log.info('Running on azure (%s, %s)' % (machine_name, self.cloud_metadata['compute']['location'])) - dbconnection1, dbconnection2 = get_connection(self.cloud_metadata is None, self.user_dir) + else: + machine_name = socket.gethostname() + runner.log.info('Running on local (%s)' % machine_name) + dbconnection2, dbconnection2 = get_connection(self.is_cloud_vm, self.user_dir) rates_db = RatesPersistence(dbconnection2) rates_db.create() - nonce_db = NoncePersistence(dbconnection1) + nonce_db = NoncePersistence(dbconnection2) nonce_db.create() - contracts_db = ContractPersistence(dbconnection1) + contracts_db = ContractPersistence(dbconnection2) contracts_db.create() - funds_db = FundsPersistence(dbconnection1) + funds_db = FundsPersistence(dbconnection2) funds_db.create() - counts_db = CountsPersistence(dbconnection1) + counts_db = CountsPersistence(dbconnection2) counts_db.create() - results_db = ResultsPersistence(dbconnection1) + results_db = ResultsPersistence(dbconnection2) results_db.create() eth_price = self.get_eth_price() @@ -177,7 +178,7 @@ def setup(self, runner): funds_db.close() counts_db.close() results_db.close() - dbconnection1.connection.close() + dbconnection2.connection.close() dbconnection2.connection.close() def run_ganache(self, runner): diff --git a/src/python/ten/test/basetest.py b/src/python/ten/test/basetest.py index 4dfde434..3577d382 100644 --- a/src/python/ten/test/basetest.py +++ b/src/python/ten/test/basetest.py @@ -34,13 +34,13 @@ def __init__(self, descriptor, outsubdir, runner): self.log.info('Running test in thread %s', threading.currentThread().getName()) # every test has its own connection to the dbs - self.dbconnection1, self.dbconnection2 = get_connection(is_local=runner.ten_runner.cloud_metadata is None, db_dir=runner.ten_runner.user_dir) + self.dbconnection2, self.dbconnection2 = get_connection(is_local=runner.ten_runner.is_cloud_vm, db_dir=runner.ten_runner.user_dir) self.rates_db = RatesPersistence(self.dbconnection2) - self.nonce_db = NoncePersistence(self.dbconnection1) - self.contract_db = ContractPersistence(self.dbconnection1) - self.funds_db = FundsPersistence(self.dbconnection1) - self.counts_db = CountsPersistence(self.dbconnection1) - self.results_db = ResultsPersistence(self.dbconnection1) + self.nonce_db = NoncePersistence(self.dbconnection2) + self.contract_db = ContractPersistence(self.dbconnection2) + self.funds_db = FundsPersistence(self.dbconnection2) + self.counts_db = CountsPersistence(self.dbconnection2) + self.results_db = ResultsPersistence(self.dbconnection2) self.addCleanupFunction(self.close_db) # every test has a unique connection for the funded account @@ -79,7 +79,7 @@ def close_db(self): self.funds_db.close() self.counts_db.close() self.results_db.close() - self.dbconnection1.connection.close() + self.dbconnection2.connection.close() self.dbconnection2.connection.close() def drain_ephemeral_pks(self): diff --git a/src/python/ten/test/persistence/__init__.py b/src/python/ten/test/persistence/__init__.py index f632fe71..932605aa 100644 --- a/src/python/ten/test/persistence/__init__.py +++ b/src/python/ten/test/persistence/__init__.py @@ -11,18 +11,18 @@ def normalise(statement, _type): return statement if _type != 'mysql' else statement.replace('?', '%s') -def get_connection(is_local, db_dir): - #if is_local: - db = os.path.join(db_dir, 'ten-test.db') - connection1 = sqlite3.connect(db) - #else: +def get_connection(is_cloud_vm, db_dir): props = Properties() config = { - 'host': props.persistence_host(), - 'user': props.persistence_user(), - 'password': props.persistence_password(), - 'database': props.persistence_database(), - 'connection_timeout': 10 + 'host': props.persistence_host(), + 'user': props.persistence_user(), + 'password': props.persistence_password(), + 'database': props.persistence_database(), + 'connection_timeout': 10 } - connection2 = mysql.connector.connect(**config) - return DBConnection(connection1, 'sqlite3'), DBConnection(connection2, 'mysql') \ No newline at end of file + connection1 = mysql.connector.connect(**config) + + db = os.path.join(db_dir, 'ten-test.db') + connection2 = sqlite3.connect(db) + + return DBConnection(connection1, 'mysql'), DBConnection(connection2, 'sqlite3') \ No newline at end of file diff --git a/src/python/ten/test/persistence/contract.py b/src/python/ten/test/persistence/contract.py index 8d412ce7..25edc071 100644 --- a/src/python/ten/test/persistence/contract.py +++ b/src/python/ten/test/persistence/contract.py @@ -1,8 +1,13 @@ +from ten.test.persistence import normalise + class ContractPersistence: """Abstracts the persistence of contract addresses into a local database. """ SQL_CREATE = "CREATE TABLE IF NOT EXISTS contracts " \ - "(name TEXT, environment TEXT, address INTEGER, abi STRING, " \ + "(name VARCHAR(64), " \ + "environment VARCHAR(64), " \ + "address INTEGER, " \ + "abi MEDIUMTEXT, " \ "PRIMARY KEY (name, environment))" SQL_INSERT = "INSERT OR REPLACE INTO contracts VALUES (?, ?, ?, ?)" SQL_DELETE = "DELETE from contracts WHERE environment=?" @@ -19,6 +24,13 @@ class ContractPersistence: def __init__(self, dbconnection): """Instantiate an instance.""" self.dbconnection = dbconnection + self.insert = normalise(self.SQL_INSERT, dbconnection.type) + self.delete = normalise(self.SQL_DELETE, dbconnection.type) + self.select = normalise(self.SQL_SELECT, dbconnection.type) + self.crt_params = normalise(self.SQL_CRT_PARAMS, dbconnection.type) + self.ins_params = normalise(self.SQL_INS_PARAMS, dbconnection.type) + self.del_params = normalise(self.SQL_DEL_PARAMS, dbconnection.type) + self.sel_params = normalise(self.SQL_SEL_PARAMS, dbconnection.type) self.cursor = self.dbconnection.connection.cursor() def create(self): @@ -32,30 +44,30 @@ def close(self): def delete_environment(self, environment): """Delete all stored contract details for a particular environment.""" - self.cursor.execute(self.SQL_DELETE, (environment, )) - self.cursor.execute(self.SQL_DEL_PARAMS, (environment, )) + self.cursor.execute(self.delete, (environment, )) + self.cursor.execute(self.del_params, (environment, )) self.dbconnection.connection.commit() def insert_contract(self, name, environment, address, abi): """Insert a new contract into the persistence. """ - self.cursor.execute(self.SQL_INSERT, (name, environment, address, abi)) + self.cursor.execute(self.insert, (name, environment, address, abi)) self.dbconnection.connection.commit() def get_contract(self, name, environment): """Return the address and abi for a particular deployed contract. """ - self.cursor.execute(self.SQL_SELECT, (name, environment)) + self.cursor.execute(self.select, (name, environment)) cursor = self.cursor.fetchall() if len(cursor) > 0: return cursor[0][0], cursor[0][1] return None, None def insert_param(self, address, environment, key, value): """Insert a parameter for a named contract. """ - self.cursor.execute(self.SQL_INS_PARAMS, (address, environment, key, value)) + self.cursor.execute(self.ins_params, (address, environment, key, value)) self.dbconnection.connection.commit() def get_param(self, address, environment, key): """Return the address and abi for a particular deployed contract. """ - self.cursor.execute(self.SQL_SEL_PARAMS, (address, environment, key)) + self.cursor.execute(self.sel_params, (address, environment, key)) cursor = self.cursor.fetchall() if len(cursor) > 0: return cursor[0][0] return None \ No newline at end of file diff --git a/src/python/ten/test/persistence/counts.py b/src/python/ten/test/persistence/counts.py index eb39d0f2..59d5d6ed 100644 --- a/src/python/ten/test/persistence/counts.py +++ b/src/python/ten/test/persistence/counts.py @@ -1,8 +1,14 @@ +from ten.test.persistence import normalise + class CountsPersistence: """Abstracts the persistence of transaction counts across accounts into a local database. """ SQL_CREATE = "CREATE TABLE IF NOT EXISTS counts " \ - "(name TEXT, address INTEGER, environment TEXT, time INTEGER, count TEXT, " \ + "(name VARCHAR(64), " \ + "address VARCHAR(64), " \ + "environment VARCHAR(64), " \ + "time INTEGER, " \ + "count INTEGER, " \ "PRIMARY KEY (name, environment, time))" SQL_INSERT = "INSERT INTO counts VALUES (?, ?, ?, ?, ?)" SQL_DELETE = "DELETE from counts WHERE environment=?" @@ -12,6 +18,10 @@ class CountsPersistence: def __init__(self, dbconnection): """Instantiate an instance.""" self.dbconnection = dbconnection + self.insert = normalise(self.SQL_INSERT, dbconnection.type) + self.delete = normalise(self.SQL_DELETE, dbconnection.type) + self.select_three = normalise(self.SQL_SELECT_THREE, dbconnection.type) + self.select_hour = normalise(self.SQL_SELECT_HOUR, dbconnection.type) self.cursor = self.dbconnection.connection.cursor() def create(self): @@ -24,20 +34,20 @@ def close(self): def delete_environment(self, environment): """Delete all stored contract details for a particular environment.""" - self.cursor.execute(self.SQL_DELETE, (environment, )) + self.cursor.execute(self.delete, (environment, )) self.dbconnection.connection.commit() def insert_count(self, name, address, environment, time, count): """Insert a new counts entry for a particular logical account.""" - self.cursor.execute(self.SQL_INSERT, (name, address, environment, time, str(count))) + self.cursor.execute(self.insert, (name, address, environment, time, str(count))) self.dbconnection.connection.commit() def get_last_three_counts(self, name, environment): """Return the transaction count with time for a particular logical account.""" - self.cursor.execute(self.SQL_SELECT_THREE, (name, environment)) + self.cursor.execute(self.select_three, (name, environment)) return self.cursor.fetchall() def get_last_hour(self, name, environment, time): """Return the transaction count with time for a particular logical account.""" - self.cursor.execute(self.SQL_SELECT_HOUR, (name, environment, time)) + self.cursor.execute(self.select_hour, (name, environment, time)) return self.cursor.fetchall() \ No newline at end of file diff --git a/src/python/ten/test/persistence/funds.py b/src/python/ten/test/persistence/funds.py index f65cb542..d887997b 100644 --- a/src/python/ten/test/persistence/funds.py +++ b/src/python/ten/test/persistence/funds.py @@ -1,8 +1,14 @@ +from ten.test.persistence import normalise + class FundsPersistence: """Abstracts the persistence of funds across accounts into a local database. """ SQL_CREATE = "CREATE TABLE IF NOT EXISTS funds " \ - "(name TEXT, address INTEGER, environment TEXT, time INTEGER, balance TEXT, " \ + "(name VARCHAR(64), " \ + "address VARCHAR(64), " \ + "environment VARCHAR(64), " \ + "time INTEGER, " \ + "balance INTEGER, " \ "PRIMARY KEY (name, environment, time))" SQL_INSERT = "INSERT INTO funds VALUES (?, ?, ?, ?, ?)" SQL_DELETE = "DELETE from funds WHERE environment=?" @@ -11,6 +17,9 @@ class FundsPersistence: def __init__(self, dbconnection): """Instantiate an instance.""" self.dbconnection = dbconnection + self.insert = normalise(self.SQL_INSERT, dbconnection.type) + self.delete = normalise(self.SQL_DELETE, dbconnection.type) + self.select = normalise(self.SQL_SELECT, dbconnection.type) self.cursor = self.dbconnection.connection.cursor() def create(self): @@ -23,16 +32,16 @@ def close(self): def delete_environment(self, environment): """Delete all stored details for a particular environment.""" - self.cursor.execute(self.SQL_DELETE, (environment, )) + self.cursor.execute(self.delete, (environment, )) self.dbconnection.connection.commit() def insert_funds(self, name, address, environment, time, balance): """Insert a new funds entry for a particular logical account.""" - self.cursor.execute(self.SQL_INSERT, (name, address, environment, time, str(balance))) + self.cursor.execute(self.insert, (name, address, environment, time, str(balance))) self.dbconnection.connection.commit() def get_funds(self, name, environment): """Return the funds with time for a particular logical account.""" - self.cursor.execute(self.SQL_SELECT, (name, environment)) + self.cursor.execute(self.select, (name, environment)) return self.cursor.fetchall() diff --git a/src/python/ten/test/persistence/nonce.py b/src/python/ten/test/persistence/nonce.py index cb0e06aa..1f8cdfcd 100644 --- a/src/python/ten/test/persistence/nonce.py +++ b/src/python/ten/test/persistence/nonce.py @@ -1,7 +1,13 @@ +from ten.test.persistence import normalise + class NoncePersistence: """Abstracts the persistence of nonces into a local database. """ - SQL_CREATE = "CREATE TABLE IF NOT EXISTS nonce_db (account TEXT, environment TEXT, nonce INTEGER, status STRING)" + SQL_CREATE = "CREATE TABLE IF NOT EXISTS nonce_db " \ + "(account VARCHAR(64), " \ + "environment VARCHAR(64), " \ + "nonce INTEGER, " \ + "status VARCHAR(64))" SQL_INSERT = "INSERT INTO nonce_db VALUES (?, ?, ?, ?)" SQL_UPDATE = "UPDATE nonce_db SET status=? WHERE account=? AND environment=? AND nonce=?" SQL_DELETE = "DELETE from nonce_db WHERE account=? AND environment=?" @@ -14,6 +20,14 @@ class NoncePersistence: def __init__(self, dbconnection): """Instantiate an instance.""" self.dbconnection = dbconnection + self.insert = normalise(self.SQL_INSERT, dbconnection.type) + self.update = normalise(self.SQL_UPDATE, dbconnection.type) + self.delete = normalise(self.SQL_DELETE, dbconnection.type) + self.delfro = normalise(self.SQL_DELFRO, dbconnection.type) + self.latest = normalise(self.SQL_LATEST, dbconnection.type) + self.delenv = normalise(self.SQL_DELENV, dbconnection.type) + self.accnts = normalise(self.SQL_ACCNTS, dbconnection.type) + self.delent = normalise(self.SQL_DELENT, dbconnection.type) self.cursor = self.dbconnection.connection.cursor() def create(self): @@ -46,42 +60,42 @@ def get_next_nonce(self, test, web3, account, environment, persist_nonce=True, l def insert(self, account, environment, nonce, status='PENDING'): """Insert a new nonce into the persistence. """ - self.cursor.execute(self.SQL_INSERT, (account, environment, nonce, status)) + self.cursor.execute(self.insert, (account, environment, nonce, status)) self.dbconnection.connection.commit() def update(self, account, environment, nonce, status): """Update the status of a transaction for a given nonce into the persistence. """ - self.cursor.execute(self.SQL_UPDATE, (status, account, environment, nonce)) + self.cursor.execute(self.update, (status, account, environment, nonce)) self.dbconnection.connection.commit() def delete(self, account, environment): """Delete all nonce entries in the persistence for a given account and environment. """ - self.cursor.execute(self.SQL_DELETE, (account, environment)) + self.cursor.execute(self.delete, (account, environment)) self.dbconnection.connection.commit() def delete_from(self, account, environment, nonce): """Delete all nonce entries in the persistence for a given account and environment. """ - self.cursor.execute(self.SQL_DELFRO, (account, environment, nonce)) + self.cursor.execute(self.delfro, (account, environment, nonce)) self.dbconnection.connection.commit() def delete_environment(self, environment): """Delete all nonce entries for all accounts for a given environment. """ - self.cursor.execute(self.SQL_DELENV, (environment, )) + self.cursor.execute(self.delenv, (environment, )) self.dbconnection.connection.commit() def delete_entries(self, account, environment, nonce): """Delete all nonce entries in the persistence for a given account and environment and nonce. """ - self.cursor.execute(self.SQL_DELENT, (account, environment, nonce)) + self.cursor.execute(self.delent, (account, environment, nonce)) self.dbconnection.connection.commit() def get_accounts(self, environment): """Return a list of all accounts with persisted values for a given environment. """ - self.cursor.execute(self.SQL_ACCNTS, (environment, )) + self.cursor.execute(self.accnts, (environment, )) return self.cursor.fetchall() def get_latest_nonce(self, account, environment): """Get the latest nonce for a given account and environment. """ - self.cursor.execute(self.SQL_LATEST, (account, environment)) + self.cursor.execute(self.latest, (account, environment)) try: result = self.cursor.fetchone()[0] return int(result) diff --git a/src/python/ten/test/persistence/rates.py b/src/python/ten/test/persistence/rates.py index ecfb8250..43717fcf 100644 --- a/src/python/ten/test/persistence/rates.py +++ b/src/python/ten/test/persistence/rates.py @@ -8,7 +8,7 @@ class RatesPersistence: "(crypto VARCHAR(3), " \ "currency VARCHAR(3), " \ "time INTEGER, " \ - "rate VARCHAR(255), " \ + "rate REAL, " \ "PRIMARY KEY (crypto, currency, time))" SQL_INSERT = "INSERT INTO rates VALUES (?, ?, ?, ?);" SQL_DELETE = "DELETE from rates WHERE crypto=?;" @@ -37,7 +37,7 @@ def delete_crypto(self, crypto): def insert_rates(self, crypto, currency, time, rate): """Insert a new rate for a particular crypto and currency.""" - self.cursor.execute(self.insert, (crypto, currency, time, str(rate))) + self.cursor.execute(self.insert, (crypto, currency, time, rate)) self.dbconnection.connection.commit() def get_latest_rate(self, crypto, currency): diff --git a/src/python/ten/test/persistence/results.py b/src/python/ten/test/persistence/results.py index 9e2f2a34..a076d86b 100644 --- a/src/python/ten/test/persistence/results.py +++ b/src/python/ten/test/persistence/results.py @@ -1,8 +1,14 @@ +from ten.test.persistence import normalise + + class ResultsPersistence: """Abstracts the persistence of performance results into a local database. """ SQL_CREATE = "CREATE TABLE IF NOT EXISTS results " \ - "(test TEXT, environment TEXT, time INTEGER, result REAL, " \ + "(test VARCHAR(64), " \ + "environment VARCHAR(64), " \ + "time INTEGER, " \ + "result REAL, " \ "PRIMARY KEY (test, environment, time))" SQL_INSERT = "INSERT INTO results VALUES (?, ?, ?, ?)" SQL_DELETE = "DELETE from results WHERE environment=?" @@ -11,6 +17,9 @@ class ResultsPersistence: def __init__(self, dbconnection): """Instantiate an instance.""" self.dbconnection = dbconnection + self.insert = normalise(self.SQL_INSERT, dbconnection.type) + self.delete = normalise(self.SQL_DELETE, dbconnection.type) + self.select = normalise(self.SQL_SELECT, dbconnection.type) self.cursor = self.dbconnection.connection.cursor() def create(self): @@ -23,16 +32,16 @@ def close(self): def delete_environment(self, environment): """Delete all stored performance results for a particular environment.""" - self.cursor.execute(self.SQL_DELETE, (environment, )) + self.cursor.execute(self.delete, (environment, )) self.dbconnection.connection.commit() def insert_result(self, test, environment, time, result): """Insert a new performance result into the persistence. """ - self.cursor.execute(self.SQL_INSERT, (test, environment, time, result)) + self.cursor.execute(self.insert, (test, environment, time, result)) self.dbconnection.connection.commit() def get_results(self, test, environment): """Return the performance results for a particular test and environment. """ - self.cursor.execute(self.SQL_SELECT, (test, environment)) + self.cursor.execute(self.select, (test, environment)) return self.cursor.fetchall() From c731f4f5009590d90f86f53ceb85053c75273d23 Mon Sep 17 00:00:00 2001 From: Moray Grieve Date: Tue, 7 Jan 2025 11:04:51 +0000 Subject: [PATCH 17/22] Add in the host --- src/python/ten/test/baserunner.py | 26 ++++----- src/python/ten/test/basetest.py | 14 ++--- src/python/ten/test/persistence/__init__.py | 1 + src/python/ten/test/persistence/contract.py | 63 +++++++++++++-------- src/python/ten/test/persistence/counts.py | 37 +++++++----- src/python/ten/test/persistence/funds.py | 10 +++- src/python/ten/test/persistence/nonce.py | 47 ++++++++------- src/python/ten/test/persistence/rates.py | 10 +++- src/python/ten/test/persistence/results.py | 28 +++++---- 9 files changed, 143 insertions(+), 93 deletions(-) diff --git a/src/python/ten/test/baserunner.py b/src/python/ten/test/baserunner.py index c27c4fec..4096d475 100644 --- a/src/python/ten/test/baserunner.py +++ b/src/python/ten/test/baserunner.py @@ -66,25 +66,19 @@ def setup(self, runner): # set up the persistence layer based on if we are running in the cloud, or locally if self.is_cloud_vm: - machine_name = self.cloud_metadata['compute']['name'] - runner.log.info('Running on azure (%s, %s)' % (machine_name, self.cloud_metadata['compute']['location'])) + self.machine_name = self.cloud_metadata['compute']['name'] + runner.log.info('Running on azure (%s, %s)' % (self.machine_name, self.cloud_metadata['compute']['location'])) else: - machine_name = socket.gethostname() - runner.log.info('Running on local (%s)' % machine_name) + self.machine_name = socket.gethostname() + runner.log.info('Running on local (%s)' % self.machine_name) dbconnection2, dbconnection2 = get_connection(self.is_cloud_vm, self.user_dir) - rates_db = RatesPersistence(dbconnection2) - rates_db.create() - nonce_db = NoncePersistence(dbconnection2) - nonce_db.create() - contracts_db = ContractPersistence(dbconnection2) - contracts_db.create() - funds_db = FundsPersistence(dbconnection2) - funds_db.create() - counts_db = CountsPersistence(dbconnection2) - counts_db.create() - results_db = ResultsPersistence(dbconnection2) - results_db.create() + rates_db = RatesPersistence.init(dbconnection2, self.machine_name) + nonce_db = NoncePersistence.init(dbconnection2, self.machine_name) + contracts_db = ContractPersistence.init(dbconnection2, self.machine_name) + funds_db = FundsPersistence.init(dbconnection2, self.machine_name) + counts_db = CountsPersistence.init(dbconnection2, self.machine_name) + results_db = ResultsPersistence.init(dbconnection2, self.machine_name) eth_price = self.get_eth_price() if eth_price is not None: diff --git a/src/python/ten/test/basetest.py b/src/python/ten/test/basetest.py index 3577d382..bdb74ae1 100644 --- a/src/python/ten/test/basetest.py +++ b/src/python/ten/test/basetest.py @@ -34,13 +34,13 @@ def __init__(self, descriptor, outsubdir, runner): self.log.info('Running test in thread %s', threading.currentThread().getName()) # every test has its own connection to the dbs - self.dbconnection2, self.dbconnection2 = get_connection(is_local=runner.ten_runner.is_cloud_vm, db_dir=runner.ten_runner.user_dir) - self.rates_db = RatesPersistence(self.dbconnection2) - self.nonce_db = NoncePersistence(self.dbconnection2) - self.contract_db = ContractPersistence(self.dbconnection2) - self.funds_db = FundsPersistence(self.dbconnection2) - self.counts_db = CountsPersistence(self.dbconnection2) - self.results_db = ResultsPersistence(self.dbconnection2) + self.dbconnection2, self.dbconnection2 = get_connection(is_cloud_vm=runner.ten_runner.is_cloud_vm, db_dir=runner.ten_runner.user_dir) + self.rates_db = RatesPersistence(self.dbconnection2, runner.ten_runner.machine_name) + self.nonce_db = NoncePersistence(self.dbconnection2, runner.ten_runner.machine_name) + self.contract_db = ContractPersistence(self.dbconnection2, runner.ten_runner.machine_name) + self.funds_db = FundsPersistence(self.dbconnection2, runner.ten_runner.machine_name) + self.counts_db = CountsPersistence(self.dbconnection2, runner.ten_runner.machine_name) + self.results_db = ResultsPersistence(self.dbconnection2, runner.ten_runner.machine_name) self.addCleanupFunction(self.close_db) # every test has a unique connection for the funded account diff --git a/src/python/ten/test/persistence/__init__.py b/src/python/ten/test/persistence/__init__.py index 932605aa..c4dadc35 100644 --- a/src/python/ten/test/persistence/__init__.py +++ b/src/python/ten/test/persistence/__init__.py @@ -12,6 +12,7 @@ def normalise(statement, _type): def get_connection(is_cloud_vm, db_dir): + if is_cloud_vm: pass props = Properties() config = { 'host': props.persistence_host(), diff --git a/src/python/ten/test/persistence/contract.py b/src/python/ten/test/persistence/contract.py index 25edc071..3e53b2b9 100644 --- a/src/python/ten/test/persistence/contract.py +++ b/src/python/ten/test/persistence/contract.py @@ -1,42 +1,55 @@ from ten.test.persistence import normalise + class ContractPersistence: """Abstracts the persistence of contract addresses into a local database. """ SQL_CREATE = "CREATE TABLE IF NOT EXISTS contracts " \ - "(name VARCHAR(64), " \ + "(host VARCHAR(64), " \ + "name VARCHAR(64), " \ "environment VARCHAR(64), " \ - "address INTEGER, " \ + "address VARCHAR(64), " \ "abi MEDIUMTEXT, " \ - "PRIMARY KEY (name, environment))" - SQL_INSERT = "INSERT OR REPLACE INTO contracts VALUES (?, ?, ?, ?)" - SQL_DELETE = "DELETE from contracts WHERE environment=?" - SQL_SELECT = "SELECT address, abi FROM contracts WHERE name=? AND environment=? ORDER BY name DESC LIMIT 1" + "PRIMARY KEY (host, name, environment))" + SQL_INSERT = "INSERT OR REPLACE INTO contracts VALUES (?, ?, ?, ?, ?)" + SQL_DELETE = "DELETE from contracts WHERE host=? AND environment=?" + SQL_SELECT = "SELECT address, abi FROM contracts WHERE host=? AND name=? AND environment=? ORDER BY name DESC LIMIT 1" + + SQL_CRTPRM = "CREATE TABLE IF NOT EXISTS params " \ + "(host VARCHAR(64), " \ + "address VARCHAR(64), " \ + "environment VARCHAR(64), " \ + "key VARCHAR(64), " \ + "value VARCHAR(64), " \ + "PRIMARY KEY (host, address, environment, key))" + SQL_INSPRM = "INSERT OR REPLACE INTO params VALUES (?, ?, ?, ?, ?)" + SQL_DELPRM = "DELETE from params WHERE host=? AND environment=?" + SQL_SELPRM = "SELECT value FROM params WHERE host=? AND address=? AND environment=? AND key=? " \ + "ORDER BY address DESC LIMIT 1" - SQL_CRT_PARAMS = "CREATE TABLE IF NOT EXISTS params " \ - "(address INTEGER, environment TEXT, key STRING, value STRING, " \ - "PRIMARY KEY (address, environment, key))" - SQL_INS_PARAMS = "INSERT OR REPLACE INTO params VALUES (?, ?, ?, ?)" - SQL_DEL_PARAMS = "DELETE from params WHERE environment=?" - SQL_SEL_PARAMS = "SELECT value FROM params WHERE address=? AND environment=? AND key=? " \ - "ORDER BY address DESC LIMIT 1" + @classmethod + def init(cls, host, dbconnection): + instance = ContractPersistence(host, dbconnection) + instance.create() + return instance - def __init__(self, dbconnection): + def __init__(self, host, dbconnection): """Instantiate an instance.""" + self.host = host self.dbconnection = dbconnection self.insert = normalise(self.SQL_INSERT, dbconnection.type) self.delete = normalise(self.SQL_DELETE, dbconnection.type) self.select = normalise(self.SQL_SELECT, dbconnection.type) - self.crt_params = normalise(self.SQL_CRT_PARAMS, dbconnection.type) - self.ins_params = normalise(self.SQL_INS_PARAMS, dbconnection.type) - self.del_params = normalise(self.SQL_DEL_PARAMS, dbconnection.type) - self.sel_params = normalise(self.SQL_SEL_PARAMS, dbconnection.type) + self.crtprm = normalise(self.SQL_CRTPRM, dbconnection.type) + self.insprm = normalise(self.SQL_INSPRM, dbconnection.type) + self.delprm = normalise(self.SQL_DELPRM, dbconnection.type) + self.selprm = normalise(self.SQL_SELPRM, dbconnection.type) self.cursor = self.dbconnection.connection.cursor() def create(self): """Create the cursor to the underlying persistence.""" self.cursor.execute(self.SQL_CREATE) - self.cursor.execute(self.SQL_CRT_PARAMS) + self.cursor.execute(self.SQL_CRTPRM) def close(self): """Close the connection to the underlying persistence.""" @@ -44,30 +57,30 @@ def close(self): def delete_environment(self, environment): """Delete all stored contract details for a particular environment.""" - self.cursor.execute(self.delete, (environment, )) - self.cursor.execute(self.del_params, (environment, )) + self.cursor.execute(self.delete, (self.host, environment)) + self.cursor.execute(self.delprm, (self.host, environment)) self.dbconnection.connection.commit() def insert_contract(self, name, environment, address, abi): """Insert a new contract into the persistence. """ - self.cursor.execute(self.insert, (name, environment, address, abi)) + self.cursor.execute(self.insert, (self.host, name, environment, address, abi)) self.dbconnection.connection.commit() def get_contract(self, name, environment): """Return the address and abi for a particular deployed contract. """ - self.cursor.execute(self.select, (name, environment)) + self.cursor.execute(self.select, (self.host, name, environment)) cursor = self.cursor.fetchall() if len(cursor) > 0: return cursor[0][0], cursor[0][1] return None, None def insert_param(self, address, environment, key, value): """Insert a parameter for a named contract. """ - self.cursor.execute(self.ins_params, (address, environment, key, value)) + self.cursor.execute(self.insprm, (self.host, address, environment, key, value)) self.dbconnection.connection.commit() def get_param(self, address, environment, key): """Return the address and abi for a particular deployed contract. """ - self.cursor.execute(self.sel_params, (address, environment, key)) + self.cursor.execute(self.selprm, (self.host, address, environment, key)) cursor = self.cursor.fetchall() if len(cursor) > 0: return cursor[0][0] return None \ No newline at end of file diff --git a/src/python/ten/test/persistence/counts.py b/src/python/ten/test/persistence/counts.py index 59d5d6ed..f2af9319 100644 --- a/src/python/ten/test/persistence/counts.py +++ b/src/python/ten/test/persistence/counts.py @@ -1,27 +1,36 @@ from ten.test.persistence import normalise + class CountsPersistence: """Abstracts the persistence of transaction counts across accounts into a local database. """ SQL_CREATE = "CREATE TABLE IF NOT EXISTS counts " \ - "(name VARCHAR(64), " \ + "(host VARCHAR(64), " \ + "name VARCHAR(64), " \ "address VARCHAR(64), " \ "environment VARCHAR(64), " \ "time INTEGER, " \ "count INTEGER, " \ - "PRIMARY KEY (name, environment, time))" - SQL_INSERT = "INSERT INTO counts VALUES (?, ?, ?, ?, ?)" - SQL_DELETE = "DELETE from counts WHERE environment=?" - SQL_SELECT_THREE = "SELECT time, count FROM counts WHERE name=? and environment=? ORDER BY time DESC LIMIT 3" - SQL_SELECT_HOUR = "SELECT time, count FROM counts WHERE name=? and environment=? and time >= ? ORDER BY time DESC" - - def __init__(self, dbconnection): + "PRIMARY KEY (host, name, environment, time))" + SQL_INSERT = "INSERT INTO counts VALUES (?, ?, ?, ?, ?, ?)" + SQL_DELETE = "DELETE from counts WHERE host=? AND environment=?" + SQL_SELTHR = "SELECT time, count FROM counts WHERE host=? AND name=? and environment=? ORDER BY time DESC LIMIT 3" + SQL_SELHOR = "SELECT time, count FROM counts WHERE host=? AND name=? and environment=? and time >= ? ORDER BY time DESC" + + @classmethod + def init(cls, host, dbconnection): + instance = CountsPersistence(host, dbconnection) + instance.create() + return instance + + def __init__(self, host, dbconnection): """Instantiate an instance.""" + self.host = host self.dbconnection = dbconnection self.insert = normalise(self.SQL_INSERT, dbconnection.type) self.delete = normalise(self.SQL_DELETE, dbconnection.type) - self.select_three = normalise(self.SQL_SELECT_THREE, dbconnection.type) - self.select_hour = normalise(self.SQL_SELECT_HOUR, dbconnection.type) + self.select_three = normalise(self.SQL_SELTHR, dbconnection.type) + self.select_hour = normalise(self.SQL_SELHOR, dbconnection.type) self.cursor = self.dbconnection.connection.cursor() def create(self): @@ -34,20 +43,20 @@ def close(self): def delete_environment(self, environment): """Delete all stored contract details for a particular environment.""" - self.cursor.execute(self.delete, (environment, )) + self.cursor.execute(self.delete, (self.host, environment)) self.dbconnection.connection.commit() def insert_count(self, name, address, environment, time, count): """Insert a new counts entry for a particular logical account.""" - self.cursor.execute(self.insert, (name, address, environment, time, str(count))) + self.cursor.execute(self.insert, (self.host, name, address, environment, time, str(count))) self.dbconnection.connection.commit() def get_last_three_counts(self, name, environment): """Return the transaction count with time for a particular logical account.""" - self.cursor.execute(self.select_three, (name, environment)) + self.cursor.execute(self.select_three, (self.host, name, environment)) return self.cursor.fetchall() def get_last_hour(self, name, environment, time): """Return the transaction count with time for a particular logical account.""" - self.cursor.execute(self.select_hour, (name, environment, time)) + self.cursor.execute(self.select_hour, (self.host, name, environment, time)) return self.cursor.fetchall() \ No newline at end of file diff --git a/src/python/ten/test/persistence/funds.py b/src/python/ten/test/persistence/funds.py index d887997b..c1a86544 100644 --- a/src/python/ten/test/persistence/funds.py +++ b/src/python/ten/test/persistence/funds.py @@ -1,5 +1,6 @@ from ten.test.persistence import normalise + class FundsPersistence: """Abstracts the persistence of funds across accounts into a local database. """ @@ -14,8 +15,15 @@ class FundsPersistence: SQL_DELETE = "DELETE from funds WHERE environment=?" SQL_SELECT = "SELECT time, balance FROM funds WHERE name=? and environment=? ORDER BY time DESC" - def __init__(self, dbconnection): + @classmethod + def init(cls, host, dbconnection): + instance = FundsPersistence(host, dbconnection) + instance.create() + return instance + + def __init__(self, host, dbconnection): """Instantiate an instance.""" + self.host = host self.dbconnection = dbconnection self.insert = normalise(self.SQL_INSERT, dbconnection.type) self.delete = normalise(self.SQL_DELETE, dbconnection.type) diff --git a/src/python/ten/test/persistence/nonce.py b/src/python/ten/test/persistence/nonce.py index 1f8cdfcd..8c825b62 100644 --- a/src/python/ten/test/persistence/nonce.py +++ b/src/python/ten/test/persistence/nonce.py @@ -1,24 +1,33 @@ from ten.test.persistence import normalise + class NoncePersistence: """Abstracts the persistence of nonces into a local database. """ SQL_CREATE = "CREATE TABLE IF NOT EXISTS nonce_db " \ - "(account VARCHAR(64), " \ + "(host VARCHAR(64), " \ + "account VARCHAR(64), " \ "environment VARCHAR(64), " \ "nonce INTEGER, " \ "status VARCHAR(64))" - SQL_INSERT = "INSERT INTO nonce_db VALUES (?, ?, ?, ?)" - SQL_UPDATE = "UPDATE nonce_db SET status=? WHERE account=? AND environment=? AND nonce=?" - SQL_DELETE = "DELETE from nonce_db WHERE account=? AND environment=?" - SQL_DELFRO = "DELETE from nonce_db WHERE account=? AND environment=? AND nonce>=?" - SQL_LATEST = "SELECT nonce FROM nonce_db WHERE account=? AND environment=? ORDER BY nonce DESC LIMIT 1" - SQL_DELENV = "DELETE from nonce_db WHERE environment=?" - SQL_ACCNTS = "SELECT DISTINCT account from nonce_db where environment=?" - SQL_DELENT = "DELETE from nonce_db WHERE account=? AND environment=? AND nonce=?" - - def __init__(self, dbconnection): + SQL_INSERT = "INSERT INTO nonce_db VALUES (?, ?, ?, ?, ?)" + SQL_UPDATE = "UPDATE nonce_db SET status=? WHERE host=? AND account=? AND environment=? AND nonce=?" + SQL_DELETE = "DELETE from nonce_db WHERE host=? AND account=? AND environment=?" + SQL_DELFRO = "DELETE from nonce_db WHERE host=? AND account=? AND environment=? AND nonce>=?" + SQL_LATEST = "SELECT nonce FROM nonce_db WHERE host=? AND account=? AND environment=? ORDER BY nonce DESC LIMIT 1" + SQL_DELENV = "DELETE from nonce_db WHERE host=? AND environment=?" + SQL_ACCNTS = "SELECT DISTINCT account from nonce_db WHERE host=? AND environment=?" + SQL_DELENT = "DELETE from nonce_db WHERE host=? AND account=? AND environment=? AND nonce=?" + + @classmethod + def init(cls, host, dbconnection): + instance = NoncePersistence(host, dbconnection) + instance.create() + return instance + + def __init__(self, host, dbconnection): """Instantiate an instance.""" + self.host = host self.dbconnection = dbconnection self.insert = normalise(self.SQL_INSERT, dbconnection.type) self.update = normalise(self.SQL_UPDATE, dbconnection.type) @@ -60,42 +69,42 @@ def get_next_nonce(self, test, web3, account, environment, persist_nonce=True, l def insert(self, account, environment, nonce, status='PENDING'): """Insert a new nonce into the persistence. """ - self.cursor.execute(self.insert, (account, environment, nonce, status)) + self.cursor.execute(self.insert, (self.host, account, environment, nonce, status)) self.dbconnection.connection.commit() def update(self, account, environment, nonce, status): """Update the status of a transaction for a given nonce into the persistence. """ - self.cursor.execute(self.update, (status, account, environment, nonce)) + self.cursor.execute(self.update, (self.host, status, account, environment, nonce)) self.dbconnection.connection.commit() def delete(self, account, environment): """Delete all nonce entries in the persistence for a given account and environment. """ - self.cursor.execute(self.delete, (account, environment)) + self.cursor.execute(self.delete, (self.host, account, environment)) self.dbconnection.connection.commit() def delete_from(self, account, environment, nonce): """Delete all nonce entries in the persistence for a given account and environment. """ - self.cursor.execute(self.delfro, (account, environment, nonce)) + self.cursor.execute(self.delfro, (self.host, account, environment, nonce)) self.dbconnection.connection.commit() def delete_environment(self, environment): """Delete all nonce entries for all accounts for a given environment. """ - self.cursor.execute(self.delenv, (environment, )) + self.cursor.execute(self.delenv, (self.host, environment, )) self.dbconnection.connection.commit() def delete_entries(self, account, environment, nonce): """Delete all nonce entries in the persistence for a given account and environment and nonce. """ - self.cursor.execute(self.delent, (account, environment, nonce)) + self.cursor.execute(self.delent, (self.host, account, environment, nonce)) self.dbconnection.connection.commit() def get_accounts(self, environment): """Return a list of all accounts with persisted values for a given environment. """ - self.cursor.execute(self.accnts, (environment, )) + self.cursor.execute(self.accnts, (self.host, environment)) return self.cursor.fetchall() def get_latest_nonce(self, account, environment): """Get the latest nonce for a given account and environment. """ - self.cursor.execute(self.latest, (account, environment)) + self.cursor.execute(self.latest, (self.host, account, environment)) try: result = self.cursor.fetchone()[0] return int(result) diff --git a/src/python/ten/test/persistence/rates.py b/src/python/ten/test/persistence/rates.py index 43717fcf..b32d205a 100644 --- a/src/python/ten/test/persistence/rates.py +++ b/src/python/ten/test/persistence/rates.py @@ -14,8 +14,15 @@ class RatesPersistence: SQL_DELETE = "DELETE from rates WHERE crypto=?;" SQL_SELECT = "SELECT time, rate FROM rates WHERE crypto=? and currency=? ORDER BY time DESC LIMIT 1;" - def __init__(self, dbconnection): + @classmethod + def init(cls, host, dbconnection): + instance = RatesPersistence(host, dbconnection) + instance.create() + return instance + + def __init__(self, host, dbconnection): """Instantiate an instance.""" + self.host = host self.dbconnection = dbconnection self.insert = normalise(self.SQL_INSERT, dbconnection.type) self.delete = normalise(self.SQL_DELETE, dbconnection.type) @@ -25,6 +32,7 @@ def __init__(self, dbconnection): def create(self): """Create the cursor to the underlying persistence.""" self.cursor.execute(self.SQL_CREATE) + return self def close(self): """Close the connection to the underlying persistence.""" diff --git a/src/python/ten/test/persistence/results.py b/src/python/ten/test/persistence/results.py index a076d86b..001c7a4c 100644 --- a/src/python/ten/test/persistence/results.py +++ b/src/python/ten/test/persistence/results.py @@ -5,17 +5,25 @@ class ResultsPersistence: """Abstracts the persistence of performance results into a local database. """ SQL_CREATE = "CREATE TABLE IF NOT EXISTS results " \ - "(test VARCHAR(64), " \ + "(host VARCHAR(64), " \ + "test VARCHAR(64), " \ "environment VARCHAR(64), " \ "time INTEGER, " \ "result REAL, " \ - "PRIMARY KEY (test, environment, time))" - SQL_INSERT = "INSERT INTO results VALUES (?, ?, ?, ?)" - SQL_DELETE = "DELETE from results WHERE environment=?" - SQL_SELECT = "SELECT time, result FROM results WHERE test=? AND environment=? ORDER BY time ASC" - - def __init__(self, dbconnection): + "PRIMARY KEY (host, test, environment, time))" + SQL_INSERT = "INSERT INTO results VALUES (?, ?, ?, ?, ?)" + SQL_DELETE = "DELETE from results WHERE host=? and environment=?" + SQL_SELECT = "SELECT time, result FROM results WHERE host=? AND test=? AND environment=? ORDER BY time ASC" + + @classmethod + def init(cls, host, dbconnection): + instance = ResultsPersistence(host, dbconnection) + instance.create() + return instance + + def __init__(self, host, dbconnection): """Instantiate an instance.""" + self.host = host self.dbconnection = dbconnection self.insert = normalise(self.SQL_INSERT, dbconnection.type) self.delete = normalise(self.SQL_DELETE, dbconnection.type) @@ -32,16 +40,16 @@ def close(self): def delete_environment(self, environment): """Delete all stored performance results for a particular environment.""" - self.cursor.execute(self.delete, (environment, )) + self.cursor.execute(self.delete, (self.host, environment)) self.dbconnection.connection.commit() def insert_result(self, test, environment, time, result): """Insert a new performance result into the persistence. """ - self.cursor.execute(self.insert, (test, environment, time, result)) + self.cursor.execute(self.insert, (self.host, test, environment, time, result)) self.dbconnection.connection.commit() def get_results(self, test, environment): """Return the performance results for a particular test and environment. """ - self.cursor.execute(self.select, (test, environment)) + self.cursor.execute(self.select, (self.host, test, environment)) return self.cursor.fetchall() From 660e8030a425798c766ca6d58a866ecd268cbadd Mon Sep 17 00:00:00 2001 From: Moray Grieve Date: Tue, 7 Jan 2025 11:49:22 +0000 Subject: [PATCH 18/22] Fixes for host --- src/python/ten/test/baserunner.py | 19 ++++++------ src/python/ten/test/basetest.py | 17 +++++------ src/python/ten/test/persistence/contract.py | 21 +++++++------- src/python/ten/test/persistence/counts.py | 16 +++++------ src/python/ten/test/persistence/funds.py | 12 ++++---- src/python/ten/test/persistence/nonce.py | 32 ++++++++++----------- src/python/ten/test/persistence/rates.py | 12 ++++---- src/python/ten/test/persistence/results.py | 12 ++++---- 8 files changed, 71 insertions(+), 70 deletions(-) diff --git a/src/python/ten/test/baserunner.py b/src/python/ten/test/baserunner.py index 4096d475..8249f570 100644 --- a/src/python/ten/test/baserunner.py +++ b/src/python/ten/test/baserunner.py @@ -71,14 +71,14 @@ def setup(self, runner): else: self.machine_name = socket.gethostname() runner.log.info('Running on local (%s)' % self.machine_name) - dbconnection2, dbconnection2 = get_connection(self.is_cloud_vm, self.user_dir) - - rates_db = RatesPersistence.init(dbconnection2, self.machine_name) - nonce_db = NoncePersistence.init(dbconnection2, self.machine_name) - contracts_db = ContractPersistence.init(dbconnection2, self.machine_name) - funds_db = FundsPersistence.init(dbconnection2, self.machine_name) - counts_db = CountsPersistence.init(dbconnection2, self.machine_name) - results_db = ResultsPersistence.init(dbconnection2, self.machine_name) + dbconnection1, dbconnection2 = get_connection(self.is_cloud_vm, self.user_dir) + dbconnection = dbconnection1 + rates_db = RatesPersistence.init(self.machine_name, dbconnection) + nonce_db = NoncePersistence.init(self.machine_name, dbconnection) + contracts_db = ContractPersistence.init(self.machine_name, dbconnection) + funds_db = FundsPersistence.init(self.machine_name, dbconnection) + counts_db = CountsPersistence.init(self.machine_name, dbconnection) + results_db = ResultsPersistence.init(self.machine_name, dbconnection) eth_price = self.get_eth_price() if eth_price is not None: @@ -172,7 +172,7 @@ def setup(self, runner): funds_db.close() counts_db.close() results_db.close() - dbconnection2.connection.close() + dbconnection1.connection.close() dbconnection2.connection.close() def run_ganache(self, runner): @@ -317,6 +317,7 @@ def __set_contract_addresses(self, runner): contracts = config["PublicSystemContracts"] Properties.L2PublicCallbacks = self.__get_contract(contracts, "PublicCallbacks") elif 'error' in response.json(): + runner.log.warn('Error getting contract address from ten_config') runner.log.error(response.json()['error']['message']) def __get_contract(self, contracts, key): diff --git a/src/python/ten/test/basetest.py b/src/python/ten/test/basetest.py index bdb74ae1..f8b458e2 100644 --- a/src/python/ten/test/basetest.py +++ b/src/python/ten/test/basetest.py @@ -34,13 +34,14 @@ def __init__(self, descriptor, outsubdir, runner): self.log.info('Running test in thread %s', threading.currentThread().getName()) # every test has its own connection to the dbs - self.dbconnection2, self.dbconnection2 = get_connection(is_cloud_vm=runner.ten_runner.is_cloud_vm, db_dir=runner.ten_runner.user_dir) - self.rates_db = RatesPersistence(self.dbconnection2, runner.ten_runner.machine_name) - self.nonce_db = NoncePersistence(self.dbconnection2, runner.ten_runner.machine_name) - self.contract_db = ContractPersistence(self.dbconnection2, runner.ten_runner.machine_name) - self.funds_db = FundsPersistence(self.dbconnection2, runner.ten_runner.machine_name) - self.counts_db = CountsPersistence(self.dbconnection2, runner.ten_runner.machine_name) - self.results_db = ResultsPersistence(self.dbconnection2, runner.ten_runner.machine_name) + self.dbconnection1, self.dbconnection2 = get_connection(is_cloud_vm=runner.ten_runner.is_cloud_vm, db_dir=runner.ten_runner.user_dir) + dbconnection = self.dbconnection1 + self.rates_db = RatesPersistence(runner.ten_runner.machine_name, dbconnection) + self.nonce_db = NoncePersistence(runner.ten_runner.machine_name, dbconnection) + self.contract_db = ContractPersistence(runner.ten_runner.machine_name, dbconnection) + self.funds_db = FundsPersistence(runner.ten_runner.machine_name, dbconnection) + self.counts_db = CountsPersistence(runner.ten_runner.machine_name, dbconnection) + self.results_db = ResultsPersistence(runner.ten_runner.machine_name, dbconnection) self.addCleanupFunction(self.close_db) # every test has a unique connection for the funded account @@ -79,7 +80,7 @@ def close_db(self): self.funds_db.close() self.counts_db.close() self.results_db.close() - self.dbconnection2.connection.close() + self.dbconnection1.connection.close() self.dbconnection2.connection.close() def drain_ephemeral_pks(self): diff --git a/src/python/ten/test/persistence/contract.py b/src/python/ten/test/persistence/contract.py index 3e53b2b9..7818a91a 100644 --- a/src/python/ten/test/persistence/contract.py +++ b/src/python/ten/test/persistence/contract.py @@ -19,12 +19,12 @@ class ContractPersistence: "(host VARCHAR(64), " \ "address VARCHAR(64), " \ "environment VARCHAR(64), " \ - "key VARCHAR(64), " \ - "value VARCHAR(64), " \ - "PRIMARY KEY (host, address, environment, key))" + "param_key VARCHAR(64), " \ + "param_val VARCHAR(64), " \ + "PRIMARY KEY (host, address, environment, param_key))" SQL_INSPRM = "INSERT OR REPLACE INTO params VALUES (?, ?, ?, ?, ?)" SQL_DELPRM = "DELETE from params WHERE host=? AND environment=?" - SQL_SELPRM = "SELECT value FROM params WHERE host=? AND address=? AND environment=? AND key=? " \ + SQL_SELPRM = "SELECT param_val FROM params WHERE host=? AND address=? AND environment=? AND param_key=? " \ "ORDER BY address DESC LIMIT 1" @classmethod @@ -37,10 +37,9 @@ def __init__(self, host, dbconnection): """Instantiate an instance.""" self.host = host self.dbconnection = dbconnection - self.insert = normalise(self.SQL_INSERT, dbconnection.type) - self.delete = normalise(self.SQL_DELETE, dbconnection.type) - self.select = normalise(self.SQL_SELECT, dbconnection.type) - self.crtprm = normalise(self.SQL_CRTPRM, dbconnection.type) + self.sqlins = normalise(self.SQL_INSERT, dbconnection.type) + self.sqldel = normalise(self.SQL_DELETE, dbconnection.type) + self.sqlsel = normalise(self.SQL_SELECT, dbconnection.type) self.insprm = normalise(self.SQL_INSPRM, dbconnection.type) self.delprm = normalise(self.SQL_DELPRM, dbconnection.type) self.selprm = normalise(self.SQL_SELPRM, dbconnection.type) @@ -57,18 +56,18 @@ def close(self): def delete_environment(self, environment): """Delete all stored contract details for a particular environment.""" - self.cursor.execute(self.delete, (self.host, environment)) + self.cursor.execute(self.sqldel, (self.host, environment)) self.cursor.execute(self.delprm, (self.host, environment)) self.dbconnection.connection.commit() def insert_contract(self, name, environment, address, abi): """Insert a new contract into the persistence. """ - self.cursor.execute(self.insert, (self.host, name, environment, address, abi)) + self.cursor.execute(self.sqlins, (self.host, name, environment, address, abi)) self.dbconnection.connection.commit() def get_contract(self, name, environment): """Return the address and abi for a particular deployed contract. """ - self.cursor.execute(self.select, (self.host, name, environment)) + self.cursor.execute(self.sqlsel, (self.host, name, environment)) cursor = self.cursor.fetchall() if len(cursor) > 0: return cursor[0][0], cursor[0][1] return None, None diff --git a/src/python/ten/test/persistence/counts.py b/src/python/ten/test/persistence/counts.py index f2af9319..ee0f7dda 100644 --- a/src/python/ten/test/persistence/counts.py +++ b/src/python/ten/test/persistence/counts.py @@ -27,10 +27,10 @@ def __init__(self, host, dbconnection): """Instantiate an instance.""" self.host = host self.dbconnection = dbconnection - self.insert = normalise(self.SQL_INSERT, dbconnection.type) - self.delete = normalise(self.SQL_DELETE, dbconnection.type) - self.select_three = normalise(self.SQL_SELTHR, dbconnection.type) - self.select_hour = normalise(self.SQL_SELHOR, dbconnection.type) + self.sqlins = normalise(self.SQL_INSERT, dbconnection.type) + self.sqldel = normalise(self.SQL_DELETE, dbconnection.type) + self.sqlthr = normalise(self.SQL_SELTHR, dbconnection.type) + self.sqlhor = normalise(self.SQL_SELHOR, dbconnection.type) self.cursor = self.dbconnection.connection.cursor() def create(self): @@ -43,20 +43,20 @@ def close(self): def delete_environment(self, environment): """Delete all stored contract details for a particular environment.""" - self.cursor.execute(self.delete, (self.host, environment)) + self.cursor.execute(self.sqldel, (self.host, environment)) self.dbconnection.connection.commit() def insert_count(self, name, address, environment, time, count): """Insert a new counts entry for a particular logical account.""" - self.cursor.execute(self.insert, (self.host, name, address, environment, time, str(count))) + self.cursor.execute(self.sqlins, (self.host, name, address, environment, time, str(count))) self.dbconnection.connection.commit() def get_last_three_counts(self, name, environment): """Return the transaction count with time for a particular logical account.""" - self.cursor.execute(self.select_three, (self.host, name, environment)) + self.cursor.execute(self.sqlthr, (self.host, name, environment)) return self.cursor.fetchall() def get_last_hour(self, name, environment, time): """Return the transaction count with time for a particular logical account.""" - self.cursor.execute(self.select_hour, (self.host, name, environment, time)) + self.cursor.execute(self.sqlhor, (self.host, name, environment, time)) return self.cursor.fetchall() \ No newline at end of file diff --git a/src/python/ten/test/persistence/funds.py b/src/python/ten/test/persistence/funds.py index c1a86544..5d67ec46 100644 --- a/src/python/ten/test/persistence/funds.py +++ b/src/python/ten/test/persistence/funds.py @@ -25,9 +25,9 @@ def __init__(self, host, dbconnection): """Instantiate an instance.""" self.host = host self.dbconnection = dbconnection - self.insert = normalise(self.SQL_INSERT, dbconnection.type) - self.delete = normalise(self.SQL_DELETE, dbconnection.type) - self.select = normalise(self.SQL_SELECT, dbconnection.type) + self.sqlins = normalise(self.SQL_INSERT, dbconnection.type) + self.sqldel = normalise(self.SQL_DELETE, dbconnection.type) + self.sqlsel = normalise(self.SQL_SELECT, dbconnection.type) self.cursor = self.dbconnection.connection.cursor() def create(self): @@ -40,16 +40,16 @@ def close(self): def delete_environment(self, environment): """Delete all stored details for a particular environment.""" - self.cursor.execute(self.delete, (environment, )) + self.cursor.execute(self.sqldel, (environment, )) self.dbconnection.connection.commit() def insert_funds(self, name, address, environment, time, balance): """Insert a new funds entry for a particular logical account.""" - self.cursor.execute(self.insert, (name, address, environment, time, str(balance))) + self.cursor.execute(self.sqlins, (name, address, environment, time, str(balance))) self.dbconnection.connection.commit() def get_funds(self, name, environment): """Return the funds with time for a particular logical account.""" - self.cursor.execute(self.select, (name, environment)) + self.cursor.execute(self.sqlsel, (name, environment)) return self.cursor.fetchall() diff --git a/src/python/ten/test/persistence/nonce.py b/src/python/ten/test/persistence/nonce.py index 8c825b62..f1394b74 100644 --- a/src/python/ten/test/persistence/nonce.py +++ b/src/python/ten/test/persistence/nonce.py @@ -4,20 +4,20 @@ class NoncePersistence: """Abstracts the persistence of nonces into a local database. """ - SQL_CREATE = "CREATE TABLE IF NOT EXISTS nonce_db " \ + SQL_CREATE = "CREATE TABLE IF NOT EXISTS nonces " \ "(host VARCHAR(64), " \ "account VARCHAR(64), " \ "environment VARCHAR(64), " \ "nonce INTEGER, " \ "status VARCHAR(64))" - SQL_INSERT = "INSERT INTO nonce_db VALUES (?, ?, ?, ?, ?)" - SQL_UPDATE = "UPDATE nonce_db SET status=? WHERE host=? AND account=? AND environment=? AND nonce=?" - SQL_DELETE = "DELETE from nonce_db WHERE host=? AND account=? AND environment=?" - SQL_DELFRO = "DELETE from nonce_db WHERE host=? AND account=? AND environment=? AND nonce>=?" - SQL_LATEST = "SELECT nonce FROM nonce_db WHERE host=? AND account=? AND environment=? ORDER BY nonce DESC LIMIT 1" - SQL_DELENV = "DELETE from nonce_db WHERE host=? AND environment=?" - SQL_ACCNTS = "SELECT DISTINCT account from nonce_db WHERE host=? AND environment=?" - SQL_DELENT = "DELETE from nonce_db WHERE host=? AND account=? AND environment=? AND nonce=?" + SQL_INSERT = "INSERT INTO nonces VALUES (?, ?, ?, ?, ?)" + SQL_UPDATE = "UPDATE nonces SET status=? WHERE host=? AND account=? AND environment=? AND nonce=?" + SQL_DELETE = "DELETE from nonces WHERE host=? AND account=? AND environment=?" + SQL_DELFRO = "DELETE from nonces WHERE host=? AND account=? AND environment=? AND nonce>=?" + SQL_LATEST = "SELECT nonce FROM nonces WHERE host=? AND account=? AND environment=? ORDER BY nonce DESC LIMIT 1" + SQL_DELENV = "DELETE from nonces WHERE host=? AND environment=?" + SQL_ACCNTS = "SELECT DISTINCT account from nonces WHERE host=? AND environment=?" + SQL_DELENT = "DELETE from nonces WHERE host=? AND account=? AND environment=? AND nonce=?" @classmethod def init(cls, host, dbconnection): @@ -29,9 +29,9 @@ def __init__(self, host, dbconnection): """Instantiate an instance.""" self.host = host self.dbconnection = dbconnection - self.insert = normalise(self.SQL_INSERT, dbconnection.type) - self.update = normalise(self.SQL_UPDATE, dbconnection.type) - self.delete = normalise(self.SQL_DELETE, dbconnection.type) + self.sqlins = normalise(self.SQL_INSERT, dbconnection.type) + self.sqlupd = normalise(self.SQL_UPDATE, dbconnection.type) + self.sqldel = normalise(self.SQL_DELETE, dbconnection.type) self.delfro = normalise(self.SQL_DELFRO, dbconnection.type) self.latest = normalise(self.SQL_LATEST, dbconnection.type) self.delenv = normalise(self.SQL_DELENV, dbconnection.type) @@ -61,7 +61,7 @@ def get_next_nonce(self, test, web3, account, environment, persist_nonce=True, l nonce = transaction_count if persist_nonce: nonce = 0 if persisted_nonce is None else persisted_nonce+1 # we have to believe the local store - test.nonce_db.insert(account, test.env, nonce) + self.insert(account, test.env, nonce) if log: test.log.info("Account %s count %d using nonce from persistence as %d", account, transaction_count, nonce) else: if log: test.log.info("Account %s using nonce from transaction count as %d", account, nonce) @@ -69,17 +69,17 @@ def get_next_nonce(self, test, web3, account, environment, persist_nonce=True, l def insert(self, account, environment, nonce, status='PENDING'): """Insert a new nonce into the persistence. """ - self.cursor.execute(self.insert, (self.host, account, environment, nonce, status)) + self.cursor.execute(self.sqlins, (self.host, account, environment, nonce, status)) self.dbconnection.connection.commit() def update(self, account, environment, nonce, status): """Update the status of a transaction for a given nonce into the persistence. """ - self.cursor.execute(self.update, (self.host, status, account, environment, nonce)) + self.cursor.execute(self.sqlupd, (self.host, status, account, environment, nonce)) self.dbconnection.connection.commit() def delete(self, account, environment): """Delete all nonce entries in the persistence for a given account and environment. """ - self.cursor.execute(self.delete, (self.host, account, environment)) + self.cursor.execute(self.sqldel, (self.host, account, environment)) self.dbconnection.connection.commit() def delete_from(self, account, environment, nonce): diff --git a/src/python/ten/test/persistence/rates.py b/src/python/ten/test/persistence/rates.py index b32d205a..8ed5a71a 100644 --- a/src/python/ten/test/persistence/rates.py +++ b/src/python/ten/test/persistence/rates.py @@ -24,9 +24,9 @@ def __init__(self, host, dbconnection): """Instantiate an instance.""" self.host = host self.dbconnection = dbconnection - self.insert = normalise(self.SQL_INSERT, dbconnection.type) - self.delete = normalise(self.SQL_DELETE, dbconnection.type) - self.select = normalise(self.SQL_SELECT, dbconnection.type) + self.sqlins = normalise(self.SQL_INSERT, dbconnection.type) + self.sqldel = normalise(self.SQL_DELETE, dbconnection.type) + self.sqlsel = normalise(self.SQL_SELECT, dbconnection.type) self.cursor = self.dbconnection.connection.cursor() def create(self): @@ -40,17 +40,17 @@ def close(self): def delete_crypto(self, crypto): """Delete all stored rates for a particular crypto.""" - self.cursor.execute(self.delete, (crypto, )) + self.cursor.execute(self.sqldel, (crypto, )) self.dbconnection.connection.commit() def insert_rates(self, crypto, currency, time, rate): """Insert a new rate for a particular crypto and currency.""" - self.cursor.execute(self.insert, (crypto, currency, time, rate)) + self.cursor.execute(self.sqlins, (crypto, currency, time, rate)) self.dbconnection.connection.commit() def get_latest_rate(self, crypto, currency): """Return the latest rate for the crypto and currency.""" - self.cursor.execute(self.select, (crypto, currency)) + self.cursor.execute(self.sqlsel, (crypto, currency)) try: result = self.cursor.fetchone()[1] return float(result) diff --git a/src/python/ten/test/persistence/results.py b/src/python/ten/test/persistence/results.py index 001c7a4c..64469492 100644 --- a/src/python/ten/test/persistence/results.py +++ b/src/python/ten/test/persistence/results.py @@ -25,9 +25,9 @@ def __init__(self, host, dbconnection): """Instantiate an instance.""" self.host = host self.dbconnection = dbconnection - self.insert = normalise(self.SQL_INSERT, dbconnection.type) - self.delete = normalise(self.SQL_DELETE, dbconnection.type) - self.select = normalise(self.SQL_SELECT, dbconnection.type) + self.sqlins = normalise(self.SQL_INSERT, dbconnection.type) + self.sqldel = normalise(self.SQL_DELETE, dbconnection.type) + self.sqldel = normalise(self.SQL_SELECT, dbconnection.type) self.cursor = self.dbconnection.connection.cursor() def create(self): @@ -40,16 +40,16 @@ def close(self): def delete_environment(self, environment): """Delete all stored performance results for a particular environment.""" - self.cursor.execute(self.delete, (self.host, environment)) + self.cursor.execute(self.sqldel, (self.host, environment)) self.dbconnection.connection.commit() def insert_result(self, test, environment, time, result): """Insert a new performance result into the persistence. """ - self.cursor.execute(self.insert, (self.host, test, environment, time, result)) + self.cursor.execute(self.sqlins, (self.host, test, environment, time, result)) self.dbconnection.connection.commit() def get_results(self, test, environment): """Return the performance results for a particular test and environment. """ - self.cursor.execute(self.select, (self.host, test, environment)) + self.cursor.execute(self.sqlsel, (self.host, test, environment)) return self.cursor.fetchall() From 589c0dfec5b6f9a6e58852cd9e16cb1b35a18627 Mon Sep 17 00:00:00 2001 From: Moray Grieve Date: Tue, 7 Jan 2025 19:23:17 +0000 Subject: [PATCH 19/22] Use one connection --- src/python/ten/test/baserunner.py | 6 ++--- src/python/ten/test/basetest.py | 18 ++++++------- src/python/ten/test/persistence/__init__.py | 29 ++++++++++----------- 3 files changed, 24 insertions(+), 29 deletions(-) diff --git a/src/python/ten/test/baserunner.py b/src/python/ten/test/baserunner.py index 8249f570..96daed3e 100644 --- a/src/python/ten/test/baserunner.py +++ b/src/python/ten/test/baserunner.py @@ -71,8 +71,7 @@ def setup(self, runner): else: self.machine_name = socket.gethostname() runner.log.info('Running on local (%s)' % self.machine_name) - dbconnection1, dbconnection2 = get_connection(self.is_cloud_vm, self.user_dir) - dbconnection = dbconnection1 + dbconnection = get_connection(self.is_cloud_vm, self.user_dir) rates_db = RatesPersistence.init(self.machine_name, dbconnection) nonce_db = NoncePersistence.init(self.machine_name, dbconnection) contracts_db = ContractPersistence.init(self.machine_name, dbconnection) @@ -172,8 +171,7 @@ def setup(self, runner): funds_db.close() counts_db.close() results_db.close() - dbconnection1.connection.close() - dbconnection2.connection.close() + dbconnection.connection.close() def run_ganache(self, runner): """Run ganache for use by the tests. """ diff --git a/src/python/ten/test/basetest.py b/src/python/ten/test/basetest.py index f8b458e2..b083a0cf 100644 --- a/src/python/ten/test/basetest.py +++ b/src/python/ten/test/basetest.py @@ -34,14 +34,13 @@ def __init__(self, descriptor, outsubdir, runner): self.log.info('Running test in thread %s', threading.currentThread().getName()) # every test has its own connection to the dbs - self.dbconnection1, self.dbconnection2 = get_connection(is_cloud_vm=runner.ten_runner.is_cloud_vm, db_dir=runner.ten_runner.user_dir) - dbconnection = self.dbconnection1 - self.rates_db = RatesPersistence(runner.ten_runner.machine_name, dbconnection) - self.nonce_db = NoncePersistence(runner.ten_runner.machine_name, dbconnection) - self.contract_db = ContractPersistence(runner.ten_runner.machine_name, dbconnection) - self.funds_db = FundsPersistence(runner.ten_runner.machine_name, dbconnection) - self.counts_db = CountsPersistence(runner.ten_runner.machine_name, dbconnection) - self.results_db = ResultsPersistence(runner.ten_runner.machine_name, dbconnection) + self.dbconnection = get_connection(is_cloud_vm=runner.ten_runner.is_cloud_vm, db_dir=runner.ten_runner.user_dir) + self.rates_db = RatesPersistence(runner.ten_runner.machine_name, self.dbconnection) + self.nonce_db = NoncePersistence(runner.ten_runner.machine_name, self.dbconnection) + self.contract_db = ContractPersistence(runner.ten_runner.machine_name, self.dbconnection) + self.funds_db = FundsPersistence(runner.ten_runner.machine_name, self.dbconnection) + self.counts_db = CountsPersistence(runner.ten_runner.machine_name, self.dbconnection) + self.results_db = ResultsPersistence(runner.ten_runner.machine_name, self.dbconnection) self.addCleanupFunction(self.close_db) # every test has a unique connection for the funded account @@ -80,8 +79,7 @@ def close_db(self): self.funds_db.close() self.counts_db.close() self.results_db.close() - self.dbconnection1.connection.close() - self.dbconnection2.connection.close() + self.dbconnection.connection.close() def drain_ephemeral_pks(self): """Drain any ephemeral accounts of their funds. """ diff --git a/src/python/ten/test/persistence/__init__.py b/src/python/ten/test/persistence/__init__.py index c4dadc35..7024c531 100644 --- a/src/python/ten/test/persistence/__init__.py +++ b/src/python/ten/test/persistence/__init__.py @@ -12,18 +12,17 @@ def normalise(statement, _type): def get_connection(is_cloud_vm, db_dir): - if is_cloud_vm: pass - props = Properties() - config = { - 'host': props.persistence_host(), - 'user': props.persistence_user(), - 'password': props.persistence_password(), - 'database': props.persistence_database(), - 'connection_timeout': 10 - } - connection1 = mysql.connector.connect(**config) - - db = os.path.join(db_dir, 'ten-test.db') - connection2 = sqlite3.connect(db) - - return DBConnection(connection1, 'mysql'), DBConnection(connection2, 'sqlite3') \ No newline at end of file + if is_cloud_vm: + props = Properties() + config = { + 'host': props.persistence_host(), + 'user': props.persistence_user(), + 'password': props.persistence_password(), + 'database': props.persistence_database(), + 'connection_timeout': 10 + } + connection = mysql.connector.connect(**config) + return DBConnection(connection, 'mysql') + else: + connection = sqlite3.connect(os.path.join(db_dir, 'ten-test.db')) + return DBConnection(connection, 'sqlite3') \ No newline at end of file From 6c841d42c39e390e13b5adbc1f578d94f28d87bf Mon Sep 17 00:00:00 2001 From: Moray Grieve Date: Tue, 7 Jan 2025 19:24:42 +0000 Subject: [PATCH 20/22] Use one connection --- .github/workflows/health_check_sepolia.yml | 2 +- .github/workflows/health_check_sepolia_funds.yml | 2 +- .github/workflows/health_check_uat.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/health_check_sepolia.yml b/.github/workflows/health_check_sepolia.yml index 2024fc4a..dba81944 100644 --- a/.github/workflows/health_check_sepolia.yml +++ b/.github/workflows/health_check_sepolia.yml @@ -1,6 +1,6 @@ # Run a set of sanity tests on sepolia-testnet to ensure it is up and running # -name: '[liveness] Check sepolia health' +name: '[inspect] Check sepolia health' run-name: Check sepolia-testnet health on: schedule: diff --git a/.github/workflows/health_check_sepolia_funds.yml b/.github/workflows/health_check_sepolia_funds.yml index 4cc197d1..2c0a7fe5 100644 --- a/.github/workflows/health_check_sepolia_funds.yml +++ b/.github/workflows/health_check_sepolia_funds.yml @@ -1,6 +1,6 @@ # Check key balances on sepolia-testnet # -name: '[liveness] Check sepolia funds' +name: '[inspect] Check sepolia funds' run-name: Check sepolia-testnet funds on: schedule: diff --git a/.github/workflows/health_check_uat.yml b/.github/workflows/health_check_uat.yml index acfe3dfc..c8c615c9 100644 --- a/.github/workflows/health_check_uat.yml +++ b/.github/workflows/health_check_uat.yml @@ -1,6 +1,6 @@ # Run a set of sanity tests on uat-testnet to ensure it is up and running # -name: '[liveness] Check uat health' +name: '[inspect] Check uat health' run-name: Check uat-testnet health on: schedule: From 0eaad91104751048a10676c17c2577122a70e25c Mon Sep 17 00:00:00 2001 From: Moray Grieve Date: Tue, 7 Jan 2025 20:33:29 +0000 Subject: [PATCH 21/22] Updates --- src/python/ten/test/persistence/contract.py | 38 ++++++++++----------- src/python/ten/test/persistence/counts.py | 21 ++++++------ src/python/ten/test/persistence/funds.py | 2 +- src/python/ten/test/persistence/nonce.py | 35 +++++++++---------- src/python/ten/test/persistence/rates.py | 4 +-- 5 files changed, 48 insertions(+), 52 deletions(-) diff --git a/src/python/ten/test/persistence/contract.py b/src/python/ten/test/persistence/contract.py index 7818a91a..3e0f614e 100644 --- a/src/python/ten/test/persistence/contract.py +++ b/src/python/ten/test/persistence/contract.py @@ -4,27 +4,25 @@ class ContractPersistence: """Abstracts the persistence of contract addresses into a local database. """ - SQL_CREATE = "CREATE TABLE IF NOT EXISTS contracts " \ - "(host VARCHAR(64), " \ - "name VARCHAR(64), " \ + SQL_CREATE = "CREATE TABLE IF NOT EXISTS contract_details " \ + "(name VARCHAR(64), " \ "environment VARCHAR(64), " \ "address VARCHAR(64), " \ "abi MEDIUMTEXT, " \ - "PRIMARY KEY (host, name, environment))" - SQL_INSERT = "INSERT OR REPLACE INTO contracts VALUES (?, ?, ?, ?, ?)" - SQL_DELETE = "DELETE from contracts WHERE host=? AND environment=?" - SQL_SELECT = "SELECT address, abi FROM contracts WHERE host=? AND name=? AND environment=? ORDER BY name DESC LIMIT 1" + "PRIMARY KEY (name, environment))" + SQL_INSERT = "INSERT OR REPLACE INTO contract_details VALUES (?, ?, ?, ?)" + SQL_DELETE = "DELETE from contract_details WHERE environment=?" + SQL_SELECT = "SELECT address, abi FROM contract_details WHERE name=? AND environment=? ORDER BY name DESC LIMIT 1" - SQL_CRTPRM = "CREATE TABLE IF NOT EXISTS params " \ - "(host VARCHAR(64), " \ - "address VARCHAR(64), " \ + SQL_CRTPRM = "CREATE TABLE IF NOT EXISTS contract_params " \ + "(address VARCHAR(64), " \ "environment VARCHAR(64), " \ "param_key VARCHAR(64), " \ "param_val VARCHAR(64), " \ - "PRIMARY KEY (host, address, environment, param_key))" - SQL_INSPRM = "INSERT OR REPLACE INTO params VALUES (?, ?, ?, ?, ?)" - SQL_DELPRM = "DELETE from params WHERE host=? AND environment=?" - SQL_SELPRM = "SELECT param_val FROM params WHERE host=? AND address=? AND environment=? AND param_key=? " \ + "PRIMARY KEY (address, environment, param_key))" + SQL_INSPRM = "INSERT OR REPLACE INTO contract_params VALUES (?, ?, ?, ?)" + SQL_DELPRM = "DELETE from contract_params WHERE environment=?" + SQL_SELPRM = "SELECT param_val FROM contract_params WHERE address=? AND environment=? AND param_key=? " \ "ORDER BY address DESC LIMIT 1" @classmethod @@ -56,30 +54,30 @@ def close(self): def delete_environment(self, environment): """Delete all stored contract details for a particular environment.""" - self.cursor.execute(self.sqldel, (self.host, environment)) - self.cursor.execute(self.delprm, (self.host, environment)) + self.cursor.execute(self.sqldel, (environment, )) + self.cursor.execute(self.delprm, (environment, )) self.dbconnection.connection.commit() def insert_contract(self, name, environment, address, abi): """Insert a new contract into the persistence. """ - self.cursor.execute(self.sqlins, (self.host, name, environment, address, abi)) + self.cursor.execute(self.sqlins, (name, environment, address, abi)) self.dbconnection.connection.commit() def get_contract(self, name, environment): """Return the address and abi for a particular deployed contract. """ - self.cursor.execute(self.sqlsel, (self.host, name, environment)) + self.cursor.execute(self.sqlsel, (name, environment)) cursor = self.cursor.fetchall() if len(cursor) > 0: return cursor[0][0], cursor[0][1] return None, None def insert_param(self, address, environment, key, value): """Insert a parameter for a named contract. """ - self.cursor.execute(self.insprm, (self.host, address, environment, key, value)) + self.cursor.execute(self.insprm, (address, environment, key, value)) self.dbconnection.connection.commit() def get_param(self, address, environment, key): """Return the address and abi for a particular deployed contract. """ - self.cursor.execute(self.selprm, (self.host, address, environment, key)) + self.cursor.execute(self.selprm, (address, environment, key)) cursor = self.cursor.fetchall() if len(cursor) > 0: return cursor[0][0] return None \ No newline at end of file diff --git a/src/python/ten/test/persistence/counts.py b/src/python/ten/test/persistence/counts.py index ee0f7dda..905c1a56 100644 --- a/src/python/ten/test/persistence/counts.py +++ b/src/python/ten/test/persistence/counts.py @@ -5,17 +5,16 @@ class CountsPersistence: """Abstracts the persistence of transaction counts across accounts into a local database. """ SQL_CREATE = "CREATE TABLE IF NOT EXISTS counts " \ - "(host VARCHAR(64), " \ - "name VARCHAR(64), " \ + "(name VARCHAR(64), " \ "address VARCHAR(64), " \ "environment VARCHAR(64), " \ "time INTEGER, " \ "count INTEGER, " \ - "PRIMARY KEY (host, name, environment, time))" - SQL_INSERT = "INSERT INTO counts VALUES (?, ?, ?, ?, ?, ?)" - SQL_DELETE = "DELETE from counts WHERE host=? AND environment=?" - SQL_SELTHR = "SELECT time, count FROM counts WHERE host=? AND name=? and environment=? ORDER BY time DESC LIMIT 3" - SQL_SELHOR = "SELECT time, count FROM counts WHERE host=? AND name=? and environment=? and time >= ? ORDER BY time DESC" + "PRIMARY KEY (name, environment, time))" + SQL_INSERT = "INSERT INTO counts VALUES (?, ?, ?, ?, ?)" + SQL_DELETE = "DELETE from counts WHERE environment=?" + SQL_SELTHR = "SELECT time, count FROM counts WHERE name=? and environment=? ORDER BY time DESC LIMIT 3" + SQL_SELHOR = "SELECT time, count FROM counts WHERE name=? and environment=? and time >= ? ORDER BY time DESC" @classmethod def init(cls, host, dbconnection): @@ -43,20 +42,20 @@ def close(self): def delete_environment(self, environment): """Delete all stored contract details for a particular environment.""" - self.cursor.execute(self.sqldel, (self.host, environment)) + self.cursor.execute(self.sqldel, (environment,)) self.dbconnection.connection.commit() def insert_count(self, name, address, environment, time, count): """Insert a new counts entry for a particular logical account.""" - self.cursor.execute(self.sqlins, (self.host, name, address, environment, time, str(count))) + self.cursor.execute(self.sqlins, (name, address, environment, time, str(count))) self.dbconnection.connection.commit() def get_last_three_counts(self, name, environment): """Return the transaction count with time for a particular logical account.""" - self.cursor.execute(self.sqlthr, (self.host, name, environment)) + self.cursor.execute(self.sqlthr, (name, environment)) return self.cursor.fetchall() def get_last_hour(self, name, environment, time): """Return the transaction count with time for a particular logical account.""" - self.cursor.execute(self.sqlhor, (self.host, name, environment, time)) + self.cursor.execute(self.sqlhor, (name, environment, time)) return self.cursor.fetchall() \ No newline at end of file diff --git a/src/python/ten/test/persistence/funds.py b/src/python/ten/test/persistence/funds.py index 5d67ec46..27d50a56 100644 --- a/src/python/ten/test/persistence/funds.py +++ b/src/python/ten/test/persistence/funds.py @@ -9,7 +9,7 @@ class FundsPersistence: "address VARCHAR(64), " \ "environment VARCHAR(64), " \ "time INTEGER, " \ - "balance INTEGER, " \ + "balance VARCHAR(64), " \ "PRIMARY KEY (name, environment, time))" SQL_INSERT = "INSERT INTO funds VALUES (?, ?, ?, ?, ?)" SQL_DELETE = "DELETE from funds WHERE environment=?" diff --git a/src/python/ten/test/persistence/nonce.py b/src/python/ten/test/persistence/nonce.py index f1394b74..e0a88a03 100644 --- a/src/python/ten/test/persistence/nonce.py +++ b/src/python/ten/test/persistence/nonce.py @@ -5,19 +5,18 @@ class NoncePersistence: """Abstracts the persistence of nonces into a local database. """ SQL_CREATE = "CREATE TABLE IF NOT EXISTS nonces " \ - "(host VARCHAR(64), " \ - "account VARCHAR(64), " \ + "(account VARCHAR(64), " \ "environment VARCHAR(64), " \ "nonce INTEGER, " \ "status VARCHAR(64))" - SQL_INSERT = "INSERT INTO nonces VALUES (?, ?, ?, ?, ?)" - SQL_UPDATE = "UPDATE nonces SET status=? WHERE host=? AND account=? AND environment=? AND nonce=?" - SQL_DELETE = "DELETE from nonces WHERE host=? AND account=? AND environment=?" - SQL_DELFRO = "DELETE from nonces WHERE host=? AND account=? AND environment=? AND nonce>=?" - SQL_LATEST = "SELECT nonce FROM nonces WHERE host=? AND account=? AND environment=? ORDER BY nonce DESC LIMIT 1" - SQL_DELENV = "DELETE from nonces WHERE host=? AND environment=?" - SQL_ACCNTS = "SELECT DISTINCT account from nonces WHERE host=? AND environment=?" - SQL_DELENT = "DELETE from nonces WHERE host=? AND account=? AND environment=? AND nonce=?" + SQL_INSERT = "INSERT INTO nonces VALUES ?, ?, ?, ?)" + SQL_UPDATE = "UPDATE nonces SET status=? WHERE account=? AND environment=? AND nonce=?" + SQL_DELETE = "DELETE from nonces WHERE account=? AND environment=?" + SQL_DELFRO = "DELETE from nonces WHERE account=? AND environment=? AND nonce>=?" + SQL_LATEST = "SELECT nonce FROM nonces WHERE account=? AND environment=? ORDER BY nonce DESC LIMIT 1" + SQL_DELENV = "DELETE from nonces WHERE environment=?" + SQL_ACCNTS = "SELECT DISTINCT account from nonces WHERE environment=?" + SQL_DELENT = "DELETE from nonces WHERE account=? AND environment=? AND nonce=?" @classmethod def init(cls, host, dbconnection): @@ -69,42 +68,42 @@ def get_next_nonce(self, test, web3, account, environment, persist_nonce=True, l def insert(self, account, environment, nonce, status='PENDING'): """Insert a new nonce into the persistence. """ - self.cursor.execute(self.sqlins, (self.host, account, environment, nonce, status)) + self.cursor.execute(self.sqlins, (account, environment, nonce, status)) self.dbconnection.connection.commit() def update(self, account, environment, nonce, status): """Update the status of a transaction for a given nonce into the persistence. """ - self.cursor.execute(self.sqlupd, (self.host, status, account, environment, nonce)) + self.cursor.execute(self.sqlupd, (status, account, environment, nonce)) self.dbconnection.connection.commit() def delete(self, account, environment): """Delete all nonce entries in the persistence for a given account and environment. """ - self.cursor.execute(self.sqldel, (self.host, account, environment)) + self.cursor.execute(self.sqldel, (account, environment)) self.dbconnection.connection.commit() def delete_from(self, account, environment, nonce): """Delete all nonce entries in the persistence for a given account and environment. """ - self.cursor.execute(self.delfro, (self.host, account, environment, nonce)) + self.cursor.execute(self.delfro, (account, environment, nonce)) self.dbconnection.connection.commit() def delete_environment(self, environment): """Delete all nonce entries for all accounts for a given environment. """ - self.cursor.execute(self.delenv, (self.host, environment, )) + self.cursor.execute(self.delenv, (environment, )) self.dbconnection.connection.commit() def delete_entries(self, account, environment, nonce): """Delete all nonce entries in the persistence for a given account and environment and nonce. """ - self.cursor.execute(self.delent, (self.host, account, environment, nonce)) + self.cursor.execute(self.delent, (account, environment, nonce)) self.dbconnection.connection.commit() def get_accounts(self, environment): """Return a list of all accounts with persisted values for a given environment. """ - self.cursor.execute(self.accnts, (self.host, environment)) + self.cursor.execute(self.accnts, (environment)) return self.cursor.fetchall() def get_latest_nonce(self, account, environment): """Get the latest nonce for a given account and environment. """ - self.cursor.execute(self.latest, (self.host, account, environment)) + self.cursor.execute(self.latest, (account, environment)) try: result = self.cursor.fetchone()[0] return int(result) diff --git a/src/python/ten/test/persistence/rates.py b/src/python/ten/test/persistence/rates.py index 8ed5a71a..7875658a 100644 --- a/src/python/ten/test/persistence/rates.py +++ b/src/python/ten/test/persistence/rates.py @@ -8,7 +8,7 @@ class RatesPersistence: "(crypto VARCHAR(3), " \ "currency VARCHAR(3), " \ "time INTEGER, " \ - "rate REAL, " \ + "rate VARCHAR(64), " \ "PRIMARY KEY (crypto, currency, time))" SQL_INSERT = "INSERT INTO rates VALUES (?, ?, ?, ?);" SQL_DELETE = "DELETE from rates WHERE crypto=?;" @@ -45,7 +45,7 @@ def delete_crypto(self, crypto): def insert_rates(self, crypto, currency, time, rate): """Insert a new rate for a particular crypto and currency.""" - self.cursor.execute(self.sqlins, (crypto, currency, time, rate)) + self.cursor.execute(self.sqlins, (crypto, currency, time, str(rate))) self.dbconnection.connection.commit() def get_latest_rate(self, crypto, currency): From 8bae841244c1d0c0087ee73a04be9ed0024b5820 Mon Sep 17 00:00:00 2001 From: Moray Grieve Date: Wed, 8 Jan 2025 10:28:12 +0000 Subject: [PATCH 22/22] Error in insert --- src/python/ten/test/persistence/nonce.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/ten/test/persistence/nonce.py b/src/python/ten/test/persistence/nonce.py index e0a88a03..78abe60a 100644 --- a/src/python/ten/test/persistence/nonce.py +++ b/src/python/ten/test/persistence/nonce.py @@ -9,7 +9,7 @@ class NoncePersistence: "environment VARCHAR(64), " \ "nonce INTEGER, " \ "status VARCHAR(64))" - SQL_INSERT = "INSERT INTO nonces VALUES ?, ?, ?, ?)" + SQL_INSERT = "INSERT INTO nonces VALUES (?, ?, ?, ?)" SQL_UPDATE = "UPDATE nonces SET status=? WHERE account=? AND environment=? AND nonce=?" SQL_DELETE = "DELETE from nonces WHERE account=? AND environment=?" SQL_DELFRO = "DELETE from nonces WHERE account=? AND environment=? AND nonce>=?"