diff --git a/.default.properties b/.default.properties index 9b67f013..526aa05d 100644 --- a/.default.properties +++ b/.default.properties @@ -20,6 +20,12 @@ node_path = /usr/lib/node_modules:/usr/local/lib/node_modules npm=/usr/bin/npm npx = /usr/bin/npx +[persistence.db] +host = +user = +database = +password = + # Constants relating to the different environments the tests can run against namely Ten, Sepolia, Arbitrum and Ganache. # The account PKs used are randomly generated and do not relate to any real values. To use real values from a metamask # wallet, override in your ~/.tentest/user.properties file. diff --git a/.github/workflows/health_check_sepolia.yml b/.github/workflows/health_check_sepolia.yml index e870771b..dba81944 100644 --- a/.github/workflows/health_check_sepolia.yml +++ b/.github/workflows/health_check_sepolia.yml @@ -1,6 +1,6 @@ # Run a set of sanity tests on sepolia-testnet to ensure it is up and running # -name: '[health] Check sepolia health' +name: '[inspect] Check sepolia health' run-name: Check sepolia-testnet health on: schedule: diff --git a/.github/workflows/health_check_sepolia_funds.yml b/.github/workflows/health_check_sepolia_funds.yml index 67dbc22d..2c0a7fe5 100644 --- a/.github/workflows/health_check_sepolia_funds.yml +++ b/.github/workflows/health_check_sepolia_funds.yml @@ -1,6 +1,6 @@ # Check key balances on sepolia-testnet # -name: '[health] Check sepolia funds' +name: '[inspect] Check sepolia funds' run-name: Check sepolia-testnet funds on: schedule: diff --git a/.github/workflows/health_check_uat.yml b/.github/workflows/health_check_uat.yml index f3dae920..c8c615c9 100644 --- a/.github/workflows/health_check_uat.yml +++ b/.github/workflows/health_check_uat.yml @@ -1,6 +1,6 @@ # Run a set of sanity tests on uat-testnet to ensure it is up and running # -name: '[health] Check uat health' +name: '[inspect] Check uat health' run-name: Check uat-testnet health on: schedule: diff --git a/README.md b/README.md index 81d43f8e..8b27b0c4 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,7 @@ pip3 install pysys==1.6.1 pip3 install solc-select==1.0.4 pip3 install py-solc-x==2.0.2 pip3 install numpy==1.26.4 +pip3 install mysql-connector-python==9.0.0 solc-select install 0.8.15 solc-select use 0.8.15 diff --git a/src/python/ten/test/baserunner.py b/src/python/ten/test/baserunner.py index aeb7ac2a..96daed3e 100644 --- a/src/python/ten/test/baserunner.py +++ b/src/python/ten/test/baserunner.py @@ -1,4 +1,4 @@ -import os, shutil, sys, json, requests, time +import os, shutil, sys, json, requests, time, socket from collections import OrderedDict from web3 import Web3 from pathlib import Path @@ -15,6 +15,8 @@ from ten.test.persistence.results import ResultsPersistence from ten.test.persistence.contract import ContractPersistence from ten.test.utils.properties import Properties +from ten.test.persistence import get_connection +from ten.test.utils.cloud import is_cloud_vm class TenRunnerPlugin(): @@ -32,6 +34,10 @@ def __init__(self): self.env = None self.NODE_HOST = None self.balances = OrderedDict() + self.cloud_metadata = is_cloud_vm() + self.is_cloud_vm = self.cloud_metadata is not None + self.user_dir = os.path.join(str(Path.home()), '.tentest') + if not os.path.exists(self.user_dir): os.makedirs(self.user_dir) def setup(self, runner): """Set up a runner plugin to start any processes required to execute the tests. """ @@ -58,21 +64,20 @@ def setup(self, runner): if os.path.exists(runner.output): shutil.rmtree(runner.output) os.makedirs(runner.output) - # create the nonce db if it does not already exist, clean it out if using ganache - db_dir = os.path.join(str(Path.home()), '.tentest') - if not os.path.exists(db_dir): os.makedirs(db_dir) - rates_db = RatesPersistence(db_dir) - rates_db.create() - nonce_db = NoncePersistence(db_dir) - nonce_db.create() - contracts_db = ContractPersistence(db_dir) - contracts_db.create() - funds_db = FundsPersistence(db_dir) - funds_db.create() - counts_db = CountsPersistence(db_dir) - counts_db.create() - results_db = ResultsPersistence(db_dir) - results_db.create() + # set up the persistence layer based on if we are running in the cloud, or locally + if self.is_cloud_vm: + self.machine_name = self.cloud_metadata['compute']['name'] + runner.log.info('Running on azure (%s, %s)' % (self.machine_name, self.cloud_metadata['compute']['location'])) + else: + self.machine_name = socket.gethostname() + runner.log.info('Running on local (%s)' % self.machine_name) + dbconnection = get_connection(self.is_cloud_vm, self.user_dir) + rates_db = RatesPersistence.init(self.machine_name, dbconnection) + nonce_db = NoncePersistence.init(self.machine_name, dbconnection) + contracts_db = ContractPersistence.init(self.machine_name, dbconnection) + funds_db = FundsPersistence.init(self.machine_name, dbconnection) + counts_db = CountsPersistence.init(self.machine_name, dbconnection) + results_db = ResultsPersistence.init(self.machine_name, dbconnection) eth_price = self.get_eth_price() if eth_price is not None: @@ -160,9 +165,13 @@ def setup(self, runner): runner.cleanup() sys.exit(1) + rates_db.close() nonce_db.close() contracts_db.close() funds_db.close() + counts_db.close() + results_db.close() + dbconnection.connection.close() def run_ganache(self, runner): """Run ganache for use by the tests. """ @@ -306,6 +315,7 @@ def __set_contract_addresses(self, runner): contracts = config["PublicSystemContracts"] Properties.L2PublicCallbacks = self.__get_contract(contracts, "PublicCallbacks") elif 'error' in response.json(): + runner.log.warn('Error getting contract address from ten_config') runner.log.error(response.json()['error']['message']) def __get_contract(self, contracts, key): diff --git a/src/python/ten/test/basetest.py b/src/python/ten/test/basetest.py index e7e96316..b083a0cf 100644 --- a/src/python/ten/test/basetest.py +++ b/src/python/ten/test/basetest.py @@ -1,7 +1,6 @@ import os, copy, sys, json, secrets, re import threading, requests from web3 import Web3 -from pathlib import Path from pysys.basetest import BaseTest from pysys.constants import PROJECT, BACKGROUND, FAILED from pysys.constants import LOG_TRACEBACK @@ -19,6 +18,7 @@ from ten.test.networks.arbitrum import ArbitrumSepolia from ten.test.networks.sepolia import Sepolia from ten.test.networks.ten import Ten, TenL1Geth, TenL1Sepolia +from ten.test.persistence import get_connection class GenericNetworkTest(BaseTest): @@ -33,14 +33,14 @@ def __init__(self, descriptor, outsubdir, runner): self.block_time = Properties().block_time_secs(self.env) self.log.info('Running test in thread %s', threading.currentThread().getName()) - # every test has its own connection to the nonce and contract db - db_dir = os.path.join(str(Path.home()), '.tentest') - self.rates_db = RatesPersistence(db_dir) - self.nonce_db = NoncePersistence(db_dir) - self.contract_db = ContractPersistence(db_dir) - self.funds_db = FundsPersistence(db_dir) - self.counts_db = CountsPersistence(db_dir) - self.results_db = ResultsPersistence(db_dir) + # every test has its own connection to the dbs + self.dbconnection = get_connection(is_cloud_vm=runner.ten_runner.is_cloud_vm, db_dir=runner.ten_runner.user_dir) + self.rates_db = RatesPersistence(runner.ten_runner.machine_name, self.dbconnection) + self.nonce_db = NoncePersistence(runner.ten_runner.machine_name, self.dbconnection) + self.contract_db = ContractPersistence(runner.ten_runner.machine_name, self.dbconnection) + self.funds_db = FundsPersistence(runner.ten_runner.machine_name, self.dbconnection) + self.counts_db = CountsPersistence(runner.ten_runner.machine_name, self.dbconnection) + self.results_db = ResultsPersistence(runner.ten_runner.machine_name, self.dbconnection) self.addCleanupFunction(self.close_db) # every test has a unique connection for the funded account @@ -72,9 +72,14 @@ def __test_cost(self): self.log.info(" %s: %s%.3f USD", 'Test cost', sign, self.eth_price*change, extra=BaseLogFormatter.tag(LOG_TRACEBACK, 0)) def close_db(self): - """Close the connection to the nonce database on completion. """ + """Close the connection to the databases on completion. """ + self.rates_db.close() self.nonce_db.close() self.contract_db.close() + self.funds_db.close() + self.counts_db.close() + self.results_db.close() + self.dbconnection.connection.close() def drain_ephemeral_pks(self): """Drain any ephemeral accounts of their funds. """ diff --git a/src/python/ten/test/persistence/__init__.py b/src/python/ten/test/persistence/__init__.py index 77f8f005..7024c531 100644 --- a/src/python/ten/test/persistence/__init__.py +++ b/src/python/ten/test/persistence/__init__.py @@ -1,2 +1,28 @@ """Package of persistence for transaction counts. """ +import os, sqlite3, mysql.connector +from collections import namedtuple +from ten.test.utils.properties import Properties + +DBConnection = namedtuple('DBConnection', 'connection type') + + +def normalise(statement, _type): + return statement if _type != 'mysql' else statement.replace('?', '%s') + + +def get_connection(is_cloud_vm, db_dir): + if is_cloud_vm: + props = Properties() + config = { + 'host': props.persistence_host(), + 'user': props.persistence_user(), + 'password': props.persistence_password(), + 'database': props.persistence_database(), + 'connection_timeout': 10 + } + connection = mysql.connector.connect(**config) + return DBConnection(connection, 'mysql') + else: + connection = sqlite3.connect(os.path.join(db_dir, 'ten-test.db')) + return DBConnection(connection, 'sqlite3') \ No newline at end of file diff --git a/src/python/ten/test/persistence/contract.py b/src/python/ten/test/persistence/contract.py index 53783274..3e0f614e 100644 --- a/src/python/ten/test/persistence/contract.py +++ b/src/python/ten/test/persistence/contract.py @@ -1,65 +1,83 @@ -import sqlite3, os +from ten.test.persistence import normalise class ContractPersistence: """Abstracts the persistence of contract addresses into a local database. """ - SQL_CREATE = "CREATE TABLE IF NOT EXISTS contracts " \ - "(name TEXT, environment TEXT, address INTEGER, abi STRING, " \ + SQL_CREATE = "CREATE TABLE IF NOT EXISTS contract_details " \ + "(name VARCHAR(64), " \ + "environment VARCHAR(64), " \ + "address VARCHAR(64), " \ + "abi MEDIUMTEXT, " \ "PRIMARY KEY (name, environment))" - SQL_INSERT = "INSERT OR REPLACE INTO contracts VALUES (?, ?, ?, ?)" - SQL_DELETE = "DELETE from contracts WHERE environment=?" - SQL_SELECT = "SELECT address, abi FROM contracts WHERE name=? AND environment=? ORDER BY name DESC LIMIT 1" + SQL_INSERT = "INSERT OR REPLACE INTO contract_details VALUES (?, ?, ?, ?)" + SQL_DELETE = "DELETE from contract_details WHERE environment=?" + SQL_SELECT = "SELECT address, abi FROM contract_details WHERE name=? AND environment=? ORDER BY name DESC LIMIT 1" - SQL_CRT_PARAMS = "CREATE TABLE IF NOT EXISTS params " \ - "(address INTEGER, environment TEXT, key STRING, value STRING, " \ - "PRIMARY KEY (address, environment, key))" - SQL_INS_PARAMS = "INSERT OR REPLACE INTO params VALUES (?, ?, ?, ?)" - SQL_DEL_PARAMS = "DELETE from params WHERE environment=?" - SQL_SEL_PARAMS = "SELECT value FROM params WHERE address=? AND environment=? AND key=? " \ - "ORDER BY address DESC LIMIT 1" + SQL_CRTPRM = "CREATE TABLE IF NOT EXISTS contract_params " \ + "(address VARCHAR(64), " \ + "environment VARCHAR(64), " \ + "param_key VARCHAR(64), " \ + "param_val VARCHAR(64), " \ + "PRIMARY KEY (address, environment, param_key))" + SQL_INSPRM = "INSERT OR REPLACE INTO contract_params VALUES (?, ?, ?, ?)" + SQL_DELPRM = "DELETE from contract_params WHERE environment=?" + SQL_SELPRM = "SELECT param_val FROM contract_params WHERE address=? AND environment=? AND param_key=? " \ + "ORDER BY address DESC LIMIT 1" - def __init__(self, db_dir): + @classmethod + def init(cls, host, dbconnection): + instance = ContractPersistence(host, dbconnection) + instance.create() + return instance + + def __init__(self, host, dbconnection): """Instantiate an instance.""" - self.db = os.path.join(db_dir, 'contracts.db') - self.connection = sqlite3.connect(self.db) - self.cursor = self.connection.cursor() + self.host = host + self.dbconnection = dbconnection + self.sqlins = normalise(self.SQL_INSERT, dbconnection.type) + self.sqldel = normalise(self.SQL_DELETE, dbconnection.type) + self.sqlsel = normalise(self.SQL_SELECT, dbconnection.type) + self.insprm = normalise(self.SQL_INSPRM, dbconnection.type) + self.delprm = normalise(self.SQL_DELPRM, dbconnection.type) + self.selprm = normalise(self.SQL_SELPRM, dbconnection.type) + self.cursor = self.dbconnection.connection.cursor() def create(self): """Create the cursor to the underlying persistence.""" self.cursor.execute(self.SQL_CREATE) - self.cursor.execute(self.SQL_CRT_PARAMS) + self.cursor.execute(self.SQL_CRTPRM) def close(self): """Close the connection to the underlying persistence.""" - self.connection.close() + self.cursor.close() def delete_environment(self, environment): """Delete all stored contract details for a particular environment.""" - self.cursor.execute(self.SQL_DELETE, (environment, )) - self.cursor.execute(self.SQL_DEL_PARAMS, (environment, )) - self.connection.commit() + self.cursor.execute(self.sqldel, (environment, )) + self.cursor.execute(self.delprm, (environment, )) + self.dbconnection.connection.commit() def insert_contract(self, name, environment, address, abi): """Insert a new contract into the persistence. """ - self.cursor.execute(self.SQL_INSERT, (name, environment, address, abi)) - self.connection.commit() + self.cursor.execute(self.sqlins, (name, environment, address, abi)) + self.dbconnection.connection.commit() def get_contract(self, name, environment): """Return the address and abi for a particular deployed contract. """ - self.cursor.execute(self.SQL_SELECT, (name, environment)) + self.cursor.execute(self.sqlsel, (name, environment)) cursor = self.cursor.fetchall() if len(cursor) > 0: return cursor[0][0], cursor[0][1] return None, None def insert_param(self, address, environment, key, value): """Insert a parameter for a named contract. """ - self.cursor.execute(self.SQL_INS_PARAMS, (address, environment, key, value)) - self.connection.commit() + self.cursor.execute(self.insprm, (address, environment, key, value)) + self.dbconnection.connection.commit() def get_param(self, address, environment, key): """Return the address and abi for a particular deployed contract. """ - self.cursor.execute(self.SQL_SEL_PARAMS, (address, environment, key)) + self.cursor.execute(self.selprm, (address, environment, key)) cursor = self.cursor.fetchall() if len(cursor) > 0: return cursor[0][0] return None \ No newline at end of file diff --git a/src/python/ten/test/persistence/counts.py b/src/python/ten/test/persistence/counts.py index 3e749d74..905c1a56 100644 --- a/src/python/ten/test/persistence/counts.py +++ b/src/python/ten/test/persistence/counts.py @@ -1,22 +1,36 @@ -import sqlite3, os +from ten.test.persistence import normalise class CountsPersistence: """Abstracts the persistence of transaction counts across accounts into a local database. """ SQL_CREATE = "CREATE TABLE IF NOT EXISTS counts " \ - "(name TEXT, address INTEGER, environment TEXT, time INTEGER, count TEXT, " \ + "(name VARCHAR(64), " \ + "address VARCHAR(64), " \ + "environment VARCHAR(64), " \ + "time INTEGER, " \ + "count INTEGER, " \ "PRIMARY KEY (name, environment, time))" SQL_INSERT = "INSERT INTO counts VALUES (?, ?, ?, ?, ?)" SQL_DELETE = "DELETE from counts WHERE environment=?" - SQL_SELECT_THREE = "SELECT time, count FROM counts WHERE name=? and environment=? ORDER BY time DESC LIMIT 3" - SQL_SELECT_HOUR = "SELECT time, count FROM counts WHERE name=? and environment=? and time >= ? ORDER BY time DESC" + SQL_SELTHR = "SELECT time, count FROM counts WHERE name=? and environment=? ORDER BY time DESC LIMIT 3" + SQL_SELHOR = "SELECT time, count FROM counts WHERE name=? and environment=? and time >= ? ORDER BY time DESC" - def __init__(self, db_dir): + @classmethod + def init(cls, host, dbconnection): + instance = CountsPersistence(host, dbconnection) + instance.create() + return instance + + def __init__(self, host, dbconnection): """Instantiate an instance.""" - self.db = os.path.join(db_dir, 'counts.db') - self.connection = sqlite3.connect(self.db) - self.cursor = self.connection.cursor() + self.host = host + self.dbconnection = dbconnection + self.sqlins = normalise(self.SQL_INSERT, dbconnection.type) + self.sqldel = normalise(self.SQL_DELETE, dbconnection.type) + self.sqlthr = normalise(self.SQL_SELTHR, dbconnection.type) + self.sqlhor = normalise(self.SQL_SELHOR, dbconnection.type) + self.cursor = self.dbconnection.connection.cursor() def create(self): """Create the cursor to the underlying persistence.""" @@ -24,24 +38,24 @@ def create(self): def close(self): """Close the connection to the underlying persistence.""" - self.connection.close() + self.cursor.close() def delete_environment(self, environment): """Delete all stored contract details for a particular environment.""" - self.cursor.execute(self.SQL_DELETE, (environment, )) - self.connection.commit() + self.cursor.execute(self.sqldel, (environment,)) + self.dbconnection.connection.commit() def insert_count(self, name, address, environment, time, count): """Insert a new counts entry for a particular logical account.""" - self.cursor.execute(self.SQL_INSERT, (name, address, environment, time, str(count))) - self.connection.commit() + self.cursor.execute(self.sqlins, (name, address, environment, time, str(count))) + self.dbconnection.connection.commit() def get_last_three_counts(self, name, environment): """Return the transaction count with time for a particular logical account.""" - self.cursor.execute(self.SQL_SELECT_THREE, (name, environment)) + self.cursor.execute(self.sqlthr, (name, environment)) return self.cursor.fetchall() def get_last_hour(self, name, environment, time): """Return the transaction count with time for a particular logical account.""" - self.cursor.execute(self.SQL_SELECT_HOUR, (name, environment, time)) + self.cursor.execute(self.sqlhor, (name, environment, time)) return self.cursor.fetchall() \ No newline at end of file diff --git a/src/python/ten/test/persistence/funds.py b/src/python/ten/test/persistence/funds.py index 80f28133..27d50a56 100644 --- a/src/python/ten/test/persistence/funds.py +++ b/src/python/ten/test/persistence/funds.py @@ -1,21 +1,34 @@ -import sqlite3, os +from ten.test.persistence import normalise class FundsPersistence: """Abstracts the persistence of funds across accounts into a local database. """ SQL_CREATE = "CREATE TABLE IF NOT EXISTS funds " \ - "(name TEXT, address INTEGER, environment TEXT, time INTEGER, balance TEXT, " \ + "(name VARCHAR(64), " \ + "address VARCHAR(64), " \ + "environment VARCHAR(64), " \ + "time INTEGER, " \ + "balance VARCHAR(64), " \ "PRIMARY KEY (name, environment, time))" SQL_INSERT = "INSERT INTO funds VALUES (?, ?, ?, ?, ?)" SQL_DELETE = "DELETE from funds WHERE environment=?" SQL_SELECT = "SELECT time, balance FROM funds WHERE name=? and environment=? ORDER BY time DESC" - def __init__(self, db_dir): + @classmethod + def init(cls, host, dbconnection): + instance = FundsPersistence(host, dbconnection) + instance.create() + return instance + + def __init__(self, host, dbconnection): """Instantiate an instance.""" - self.db = os.path.join(db_dir, 'funds.db') - self.connection = sqlite3.connect(self.db) - self.cursor = self.connection.cursor() + self.host = host + self.dbconnection = dbconnection + self.sqlins = normalise(self.SQL_INSERT, dbconnection.type) + self.sqldel = normalise(self.SQL_DELETE, dbconnection.type) + self.sqlsel = normalise(self.SQL_SELECT, dbconnection.type) + self.cursor = self.dbconnection.connection.cursor() def create(self): """Create the cursor to the underlying persistence.""" @@ -23,20 +36,20 @@ def create(self): def close(self): """Close the connection to the underlying persistence.""" - self.connection.close() + self.cursor.close() def delete_environment(self, environment): """Delete all stored details for a particular environment.""" - self.cursor.execute(self.SQL_DELETE, (environment, )) - self.connection.commit() + self.cursor.execute(self.sqldel, (environment, )) + self.dbconnection.connection.commit() def insert_funds(self, name, address, environment, time, balance): """Insert a new funds entry for a particular logical account.""" - self.cursor.execute(self.SQL_INSERT, (name, address, environment, time, str(balance))) - self.connection.commit() + self.cursor.execute(self.sqlins, (name, address, environment, time, str(balance))) + self.dbconnection.connection.commit() def get_funds(self, name, environment): """Return the funds with time for a particular logical account.""" - self.cursor.execute(self.SQL_SELECT, (name, environment)) + self.cursor.execute(self.sqlsel, (name, environment)) return self.cursor.fetchall() diff --git a/src/python/ten/test/persistence/nonce.py b/src/python/ten/test/persistence/nonce.py index c65fdc5a..78abe60a 100644 --- a/src/python/ten/test/persistence/nonce.py +++ b/src/python/ten/test/persistence/nonce.py @@ -1,24 +1,42 @@ -import sqlite3, os +from ten.test.persistence import normalise class NoncePersistence: """Abstracts the persistence of nonces into a local database. """ - SQL_CREATE = "CREATE TABLE IF NOT EXISTS nonce_db (account TEXT, environment TEXT, nonce INTEGER, status STRING)" - SQL_INSERT = "INSERT INTO nonce_db VALUES (?, ?, ?, ?)" - SQL_UPDATE = "UPDATE nonce_db SET status=? WHERE account=? AND environment=? AND nonce=?" - SQL_DELETE = "DELETE from nonce_db WHERE account=? AND environment=?" - SQL_DELFRO = "DELETE from nonce_db WHERE account=? AND environment=? AND nonce>=?" - SQL_LATEST = "SELECT nonce FROM nonce_db WHERE account=? AND environment=? ORDER BY nonce DESC LIMIT 1" - SQL_DELENV = "DELETE from nonce_db WHERE environment=?" - SQL_ACCNTS = "SELECT DISTINCT account from nonce_db where environment=?" - SQL_DELENT = "DELETE from nonce_db WHERE account=? AND environment=? AND nonce=?" - - def __init__(self, db_dir): - """Instantiate an instance. """ - self.db = os.path.join(db_dir, 'nonce.db') - self.connection = sqlite3.connect(self.db) - self.cursor = self.connection.cursor() + SQL_CREATE = "CREATE TABLE IF NOT EXISTS nonces " \ + "(account VARCHAR(64), " \ + "environment VARCHAR(64), " \ + "nonce INTEGER, " \ + "status VARCHAR(64))" + SQL_INSERT = "INSERT INTO nonces VALUES (?, ?, ?, ?)" + SQL_UPDATE = "UPDATE nonces SET status=? WHERE account=? AND environment=? AND nonce=?" + SQL_DELETE = "DELETE from nonces WHERE account=? AND environment=?" + SQL_DELFRO = "DELETE from nonces WHERE account=? AND environment=? AND nonce>=?" + SQL_LATEST = "SELECT nonce FROM nonces WHERE account=? AND environment=? ORDER BY nonce DESC LIMIT 1" + SQL_DELENV = "DELETE from nonces WHERE environment=?" + SQL_ACCNTS = "SELECT DISTINCT account from nonces WHERE environment=?" + SQL_DELENT = "DELETE from nonces WHERE account=? AND environment=? AND nonce=?" + + @classmethod + def init(cls, host, dbconnection): + instance = NoncePersistence(host, dbconnection) + instance.create() + return instance + + def __init__(self, host, dbconnection): + """Instantiate an instance.""" + self.host = host + self.dbconnection = dbconnection + self.sqlins = normalise(self.SQL_INSERT, dbconnection.type) + self.sqlupd = normalise(self.SQL_UPDATE, dbconnection.type) + self.sqldel = normalise(self.SQL_DELETE, dbconnection.type) + self.delfro = normalise(self.SQL_DELFRO, dbconnection.type) + self.latest = normalise(self.SQL_LATEST, dbconnection.type) + self.delenv = normalise(self.SQL_DELENV, dbconnection.type) + self.accnts = normalise(self.SQL_ACCNTS, dbconnection.type) + self.delent = normalise(self.SQL_DELENT, dbconnection.type) + self.cursor = self.dbconnection.connection.cursor() def create(self): """Create the cursor to the underlying persistence. """ @@ -26,7 +44,7 @@ def create(self): def close(self): """Close the connection to the underlying persistence. """ - self.connection.close() + self.cursor.close() def get_next_nonce(self, test, web3, account, environment, persist_nonce=True, log=True): """Get the next nonce to use in a transaction. @@ -42,7 +60,7 @@ def get_next_nonce(self, test, web3, account, environment, persist_nonce=True, l nonce = transaction_count if persist_nonce: nonce = 0 if persisted_nonce is None else persisted_nonce+1 # we have to believe the local store - test.nonce_db.insert(account, test.env, nonce) + self.insert(account, test.env, nonce) if log: test.log.info("Account %s count %d using nonce from persistence as %d", account, transaction_count, nonce) else: if log: test.log.info("Account %s using nonce from transaction count as %d", account, nonce) @@ -50,42 +68,42 @@ def get_next_nonce(self, test, web3, account, environment, persist_nonce=True, l def insert(self, account, environment, nonce, status='PENDING'): """Insert a new nonce into the persistence. """ - self.cursor.execute(self.SQL_INSERT, (account, environment, nonce, status)) - self.connection.commit() + self.cursor.execute(self.sqlins, (account, environment, nonce, status)) + self.dbconnection.connection.commit() def update(self, account, environment, nonce, status): """Update the status of a transaction for a given nonce into the persistence. """ - self.cursor.execute(self.SQL_UPDATE, (status, account, environment, nonce)) - self.connection.commit() + self.cursor.execute(self.sqlupd, (status, account, environment, nonce)) + self.dbconnection.connection.commit() def delete(self, account, environment): """Delete all nonce entries in the persistence for a given account and environment. """ - self.cursor.execute(self.SQL_DELETE, (account, environment)) - self.connection.commit() + self.cursor.execute(self.sqldel, (account, environment)) + self.dbconnection.connection.commit() def delete_from(self, account, environment, nonce): """Delete all nonce entries in the persistence for a given account and environment. """ - self.cursor.execute(self.SQL_DELFRO, (account, environment, nonce)) - self.connection.commit() + self.cursor.execute(self.delfro, (account, environment, nonce)) + self.dbconnection.connection.commit() def delete_environment(self, environment): """Delete all nonce entries for all accounts for a given environment. """ - self.cursor.execute(self.SQL_DELENV, (environment, )) - self.connection.commit() + self.cursor.execute(self.delenv, (environment, )) + self.dbconnection.connection.commit() def delete_entries(self, account, environment, nonce): """Delete all nonce entries in the persistence for a given account and environment and nonce. """ - self.cursor.execute(self.SQL_DELENT, (account, environment, nonce)) - self.connection.commit() + self.cursor.execute(self.delent, (account, environment, nonce)) + self.dbconnection.connection.commit() def get_accounts(self, environment): """Return a list of all accounts with persisted values for a given environment. """ - self.cursor.execute(self.SQL_ACCNTS, (environment, )) + self.cursor.execute(self.accnts, (environment)) return self.cursor.fetchall() def get_latest_nonce(self, account, environment): """Get the latest nonce for a given account and environment. """ - self.cursor.execute(self.SQL_LATEST, (account, environment)) + self.cursor.execute(self.latest, (account, environment)) try: result = self.cursor.fetchone()[0] return int(result) diff --git a/src/python/ten/test/persistence/rates.py b/src/python/ten/test/persistence/rates.py index 4cc7f826..7875658a 100644 --- a/src/python/ten/test/persistence/rates.py +++ b/src/python/ten/test/persistence/rates.py @@ -1,43 +1,56 @@ -import sqlite3, os +from ten.test.persistence import normalise class RatesPersistence: """Abstracts the persistence of rates across cryptos into a local database. """ SQL_CREATE = "CREATE TABLE IF NOT EXISTS rates " \ - "(crypto TEXT, currency INTEGER, time INTEGER, rate TEXT, " \ + "(crypto VARCHAR(3), " \ + "currency VARCHAR(3), " \ + "time INTEGER, " \ + "rate VARCHAR(64), " \ "PRIMARY KEY (crypto, currency, time))" - SQL_INSERT = "INSERT INTO rates VALUES (?, ?, ?, ?)" - SQL_DELETE = "DELETE from rates WHERE crypto=?" - SQL_SELECT = "SELECT time, rate FROM rates WHERE crypto=? and currency=? ORDER BY time DESC LIMIT 1" + SQL_INSERT = "INSERT INTO rates VALUES (?, ?, ?, ?);" + SQL_DELETE = "DELETE from rates WHERE crypto=?;" + SQL_SELECT = "SELECT time, rate FROM rates WHERE crypto=? and currency=? ORDER BY time DESC LIMIT 1;" - def __init__(self, db_dir): + @classmethod + def init(cls, host, dbconnection): + instance = RatesPersistence(host, dbconnection) + instance.create() + return instance + + def __init__(self, host, dbconnection): """Instantiate an instance.""" - self.db = os.path.join(db_dir, 'rates.db') - self.connection = sqlite3.connect(self.db) - self.cursor = self.connection.cursor() + self.host = host + self.dbconnection = dbconnection + self.sqlins = normalise(self.SQL_INSERT, dbconnection.type) + self.sqldel = normalise(self.SQL_DELETE, dbconnection.type) + self.sqlsel = normalise(self.SQL_SELECT, dbconnection.type) + self.cursor = self.dbconnection.connection.cursor() def create(self): """Create the cursor to the underlying persistence.""" self.cursor.execute(self.SQL_CREATE) + return self def close(self): """Close the connection to the underlying persistence.""" - self.connection.close() + self.cursor.close() def delete_crypto(self, crypto): """Delete all stored rates for a particular crypto.""" - self.cursor.execute(self.SQL_DELETE, (crypto, )) - self.connection.commit() + self.cursor.execute(self.sqldel, (crypto, )) + self.dbconnection.connection.commit() def insert_rates(self, crypto, currency, time, rate): """Insert a new rate for a particular crypto and currency.""" - self.cursor.execute(self.SQL_INSERT, (crypto, currency, time, str(rate))) - self.connection.commit() + self.cursor.execute(self.sqlins, (crypto, currency, time, str(rate))) + self.dbconnection.connection.commit() def get_latest_rate(self, crypto, currency): """Return the latest rate for the crypto and currency.""" - self.cursor.execute(self.SQL_SELECT, (crypto, currency)) + self.cursor.execute(self.sqlsel, (crypto, currency)) try: result = self.cursor.fetchone()[1] return float(result) @@ -45,3 +58,5 @@ def get_latest_rate(self, crypto, currency): return None + + diff --git a/src/python/ten/test/persistence/results.py b/src/python/ten/test/persistence/results.py index 5c0f680e..64469492 100644 --- a/src/python/ten/test/persistence/results.py +++ b/src/python/ten/test/persistence/results.py @@ -1,21 +1,34 @@ -import sqlite3, os +from ten.test.persistence import normalise class ResultsPersistence: """Abstracts the persistence of performance results into a local database. """ SQL_CREATE = "CREATE TABLE IF NOT EXISTS results " \ - "(test TEXT, environment TEXT, time INTEGER, result REAL, " \ - "PRIMARY KEY (test, environment, time))" - SQL_INSERT = "INSERT INTO results VALUES (?, ?, ?, ?)" - SQL_DELETE = "DELETE from results WHERE environment=?" - SQL_SELECT = "SELECT time, result FROM results WHERE test=? AND environment=? ORDER BY time ASC" - - def __init__(self, db_dir): + "(host VARCHAR(64), " \ + "test VARCHAR(64), " \ + "environment VARCHAR(64), " \ + "time INTEGER, " \ + "result REAL, " \ + "PRIMARY KEY (host, test, environment, time))" + SQL_INSERT = "INSERT INTO results VALUES (?, ?, ?, ?, ?)" + SQL_DELETE = "DELETE from results WHERE host=? and environment=?" + SQL_SELECT = "SELECT time, result FROM results WHERE host=? AND test=? AND environment=? ORDER BY time ASC" + + @classmethod + def init(cls, host, dbconnection): + instance = ResultsPersistence(host, dbconnection) + instance.create() + return instance + + def __init__(self, host, dbconnection): """Instantiate an instance.""" - self.db = os.path.join(db_dir, 'results.db') - self.connection = sqlite3.connect(self.db) - self.cursor = self.connection.cursor() + self.host = host + self.dbconnection = dbconnection + self.sqlins = normalise(self.SQL_INSERT, dbconnection.type) + self.sqldel = normalise(self.SQL_DELETE, dbconnection.type) + self.sqldel = normalise(self.SQL_SELECT, dbconnection.type) + self.cursor = self.dbconnection.connection.cursor() def create(self): """Create the cursor to the underlying persistence.""" @@ -23,20 +36,20 @@ def create(self): def close(self): """Close the connection to the underlying persistence.""" - self.connection.close() + self.cursor.close() def delete_environment(self, environment): """Delete all stored performance results for a particular environment.""" - self.cursor.execute(self.SQL_DELETE, (environment, )) - self.connection.commit() + self.cursor.execute(self.sqldel, (self.host, environment)) + self.dbconnection.connection.commit() def insert_result(self, test, environment, time, result): """Insert a new performance result into the persistence. """ - self.cursor.execute(self.SQL_INSERT, (test, environment, time, result)) - self.connection.commit() + self.cursor.execute(self.sqlins, (self.host, test, environment, time, result)) + self.dbconnection.connection.commit() def get_results(self, test, environment): """Return the performance results for a particular test and environment. """ - self.cursor.execute(self.SQL_SELECT, (test, environment)) + self.cursor.execute(self.sqlsel, (self.host, test, environment)) return self.cursor.fetchall() diff --git a/src/python/ten/test/utils/cloud.py b/src/python/ten/test/utils/cloud.py new file mode 100644 index 00000000..b8c9e98e --- /dev/null +++ b/src/python/ten/test/utils/cloud.py @@ -0,0 +1,14 @@ +import requests + +def is_cloud_vm(): + metadata_url = "http://169.254.169.254/metadata/instance?api-version=2021-02-01" + headers = {"Metadata": "true"} + + try: + response = requests.get(metadata_url, headers=headers, timeout=3) + if response.status_code == 200: + return response.json() + else: + return None + except requests.exceptions.RequestException as e: + return None diff --git a/src/python/ten/test/utils/properties.py b/src/python/ten/test/utils/properties.py index 6b20673e..cc50efcd 100644 --- a/src/python/ten/test/utils/properties.py +++ b/src/python/ten/test/utils/properties.py @@ -77,6 +77,12 @@ def npx_binary(self): raise FileNotFoundException('npx binary not found at default location %s' % path) return path + # persistence + def persistence_host(self): return self.get('persistence.db', 'host') + def persistence_user(self): return self.get('persistence.db', 'user') + def persistence_database(self): return self.get('persistence.db', 'database') + def persistence_password(self): return self.get('persistence.db', 'password') + # common to all environments def block_time_secs(self, key): return self.get('env.'+key, 'BlockTimeSecs') diff --git a/utils/docker/image.Dockerfile b/utils/docker/image.Dockerfile index 80d7c7fa..8df09c81 100644 --- a/utils/docker/image.Dockerfile +++ b/utils/docker/image.Dockerfile @@ -27,6 +27,7 @@ RUN python3 -m pip install web3==6.13.0 RUN python3 -m pip install pysys==1.6.1 RUN python3 -m pip install py-solc-x RUN python3 -m pip install numpy==1.24.4 +RUN python3 -m pip install mysql-connector-python==9.0.0 RUN mkdir /home/ten-test RUN mkdir /home/go-ten diff --git a/utils/github/install.sh b/utils/github/install.sh index ca19b489..01e88734 100755 --- a/utils/github/install.sh +++ b/utils/github/install.sh @@ -24,6 +24,7 @@ python3 -m pip install pysys==1.6.1 python3 -m pip install solc-select python3 -m pip install py-solc-x python3 -m pip install numpy==1.24.4 +python3 -m pip install mysql-connector-python==9.0.0 snap install go --classic curl -fsSL https://get.docker.com -o get-docker.sh sh ./get-docker.sh