Remote db

.default.properties

@@ -20,6 +20,12 @@ node_path = /usr/lib/node_modules:/usr/local/lib/node_modules
 npm=/usr/bin/npm
 npx = /usr/bin/npx
 
+[persistence.db]
+host = <set in user.properties>
+user = <set in user.properties>
+database = <set in user.properties>
+password = <set in user.properties>
+
 # Constants relating to the different environments the tests can run against namely Ten, Sepolia, Arbitrum and Ganache.
 # The account PKs used are randomly generated and do not relate to any real values. To use real values from a metamask
 # wallet, override in your ~/.tentest/user.properties file.

.github/workflows/health_check_sepolia.yml

@@ -1,6 +1,6 @@
 # Run a set of sanity tests on sepolia-testnet to ensure it is up and running
 #
-name: '[health] Check sepolia health'
+name: '[inspect] Check sepolia health'
 run-name: Check sepolia-testnet health
 on:
   schedule:

.github/workflows/health_check_sepolia_funds.yml

@@ -1,6 +1,6 @@
 # Check key balances on sepolia-testnet
 #
-name: '[health] Check sepolia funds'
+name: '[inspect] Check sepolia funds'
 run-name: Check sepolia-testnet funds
 on:
   schedule:

.github/workflows/health_check_uat.yml

@@ -1,6 +1,6 @@
 # Run a set of sanity tests on uat-testnet to ensure it is up and running
 #
-name: '[health] Check uat health'
+name: '[inspect] Check uat health'
 run-name: Check uat-testnet health
 on:
   schedule:

README.md

@@ -104,6 +104,7 @@ pip3 install pysys==1.6.1
 pip3 install solc-select==1.0.4 
 pip3 install py-solc-x==2.0.2
 pip3 install numpy==1.26.4
+pip3 install mysql-connector-python==9.0.0
 
 solc-select install 0.8.15
 solc-select use 0.8.15

src/python/ten/test/baserunner.py

@@ -1,4 +1,4 @@
-import os, shutil, sys, json, requests, time
+import os, shutil, sys, json, requests, time, socket
 from collections import OrderedDict
 from web3 import Web3
 from pathlib import Path
@@ -15,6 +15,8 @@
 from ten.test.persistence.results import ResultsPersistence
 from ten.test.persistence.contract import ContractPersistence
 from ten.test.utils.properties import Properties
+from ten.test.persistence import get_connection
+from ten.test.utils.cloud import is_cloud_vm
 
 
 class TenRunnerPlugin():
@@ -32,6 +34,10 @@ def __init__(self):
         self.env = None
         self.NODE_HOST = None
         self.balances = OrderedDict()
+        self.cloud_metadata = is_cloud_vm()
+        self.is_cloud_vm = self.cloud_metadata is not None
+        self.user_dir = os.path.join(str(Path.home()), '.tentest')
+        if not os.path.exists(self.user_dir): os.makedirs(self.user_dir)
 
     def setup(self, runner):
         """Set up a runner plugin to start any processes required to execute the tests. """
@@ -58,21 +64,20 @@ def setup(self, runner):
         if os.path.exists(runner.output): shutil.rmtree(runner.output)
         os.makedirs(runner.output)
 
-        # create the nonce db if it does not already exist, clean it out if using ganache
-        db_dir = os.path.join(str(Path.home()), '.tentest')
-        if not os.path.exists(db_dir): os.makedirs(db_dir)
-        rates_db = RatesPersistence(db_dir)
-        rates_db.create()
-        nonce_db = NoncePersistence(db_dir)
-        nonce_db.create()
-        contracts_db = ContractPersistence(db_dir)
-        contracts_db.create()
-        funds_db = FundsPersistence(db_dir)
-        funds_db.create()
-        counts_db = CountsPersistence(db_dir)
-        counts_db.create()
-        results_db = ResultsPersistence(db_dir)
-        results_db.create()
+        # set up the persistence layer based on if we are running in the cloud, or locally
+        if self.is_cloud_vm:
+            self.machine_name = self.cloud_metadata['compute']['name']
+            runner.log.info('Running on azure (%s, %s)' % (self.machine_name, self.cloud_metadata['compute']['location']))
+        else:
+            self.machine_name = socket.gethostname()
+            runner.log.info('Running on local (%s)' % self.machine_name)
+        dbconnection = get_connection(self.is_cloud_vm, self.user_dir)
+        rates_db = RatesPersistence.init(self.machine_name, dbconnection)
+        nonce_db = NoncePersistence.init(self.machine_name, dbconnection)
+        contracts_db = ContractPersistence.init(self.machine_name, dbconnection)
+        funds_db = FundsPersistence.init(self.machine_name, dbconnection)
+        counts_db = CountsPersistence.init(self.machine_name, dbconnection)
+        results_db = ResultsPersistence.init(self.machine_name, dbconnection)
 
         eth_price = self.get_eth_price()
         if eth_price is not None:
@@ -160,9 +165,13 @@ def setup(self, runner):
             runner.cleanup()
             sys.exit(1)
 
+        rates_db.close()
         nonce_db.close()
         contracts_db.close()
         funds_db.close()
+        counts_db.close()
+        results_db.close()
+        dbconnection.connection.close()
 
     def run_ganache(self, runner):
         """Run ganache for use by the tests. """
@@ -306,6 +315,7 @@ def __set_contract_addresses(self, runner):
                 contracts = config["PublicSystemContracts"]
                 Properties.L2PublicCallbacks = self.__get_contract(contracts, "PublicCallbacks")
         elif 'error' in response.json():
+            runner.log.warn('Error getting contract address from ten_config')
             runner.log.error(response.json()['error']['message'])
 
     def __get_contract(self, contracts, key):

src/python/ten/test/basetest.py

@@ -1,7 +1,6 @@
 import os, copy, sys, json, secrets, re
 import threading, requests
 from web3 import Web3
-from pathlib import Path
 from pysys.basetest import BaseTest
 from pysys.constants import PROJECT, BACKGROUND, FAILED
 from pysys.constants import LOG_TRACEBACK
@@ -19,6 +18,7 @@
 from ten.test.networks.arbitrum import ArbitrumSepolia
 from ten.test.networks.sepolia import Sepolia
 from ten.test.networks.ten import Ten, TenL1Geth, TenL1Sepolia
+from ten.test.persistence import get_connection
 
 
 class GenericNetworkTest(BaseTest):
@@ -33,14 +33,14 @@ def __init__(self, descriptor, outsubdir, runner):
         self.block_time = Properties().block_time_secs(self.env)
         self.log.info('Running test in thread %s', threading.currentThread().getName())
 
-        # every test has its own connection to the nonce and contract db
-        db_dir = os.path.join(str(Path.home()), '.tentest')
-        self.rates_db = RatesPersistence(db_dir)
-        self.nonce_db = NoncePersistence(db_dir)
-        self.contract_db = ContractPersistence(db_dir)
-        self.funds_db = FundsPersistence(db_dir)
-        self.counts_db = CountsPersistence(db_dir)
-        self.results_db = ResultsPersistence(db_dir)
+        # every test has its own connection to the dbs
+        self.dbconnection = get_connection(is_cloud_vm=runner.ten_runner.is_cloud_vm, db_dir=runner.ten_runner.user_dir)
+        self.rates_db = RatesPersistence(runner.ten_runner.machine_name, self.dbconnection)
+        self.nonce_db = NoncePersistence(runner.ten_runner.machine_name, self.dbconnection)
+        self.contract_db = ContractPersistence(runner.ten_runner.machine_name, self.dbconnection)
+        self.funds_db = FundsPersistence(runner.ten_runner.machine_name, self.dbconnection)
+        self.counts_db = CountsPersistence(runner.ten_runner.machine_name, self.dbconnection)
+        self.results_db = ResultsPersistence(runner.ten_runner.machine_name, self.dbconnection)
         self.addCleanupFunction(self.close_db)
 
         # every test has a unique connection for the funded account
@@ -72,9 +72,14 @@ def __test_cost(self):
             self.log.info("  %s: %s%.3f USD", 'Test cost', sign, self.eth_price*change, extra=BaseLogFormatter.tag(LOG_TRACEBACK, 0))
 
     def close_db(self):
-        """Close the connection to the nonce database on completion. """
+        """Close the connection to the databases on completion. """
+        self.rates_db.close()
         self.nonce_db.close()
         self.contract_db.close()
+        self.funds_db.close()
+        self.counts_db.close()
+        self.results_db.close()
+        self.dbconnection.connection.close()
 
     def drain_ephemeral_pks(self):
         """Drain any ephemeral accounts of their funds. """

src/python/ten/test/persistence/__init__.py

@@ -1,2 +1,28 @@
 """Package of persistence for transaction counts.
 """
+import os, sqlite3, mysql.connector
+from collections import namedtuple
+from ten.test.utils.properties import Properties
+
+DBConnection = namedtuple('DBConnection', 'connection type')
+
+
+def normalise(statement, _type):
+    return statement if _type != 'mysql' else statement.replace('?', '%s')
+
+
+def get_connection(is_cloud_vm, db_dir):
+    if is_cloud_vm:
+        props = Properties()
+        config = {
+            'host': props.persistence_host(),
+            'user': props.persistence_user(),
+            'password': props.persistence_password(),
+            'database': props.persistence_database(),
+            'connection_timeout': 10
+        }
+        connection = mysql.connector.connect(**config)
+        return DBConnection(connection, 'mysql')
+    else:
+        connection = sqlite3.connect(os.path.join(db_dir, 'ten-test.db'))
+        return DBConnection(connection, 'sqlite3')

src/python/ten/test/persistence/contract.py

@@ -1,65 +1,83 @@
-import sqlite3, os
+from ten.test.persistence import normalise
 
 
 class ContractPersistence:
     """Abstracts the persistence of contract addresses into a local database. """
 
-    SQL_CREATE = "CREATE TABLE IF NOT EXISTS contracts " \
-                 "(name TEXT, environment TEXT, address INTEGER, abi STRING, " \
+    SQL_CREATE = "CREATE TABLE IF NOT EXISTS contract_details " \
+                 "(name VARCHAR(64), " \
+                 "environment VARCHAR(64), " \
+                 "address VARCHAR(64), " \
+                 "abi MEDIUMTEXT, " \
                  "PRIMARY KEY (name, environment))"
-    SQL_INSERT = "INSERT OR REPLACE INTO contracts VALUES (?, ?, ?, ?)"
-    SQL_DELETE = "DELETE from contracts WHERE environment=?"
-    SQL_SELECT = "SELECT address, abi FROM contracts WHERE name=? AND environment=? ORDER BY name DESC LIMIT 1"
+    SQL_INSERT = "INSERT OR REPLACE INTO contract_details VALUES (?, ?, ?, ?)"
+    SQL_DELETE = "DELETE from contract_details WHERE environment=?"
+    SQL_SELECT = "SELECT address, abi FROM contract_details WHERE name=? AND environment=? ORDER BY name DESC LIMIT 1"
 
-    SQL_CRT_PARAMS = "CREATE TABLE IF NOT EXISTS params " \
-                     "(address INTEGER, environment TEXT, key STRING, value STRING, " \
-                     "PRIMARY KEY (address, environment, key))"
-    SQL_INS_PARAMS = "INSERT OR REPLACE INTO params VALUES (?, ?, ?, ?)"
-    SQL_DEL_PARAMS = "DELETE from params WHERE environment=?"
-    SQL_SEL_PARAMS = "SELECT value FROM params WHERE address=? AND environment=? AND key=? " \
-                     "ORDER BY address DESC LIMIT 1"
+    SQL_CRTPRM = "CREATE TABLE IF NOT EXISTS contract_params " \
+                 "(address VARCHAR(64), " \
+                 "environment VARCHAR(64), " \
+                 "param_key VARCHAR(64), " \
+                 "param_val VARCHAR(64), " \
+                 "PRIMARY KEY (address, environment, param_key))"
+    SQL_INSPRM = "INSERT OR REPLACE INTO contract_params VALUES (?, ?, ?, ?)"
+    SQL_DELPRM = "DELETE from contract_params WHERE environment=?"
+    SQL_SELPRM = "SELECT param_val FROM contract_params WHERE address=? AND environment=? AND param_key=? " \
+                 "ORDER BY address DESC LIMIT 1"
 
-    def __init__(self, db_dir):
+    @classmethod
+    def init(cls, host, dbconnection):
+        instance = ContractPersistence(host, dbconnection)
+        instance.create()
+        return instance
+
+    def __init__(self, host, dbconnection):
         """Instantiate an instance."""
-        self.db = os.path.join(db_dir, 'contracts.db')
-        self.connection = sqlite3.connect(self.db)
-        self.cursor = self.connection.cursor()
+        self.host = host
+        self.dbconnection = dbconnection
+        self.sqlins = normalise(self.SQL_INSERT, dbconnection.type)
+        self.sqldel = normalise(self.SQL_DELETE, dbconnection.type)
+        self.sqlsel = normalise(self.SQL_SELECT, dbconnection.type)
+        self.insprm = normalise(self.SQL_INSPRM, dbconnection.type)
+        self.delprm = normalise(self.SQL_DELPRM, dbconnection.type)
+        self.selprm = normalise(self.SQL_SELPRM, dbconnection.type)
+        self.cursor = self.dbconnection.connection.cursor()
 
     def create(self):
         """Create the cursor to the underlying persistence."""
         self.cursor.execute(self.SQL_CREATE)
-        self.cursor.execute(self.SQL_CRT_PARAMS)
+        self.cursor.execute(self.SQL_CRTPRM)
 
     def close(self):
         """Close the connection to the underlying persistence."""
-        self.connection.close()
+        self.cursor.close()
 
     def delete_environment(self, environment):
         """Delete all stored contract details for a particular environment."""
-        self.cursor.execute(self.SQL_DELETE, (environment, ))
-        self.cursor.execute(self.SQL_DEL_PARAMS, (environment, ))
-        self.connection.commit()
+        self.cursor.execute(self.sqldel, (environment, ))
+        self.cursor.execute(self.delprm, (environment, ))
+        self.dbconnection.connection.commit()
 
     def insert_contract(self, name, environment, address, abi):
         """Insert a new contract into the persistence. """
-        self.cursor.execute(self.SQL_INSERT, (name, environment, address, abi))
-        self.connection.commit()
+        self.cursor.execute(self.sqlins, (name, environment, address, abi))
+        self.dbconnection.connection.commit()
 
     def get_contract(self, name, environment):
         """Return the address and abi for a particular deployed contract. """
-        self.cursor.execute(self.SQL_SELECT, (name, environment))
+        self.cursor.execute(self.sqlsel, (name, environment))
         cursor = self.cursor.fetchall()
         if len(cursor) > 0: return cursor[0][0], cursor[0][1]
         return None, None
 
     def insert_param(self, address, environment, key, value):
         """Insert a parameter for a named contract. """
-        self.cursor.execute(self.SQL_INS_PARAMS, (address, environment, key, value))
-        self.connection.commit()
+        self.cursor.execute(self.insprm, (address, environment, key, value))
+        self.dbconnection.connection.commit()
 
     def get_param(self, address, environment, key):
         """Return the address and abi for a particular deployed contract. """
-        self.cursor.execute(self.SQL_SEL_PARAMS, (address, environment, key))
+        self.cursor.execute(self.selprm, (address, environment, key))
         cursor = self.cursor.fetchall()
         if len(cursor) > 0: return cursor[0][0]
         return None