Nonce persistence remains local for now

src/python/ten/test/baserunner.py

@@ -15,7 +15,6 @@
 from ten.test.persistence.results import ResultsPersistence
 from ten.test.persistence.contract import ContractPersistence
 from ten.test.utils.properties import Properties
-from ten.test.persistence import get_connection
 from ten.test.utils.cloud import is_cloud_vm
 
 
@@ -71,13 +70,13 @@ def setup(self, runner):
         else:
             self.machine_name = socket.gethostname()
             runner.log.info('Running on local (%s)' % self.machine_name)
-        dbconnection = get_connection(self.is_cloud_vm, self.user_dir)
-        rates_db = RatesPersistence.init(self.machine_name, dbconnection)
-        nonce_db = NoncePersistence.init(self.machine_name, dbconnection)
-        contracts_db = ContractPersistence.init(self.machine_name, dbconnection)
-        funds_db = FundsPersistence.init(self.machine_name, dbconnection)
-        counts_db = CountsPersistence.init(self.machine_name, dbconnection)
-        results_db = ResultsPersistence.init(self.machine_name, dbconnection)
+
+        rates_db = RatesPersistence.init(self.user_dir, self.machine_name, self.is_cloud_vm)
+        nonce_db = NoncePersistence.init(self.user_dir, self.machine_name, self.is_cloud_vm)
+        contracts_db = ContractPersistence.init(self.user_dir, self.machine_name, self.is_cloud_vm)
+        funds_db = FundsPersistence.init(self.user_dir, self.machine_name, self.is_cloud_vm)
+        counts_db = CountsPersistence.init(self.user_dir, self.machine_name, self.is_cloud_vm)
+        results_db = ResultsPersistence.init(self.user_dir, self.machine_name, self.is_cloud_vm)
 
         eth_price = self.get_eth_price()
         if eth_price is not None:
@@ -171,7 +170,6 @@ def setup(self, runner):
         funds_db.close()
         counts_db.close()
         results_db.close()
-        dbconnection.connection.close()
 
     def run_ganache(self, runner):
         """Run ganache for use by the tests. """

src/python/ten/test/basetest.py

@@ -32,15 +32,17 @@ def __init__(self, descriptor, outsubdir, runner):
         self.env = self.mode
         self.block_time = Properties().block_time_secs(self.env)
         self.log.info('Running test in thread %s', threading.currentThread().getName())
-
-        # every test has its own connection to the dbs
-        self.dbconnection = get_connection(is_cloud_vm=runner.ten_runner.is_cloud_vm, db_dir=runner.ten_runner.user_dir)
-        self.rates_db = RatesPersistence(runner.ten_runner.machine_name, self.dbconnection)
-        self.nonce_db = NoncePersistence(runner.ten_runner.machine_name, self.dbconnection)
-        self.contract_db = ContractPersistence(runner.ten_runner.machine_name, self.dbconnection)
-        self.funds_db = FundsPersistence(runner.ten_runner.machine_name, self.dbconnection)
-        self.counts_db = CountsPersistence(runner.ten_runner.machine_name, self.dbconnection)
-        self.results_db = ResultsPersistence(runner.ten_runner.machine_name, self.dbconnection)
+        self.user_dir = runner.ten_runners.user_dir
+        self.machine_name = runner.ten_runners.machine_name
+        self.is_cloud_vm = runner.ten_runners.is_cloud_vm
+
+        # every test has its own connection to the dbs)
+        self.rates_db = RatesPersistence(self.user_dir, self.machine_name, self.is_cloud_vm)
+        self.nonce_db = NoncePersistence(self.user_dir, self.machine_name, self.is_cloud_vm)
+        self.contract_db = ContractPersistence(self.user_dir, self.machine_name, self.is_cloud_vm)
+        self.funds_db = FundsPersistence(self.user_dir, self.machine_name, self.is_cloud_vm)
+        self.counts_db = CountsPersistence(self.user_dir, self.machine_name, self.is_cloud_vm)
+        self.results_db = ResultsPersistence(self.user_dir, self.machine_name, self.is_cloud_vm)
         self.addCleanupFunction(self.close_db)
 
         # every test has a unique connection for the funded account
@@ -79,7 +81,6 @@ def close_db(self):
         self.funds_db.close()
         self.counts_db.close()
         self.results_db.close()
-        self.dbconnection.connection.close()
 
     def drain_ephemeral_pks(self):
         """Drain any ephemeral accounts of their funds. """

src/python/ten/test/persistence/__init__.py

@@ -11,18 +11,25 @@ def normalise(statement, _type):
     return statement if _type != 'mysql' else statement.replace('?', '%s')
 
 
-def get_connection(is_cloud_vm, db_dir):
-    # if is_cloud_vm:
-    #     props = Properties()
-    #     config = {
-    #         'host': props.persistence_host(),
-    #         'user': props.persistence_user(),
-    #         'password': props.persistence_password(),
-    #         'database': props.persistence_database(),
-    #         'connection_timeout': 10
-    #     }
-    #     connection = mysql.connector.connect(**config)
-    #     return DBConnection(connection, 'mysql')
-    # else:
-    connection = sqlite3.connect(os.path.join(db_dir, 'ten-test.db'))
+def get_connection(is_cloud_vm, user_dir, db):
+    '''Get a connection to a db, remote if running in the cloud, local if not.'''
+    if is_cloud_vm:
+        props = Properties()
+        config = {
+            'host': props.persistence_host(),
+            'user': props.persistence_user(),
+            'password': props.persistence_password(),
+            'database': props.persistence_database(),
+            'connection_timeout': 10
+        }
+        connection = mysql.connector.connect(**config)
+        return DBConnection(connection, 'mysql')
+    else:
+        connection = sqlite3.connect(os.path.join(user_dir, db))
+        return DBConnection(connection, 'sqlite3')
+
+
+def get_local_connection(user_dir, db):
+    '''Get a connection to a db, always local.'''
+    connection = sqlite3.connect(os.path.join(user_dir, db))
     return DBConnection(connection, 'sqlite3')

src/python/ten/test/persistence/contract.py

@@ -1,8 +1,14 @@
 from ten.test.persistence import normalise
+from ten.test.persistence import get_connection
 
 
 class ContractPersistence:
-    """Abstracts the persistence of contract addresses into a local database. """
+    """Abstracts the persistence of contract addresses and params into a local database.
+
+    Since a deployed contract is a property of the environment, not the test runner, it should be sharable across
+    different test runners when running in the cloud, so the persistence is externalised into a mysql server under these
+    conditions.
+    """
 
     SQL_CREATE = "CREATE TABLE IF NOT EXISTS contract_details " \
                  "(name VARCHAR(64), " \
@@ -26,21 +32,21 @@ class ContractPersistence:
                  "ORDER BY address DESC LIMIT 1"
 
     @classmethod
-    def init(cls, host, dbconnection):
-        instance = ContractPersistence(host, dbconnection)
+    def init(cls, user_dir, host=None, is_cloud=None):
+        instance = ContractPersistence(user_dir, host, is_cloud)
         instance.create()
         return instance
 
-    def __init__(self, host, dbconnection):
-        """Instantiate an instance."""
+    def __init__(self, user_dir, host=None, is_cloud=None):
+        """Instantiate an instance (mysql server if on azure, sqlite3 if not)"""
         self.host = host
-        self.dbconnection = dbconnection
-        self.sqlins = normalise(self.SQL_INSERT, dbconnection.type)
-        self.sqldel = normalise(self.SQL_DELETE, dbconnection.type)
-        self.sqlsel = normalise(self.SQL_SELECT, dbconnection.type)
-        self.insprm = normalise(self.SQL_INSPRM, dbconnection.type)
-        self.delprm = normalise(self.SQL_DELPRM, dbconnection.type)
-        self.selprm = normalise(self.SQL_SELPRM, dbconnection.type)
+        self.dbconnection = get_connection(is_cloud, user_dir, 'ten-test.db')
+        self.sqlins = normalise(self.SQL_INSERT, self.dbconnection.type)
+        self.sqldel = normalise(self.SQL_DELETE, self.dbconnection.type)
+        self.sqlsel = normalise(self.SQL_SELECT, self.dbconnection.type)
+        self.insprm = normalise(self.SQL_INSPRM, self.dbconnection.type)
+        self.delprm = normalise(self.SQL_DELPRM, self.dbconnection.type)
+        self.selprm = normalise(self.SQL_SELPRM, self.dbconnection.type)
         self.cursor = self.dbconnection.connection.cursor()
 
     def create(self):
@@ -51,6 +57,7 @@ def create(self):
     def close(self):
         """Close the connection to the underlying persistence."""
         self.cursor.close()
+        self.dbconnection.connection.close()
 
     def delete_environment(self, environment):
         """Delete all stored contract details for a particular environment."""

src/python/ten/test/persistence/counts.py

@@ -1,8 +1,15 @@
 from ten.test.persistence import normalise
+from ten.test.persistence import get_connection
 
 
 class CountsPersistence:
-    """Abstracts the persistence of transaction counts across accounts into a local database. """
+    """Abstracts the persistence of transaction counts across accounts into a local database.
+
+    This is only really used to persist the tx count of the L1 sequencer, as it means we can ensure that it is writing
+    roll-ups to the L1. Since this is a property of the environment, not the test runner, it should be sharable across
+    different test runners when running in the cloud, so the persistence is externalised into a mysql server under these
+    conditions.
+    """
 
     SQL_CREATE = "CREATE TABLE IF NOT EXISTS counts " \
                  "(name VARCHAR(64), " \
@@ -17,19 +24,19 @@ class CountsPersistence:
     SQL_SELHOR = "SELECT time, count FROM counts WHERE name=? and environment=? and time >= ? ORDER BY time DESC"
 
     @classmethod
-    def init(cls, host, dbconnection):
-        instance = CountsPersistence(host, dbconnection)
+    def init(cls, user_dir, host=None, is_cloud=None):
+        instance = CountsPersistence(user_dir, host, is_cloud)
         instance.create()
         return instance
 
-    def __init__(self, host, dbconnection):
-        """Instantiate an instance."""
+    def __init__(self, user_dir, host=None, is_cloud=None):
+        """Instantiate an instance (mysql server if on azure, sqlite3 if not)"""
         self.host = host
-        self.dbconnection = dbconnection
-        self.sqlins = normalise(self.SQL_INSERT, dbconnection.type)
-        self.sqldel = normalise(self.SQL_DELETE, dbconnection.type)
-        self.sqlthr = normalise(self.SQL_SELTHR, dbconnection.type)
-        self.sqlhor = normalise(self.SQL_SELHOR, dbconnection.type)
+        self.dbconnection = get_connection(is_cloud, user_dir, 'ten-test.db')
+        self.sqlins = normalise(self.SQL_INSERT, self.dbconnection.type)
+        self.sqldel = normalise(self.SQL_DELETE, self.dbconnection.type)
+        self.sqlthr = normalise(self.SQL_SELTHR, self.dbconnection.type)
+        self.sqlhor = normalise(self.SQL_SELHOR, self.dbconnection.type)
         self.cursor = self.dbconnection.connection.cursor()
 
     def create(self):
@@ -39,6 +46,7 @@ def create(self):
     def close(self):
         """Close the connection to the underlying persistence."""
         self.cursor.close()
+        self.dbconnection.connection.close()
 
     def delete_environment(self, environment):
         """Delete all stored contract details for a particular environment."""

src/python/ten/test/persistence/funds.py

@@ -1,8 +1,13 @@
 from ten.test.persistence import normalise
+from ten.test.persistence import get_connection
 
 
 class FundsPersistence:
-    """Abstracts the persistence of funds across accounts into a local database. """
+    """Abstracts the persistence of funds across accounts into a local database.
+
+    Since this is a property of the environment, not the test runner, it should be sharable across different test
+    runners when running in the cloud, so the persistence is externalised into a mysql server under these conditions.
+    """
 
     SQL_CREATE = "CREATE TABLE IF NOT EXISTS funds " \
                  "(name VARCHAR(64), " \
@@ -16,18 +21,18 @@ class FundsPersistence:
     SQL_SELECT = "SELECT time, balance FROM funds WHERE name=? and environment=? ORDER BY time DESC"
 
     @classmethod
-    def init(cls, host, dbconnection):
-        instance = FundsPersistence(host, dbconnection)
+    def init(cls, user_dir, host=None, is_cloud=None):
+        instance = FundsPersistence(user_dir, host, is_cloud)
         instance.create()
         return instance
 
-    def __init__(self, host, dbconnection):
-        """Instantiate an instance."""
+    def __init__(self, user_dir, host=None, is_cloud=None):
+        """Instantiate an instance (mysql server if on azure, sqlite3 if not)"""
         self.host = host
-        self.dbconnection = dbconnection
-        self.sqlins = normalise(self.SQL_INSERT, dbconnection.type)
-        self.sqldel = normalise(self.SQL_DELETE, dbconnection.type)
-        self.sqlsel = normalise(self.SQL_SELECT, dbconnection.type)
+        self.dbconnection = get_connection(is_cloud, user_dir, 'ten-test.db')
+        self.sqlins = normalise(self.SQL_INSERT, self.dbconnection.type)
+        self.sqldel = normalise(self.SQL_DELETE, self.dbconnection.type)
+        self.sqlsel = normalise(self.SQL_SELECT, self.dbconnection.type)
         self.cursor = self.dbconnection.connection.cursor()
 
     def create(self):
@@ -37,6 +42,7 @@ def create(self):
     def close(self):
         """Close the connection to the underlying persistence."""
         self.cursor.close()
+        self.dbconnection.connection.close()
 
     def delete_environment(self, environment):
         """Delete all stored details for a particular environment."""

src/python/ten/test/persistence/nonce.py

@@ -1,41 +1,41 @@
 from ten.test.persistence import normalise
+from ten.test.persistence import get_local_connection
 
 
 class NoncePersistence:
     """Abstracts the persistence of nonces into a local database. """
 
-    SQL_CREATE = "CREATE TABLE IF NOT EXISTS nonces " \
+    SQL_CREATE = "CREATE TABLE IF NOT EXISTS nonce_db " \
                  "(account VARCHAR(64), " \
                  "environment VARCHAR(64), " \
                  "nonce INTEGER, " \
                  "status VARCHAR(64))"
-    SQL_INSERT = "INSERT INTO nonces VALUES (?, ?, ?, ?)"
-    SQL_UPDATE = "UPDATE nonces SET status=? WHERE account=? AND environment=? AND nonce=?"
-    SQL_DELETE = "DELETE from nonces WHERE account=? AND environment=?"
-    SQL_DELFRO = "DELETE from nonces WHERE account=? AND environment=? AND nonce>=?"
-    SQL_LATEST = "SELECT nonce FROM nonces WHERE account=? AND environment=? ORDER BY nonce DESC LIMIT 1"
-    SQL_DELENV = "DELETE from nonces WHERE environment=?"
-    SQL_ACCNTS = "SELECT DISTINCT account from nonces WHERE environment=?"
-    SQL_DELENT = "DELETE from nonces WHERE account=? AND environment=? AND nonce=?"
+    SQL_INSERT = "INSERT INTO nonce_db VALUES (?, ?, ?, ?)"
+    SQL_UPDATE = "UPDATE nonce_db SET status=? WHERE account=? AND environment=? AND nonce=?"
+    SQL_DELETE = "DELETE from nonce_db WHERE account=? AND environment=?"
+    SQL_DELFRO = "DELETE from nonce_db WHERE account=? AND environment=? AND nonce>=?"
+    SQL_LATEST = "SELECT nonce FROM nonce_db WHERE account=? AND environment=? ORDER BY nonce DESC LIMIT 1"
+    SQL_DELENV = "DELETE from nonce_db WHERE environment=?"
+    SQL_ACCNTS = "SELECT DISTINCT account from nonce_db WHERE environment=?"
+    SQL_DELENT = "DELETE from nonce_db WHERE account=? AND environment=? AND nonce=?"
 
     @classmethod
-    def init(cls, host, dbconnection):
-        instance = NoncePersistence(host, dbconnection)
+    def init(cls, user_dir, host=None, is_cloud=None):
+        instance = NoncePersistence(user_dir, host, is_cloud)
         instance.create()
         return instance
 
-    def __init__(self, host, dbconnection):
-        """Instantiate an instance."""
-        self.host = host
-        self.dbconnection = dbconnection
-        self.sqlins = normalise(self.SQL_INSERT, dbconnection.type)
-        self.sqlupd = normalise(self.SQL_UPDATE, dbconnection.type)
-        self.sqldel = normalise(self.SQL_DELETE, dbconnection.type)
-        self.delfro = normalise(self.SQL_DELFRO, dbconnection.type)
-        self.latest = normalise(self.SQL_LATEST, dbconnection.type)
-        self.delenv = normalise(self.SQL_DELENV, dbconnection.type)
-        self.accnts = normalise(self.SQL_ACCNTS, dbconnection.type)
-        self.delent = normalise(self.SQL_DELENT, dbconnection.type)
+    def __init__(self, user_dir, host=None, is_cloud=None):
+        """Instantiate an instance (always local)"""
+        self.dbconnection = get_local_connection(user_dir, 'nonce.db')
+        self.sqlins = normalise(self.SQL_INSERT, self.dbconnection.type)
+        self.sqlupd = normalise(self.SQL_UPDATE, self.dbconnection.type)
+        self.sqldel = normalise(self.SQL_DELETE, self.dbconnection.type)
+        self.delfro = normalise(self.SQL_DELFRO, self.dbconnection.type)
+        self.latest = normalise(self.SQL_LATEST, self.dbconnection.type)
+        self.delenv = normalise(self.SQL_DELENV, self.dbconnection.type)
+        self.accnts = normalise(self.SQL_ACCNTS, self.dbconnection.type)
+        self.delent = normalise(self.SQL_DELENT, self.dbconnection.type)
         self.cursor = self.dbconnection.connection.cursor()
 
     def create(self):
@@ -45,6 +45,7 @@ def create(self):
     def close(self):
         """Close the connection to the underlying persistence. """
         self.cursor.close()
+        self.dbconnection.connection.close()
 
     def get_next_nonce(self, test, web3, account, environment, persist_nonce=True, log=True):
         """Get the next nonce to use in a transaction.

src/python/ten/test/persistence/rates.py

@@ -1,8 +1,13 @@
 from ten.test.persistence import normalise
+from ten.test.persistence import get_connection
 
 
 class RatesPersistence:
-    """Abstracts the persistence of rates across cryptos into a local database. """
+    """Abstracts the persistence of rates across cryptos into a local database.
+
+    Since this is an absolute property, not one of the test runner, it should be sharable across different test runners
+    when running in the cloud, so the persistence is externalised into a mysql server under these conditions.
+    """
 
     SQL_CREATE = "CREATE TABLE IF NOT EXISTS rates " \
                  "(crypto VARCHAR(3), " \
@@ -15,28 +20,28 @@ class RatesPersistence:
     SQL_SELECT = "SELECT time, rate FROM rates WHERE crypto=? and currency=? ORDER BY time DESC LIMIT 1;"
 
     @classmethod
-    def init(cls, host, dbconnection):
-        instance = RatesPersistence(host, dbconnection)
+    def init(cls, user_dir, host=None, is_cloud=None):
+        instance = RatesPersistence(user_dir, host, is_cloud)
         instance.create()
         return instance
 
-    def __init__(self, host, dbconnection):
-        """Instantiate an instance."""
+    def __init__(self, user_dir, host=None, is_cloud=None):
+        """Instantiate an instance (mysql server if on azure, sqlite3 if not)"""
         self.host = host
-        self.dbconnection = dbconnection
-        self.sqlins = normalise(self.SQL_INSERT, dbconnection.type)
-        self.sqldel = normalise(self.SQL_DELETE, dbconnection.type)
-        self.sqlsel = normalise(self.SQL_SELECT, dbconnection.type)
+        self.dbconnection = get_connection(is_cloud, user_dir, 'ten-test.db')
+        self.sqlins = normalise(self.SQL_INSERT, self.dbconnection.type)
+        self.sqldel = normalise(self.SQL_DELETE, self.dbconnection.type)
+        self.sqlsel = normalise(self.SQL_SELECT, self.dbconnection.type)
         self.cursor = self.dbconnection.connection.cursor()
 
     def create(self):
         """Create the cursor to the underlying persistence."""
         self.cursor.execute(self.SQL_CREATE)
-        return self
 
     def close(self):
         """Close the connection to the underlying persistence."""
         self.cursor.close()
+        self.dbconnection.connection.close()
 
     def delete_crypto(self, crypto):
         """Delete all stored rates for a particular crypto."""