Improved system tests

.env

@@ -12,3 +12,7 @@ export SPONSOR_SECRET_KEY=413f42575f7f26fad3317a778771212fdb80245850981e48b58a4f
 
 # For the Github workflows, a repository secret (private deep history instance) is being used.
 export MAINNET_PROXY_URL="https://gateway.multiversx.com"
+
+# For internal testing:
+export INTERNAL_TESTNET_PROXY_URL=""
+export INTERNAL_TESTNET_EXPLORER_URL=""

.github/workflows/check_with_mesh_cli.yml

@@ -1,4 +1,4 @@
-name: Check with Mesh CLI
+name: Check with Mesh CLI (testnet)
 
 on:
   pull_request:

.github/workflows/check_with_mesh_cli_on_localnet.yml

@@ -0,0 +1,53 @@
+name: Check with Mesh CLI (localnet)
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+
+      - uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: |
+          pipx install multiversx-sdk-cli --force
+          pip install -r ./requirements-dev.txt
+          curl -sSfL https://raw.githubusercontent.com/coinbase/mesh-cli/master/scripts/install.sh | sh -s -- -b "$HOME/.local/bin"
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+      - name: Build
+        run: |
+          cd $GITHUB_WORKSPACE/cmd/rosetta && go build .
+          cd $GITHUB_WORKSPACE/systemtests && go build ./proxyToObserverAdapter.go
+
+      - name: Set up MultiversX localnet
+        run: |
+          mkdir -p ~/localnet && cd ~/localnet
+          mxpy localnet setup --configfile=${GITHUB_WORKSPACE}/systemtests/localnet.toml
+
+          # Start the localnet and store the PID
+          nohup mxpy localnet start --configfile=${GITHUB_WORKSPACE}/systemtests/localnet.toml > localnet.log 2>&1 & echo $! > localnet.pid
+          sleep 60 # Allow time for the localnet to start
+
+      - name: Generate testdata
+        run: |
+          source .env
+          PYTHONPATH=. python3 ./systemtests/generate_testdata_on_network.py setup --network localnet
+          PYTHONPATH=. python3 ./systemtests/generate_testdata_on_network.py run --network localnet
+
+      - name: check:data
+        run: |
+          source .env
+          PYTHONPATH=. python3 ./systemtests/check_with_mesh_cli.py --mode=data --network=localnet
+
+      - name: Stop MultiversX localnet
+        if: success() || failure()
+        run: |
+          kill $(cat localnet.pid) || echo "Testnet already stopped"

.gitignore

@@ -3,6 +3,7 @@ cmd/rosetta/logs/**
 systemtests/logs/**
 systemtests/memento/**
 systemtests/**/check-data/**
+systemtests/localnet/**
 logs/**
 
 **/__pycache__/**

README.md

@@ -255,6 +255,9 @@ Run the checks:
 
 ```
 PYTHONPATH=. python3 ./systemtests/check_with_mesh_cli.py --mode=data --network=testnet
+
+# Or, continuously:
+while PYTHONPATH=. python3 ./systemtests/check_with_mesh_cli.py --mode=data --network=testnet; do sleep 5; done
 ```
 
 ### Check:construction

systemtests/config.py

@@ -19,10 +19,12 @@ class Configuration:
     check_data_configuration_file: str
     check_data_directory: str
     check_data_num_blocks: int
-    explorer_url: str
+    view_url: str
+    custom_currency_issue_cost: int = 50000000000000000
     memento_file: str = ""
     sponsor_secret_key: bytes = bytes.fromhex(os.environ.get("SPONSOR_SECRET_KEY", ""))
     users_mnemonic: str = os.environ.get("USERS_MNEMONIC", "")
+    num_users: int = 128
 
 
 CONFIGURATIONS = {
@@ -36,14 +38,14 @@ class Configuration:
         observer_url="",
         proxy_url=os.environ.get("MAINNET_PROXY_URL", "https://gateway.multiversx.com"),
         activation_epoch_sirius=1265,
-        activation_epoch_spica=4294967295,
+        activation_epoch_spica=1538,
         check_construction_native_configuration_file="",
         check_construction_custom_configuration_file="",
         check_data_configuration_file="systemtests/mesh_cli_config/check-data.json",
         check_data_directory="systemtests/mainnet-data",
         check_data_num_blocks=4096,
         memento_file="systemtests/memento/mainnet.json",
-        explorer_url="https://explorer.multiversx.com",
+        view_url="https://explorer.multiversx.com/transactions/{hash}",
     ),
     "devnet": Configuration(
         network_shard=0,
@@ -55,14 +57,14 @@ class Configuration:
         observer_url="",
         proxy_url="https://devnet-gateway.multiversx.com",
         activation_epoch_sirius=629,
-        activation_epoch_spica=4294967295,
+        activation_epoch_spica=2327,
         check_construction_native_configuration_file="systemtests/mesh_cli_config/devnet-construction-native.json",
         check_construction_custom_configuration_file="systemtests/mesh_cli_config/devnet-construction-custom.json",
         check_data_configuration_file="systemtests/mesh_cli_config/check-data.json",
         check_data_directory="systemtests/devnet-data",
         check_data_num_blocks=0,
         memento_file="systemtests/memento/devnet.json",
-        explorer_url="https://devnet-explorer.multiversx.com",
+        view_url="https://devnet-explorer.multiversx.com/transactions/{hash}",
     ),
     "testnet": Configuration(
         network_shard=0,
@@ -81,25 +83,46 @@ class Configuration:
         check_data_directory="systemtests/testnet-data",
         check_data_num_blocks=0,
         memento_file="systemtests/memento/testnet.json",
-        explorer_url="https://testnet-explorer.multiversx.com",
+        view_url="https://testnet-explorer.multiversx.com/transactions/{hash}",
     ),
     "localnet": Configuration(
         network_shard=0,
         network_id="localnet",
         network_name="untitled",
         native_currency="EGLD",
         config_file_custom_currencies="systemtests/rosetta_config/localnet-custom-currencies.json",
-        num_historical_epochs=2,
+        num_historical_epochs=3,
         observer_url="",
         proxy_url="http://localhost:7950",
         activation_epoch_sirius=1,
-        activation_epoch_spica=4294967295,
+        activation_epoch_spica=4,
         check_construction_native_configuration_file="systemtests/mesh_cli_config/localnet-construction-native.json",
         check_construction_custom_configuration_file="systemtests/mesh_cli_config/localnet-construction-custom.json",
         check_data_configuration_file="systemtests/mesh_cli_config/check-data.json",
         check_data_directory="systemtests/localnet-data",
         check_data_num_blocks=0,
         memento_file="systemtests/memento/localnet.json",
-        explorer_url="",
+        view_url="http://localhost:7950/transaction/{hash}?withResults=true&withLogs=true",
+        custom_currency_issue_cost=5000000000000000000
+    ),
+    "internal": Configuration(
+        network_shard=0,
+        network_id="1",
+        network_name="untitled",
+        native_currency="EGLD",
+        config_file_custom_currencies="systemtests/rosetta_config/internal-custom-currencies.json",
+        num_historical_epochs=1,
+        observer_url="",
+        proxy_url=os.environ.get("INTERNAL_TESTNET_PROXY_URL", ""),
+        activation_epoch_sirius=1,
+        activation_epoch_spica=4,
+        check_construction_native_configuration_file="systemtests/mesh_cli_config/internal-construction-native.json",
+        check_construction_custom_configuration_file="systemtests/mesh_cli_config/internal-construction-custom.json",
+        check_data_configuration_file="systemtests/mesh_cli_config/check-data.json",
+        check_data_directory="systemtests/internal-data",
+        check_data_num_blocks=0,
+        memento_file="systemtests/memento/internal.json",
+        view_url=f"{os.environ.get('INTERNAL_TESTNET_EXPLORER_URL', '')}/transactions/{{hash}}",
+        custom_currency_issue_cost=5000000000000000000
     ),
 }