From 221b69c9cf76ea7cd08148dc0a06267d03f728b8 Mon Sep 17 00:00:00 2001
From: Darren Kelly <107671032+darrenvechain@users.noreply.github.com>
Date: Thu, 21 Nov 2024 09:15:56 +0000
Subject: [PATCH 01/15] fix(documentation): use absolute links in markdown
(#889)
---
README.md | 14 +++++++-------
api/doc/README.md | 4 ++--
docs/CONTRIBUTING.md | 2 +-
docs/hosting-a-node.md | 2 +-
docs/usage.md | 4 ++--
5 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/README.md b/README.md
index 50ee4e85c..c41d0923a 100644
--- a/README.md
+++ b/README.md
@@ -1,8 +1,8 @@
-
- 
+
+ 
@@ -44,9 +44,9 @@ ___
## Documentation
-- [Build](./docs/build.md) - How to build the `thor` binary.
-- [Usage](./docs/usage.md) - How to run thor with different configurations.
-- [Hosting a Node](./docs/hosting-a-node.md) - Considerations and requirements for hosting a node.
+- [Build](https://github.com/vechain/thor/blob/master/docs/build.md) - How to build the `thor` binary.
+- [Usage](https://github.com/vechain/thor/blob/master/docs/usage.md) - How to run thor with different configurations.
+- [Hosting a Node](https://github.com/vechain/thor/blob/master/docs/hosting-a-node.md) - Considerations and requirements for hosting a node.
- [Core Concepts](https://docs.vechain.org/core-concepts) - Core concepts of the VeChainThor blockchain.
- [API Reference](https://mainnet.vechain.org) - The API reference for the VeChainThor blockchain.
@@ -67,7 +67,7 @@ To chat with other community members you can join:
-Do note that our [Code of Conduct](./docs/CODE_OF_CONDUCT.md) applies to all VeChain community channels. Users are
+Do note that our [Code of Conduct](https://github.com/vechain/thor/blob/master/docs/CODE_OF_CONDUCT.md) applies to all VeChain community channels. Users are
**highly encouraged** to read and adhere to them to avoid repercussions.
---
@@ -75,7 +75,7 @@ Do note that our [Code of Conduct](./docs/CODE_OF_CONDUCT.md) applies to all VeC
## Contributing
Contributions to VeChainThor are welcome and highly appreciated. However, before you jump right into it, we would like
-you to review our [Contribution Guidelines](./docs/CONTRIBUTING.md) to make sure you have a smooth experience
+you to review our [Contribution Guidelines](https://github.com/vechain/thor/blob/master/docs/CONTRIBUTING.md) to make sure you have a smooth experience
contributing to VeChainThor.
---
diff --git a/api/doc/README.md b/api/doc/README.md
index 2f2c0ee62..644641053 100644
--- a/api/doc/README.md
+++ b/api/doc/README.md
@@ -1,7 +1,7 @@
## Swagger
swagger-ui from https://github.com/swagger-api/swagger-ui @v5.11.2
-- Created [window-observer.js](swagger-ui/window-observer.js) to remove `Try it out` functionality for subscription endpoints
+- Created [window-observer.js](./swagger-ui/window-observer.js) to remove `Try it out` functionality for subscription endpoints
```bash
curl https://unpkg.com/swagger-ui-dist@5.11.2/swagger-ui.css > swagger-ui/swagger-ui.css
@@ -11,7 +11,7 @@ curl https://unpkg.com/swagger-ui-dist@5.11.2/swagger-ui-standalone-preset.js >
## Stoplight
Spotlight UI from https://github.com/stoplightio/elements @v8.0.3
-- Created [window-observer.js](stoplight-ui/window-observer.js) to remove `Send API Request` functionality for subscription endpoints
+- Created [window-observer.js](./stoplight-ui/window-observer.js) to remove `Send API Request` functionality for subscription endpoints
```bash
curl https://unpkg.com/@stoplight/elements@8.0.3/styles.min.css > stoplight-ui/styles.min.css
diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md
index 407dc0b2a..e5a45b559 100644
--- a/docs/CONTRIBUTING.md
+++ b/docs/CONTRIBUTING.md
@@ -1,7 +1,7 @@
# Contributing to VechainThor
Welcome to VechainThor! We appreciate your interest in contributing. By participating in this project, you agree to
-abide by our [Code of Conduct](https://github.com/vechain/thor/blob/master/CODE_OF_CONDUCT.md).
+abide by our [Code of Conduct](https://github.com/vechain/thor/blob/master/docs/CODE_OF_CONDUCT.md).
## VeChain Improvement Proposals (VIPs)
diff --git a/docs/hosting-a-node.md b/docs/hosting-a-node.md
index 1dc1e12ce..6212d7360 100644
--- a/docs/hosting-a-node.md
+++ b/docs/hosting-a-node.md
@@ -21,7 +21,7 @@ state, including the disk space required for various node types.
### Command Line Options
-Please refer to [Command Line Options](./usage.md#command-line-options) in the usage documentation to see a list of all
+Please refer to [Command Line Options](https://github.com/vechain/thor/blob/master/docs/usage.md#command-line-options) in the usage documentation to see a list of all
available options.
---
diff --git a/docs/usage.md b/docs/usage.md
index 7358e3f0e..3a3b7694a 100644
--- a/docs/usage.md
+++ b/docs/usage.md
@@ -20,7 +20,7 @@ ___
### Running from source
-- To install the `thor` binary, follow the instructions in the [build](build) guide.
+- To install the `thor` binary, follow the instructions in the [build](https://github.com/vechain/thor/blob/master/docs/build.md) guide.
Connect to vechain's mainnet:
@@ -47,7 +47,7 @@ ___
### Running a discovery node
-- To install the `disco` binary, follow the instructions in the [build](build) guide.
+- To install the `disco` binary, follow the instructions in the [build](https://github.com/vechain/thor/blob/master/docs/build.md) guide.
Start a discovery node:
From ae97fba2f93abc76050306b0fa2565c766f3b205 Mon Sep 17 00:00:00 2001
From: Pedro Gomes
Date: Mon, 25 Nov 2024 15:42:44 +0000
Subject: [PATCH 02/15] Add benchmark test to node block process (#892)
* Add benchmark test to node block process
* added file-based storage
* use tempdir
---
cmd/thor/node/node_benchmark_test.go | 547 +++++++++++++++++++++++++++
test/testchain/chain.go | 9 +-
2 files changed, 555 insertions(+), 1 deletion(-)
create mode 100644 cmd/thor/node/node_benchmark_test.go
diff --git a/cmd/thor/node/node_benchmark_test.go b/cmd/thor/node/node_benchmark_test.go
new file mode 100644
index 000000000..0f6d1f1f0
--- /dev/null
+++ b/cmd/thor/node/node_benchmark_test.go
@@ -0,0 +1,547 @@
+// Copyright (c) 2024 The VeChainThor developers
+
+// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying
+// file LICENSE or
+
+package node
+
+import (
+ "crypto/ecdsa"
+ "crypto/rand"
+ "fmt"
+ "math"
+ "math/big"
+ "path/filepath"
+ "runtime/debug"
+ "sync"
+ "testing"
+
+ "github.com/elastic/gosigar"
+ "github.com/ethereum/go-ethereum/common/fdlimit"
+ "github.com/ethereum/go-ethereum/crypto"
+ "github.com/pkg/errors"
+ "github.com/stretchr/testify/require"
+ "github.com/vechain/thor/v2/bft"
+ "github.com/vechain/thor/v2/block"
+ "github.com/vechain/thor/v2/chain"
+ "github.com/vechain/thor/v2/cmd/thor/solo"
+ "github.com/vechain/thor/v2/genesis"
+ "github.com/vechain/thor/v2/logdb"
+ "github.com/vechain/thor/v2/muxdb"
+ "github.com/vechain/thor/v2/packer"
+ "github.com/vechain/thor/v2/state"
+ "github.com/vechain/thor/v2/test/datagen"
+ "github.com/vechain/thor/v2/test/testchain"
+ "github.com/vechain/thor/v2/thor"
+ "github.com/vechain/thor/v2/tx"
+)
+
+var (
+ cachedAccounts []genesis.DevAccount
+ once sync.Once
+ blockCount = 1_000
+)
+
+func getCachedAccounts(b *testing.B) []genesis.DevAccount {
+ once.Do(func() {
+ cachedAccounts = createAccounts(b, 1_000)
+ })
+ return cachedAccounts
+}
+
+func BenchmarkBlockProcess_RandomSigners_ManyClausesPerTx_RealDB(b *testing.B) {
+ // create state accounts
+ accounts := getCachedAccounts(b)
+
+ // randomly pick a signer for signing the transactions
+ randomSignerFunc := randomPickSignerFunc(accounts, createOneClausePerTx)
+
+ // create blocks
+ blocks := createBlocks(b, blockCount, accounts, randomSignerFunc)
+
+ // create test db - will be automagically removed when the benchmark ends
+ db, err := openTempMainDB(b.TempDir())
+ require.NoError(b, err)
+
+ // run the benchmark
+ benchmarkBlockProcess(b, db, accounts, blocks)
+}
+func BenchmarkBlockProcess_RandomSigners_OneClausePerTx_RealDB(b *testing.B) {
+ // create state accounts
+ accounts := getCachedAccounts(b)
+
+ // randomly pick a signer for signing the transactions
+ randomSignerFunc := randomPickSignerFunc(accounts, createManyClausesPerTx)
+
+ // create blocks
+ blocks := createBlocks(b, blockCount, accounts, randomSignerFunc)
+
+ // create test db - will be automagically removed when the benchmark ends
+ db, err := openTempMainDB(b.TempDir())
+ require.NoError(b, err)
+
+ // run the benchmark
+ benchmarkBlockProcess(b, db, accounts, blocks)
+}
+func BenchmarkBlockProcess_ManyClausesPerTx_RealDB(b *testing.B) {
+ // create state accounts
+ accounts := getCachedAccounts(b)
+
+ // Use one signer for signing the transactions
+ singleSignerFun := randomPickSignerFunc([]genesis.DevAccount{accounts[0]}, createManyClausesPerTx)
+
+ // create blocks
+ blocks := createBlocks(b, blockCount, accounts, singleSignerFun)
+
+ // create test db - will be automagically removed when the benchmark ends
+ db, err := openTempMainDB(b.TempDir())
+ require.NoError(b, err)
+
+ // run the benchmark
+ benchmarkBlockProcess(b, db, accounts, blocks)
+}
+func BenchmarkBlockProcess_OneClausePerTx_RealDB(b *testing.B) {
+ // create state accounts
+ accounts := getCachedAccounts(b)
+
+ // Use one signer for signing the transactions
+ singleSignerFun := randomPickSignerFunc([]genesis.DevAccount{accounts[0]}, createOneClausePerTx)
+
+ // create blocks
+ blocks := createBlocks(b, blockCount, accounts, singleSignerFun)
+
+ // create test db - will be automagically removed when the benchmark ends
+ db, err := openTempMainDB(b.TempDir())
+ require.NoError(b, err)
+
+ // run the benchmark
+ benchmarkBlockProcess(b, db, accounts, blocks)
+}
+
+func BenchmarkBlockProcess_RandomSigners_ManyClausesPerTx(b *testing.B) {
+ // create state accounts
+ accounts := getCachedAccounts(b)
+
+ // randomly pick a signer for signing the transactions
+ randomSignerFunc := randomPickSignerFunc(accounts, createOneClausePerTx)
+
+ // create blocks
+ blocks := createBlocks(b, blockCount, accounts, randomSignerFunc)
+
+ // create test db
+ db := muxdb.NewMem()
+
+ // run the benchmark
+ benchmarkBlockProcess(b, db, accounts, blocks)
+}
+
+func BenchmarkBlockProcess_RandomSigners_OneClausePerTx(b *testing.B) {
+ // create state accounts
+ accounts := getCachedAccounts(b)
+
+ // randomly pick a signer for signing the transactions
+ randomSignerFunc := randomPickSignerFunc(accounts, createManyClausesPerTx)
+
+ // create blocks
+ blocks := createBlocks(b, blockCount, accounts, randomSignerFunc)
+
+ // create test db
+ db := muxdb.NewMem()
+
+ // run the benchmark
+ benchmarkBlockProcess(b, db, accounts, blocks)
+}
+
+func BenchmarkBlockProcess_ManyClausesPerTx(b *testing.B) {
+ // create state accounts
+ accounts := getCachedAccounts(b)
+
+ // Use one signer for signing the transactions
+ singleSignerFun := randomPickSignerFunc([]genesis.DevAccount{accounts[0]}, createManyClausesPerTx)
+
+ // create blocks
+ blocks := createBlocks(b, blockCount, accounts, singleSignerFun)
+
+ // create test db
+ db := muxdb.NewMem()
+
+ // run the benchmark
+ benchmarkBlockProcess(b, db, accounts, blocks)
+}
+
+func BenchmarkBlockProcess_OneClausePerTx(b *testing.B) {
+ // create state accounts
+ accounts := getCachedAccounts(b)
+
+ // Use one signer for signing the transactions
+ singleSignerFun := randomPickSignerFunc([]genesis.DevAccount{accounts[0]}, createOneClausePerTx)
+
+ // create blocks
+ blocks := createBlocks(b, blockCount, accounts, singleSignerFun)
+
+ // create test db
+ db := muxdb.NewMem()
+
+ // run the benchmark
+ benchmarkBlockProcess(b, db, accounts, blocks)
+}
+
+func benchmarkBlockProcess(b *testing.B, db *muxdb.MuxDB, accounts []genesis.DevAccount, blocks []*block.Block) {
+ // Initialize the test chain and dependencies
+ thorChain, err := createChain(db, accounts)
+ require.NoError(b, err)
+
+ proposer := &accounts[0]
+
+ engine, err := bft.NewEngine(thorChain.Repo(), thorChain.Database(), thorChain.GetForkConfig(), proposer.Address)
+ require.NoError(b, err)
+
+ node := New(
+ &Master{
+ PrivateKey: proposer.PrivateKey,
+ },
+ thorChain.Repo(),
+ engine,
+ thorChain.Stater(),
+ nil,
+ nil,
+ "",
+ nil,
+ 10_000_000,
+ true,
+ thor.NoFork,
+ )
+
+ stats := &blockStats{}
+
+ // Measure memory usage
+ b.ReportAllocs()
+
+ // Benchmark execution
+ b.ResetTimer()
+ for _, blk := range blocks {
+ _, err = node.processBlock(blk, stats)
+ if err != nil {
+ b.Fatalf("processBlock failed: %v", err)
+ }
+ }
+}
+
+func createBlocks(b *testing.B, noBlocks int, accounts []genesis.DevAccount, createTxFunc func(chain *testchain.Chain) (tx.Transactions, error)) []*block.Block {
+ proposer := &accounts[0]
+
+ // mock a fake chain for block production
+ fakeChain, err := createChain(muxdb.NewMem(), accounts)
+ require.NoError(b, err)
+
+ // pre-alloc blocks
+ var blocks []*block.Block
+ var transactions tx.Transactions
+
+ // Start from the Genesis block
+ previousBlock := fakeChain.GenesisBlock()
+ for i := 0; i < noBlocks; i++ {
+ transactions, err = createTxFunc(fakeChain)
+ require.NoError(b, err)
+ previousBlock, err = packTxsIntoBlock(
+ fakeChain,
+ proposer,
+ previousBlock,
+ transactions,
+ )
+ require.NoError(b, err)
+ blocks = append(blocks, previousBlock)
+ }
+
+ return blocks
+}
+
+func createOneClausePerTx(signerPK *ecdsa.PrivateKey, thorChain *testchain.Chain) (tx.Transactions, error) {
+ var transactions tx.Transactions
+ gasUsed := uint64(0)
+ for gasUsed < 9_500_000 {
+ toAddr := datagen.RandAddress()
+ cla := tx.NewClause(&toAddr).WithValue(big.NewInt(10000))
+ transaction := new(tx.Builder).
+ ChainTag(thorChain.Repo().ChainTag()).
+ GasPriceCoef(1).
+ Expiration(math.MaxUint32 - 1).
+ Gas(21_000).
+ Nonce(uint64(datagen.RandInt())).
+ Clause(cla).
+ BlockRef(tx.NewBlockRef(0)).
+ Build()
+
+ sig, err := crypto.Sign(transaction.SigningHash().Bytes(), signerPK)
+ if err != nil {
+ return nil, err
+ }
+ transaction = transaction.WithSignature(sig)
+
+ gasUsed += 21_000 // Gas per transaction
+ transactions = append(transactions, transaction)
+ }
+ return transactions, nil
+}
+
+func createManyClausesPerTx(signerPK *ecdsa.PrivateKey, thorChain *testchain.Chain) (tx.Transactions, error) {
+ var transactions tx.Transactions
+ gasUsed := uint64(0)
+ txGas := uint64(42_000)
+
+ transactionBuilder := new(tx.Builder).
+ ChainTag(thorChain.Repo().ChainTag()).
+ GasPriceCoef(1).
+ Expiration(math.MaxUint32 - 1).
+ Nonce(uint64(datagen.RandInt())).
+ BlockRef(tx.NewBlockRef(0))
+
+ for ; gasUsed < 9_500_000; gasUsed += txGas {
+ toAddr := datagen.RandAddress()
+ transactionBuilder.Clause(tx.NewClause(&toAddr).WithValue(big.NewInt(10000)))
+ }
+
+ transaction := transactionBuilder.Gas(gasUsed).Build()
+
+ sig, err := crypto.Sign(transaction.SigningHash().Bytes(), signerPK)
+ if err != nil {
+ return nil, err
+ }
+ transaction = transaction.WithSignature(sig)
+
+ transactions = append(transactions, transaction)
+
+ return transactions, nil
+}
+
+func packTxsIntoBlock(thorChain *testchain.Chain, proposerAccount *genesis.DevAccount, parentBlk *block.Block, transactions tx.Transactions) (*block.Block, error) {
+ p := packer.New(thorChain.Repo(), thorChain.Stater(), proposerAccount.Address, &proposerAccount.Address, thorChain.GetForkConfig())
+
+ parentSum, err := thorChain.Repo().GetBlockSummary(parentBlk.Header().ID())
+ if err != nil {
+ return nil, err
+ }
+
+ flow, err := p.Schedule(parentSum, parentBlk.Header().Timestamp()+1)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, transaction := range transactions {
+ err = flow.Adopt(transaction)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ b1, stage, receipts, err := flow.Pack(proposerAccount.PrivateKey, 0, false)
+ if err != nil {
+ return nil, err
+ }
+
+ if _, err := stage.Commit(); err != nil {
+ return nil, err
+ }
+
+ if err := thorChain.Repo().AddBlock(b1, receipts, 0); err != nil {
+ return nil, err
+ }
+
+ if err := thorChain.Repo().SetBestBlockID(b1.Header().ID()); err != nil {
+ return nil, err
+ }
+
+ return b1, nil
+}
+
+func createChain(db *muxdb.MuxDB, accounts []genesis.DevAccount) (*testchain.Chain, error) {
+ forkConfig := thor.NoFork
+ forkConfig.VIP191 = 1
+ forkConfig.BLOCKLIST = 0
+ forkConfig.VIP214 = 2
+
+ // Create the state manager (Stater) with the initialized database.
+ stater := state.NewStater(db)
+
+ authAccs := make([]genesis.Authority, 0, len(accounts))
+ stateAccs := make([]genesis.Account, 0, len(accounts))
+
+ for _, acc := range accounts {
+ authAccs = append(authAccs, genesis.Authority{
+ MasterAddress: acc.Address,
+ EndorsorAddress: acc.Address,
+ Identity: thor.BytesToBytes32([]byte("master")),
+ })
+ bal, _ := new(big.Int).SetString("1000000000000000000000000000", 10)
+ stateAccs = append(stateAccs, genesis.Account{
+ Address: acc.Address,
+ Balance: (*genesis.HexOrDecimal256)(bal),
+ Energy: (*genesis.HexOrDecimal256)(bal),
+ Code: "",
+ Storage: nil,
+ })
+ }
+ mbp := uint64(1_000)
+ genConfig := genesis.CustomGenesis{
+ LaunchTime: 1526400000,
+ GasLimit: thor.InitialGasLimit,
+ ExtraData: "",
+ ForkConfig: &forkConfig,
+ Authority: authAccs,
+ Accounts: stateAccs,
+ Params: genesis.Params{
+ MaxBlockProposers: &mbp,
+ },
+ }
+
+ builder, err := genesis.NewCustomNet(&genConfig)
+ if err != nil {
+ return nil, err
+ }
+
+ // Initialize the genesis and retrieve the genesis block
+ //gene := genesis.NewDevnet()
+ geneBlk, _, _, err := builder.Build(stater)
+ if err != nil {
+ return nil, err
+ }
+
+ // Create the repository which manages chain data, using the database and genesis block.
+ repo, err := chain.NewRepository(db, geneBlk)
+ if err != nil {
+ return nil, err
+ }
+
+ // Create an inMemory logdb
+ logDb, err := logdb.NewMem()
+ if err != nil {
+ return nil, err
+ }
+
+ return testchain.New(
+ db,
+ builder,
+ solo.NewBFTEngine(repo),
+ repo,
+ stater,
+ geneBlk,
+ logDb,
+ thor.NoFork,
+ ), nil
+}
+
+func randomPickSignerFunc(
+ accounts []genesis.DevAccount,
+ createTxFun func(signerPK *ecdsa.PrivateKey, thorChain *testchain.Chain) (tx.Transactions, error),
+) func(chain *testchain.Chain) (tx.Transactions, error) {
+ return func(chain *testchain.Chain) (tx.Transactions, error) {
+ // Ensure there are accounts available
+ if len(accounts) == 0 {
+ return nil, fmt.Errorf("no accounts available to pick a random sender")
+ }
+
+ // Securely pick a random index
+ maxLen := big.NewInt(int64(len(accounts)))
+ randomIndex, err := rand.Int(rand.Reader, maxLen)
+ if err != nil {
+ return nil, fmt.Errorf("failed to generate random index: %v", err)
+ }
+
+ // Use the selected account to create transactions
+ sender := accounts[randomIndex.Int64()]
+ return createTxFun(sender.PrivateKey, chain)
+ }
+}
+
+func createAccounts(b *testing.B, accountNo int) []genesis.DevAccount {
+ var accs []genesis.DevAccount
+
+ for i := 0; i < accountNo; i++ {
+ pk, err := crypto.GenerateKey()
+ require.NoError(b, err)
+ addr := crypto.PubkeyToAddress(pk.PublicKey)
+ accs = append(accs, genesis.DevAccount{Address: thor.Address(addr), PrivateKey: pk})
+ }
+
+ return accs
+}
+
+func openTempMainDB(dir string) (*muxdb.MuxDB, error) {
+ cacheMB := normalizeCacheSize(4096)
+
+ fdCache := suggestFDCache()
+
+ opts := muxdb.Options{
+ TrieNodeCacheSizeMB: cacheMB,
+ TrieRootCacheCapacity: 256,
+ TrieCachedNodeTTL: 30, // 5min
+ TrieLeafBankSlotCapacity: 256,
+ TrieDedupedPartitionFactor: math.MaxUint32,
+ TrieWillCleanHistory: true,
+ OpenFilesCacheCapacity: fdCache,
+ ReadCacheMB: 256, // rely on os page cache other than huge db read cache.
+ WriteBufferMB: 128,
+ }
+
+ // go-ethereum stuff
+ // Ensure Go's GC ignores the database cache for trigger percentage
+ totalCacheMB := cacheMB + opts.ReadCacheMB + opts.WriteBufferMB*2
+ gogc := math.Max(10, math.Min(100, 50/(float64(totalCacheMB)/1024)))
+
+ debug.SetGCPercent(int(gogc))
+
+ if opts.TrieWillCleanHistory {
+ opts.TrieHistPartitionFactor = 1000
+ } else {
+ opts.TrieHistPartitionFactor = 500000
+ }
+
+ db, err := muxdb.Open(filepath.Join(dir, "maindb"), &opts)
+ if err != nil {
+ return nil, errors.Wrapf(err, "open main database [%v]", dir)
+ }
+ return db, nil
+}
+
+func normalizeCacheSize(sizeMB int) int {
+ if sizeMB < 128 {
+ sizeMB = 128
+ }
+
+ var mem gosigar.Mem
+ if err := mem.Get(); err != nil {
+ fmt.Println("failed to get total mem:", "err", err)
+ } else {
+ total := int(mem.Total / 1024 / 1024)
+ half := total / 2
+
+ // limit to not less than total/2 and up to total-2GB
+ limitMB := total - 2048
+ if limitMB < half {
+ limitMB = half
+ }
+
+ if sizeMB > limitMB {
+ sizeMB = limitMB
+ fmt.Println("cache size(MB) limited", "limit", limitMB)
+ }
+ }
+ return sizeMB
+}
+
+func suggestFDCache() int {
+ limit, err := fdlimit.Current()
+ if err != nil {
+ fmt.Println("unable to get fdlimit", "error", err)
+ return 500
+ }
+ if limit <= 1024 {
+ fmt.Println("low fd limit, increase it if possible", "limit", limit)
+ }
+
+ n := limit / 2
+ if n > 5120 {
+ return 5120
+ }
+ return n
+}
diff --git a/test/testchain/chain.go b/test/testchain/chain.go
index b35687d14..af908594e 100644
--- a/test/testchain/chain.go
+++ b/test/testchain/chain.go
@@ -46,6 +46,7 @@ func New(
stater *state.Stater,
genesisBlock *block.Block,
logDB *logdb.LogDB,
+ forkConfig thor.ForkConfig,
) *Chain {
return &Chain{
db: db,
@@ -55,7 +56,7 @@ func New(
stater: stater,
genesisBlock: genesisBlock,
logDB: logDB,
- forkConfig: thor.GetForkConfig(genesisBlock.Header().ID()),
+ forkConfig: forkConfig,
}
}
@@ -87,6 +88,11 @@ func NewIntegrationTestChain() (*Chain, error) {
return nil, err
}
+ forkConfig := thor.NoFork
+ forkConfig.VIP191 = 1
+ forkConfig.BLOCKLIST = 0
+ forkConfig.VIP214 = 2
+
return New(
db,
gene,
@@ -95,6 +101,7 @@ func NewIntegrationTestChain() (*Chain, error) {
stater,
geneBlk,
logDb,
+ thor.NoFork,
), nil
}
From 198e537cf46243de4e505c822bfebd442a33d2b5 Mon Sep 17 00:00:00 2001
From: libotony
Date: Wed, 27 Nov 2024 15:31:07 +0800
Subject: [PATCH 03/15] update dependency go-ethereum (#895)
---
go.mod | 14 +++++++-------
go.sum | 30 +++++++++++++++++-------------
2 files changed, 24 insertions(+), 20 deletions(-)
diff --git a/go.mod b/go.mod
index 4ba00ec9a..f3032a927 100644
--- a/go.mod
+++ b/go.mod
@@ -18,7 +18,7 @@ require (
github.com/mattn/go-sqlite3 v1.14.22
github.com/mattn/go-tty v0.0.0-20180219170247-931426f7535a
github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c
- github.com/pkg/errors v0.8.0
+ github.com/pkg/errors v0.8.1-0.20171216070316-e881fd58d78e
github.com/pmezard/go-difflib v1.0.0
github.com/prometheus/client_golang v1.18.0
github.com/prometheus/client_model v0.5.0
@@ -27,7 +27,7 @@ require (
github.com/stretchr/testify v1.8.4
github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a
github.com/vechain/go-ecvrf v0.0.0-20220525125849-96fa0442e765
- golang.org/x/crypto v0.21.0
+ golang.org/x/crypto v0.22.0
gopkg.in/cheggaaa/pb.v1 v1.0.28
gopkg.in/urfave/cli.v1 v1.20.0
gopkg.in/yaml.v3 v3.0.1
@@ -37,6 +37,7 @@ require (
github.com/aristanetworks/goarista v0.0.0-20180222005525-c41ed3986faa // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6 // indirect
+ github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect
github.com/cespare/cp v1.1.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/deckarep/golang-set v1.7.1 // indirect
@@ -46,17 +47,16 @@ require (
github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
github.com/go-stack/stack v1.7.0 // indirect
github.com/golang/snappy v0.0.4 // indirect
- github.com/google/go-cmp v0.6.0 // indirect
github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect
github.com/huin/goupnp v0.0.0-20171109214107-dceda08e705b // indirect
- github.com/jackpal/go-nat-pmp v1.0.1 // indirect
+ github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 // indirect
github.com/mattn/go-colorable v0.0.9 // indirect
github.com/mattn/go-runewidth v0.0.4 // indirect
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/rjeczalik/notify v0.9.3 // indirect
- golang.org/x/net v0.23.0 // indirect
- golang.org/x/sys v0.18.0 // indirect
+ golang.org/x/net v0.24.0 // indirect
+ golang.org/x/sys v0.19.0 // indirect
golang.org/x/text v0.14.0 // indirect
google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/karalabe/cookiejar.v2 v2.0.0-20150724131613-8dcd6a7f4951 // indirect
@@ -64,4 +64,4 @@ require (
replace github.com/syndtr/goleveldb => github.com/vechain/goleveldb v1.0.1-0.20220809091043-51eb019c8655
-replace github.com/ethereum/go-ethereum => github.com/vechain/go-ethereum v1.8.15-0.20240528020007-2994c2a24b9c
+replace github.com/ethereum/go-ethereum => github.com/vechain/go-ethereum v1.8.15-0.20241126085506-c74017ec91b2
diff --git a/go.sum b/go.sum
index fd12a92aa..648890cc0 100644
--- a/go.sum
+++ b/go.sum
@@ -6,6 +6,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6 h1:Eey/GGQ/E5Xp1P2Lyx1qj007hLZfbi0+CoVeJruGCtI=
github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ=
+github.com/btcsuite/btcd/btcec/v2 v2.3.4 h1:3EJjcN70HCu/mwqlUsGK8GcNVyLVxFDlWurTXGPFfiQ=
+github.com/btcsuite/btcd/btcec/v2 v2.3.4/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04=
github.com/cespare/cp v1.1.1 h1:nCb6ZLdB7NRaqsm91JtQTAme2SKJzXVsdPIPkyJr1MU=
github.com/cespare/cp v1.1.1/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s=
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -23,6 +25,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/deckarep/golang-set v1.7.1 h1:SCQV0S6gTtp6itiFrTqI+pfmJ4LN85S1YzhDf9rTHJQ=
github.com/deckarep/golang-set v1.7.1/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ=
+github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0=
github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc=
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc=
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs=
@@ -84,8 +87,8 @@ github.com/huin/goupnp v0.0.0-20171109214107-dceda08e705b h1:mvnS3LbcRgdM4nBLksE
github.com/huin/goupnp v0.0.0-20171109214107-dceda08e705b/go.mod h1:MZ2ZmwcBpvOoJ22IJsc7va19ZwoheaBk43rKg12SKag=
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
-github.com/jackpal/go-nat-pmp v1.0.1 h1:i0LektDkO1QlrTm/cSuP+PyBCDnYvjPLGl4LdWEMiaA=
-github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
+github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA=
+github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
@@ -119,12 +122,13 @@ github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
-github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw=
github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro=
+github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk=
+github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0=
github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c h1:MUyE44mTvnI5A0xrxIxaMqoWFzPfQvtE2IWUollMDMs=
github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34=
-github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
-github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1-0.20171216070316-e881fd58d78e h1:osn9cOzd93npXpRuTFR/MPjiTvTSNHA7pqbXkPyLqQ4=
+github.com/pkg/errors v0.8.1-0.20171216070316-e881fd58d78e/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk=
@@ -150,8 +154,8 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/vechain/go-ecvrf v0.0.0-20220525125849-96fa0442e765 h1:jvr+TSivjObZmOKVdqlgeLtRhaDG27gE39PMuE2IJ24=
github.com/vechain/go-ecvrf v0.0.0-20220525125849-96fa0442e765/go.mod h1:cwnTMgAVzMb30xMKnGI1LdU1NjMiPllYb7i3ibj/fzE=
-github.com/vechain/go-ethereum v1.8.15-0.20240528020007-2994c2a24b9c h1:YfGsGXMNKI64gR76KumYgGnYSdAFtMA8igtmpFiBt74=
-github.com/vechain/go-ethereum v1.8.15-0.20240528020007-2994c2a24b9c/go.mod h1:EhX+lSkpNdEIxu1zOXtiFZu5nv1i8MX1mQA/qhUE+gw=
+github.com/vechain/go-ethereum v1.8.15-0.20241126085506-c74017ec91b2 h1:ch3DqXvl1ApfJut768bf5Vlhqtw+bxAWTyPDYXQkQZk=
+github.com/vechain/go-ethereum v1.8.15-0.20241126085506-c74017ec91b2/go.mod h1:yPUCNmntAh1PritrMfSi7noK+9vVPStZX3wgh3ieaY0=
github.com/vechain/goleveldb v1.0.1-0.20220809091043-51eb019c8655 h1:CbHcWpCi7wOYfpoErRABh3Slyq9vO0Ay/EHN5GuJSXQ=
github.com/vechain/goleveldb v1.0.1-0.20220809091043-51eb019c8655/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -160,8 +164,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
-golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
+golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -174,8 +178,8 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
-golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
+golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -199,8 +203,8 @@ golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
-golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
+golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
From 45ab2bde4c6af9b8e06453c54b02663e95232f0a Mon Sep 17 00:00:00 2001
From: Darren Kelly <107671032+darrenvechain@users.noreply.github.com>
Date: Fri, 29 Nov 2024 09:35:32 +0000
Subject: [PATCH 04/15] chore: update API metrics bucket and endpoint names
(#893)
* chore: update API metrics bucket and endpoint names
* fix: typo & tests
* fix: lint
* chore: add websocket total counter
* fix: txs endpoints names & ws subject
* fix: unit tests
* chore: standardise naming convention
* chore: add websocke duration & http code
* chore: add websocke duration & http code
* fix: lint issues
* fix: sync issues with metrics
* chore: update websocket durations bucket
* fix: PR comments - use sync.Once
---
api/accounts/accounts.go | 12 ++++++------
api/blocks/blocks.go | 2 +-
api/debug/debug.go | 6 +++---
api/events/events.go | 2 +-
api/metrics.go | 28 +++++++++++++++++-----------
api/metrics_test.go | 22 +++++++++++-----------
api/node/node.go | 2 +-
api/transactions/transactions.go | 6 +++---
api/transfers/transfers.go | 2 +-
metrics/telemetry.go | 18 ++++++++++++------
10 files changed, 56 insertions(+), 44 deletions(-)
diff --git a/api/accounts/accounts.go b/api/accounts/accounts.go
index 4bd940709..54058a160 100644
--- a/api/accounts/accounts.go
+++ b/api/accounts/accounts.go
@@ -358,27 +358,27 @@ func (a *Accounts) Mount(root *mux.Router, pathPrefix string) {
sub.Path("/*").
Methods(http.MethodPost).
- Name("accounts_call_batch_code").
+ Name("POST /accounts/*").
HandlerFunc(utils.WrapHandlerFunc(a.handleCallBatchCode))
sub.Path("/{address}").
Methods(http.MethodGet).
- Name("accounts_get_account").
+ Name("GET /accounts/{address}").
HandlerFunc(utils.WrapHandlerFunc(a.handleGetAccount))
sub.Path("/{address}/code").
Methods(http.MethodGet).
- Name("accounts_get_code").
+ Name("GET /accounts/{address}/code").
HandlerFunc(utils.WrapHandlerFunc(a.handleGetCode))
sub.Path("/{address}/storage/{key}").
Methods("GET").
- Name("accounts_get_storage").
+ Name("GET /accounts/{address}/storage").
HandlerFunc(utils.WrapHandlerFunc(a.handleGetStorage))
// These two methods are currently deprecated
sub.Path("").
Methods(http.MethodPost).
- Name("accounts_call_contract").
+ Name("POST /accounts").
HandlerFunc(utils.WrapHandlerFunc(a.handleCallContract))
sub.Path("/{address}").
Methods(http.MethodPost).
- Name("accounts_call_contract_address").
+ Name("POST /accounts/{address}").
HandlerFunc(utils.WrapHandlerFunc(a.handleCallContract))
}
diff --git a/api/blocks/blocks.go b/api/blocks/blocks.go
index bddb3ac12..ff86e02e6 100644
--- a/api/blocks/blocks.go
+++ b/api/blocks/blocks.go
@@ -95,6 +95,6 @@ func (b *Blocks) Mount(root *mux.Router, pathPrefix string) {
sub := root.PathPrefix(pathPrefix).Subrouter()
sub.Path("/{revision}").
Methods(http.MethodGet).
- Name("blocks_get_block").
+ Name("GET /blocks/{revision}").
HandlerFunc(utils.WrapHandlerFunc(b.handleGetBlock))
}
diff --git a/api/debug/debug.go b/api/debug/debug.go
index e84a88d57..5ff54f1dc 100644
--- a/api/debug/debug.go
+++ b/api/debug/debug.go
@@ -466,14 +466,14 @@ func (d *Debug) Mount(root *mux.Router, pathPrefix string) {
sub.Path("/tracers").
Methods(http.MethodPost).
- Name("debug_trace_clause").
+ Name("POST /debug/tracers").
HandlerFunc(utils.WrapHandlerFunc(d.handleTraceClause))
sub.Path("/tracers/call").
Methods(http.MethodPost).
- Name("debug_trace_call").
+ Name("POST /debug/tracers/call").
HandlerFunc(utils.WrapHandlerFunc(d.handleTraceCall))
sub.Path("/storage-range").
Methods(http.MethodPost).
- Name("debug_trace_storage").
+ Name("POST /debug/storage-range").
HandlerFunc(utils.WrapHandlerFunc(d.handleDebugStorage))
}
diff --git a/api/events/events.go b/api/events/events.go
index 40dff7b09..669c47b03 100644
--- a/api/events/events.go
+++ b/api/events/events.go
@@ -84,6 +84,6 @@ func (e *Events) Mount(root *mux.Router, pathPrefix string) {
sub.Path("").
Methods(http.MethodPost).
- Name("logs_filter_event").
+ Name("POST /logs/event").
HandlerFunc(utils.WrapHandlerFunc(e.handleFilter))
}
diff --git a/api/metrics.go b/api/metrics.go
index 9fd5c3d94..57631be71 100644
--- a/api/metrics.go
+++ b/api/metrics.go
@@ -19,9 +19,15 @@ import (
)
var (
+ websocketDurations = []int64{
+ 0, 1, 2, 5, 10, 25, 50, 100, 250, 500, 1_000, 2_500, 5_000, 10_000, 25_000,
+ 50_000, 100_000, 250_000, 500_000, 1000_000, 2_500_000, 5_000_000, 10_000_000,
+ }
metricHTTPReqCounter = metrics.LazyLoadCounterVec("api_request_count", []string{"name", "code", "method"})
metricHTTPReqDuration = metrics.LazyLoadHistogramVec("api_duration_ms", []string{"name", "code", "method"}, metrics.BucketHTTPReqs)
- metricActiveWebsocketCount = metrics.LazyLoadGaugeVec("api_active_websocket_count", []string{"subject"})
+ metricWebsocketDuration = metrics.LazyLoadHistogramVec("api_websocket_duration", []string{"name", "code"}, websocketDurations)
+ metricActiveWebsocketGauge = metrics.LazyLoadGaugeVec("api_active_websocket_gauge", []string{"name"})
+ metricWebsocketCounter = metrics.LazyLoadCounterVec("api_websocket_counter", []string{"name"})
)
// metricsResponseWriter is a wrapper around http.ResponseWriter that captures the status code.
@@ -62,7 +68,7 @@ func metricsMiddleware(next http.Handler) http.Handler {
var (
enabled = false
name = ""
- subscription = ""
+ subscription = false
)
// all named route will be recorded
@@ -70,24 +76,24 @@ func metricsMiddleware(next http.Handler) http.Handler {
enabled = true
name = rt.GetName()
if strings.HasPrefix(name, "subscriptions") {
- // example path: /subscriptions/txpool -> subject = txpool
- paths := strings.Split(r.URL.Path, "/")
- if len(paths) > 2 {
- subscription = paths[2]
- }
+ subscription = true
+ name = "WS " + r.URL.Path
}
}
now := time.Now()
mrw := newMetricsResponseWriter(w)
- if subscription != "" {
- metricActiveWebsocketCount().AddWithLabel(1, map[string]string{"subject": subscription})
+ if subscription {
+ metricActiveWebsocketGauge().AddWithLabel(1, map[string]string{"name": name})
+ metricWebsocketCounter().AddWithLabel(1, map[string]string{"name": name})
}
next.ServeHTTP(mrw, r)
- if subscription != "" {
- metricActiveWebsocketCount().AddWithLabel(-1, map[string]string{"subject": subscription})
+ if subscription {
+ metricActiveWebsocketGauge().AddWithLabel(-1, map[string]string{"name": name})
+ // record websocket duration in seconds, not MS
+ metricWebsocketDuration().ObserveWithLabels(time.Since(now).Milliseconds()/1000, map[string]string{"name": name, "code": strconv.Itoa(mrw.statusCode)})
} else if enabled {
metricHTTPReqCounter().AddWithLabel(1, map[string]string{"name": name, "code": strconv.Itoa(mrw.statusCode), "method": r.Method})
metricHTTPReqDuration().ObserveWithLabels(time.Since(now).Milliseconds(), map[string]string{"name": name, "code": strconv.Itoa(mrw.statusCode), "method": r.Method})
diff --git a/api/metrics_test.go b/api/metrics_test.go
index 7cb1794e4..4e4b6daaf 100644
--- a/api/metrics_test.go
+++ b/api/metrics_test.go
@@ -77,7 +77,7 @@ func TestMetricsMiddleware(t *testing.T) {
assert.Equal(t, "method", labels[1].GetName())
assert.Equal(t, "GET", labels[1].GetValue())
assert.Equal(t, "name", labels[2].GetName())
- assert.Equal(t, "accounts_get_account", labels[2].GetValue())
+ assert.Equal(t, "GET /accounts/{address}", labels[2].GetValue())
labels = m[1].GetLabel()
assert.Equal(t, 3, len(labels))
@@ -86,7 +86,7 @@ func TestMetricsMiddleware(t *testing.T) {
assert.Equal(t, "method", labels[1].GetName())
assert.Equal(t, "GET", labels[1].GetValue())
assert.Equal(t, "name", labels[2].GetName())
- assert.Equal(t, "accounts_get_account", labels[2].GetValue())
+ assert.Equal(t, "GET /accounts/{address}", labels[2].GetValue())
labels = m[2].GetLabel()
assert.Equal(t, 3, len(labels))
@@ -95,7 +95,7 @@ func TestMetricsMiddleware(t *testing.T) {
assert.Equal(t, "method", labels[1].GetName())
assert.Equal(t, "GET", labels[1].GetValue())
assert.Equal(t, "name", labels[2].GetName())
- assert.Equal(t, "accounts_get_account", labels[2].GetValue())
+ assert.Equal(t, "GET /accounts/{address}", labels[2].GetValue())
}
func TestWebsocketMetrics(t *testing.T) {
@@ -120,13 +120,13 @@ func TestWebsocketMetrics(t *testing.T) {
metrics, err := parser.TextToMetricFamilies(bytes.NewReader(body))
assert.Nil(t, err)
- m := metrics["thor_metrics_api_active_websocket_count"].GetMetric()
+ m := metrics["thor_metrics_api_active_websocket_gauge"].GetMetric()
assert.Equal(t, 1, len(m), "should be 1 metric entries")
assert.Equal(t, float64(1), m[0].GetGauge().GetValue())
labels := m[0].GetLabel()
- assert.Equal(t, "subject", labels[0].GetName())
- assert.Equal(t, "beat", labels[0].GetValue())
+ assert.Equal(t, "name", labels[0].GetName())
+ assert.Equal(t, "WS /subscriptions/beat", labels[0].GetValue())
// initiate 1 beat subscription, active websocket should be 2
conn2, _, err := websocket.DefaultDialer.Dial(u.String(), nil)
@@ -137,7 +137,7 @@ func TestWebsocketMetrics(t *testing.T) {
metrics, err = parser.TextToMetricFamilies(bytes.NewReader(body))
assert.Nil(t, err)
- m = metrics["thor_metrics_api_active_websocket_count"].GetMetric()
+ m = metrics["thor_metrics_api_active_websocket_gauge"].GetMetric()
assert.Equal(t, 1, len(m), "should be 1 metric entries")
assert.Equal(t, float64(2), m[0].GetGauge().GetValue())
@@ -151,16 +151,16 @@ func TestWebsocketMetrics(t *testing.T) {
metrics, err = parser.TextToMetricFamilies(bytes.NewReader(body))
assert.Nil(t, err)
- m = metrics["thor_metrics_api_active_websocket_count"].GetMetric()
+ m = metrics["thor_metrics_api_active_websocket_gauge"].GetMetric()
assert.Equal(t, 2, len(m), "should be 2 metric entries")
// both m[0] and m[1] should have the value of 1
assert.Equal(t, float64(2), m[0].GetGauge().GetValue())
assert.Equal(t, float64(1), m[1].GetGauge().GetValue())
- // m[1] should have the subject of block
+ // m[1] should have the name of block
labels = m[1].GetLabel()
- assert.Equal(t, "subject", labels[0].GetName())
- assert.Equal(t, "block", labels[0].GetValue())
+ assert.Equal(t, "name", labels[0].GetName())
+ assert.Equal(t, "WS /subscriptions/block", labels[0].GetValue())
}
func httpGet(t *testing.T, url string) ([]byte, int) {
diff --git a/api/node/node.go b/api/node/node.go
index 11c1ce7e1..21c69dcdf 100644
--- a/api/node/node.go
+++ b/api/node/node.go
@@ -35,6 +35,6 @@ func (n *Node) Mount(root *mux.Router, pathPrefix string) {
sub.Path("/network/peers").
Methods(http.MethodGet).
- Name("node_get_peers").
+ Name("GET /node/network/peers").
HandlerFunc(utils.WrapHandlerFunc(n.handleNetwork))
}
diff --git a/api/transactions/transactions.go b/api/transactions/transactions.go
index af32cb6da..3cf2e4d65 100644
--- a/api/transactions/transactions.go
+++ b/api/transactions/transactions.go
@@ -218,14 +218,14 @@ func (t *Transactions) Mount(root *mux.Router, pathPrefix string) {
sub.Path("").
Methods(http.MethodPost).
- Name("transactions_send_tx").
+ Name("POST /transactions").
HandlerFunc(utils.WrapHandlerFunc(t.handleSendTransaction))
sub.Path("/{id}").
Methods(http.MethodGet).
- Name("transactions_get_tx").
+ Name("GET /transactions/{id}").
HandlerFunc(utils.WrapHandlerFunc(t.handleGetTransactionByID))
sub.Path("/{id}/receipt").
Methods(http.MethodGet).
- Name("transactions_get_receipt").
+ Name("GET /transactions/{id}/receipt").
HandlerFunc(utils.WrapHandlerFunc(t.handleGetTransactionReceiptByID))
}
diff --git a/api/transfers/transfers.go b/api/transfers/transfers.go
index cad4ee6b3..25d2e2599 100644
--- a/api/transfers/transfers.go
+++ b/api/transfers/transfers.go
@@ -90,6 +90,6 @@ func (t *Transfers) Mount(root *mux.Router, pathPrefix string) {
sub.Path("").
Methods(http.MethodPost).
- Name("logs_filter_transfer").
+ Name("POST /logs/transfer").
HandlerFunc(utils.WrapHandlerFunc(t.handleFilterTransferLogs))
}
diff --git a/metrics/telemetry.go b/metrics/telemetry.go
index 9ab3bd633..1d1ee96f2 100644
--- a/metrics/telemetry.go
+++ b/metrics/telemetry.go
@@ -5,7 +5,10 @@
package metrics
-import "net/http"
+import (
+ "net/http"
+ "sync"
+)
// metrics is a singleton service that provides global access to a set of meters
// it wraps multiple implementations and defaults to a no-op implementation
@@ -30,7 +33,11 @@ func HTTPHandler() http.Handler {
// Define standard buckets for histograms
var (
Bucket10s = []int64{0, 500, 1000, 2000, 3000, 4000, 5000, 7500, 10_000}
- BucketHTTPReqs = []int64{0, 150, 300, 450, 600, 900, 1200, 1500, 3000}
+ BucketHTTPReqs = []int64{
+ 0, 1, 2, 5, 10, 20, 30, 50, 75, 100,
+ 150, 200, 300, 400, 500, 750, 1000,
+ 1500, 2000, 3000, 4000, 5000, 10000,
+ }
)
// HistogramMeter represents the type of metric that is calculated by aggregating
@@ -96,12 +103,11 @@ func GaugeVec(name string, labels []string) GaugeVecMeter {
// - it avoid metrics definition to determine the singleton to use (noop vs prometheus)
func LazyLoad[T any](f func() T) func() T {
var result T
- var loaded bool
+ var once sync.Once
return func() T {
- if !loaded {
+ once.Do(func() {
result = f()
- loaded = true
- }
+ })
return result
}
}
From 9eec212d8521e48af3ca5466573e954e24444849 Mon Sep 17 00:00:00 2001
From: Darren Kelly <107671032+darrenvechain@users.noreply.github.com>
Date: Fri, 29 Nov 2024 09:44:09 +0000
Subject: [PATCH 05/15] chore: update builtin generation (#896)
* chore: update builtin generation
* fix: update GHA
---
.github/workflows/lint-go.yaml | 7 +++++++
builtin/gen/bindata.go | 27 +++++++++------------------
builtin/gen/gen.go | 5 +++--
3 files changed, 19 insertions(+), 20 deletions(-)
diff --git a/.github/workflows/lint-go.yaml b/.github/workflows/lint-go.yaml
index ea4d2195a..0f49ceb39 100644
--- a/.github/workflows/lint-go.yaml
+++ b/.github/workflows/lint-go.yaml
@@ -17,6 +17,13 @@ jobs:
with:
go-version: '1.22'
cache: false
+
+ - name: Check `builtins` directory
+ # if it has any changes in the 'builtins' dir after running `go generate`, echo an error and fail the workflow
+ run: |
+ go generate ./builtin/gen
+ git diff --exit-code builtin/gen || (echo "\n\n\nbuiltin/gen directory is not up to date, run 'go generate ./...' to update it" && exit 1)
+
- name: golangci-lint
uses: golangci/golangci-lint-action@v6
with:
diff --git a/builtin/gen/bindata.go b/builtin/gen/bindata.go
index 8f1ffbc5d..c0724a53d 100644
--- a/builtin/gen/bindata.go
+++ b/builtin/gen/bindata.go
@@ -1,4 +1,4 @@
-// Package gen Code generated by go-bindata. (@generated) DO NOT EDIT.
+// Code generated by go-bindata. DO NOT EDIT.
// sources:
// compiled/Authority.abi
// compiled/Authority.bin-runtime
@@ -76,32 +76,21 @@ type bindataFileInfo struct {
modTime time.Time
}
-// Name return file name
func (fi bindataFileInfo) Name() string {
return fi.name
}
-
-// Size return file size
func (fi bindataFileInfo) Size() int64 {
return fi.size
}
-
-// Mode return file mode
func (fi bindataFileInfo) Mode() os.FileMode {
return fi.mode
}
-
-// Mode return file modify time
func (fi bindataFileInfo) ModTime() time.Time {
return fi.modTime
}
-
-// IsDir return file whether a directory
func (fi bindataFileInfo) IsDir() bool {
- return fi.mode&os.ModeDir != 0
+ return false
}
-
-// Sys return file is sys mode
func (fi bindataFileInfo) Sys() interface{} {
return nil
}
@@ -794,11 +783,13 @@ var _bindata = map[string]func() (*asset, error){
// directory embedded in the file by go-bindata.
// For example if you run go-bindata on data/... and data contains the
// following hierarchy:
-// data/
-// foo.txt
-// img/
-// a.png
-// b.png
+//
+// data/
+// foo.txt
+// img/
+// a.png
+// b.png
+//
// then AssetDir("data") would return []string{"foo.txt", "img"}
// AssetDir("data/img") would return []string{"a.png", "b.png"}
// AssetDir("foo.txt") and AssetDir("notexist") would return an error
diff --git a/builtin/gen/gen.go b/builtin/gen/gen.go
index d08c2f179..ce5fed8e1 100644
--- a/builtin/gen/gen.go
+++ b/builtin/gen/gen.go
@@ -6,5 +6,6 @@
package gen
//go:generate rm -rf ./compiled/
-//go:generate solc --optimize-runs 200 --overwrite --bin-runtime --abi -o ./compiled authority.sol energy.sol executor.sol extension.sol extension-v2.sol measure.sol params.sol prototype.sol
-//go:generate go-bindata -nometadata -ignore=_ -pkg gen -o bindata.go compiled/
+//go:generate docker run -v ./:/solidity ethereum/solc:0.4.24 --optimize-runs 200 --overwrite --bin-runtime --abi -o /solidity/compiled authority.sol energy.sol executor.sol extension.sol extension-v2.sol measure.sol params.sol prototype.sol
+//go:generate go run github.com/go-bindata/go-bindata/go-bindata@v1.0.0 -nometadata -ignore=_ -pkg gen -o bindata.go compiled/
+//go:generate go fmt
From f1711c314ea8cdcc2047a382dfb4373c1cdae094 Mon Sep 17 00:00:00 2001
From: Pedro Gomes
Date: Fri, 29 Nov 2024 15:39:20 +0000
Subject: [PATCH 06/15] getreceipts metrics + lint (#902)
---
.../transactions_benchmark_test.go | 536 ++++++++++++++++++
1 file changed, 536 insertions(+)
create mode 100644 api/transactions/transactions_benchmark_test.go
diff --git a/api/transactions/transactions_benchmark_test.go b/api/transactions/transactions_benchmark_test.go
new file mode 100644
index 000000000..f6f1fccd6
--- /dev/null
+++ b/api/transactions/transactions_benchmark_test.go
@@ -0,0 +1,536 @@
+// Copyright (c) 2024 The VeChainThor developers
+
+// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying
+// file LICENSE or
+
+package transactions
+
+import (
+ "crypto/ecdsa"
+ "crypto/rand"
+ "fmt"
+ "math"
+ "math/big"
+ "path/filepath"
+ "runtime/debug"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/elastic/gosigar"
+ "github.com/ethereum/go-ethereum/common/fdlimit"
+ "github.com/ethereum/go-ethereum/crypto"
+ "github.com/pkg/errors"
+ "github.com/stretchr/testify/require"
+ "github.com/vechain/thor/v2/block"
+ "github.com/vechain/thor/v2/chain"
+ "github.com/vechain/thor/v2/cmd/thor/solo"
+ "github.com/vechain/thor/v2/genesis"
+ "github.com/vechain/thor/v2/logdb"
+ "github.com/vechain/thor/v2/muxdb"
+ "github.com/vechain/thor/v2/packer"
+ "github.com/vechain/thor/v2/state"
+ "github.com/vechain/thor/v2/test/datagen"
+ "github.com/vechain/thor/v2/test/testchain"
+ "github.com/vechain/thor/v2/thor"
+ "github.com/vechain/thor/v2/tx"
+ "github.com/vechain/thor/v2/txpool"
+)
+
+var (
+ cachedAccounts []genesis.DevAccount
+ once sync.Once
+ blockCount = 1_000
+)
+
+func getCachedAccounts(b *testing.B) []genesis.DevAccount {
+ once.Do(func() {
+ now := time.Now()
+ cachedAccounts = createAccounts(b, 10_000)
+ b.Logf("Created accounts in: %f secs", time.Since(now).Seconds())
+ })
+ return cachedAccounts
+}
+
+func BenchmarkFetchTx_RealDB_RandomSigners_ManyClausesPerTx(b *testing.B) {
+ // create state accounts
+ accounts := getCachedAccounts(b)
+
+ // randomly pick a signer for signing the transactions
+ randomSignerFunc := randomPickSignerFunc(accounts, createManyClausesPerTx)
+
+ // create test db - will be automagically removed when the benchmark ends
+ db, err := openTempMainDB(b.TempDir())
+ require.NoError(b, err)
+
+ // create blocks
+ newChain, transactions := createPackedChain(b, db, blockCount, accounts, randomSignerFunc)
+
+ // shuffle the transaction into a randomized order
+ randomizedTransactions := shuffleSlice(transactions)
+ b.Logf("About to process %d txs", len(randomizedTransactions))
+
+ // run the benchmarks
+ b.Run("getTransaction", func(b *testing.B) {
+ benchmarkGetTransaction(b, newChain, randomizedTransactions)
+ })
+
+ b.Run("getReceipt", func(b *testing.B) {
+ benchmarkGetReceipt(b, newChain, randomizedTransactions)
+ })
+}
+
+func BenchmarkFetchTx_RealDB_RandomSigners_OneClausePerTx(b *testing.B) {
+ // create state accounts
+ accounts := getCachedAccounts(b)
+
+ // randomly pick a signer for signing the transactions
+ randomSignerFunc := randomPickSignerFunc(accounts, createOneClausePerTx)
+
+ // create test db - will be automagically removed when the benchmark ends
+ db, err := openTempMainDB(b.TempDir())
+ require.NoError(b, err)
+
+ // create blocks
+ newChain, transactions := createPackedChain(b, db, blockCount, accounts, randomSignerFunc)
+
+ // shuffle the transaction into a randomized order
+ randomizedTransactions := shuffleSlice(transactions)
+ b.Logf("About to process %d txs", len(randomizedTransactions))
+
+ // run the benchmarks
+ b.Run("getTransaction", func(b *testing.B) {
+ benchmarkGetTransaction(b, newChain, randomizedTransactions)
+ })
+
+ b.Run("getReceipt", func(b *testing.B) {
+ benchmarkGetReceipt(b, newChain, randomizedTransactions)
+ })
+}
+
+func BenchmarkFetchTx_RandomSigners_ManyClausesPerTx(b *testing.B) {
+ // create state accounts
+ accounts := getCachedAccounts(b)
+
+ // randomly pick a signer for signing the transactions
+ randomSignerFunc := randomPickSignerFunc(accounts, createManyClausesPerTx)
+
+ // create blocks
+ newChain, transactions := createPackedChain(b, muxdb.NewMem(), blockCount, accounts, randomSignerFunc)
+
+ // shuffle the transaction into a randomized order
+ randomizedTransactions := shuffleSlice(transactions)
+ b.Logf("About to process %d txs", len(randomizedTransactions))
+
+ // run the benchmarks
+ b.Run("getTransaction", func(b *testing.B) {
+ benchmarkGetTransaction(b, newChain, randomizedTransactions)
+ })
+
+ b.Run("getReceipt", func(b *testing.B) {
+ benchmarkGetReceipt(b, newChain, randomizedTransactions)
+ })
+}
+
+func BenchmarkFetchTx_RandomSigners_OneClausePerTx(b *testing.B) {
+ // Setup phase: Not part of the benchmark timing
+ b.StopTimer()
+
+ // create state accounts
+ accounts := getCachedAccounts(b)
+
+ // randomly pick a signer for signing the transactions
+ randomSignerFunc := randomPickSignerFunc(accounts, createOneClausePerTx)
+
+ // create blocks
+ newChain, transactions := createPackedChain(b, muxdb.NewMem(), blockCount, accounts, randomSignerFunc)
+
+ // shuffle the transaction into a randomized order
+ randomizedTransactions := shuffleSlice(transactions)
+ b.Logf("About to process %d txs", len(randomizedTransactions))
+
+ // run the benchmarks
+ b.Run("getTransaction", func(b *testing.B) {
+ benchmarkGetTransaction(b, newChain, randomizedTransactions)
+ })
+
+ b.Run("getReceipt", func(b *testing.B) {
+ benchmarkGetReceipt(b, newChain, randomizedTransactions)
+ })
+}
+
+func benchmarkGetTransaction(b *testing.B, thorChain *testchain.Chain, randTxs tx.Transactions) {
+ mempool := txpool.New(thorChain.Repo(), thorChain.Stater(), txpool.Options{Limit: 10, LimitPerAccount: 16, MaxLifetime: 10 * time.Minute})
+ transactionAPI := New(thorChain.Repo(), mempool)
+ head := thorChain.Repo().BestBlockSummary().Header.ID()
+ var err error
+
+ // Measure memory usage
+ b.ReportAllocs()
+
+ // Benchmark execution
+ b.ResetTimer()
+
+ for _, randTx := range randTxs {
+ _, err = transactionAPI.getRawTransaction(randTx.ID(), head, false)
+ if err != nil {
+ b.Fatalf("getRawTransaction failed: %v", err)
+ }
+ }
+}
+
+func benchmarkGetReceipt(b *testing.B, thorChain *testchain.Chain, randTxs tx.Transactions) {
+ mempool := txpool.New(thorChain.Repo(), thorChain.Stater(), txpool.Options{Limit: 10, LimitPerAccount: 16, MaxLifetime: 10 * time.Minute})
+ transactionAPI := New(thorChain.Repo(), mempool)
+ head := thorChain.Repo().BestBlockSummary().Header.ID()
+ var err error
+
+ // Measure memory usage
+ b.ReportAllocs()
+
+ // Benchmark execution
+ b.ResetTimer()
+
+ for _, randTx := range randTxs {
+ _, err = transactionAPI.getTransactionReceiptByID(randTx.ID(), head)
+ if err != nil {
+ b.Fatalf("getTransactionReceiptByID failed: %v", err)
+ }
+ }
+}
+
+func createPackedChain(b *testing.B, db *muxdb.MuxDB, noBlocks int, accounts []genesis.DevAccount, createTxFunc func(chain *testchain.Chain) (tx.Transactions, error)) (*testchain.Chain, tx.Transactions) {
+ proposer := &accounts[0]
+
+ // mock a fake chain for block production
+ fakeChain, err := createChain(db, accounts)
+ require.NoError(b, err)
+
+ // pre-alloc blocks
+ var transactions tx.Transactions
+
+ // Start from the Genesis block
+ previousBlock := fakeChain.GenesisBlock()
+ for i := 0; i < noBlocks; i++ {
+ newTxs, err := createTxFunc(fakeChain)
+ require.NoError(b, err)
+ previousBlock, err = packTxsIntoBlock(
+ fakeChain,
+ proposer,
+ previousBlock,
+ newTxs,
+ )
+ require.NoError(b, err)
+ transactions = append(transactions, newTxs...)
+ }
+
+ return fakeChain, transactions
+}
+
+func createOneClausePerTx(signerPK *ecdsa.PrivateKey, thorChain *testchain.Chain) (tx.Transactions, error) {
+ var transactions tx.Transactions
+ gasUsed := uint64(0)
+ for gasUsed < 9_500_000 {
+ toAddr := datagen.RandAddress()
+ cla := tx.NewClause(&toAddr).WithValue(big.NewInt(10000))
+ transaction := new(tx.Builder).
+ ChainTag(thorChain.Repo().ChainTag()).
+ GasPriceCoef(1).
+ Expiration(math.MaxUint32 - 1).
+ Gas(21_000).
+ Nonce(uint64(datagen.RandInt())).
+ Clause(cla).
+ BlockRef(tx.NewBlockRef(0)).
+ Build()
+
+ sig, err := crypto.Sign(transaction.SigningHash().Bytes(), signerPK)
+ if err != nil {
+ return nil, err
+ }
+ transaction = transaction.WithSignature(sig)
+
+ gasUsed += 21_000 // Gas per transaction
+ transactions = append(transactions, transaction)
+ }
+ return transactions, nil
+}
+
+func createManyClausesPerTx(signerPK *ecdsa.PrivateKey, thorChain *testchain.Chain) (tx.Transactions, error) {
+ var transactions tx.Transactions
+ gasUsed := uint64(0)
+ txGas := uint64(42_000)
+
+ transactionBuilder := new(tx.Builder).
+ ChainTag(thorChain.Repo().ChainTag()).
+ GasPriceCoef(1).
+ Expiration(math.MaxUint32 - 1).
+ Nonce(uint64(datagen.RandInt())).
+ BlockRef(tx.NewBlockRef(0))
+
+ for ; gasUsed < 9_500_000; gasUsed += txGas {
+ toAddr := datagen.RandAddress()
+ transactionBuilder.Clause(tx.NewClause(&toAddr).WithValue(big.NewInt(10000)))
+ }
+
+ transaction := transactionBuilder.Gas(gasUsed).Build()
+
+ sig, err := crypto.Sign(transaction.SigningHash().Bytes(), signerPK)
+ if err != nil {
+ return nil, err
+ }
+ transaction = transaction.WithSignature(sig)
+
+ transactions = append(transactions, transaction)
+
+ return transactions, nil
+}
+
+func packTxsIntoBlock(thorChain *testchain.Chain, proposerAccount *genesis.DevAccount, parentBlk *block.Block, transactions tx.Transactions) (*block.Block, error) {
+ p := packer.New(thorChain.Repo(), thorChain.Stater(), proposerAccount.Address, &proposerAccount.Address, thorChain.GetForkConfig())
+
+ parentSum, err := thorChain.Repo().GetBlockSummary(parentBlk.Header().ID())
+ if err != nil {
+ return nil, err
+ }
+
+ flow, err := p.Schedule(parentSum, parentBlk.Header().Timestamp()+1)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, transaction := range transactions {
+ err = flow.Adopt(transaction)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ b1, stage, receipts, err := flow.Pack(proposerAccount.PrivateKey, 0, false)
+ if err != nil {
+ return nil, err
+ }
+
+ if _, err := stage.Commit(); err != nil {
+ return nil, err
+ }
+
+ if err := thorChain.Repo().AddBlock(b1, receipts, 0); err != nil {
+ return nil, err
+ }
+
+ if err := thorChain.Repo().SetBestBlockID(b1.Header().ID()); err != nil {
+ return nil, err
+ }
+
+ return b1, nil
+}
+
+func createChain(db *muxdb.MuxDB, accounts []genesis.DevAccount) (*testchain.Chain, error) {
+ forkConfig := thor.NoFork
+ forkConfig.VIP191 = 1
+ forkConfig.BLOCKLIST = 0
+ forkConfig.VIP214 = 2
+
+ // Create the state manager (Stater) with the initialized database.
+ stater := state.NewStater(db)
+
+ authAccs := make([]genesis.Authority, 0, len(accounts))
+ stateAccs := make([]genesis.Account, 0, len(accounts))
+
+ for _, acc := range accounts {
+ authAccs = append(authAccs, genesis.Authority{
+ MasterAddress: acc.Address,
+ EndorsorAddress: acc.Address,
+ Identity: thor.BytesToBytes32([]byte("master")),
+ })
+ bal, _ := new(big.Int).SetString("1000000000000000000000000000", 10)
+ stateAccs = append(stateAccs, genesis.Account{
+ Address: acc.Address,
+ Balance: (*genesis.HexOrDecimal256)(bal),
+ Energy: (*genesis.HexOrDecimal256)(bal),
+ Code: "",
+ Storage: nil,
+ })
+ }
+ mbp := uint64(1_000)
+ genConfig := genesis.CustomGenesis{
+ LaunchTime: 1526400000,
+ GasLimit: thor.InitialGasLimit,
+ ExtraData: "",
+ ForkConfig: &forkConfig,
+ Authority: authAccs,
+ Accounts: stateAccs,
+ Params: genesis.Params{
+ MaxBlockProposers: &mbp,
+ },
+ }
+
+ builder, err := genesis.NewCustomNet(&genConfig)
+ if err != nil {
+ return nil, err
+ }
+
+ // Initialize the genesis and retrieve the genesis block
+ //gene := genesis.NewDevnet()
+ geneBlk, _, _, err := builder.Build(stater)
+ if err != nil {
+ return nil, err
+ }
+
+ // Create the repository which manages chain data, using the database and genesis block.
+ repo, err := chain.NewRepository(db, geneBlk)
+ if err != nil {
+ return nil, err
+ }
+
+ // Create an inMemory logdb
+ logDb, err := logdb.NewMem()
+ if err != nil {
+ return nil, err
+ }
+
+ return testchain.New(
+ db,
+ builder,
+ solo.NewBFTEngine(repo),
+ repo,
+ stater,
+ geneBlk,
+ logDb,
+ thor.NoFork,
+ ), nil
+}
+
+func randomPickSignerFunc(
+ accounts []genesis.DevAccount,
+ createTxFun func(signerPK *ecdsa.PrivateKey, thorChain *testchain.Chain) (tx.Transactions, error),
+) func(chain *testchain.Chain) (tx.Transactions, error) {
+ return func(chain *testchain.Chain) (tx.Transactions, error) {
+ // Ensure there are accounts available
+ if len(accounts) == 0 {
+ return nil, fmt.Errorf("no accounts available to pick a random sender")
+ }
+
+ // Securely pick a random index
+ maxLen := big.NewInt(int64(len(accounts)))
+ randomIndex, err := rand.Int(rand.Reader, maxLen)
+ if err != nil {
+ return nil, fmt.Errorf("failed to generate random index: %v", err)
+ }
+
+ // Use the selected account to create transactions
+ sender := accounts[randomIndex.Int64()]
+ return createTxFun(sender.PrivateKey, chain)
+ }
+}
+
+func createAccounts(b *testing.B, accountNo int) []genesis.DevAccount {
+ var accs []genesis.DevAccount
+
+ for i := 0; i < accountNo; i++ {
+ pk, err := crypto.GenerateKey()
+ require.NoError(b, err)
+ addr := crypto.PubkeyToAddress(pk.PublicKey)
+ accs = append(accs, genesis.DevAccount{Address: thor.Address(addr), PrivateKey: pk})
+ }
+
+ return accs
+}
+
+func openTempMainDB(dir string) (*muxdb.MuxDB, error) {
+ cacheMB := normalizeCacheSize(4096)
+
+ fdCache := suggestFDCache()
+
+ opts := muxdb.Options{
+ TrieNodeCacheSizeMB: cacheMB,
+ TrieRootCacheCapacity: 256,
+ TrieCachedNodeTTL: 30, // 5min
+ TrieLeafBankSlotCapacity: 256,
+ TrieDedupedPartitionFactor: math.MaxUint32,
+ TrieWillCleanHistory: true,
+ OpenFilesCacheCapacity: fdCache,
+ ReadCacheMB: 256, // rely on os page cache other than huge db read cache.
+ WriteBufferMB: 128,
+ }
+
+ // go-ethereum stuff
+ // Ensure Go's GC ignores the database cache for trigger percentage
+ totalCacheMB := cacheMB + opts.ReadCacheMB + opts.WriteBufferMB*2
+ gogc := math.Max(10, math.Min(100, 50/(float64(totalCacheMB)/1024)))
+
+ debug.SetGCPercent(int(gogc))
+
+ if opts.TrieWillCleanHistory {
+ opts.TrieHistPartitionFactor = 256
+ } else {
+ opts.TrieHistPartitionFactor = 524288
+ }
+
+ db, err := muxdb.Open(filepath.Join(dir, "maindb"), &opts)
+ if err != nil {
+ return nil, errors.Wrapf(err, "open main database [%v]", dir)
+ }
+ return db, nil
+}
+
+func normalizeCacheSize(sizeMB int) int {
+ if sizeMB < 128 {
+ sizeMB = 128
+ }
+
+ var mem gosigar.Mem
+ if err := mem.Get(); err != nil {
+ fmt.Println("failed to get total mem:", "err", err)
+ } else {
+ total := int(mem.Total / 1024 / 1024)
+ half := total / 2
+
+ // limit to not less than total/2 and up to total-2GB
+ limitMB := total - 2048
+ if limitMB < half {
+ limitMB = half
+ }
+
+ if sizeMB > limitMB {
+ sizeMB = limitMB
+ fmt.Println("cache size(MB) limited", "limit", limitMB)
+ }
+ }
+ return sizeMB
+}
+
+func suggestFDCache() int {
+ limit, err := fdlimit.Current()
+ if err != nil {
+ fmt.Println("unable to get fdlimit", "error", err)
+ return 500
+ }
+ if limit <= 1024 {
+ fmt.Println("low fd limit, increase it if possible", "limit", limit)
+ }
+
+ n := limit / 2
+ if n > 5120 {
+ return 5120
+ }
+ return n
+}
+
+func shuffleSlice(slice tx.Transactions) tx.Transactions {
+ shuffled := make(tx.Transactions, len(slice))
+ copy(shuffled, slice)
+
+ for i := len(shuffled) - 1; i > 0; i-- {
+ n, err := rand.Int(rand.Reader, big.NewInt(int64(i+1)))
+ if err != nil {
+ panic(err) // Handle errors appropriately in real code
+ }
+
+ // Swap the current element with the random index
+ j := int(n.Int64())
+ shuffled[i], shuffled[j] = shuffled[j], shuffled[i]
+ }
+
+ return shuffled
+}
From a1576965cf5cad71a78ff6d9b3ce6b389681ff5e Mon Sep 17 00:00:00 2001
From: Darren Kelly <107671032+darrenvechain@users.noreply.github.com>
Date: Mon, 2 Dec 2024 16:01:35 +0000
Subject: [PATCH 07/15] chore: add flag to enable/disable deprecated APIs
(#897)
* chore: add flag to enable/disable deprecated APIs
* chore: update for PR comments
* chore: update for PR comments
* fix: update e2e commit sha
* fix: update e2e commit sha
* fix: update flag name
---
.github/workflows/test-e2e.yaml | 4 +--
api/accounts/accounts.go | 16 ++++++---
api/accounts/accounts_test.go | 21 +++++++++--
api/api.go | 47 ++++++++++++++-----------
api/metrics_test.go | 4 +--
api/subscriptions/subscriptions.go | 27 ++++++++------
api/subscriptions/subscriptions_test.go | 19 +++++++---
cmd/thor/flags.go | 4 +++
cmd/thor/main.go | 28 +++------------
cmd/thor/utils.go | 18 ++++++++++
thorclient/api_test.go | 2 +-
11 files changed, 118 insertions(+), 72 deletions(-)
diff --git a/.github/workflows/test-e2e.yaml b/.github/workflows/test-e2e.yaml
index babbb3a39..55c82689e 100644
--- a/.github/workflows/test-e2e.yaml
+++ b/.github/workflows/test-e2e.yaml
@@ -43,8 +43,8 @@ jobs:
uses: actions/checkout@v4
with:
repository: vechain/thor-e2e-tests
- # https://github.com/vechain/thor-e2e-tests/tree/209f6ea9a81a98dc2d5e42bf036d2878c5837036
- ref: 209f6ea9a81a98dc2d5e42bf036d2878c5837036
+ # https://github.com/vechain/thor-e2e-tests/tree/8b72bedff11c9e8873d88b6e2dba356d43b56779
+ ref: 8b72bedff11c9e8873d88b6e2dba356d43b56779
- name: Download artifact
uses: actions/download-artifact@v4
diff --git a/api/accounts/accounts.go b/api/accounts/accounts.go
index 54058a160..22698bdbd 100644
--- a/api/accounts/accounts.go
+++ b/api/accounts/accounts.go
@@ -27,11 +27,12 @@ import (
)
type Accounts struct {
- repo *chain.Repository
- stater *state.Stater
- callGasLimit uint64
- forkConfig thor.ForkConfig
- bft bft.Committer
+ repo *chain.Repository
+ stater *state.Stater
+ callGasLimit uint64
+ forkConfig thor.ForkConfig
+ bft bft.Committer
+ enabledDeprecated bool
}
func New(
@@ -40,6 +41,7 @@ func New(
callGasLimit uint64,
forkConfig thor.ForkConfig,
bft bft.Committer,
+ enabledDeprecated bool,
) *Accounts {
return &Accounts{
repo,
@@ -47,6 +49,7 @@ func New(
callGasLimit,
forkConfig,
bft,
+ enabledDeprecated,
}
}
@@ -168,6 +171,9 @@ func (a *Accounts) handleGetStorage(w http.ResponseWriter, req *http.Request) er
}
func (a *Accounts) handleCallContract(w http.ResponseWriter, req *http.Request) error {
+ if !a.enabledDeprecated {
+ return utils.HTTPError(nil, http.StatusGone)
+ }
callData := &CallData{}
if err := utils.ParseJSON(req.Body, &callData); err != nil {
return utils.BadRequest(errors.WithMessage(err, "body"))
diff --git a/api/accounts/accounts_test.go b/api/accounts/accounts_test.go
index 9294723eb..8630bea4b 100644
--- a/api/accounts/accounts_test.go
+++ b/api/accounts/accounts_test.go
@@ -103,7 +103,7 @@ var (
)
func TestAccount(t *testing.T) {
- initAccountServer(t)
+ initAccountServer(t, true)
defer ts.Close()
tclient = thorclient.New(ts.URL)
@@ -126,6 +126,21 @@ func TestAccount(t *testing.T) {
}
}
+func TestDeprecated(t *testing.T) {
+ initAccountServer(t, false)
+ defer ts.Close()
+
+ tclient = thorclient.New(ts.URL)
+
+ body := &accounts.CallData{}
+
+ _, statusCode, _ := tclient.RawHTTPClient().RawHTTPPost("/accounts", body)
+ assert.Equal(t, http.StatusGone, statusCode, "invalid address")
+
+ _, statusCode, _ = tclient.RawHTTPClient().RawHTTPPost("/accounts/"+contractAddr.String(), body)
+ assert.Equal(t, http.StatusGone, statusCode, "invalid address")
+}
+
func getAccount(t *testing.T) {
_, statusCode, err := tclient.RawHTTPClient().RawHTTPGet("/accounts/" + invalidAddr)
require.NoError(t, err)
@@ -264,7 +279,7 @@ func getStorageWithNonExistingRevision(t *testing.T) {
assert.Equal(t, "revision: leveldb: not found\n", string(res), "revision not found")
}
-func initAccountServer(t *testing.T) {
+func initAccountServer(t *testing.T, enabledDeprecated bool) {
thorChain, err := testchain.NewIntegrationTestChain()
require.NoError(t, err)
@@ -291,7 +306,7 @@ func initAccountServer(t *testing.T) {
)
router := mux.NewRouter()
- accounts.New(thorChain.Repo(), thorChain.Stater(), uint64(gasLimit), thor.NoFork, thorChain.Engine()).
+ accounts.New(thorChain.Repo(), thorChain.Stater(), uint64(gasLimit), thor.NoFork, thorChain.Engine(), enabledDeprecated).
Mount(router, "/accounts")
ts = httptest.NewServer(router)
diff --git a/api/api.go b/api/api.go
index 38b412a97..0385929ec 100644
--- a/api/api.go
+++ b/api/api.go
@@ -32,6 +32,21 @@ import (
var logger = log.WithContext("pkg", "api")
+type Config struct {
+ AllowedOrigins string
+ BacktraceLimit uint32
+ CallGasLimit uint64
+ PprofOn bool
+ SkipLogs bool
+ AllowCustomTracer bool
+ EnableReqLogger bool
+ EnableMetrics bool
+ LogsLimit uint64
+ AllowedTracers []string
+ SoloMode bool
+ EnableDeprecated bool
+}
+
// New return api router
func New(
repo *chain.Repository,
@@ -41,19 +56,9 @@ func New(
bft bft.Committer,
nw node.Network,
forkConfig thor.ForkConfig,
- allowedOrigins string,
- backtraceLimit uint32,
- callGasLimit uint64,
- pprofOn bool,
- skipLogs bool,
- allowCustomTracer bool,
- enableReqLogger bool,
- enableMetrics bool,
- logsLimit uint64,
- allowedTracers []string,
- soloMode bool,
+ config Config,
) (http.HandlerFunc, func()) {
- origins := strings.Split(strings.TrimSpace(allowedOrigins), ",")
+ origins := strings.Split(strings.TrimSpace(config.AllowedOrigins), ",")
for i, o := range origins {
origins[i] = strings.ToLower(strings.TrimSpace(o))
}
@@ -71,27 +76,27 @@ func New(
http.Redirect(w, req, "doc/stoplight-ui/", http.StatusTemporaryRedirect)
})
- accounts.New(repo, stater, callGasLimit, forkConfig, bft).
+ accounts.New(repo, stater, config.CallGasLimit, forkConfig, bft, config.EnableDeprecated).
Mount(router, "/accounts")
- if !skipLogs {
- events.New(repo, logDB, logsLimit).
+ if !config.SkipLogs {
+ events.New(repo, logDB, config.LogsLimit).
Mount(router, "/logs/event")
- transfers.New(repo, logDB, logsLimit).
+ transfers.New(repo, logDB, config.LogsLimit).
Mount(router, "/logs/transfer")
}
blocks.New(repo, bft).
Mount(router, "/blocks")
transactions.New(repo, txPool).
Mount(router, "/transactions")
- debug.New(repo, stater, forkConfig, callGasLimit, allowCustomTracer, bft, allowedTracers, soloMode).
+ debug.New(repo, stater, forkConfig, config.CallGasLimit, config.AllowCustomTracer, bft, config.AllowedTracers, config.SoloMode).
Mount(router, "/debug")
node.New(nw).
Mount(router, "/node")
- subs := subscriptions.New(repo, origins, backtraceLimit, txPool)
+ subs := subscriptions.New(repo, origins, config.BacktraceLimit, txPool, config.EnableDeprecated)
subs.Mount(router, "/subscriptions")
- if pprofOn {
+ if config.PprofOn {
router.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
router.HandleFunc("/debug/pprof/profile", pprof.Profile)
router.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
@@ -99,7 +104,7 @@ func New(
router.PathPrefix("/debug/pprof/").HandlerFunc(pprof.Index)
}
- if enableMetrics {
+ if config.EnableMetrics {
router.Use(metricsMiddleware)
}
@@ -110,7 +115,7 @@ func New(
handlers.ExposedHeaders([]string{"x-genesis-id", "x-thorest-ver"}),
)(handler)
- if enableReqLogger {
+ if config.EnableReqLogger {
handler = RequestLoggerHandler(handler, logger)
}
diff --git a/api/metrics_test.go b/api/metrics_test.go
index 4e4b6daaf..9b83a08a6 100644
--- a/api/metrics_test.go
+++ b/api/metrics_test.go
@@ -48,7 +48,7 @@ func TestMetricsMiddleware(t *testing.T) {
assert.NotNil(t, err)
router := mux.NewRouter()
- acc := accounts.New(thorChain.Repo(), thorChain.Stater(), math.MaxUint64, thor.NoFork, thorChain.Engine())
+ acc := accounts.New(thorChain.Repo(), thorChain.Stater(), math.MaxUint64, thor.NoFork, thorChain.Engine(), true)
acc.Mount(router, "/accounts")
router.PathPrefix("/metrics").Handler(metrics.HTTPHandler())
router.Use(metricsMiddleware)
@@ -103,7 +103,7 @@ func TestWebsocketMetrics(t *testing.T) {
require.NoError(t, err)
router := mux.NewRouter()
- sub := subscriptions.New(thorChain.Repo(), []string{"*"}, 10, txpool.New(thorChain.Repo(), thorChain.Stater(), txpool.Options{}))
+ sub := subscriptions.New(thorChain.Repo(), []string{"*"}, 10, txpool.New(thorChain.Repo(), thorChain.Stater(), txpool.Options{}), true)
sub.Mount(router, "/subscriptions")
router.PathPrefix("/metrics").Handler(metrics.HTTPHandler())
router.Use(metricsMiddleware)
diff --git a/api/subscriptions/subscriptions.go b/api/subscriptions/subscriptions.go
index 7582da5bb..715a71308 100644
--- a/api/subscriptions/subscriptions.go
+++ b/api/subscriptions/subscriptions.go
@@ -25,14 +25,15 @@ import (
const txQueueSize = 20
type Subscriptions struct {
- backtraceLimit uint32
- repo *chain.Repository
- upgrader *websocket.Upgrader
- pendingTx *pendingTx
- done chan struct{}
- wg sync.WaitGroup
- beat2Cache *messageCache[Beat2Message]
- beatCache *messageCache[BeatMessage]
+ backtraceLimit uint32
+ enabledDeprecated bool
+ repo *chain.Repository
+ upgrader *websocket.Upgrader
+ pendingTx *pendingTx
+ done chan struct{}
+ wg sync.WaitGroup
+ beat2Cache *messageCache[Beat2Message]
+ beatCache *messageCache[BeatMessage]
}
type msgReader interface {
@@ -50,10 +51,11 @@ const (
pingPeriod = (pongWait * 7) / 10
)
-func New(repo *chain.Repository, allowedOrigins []string, backtraceLimit uint32, txpool *txpool.TxPool) *Subscriptions {
+func New(repo *chain.Repository, allowedOrigins []string, backtraceLimit uint32, txpool *txpool.TxPool, enabledDeprecated bool) *Subscriptions {
sub := &Subscriptions{
- backtraceLimit: backtraceLimit,
- repo: repo,
+ backtraceLimit: backtraceLimit,
+ repo: repo,
+ enabledDeprecated: enabledDeprecated,
upgrader: &websocket.Upgrader{
EnableCompression: true,
CheckOrigin: func(r *http.Request) bool {
@@ -195,6 +197,9 @@ func (s *Subscriptions) handleSubject(w http.ResponseWriter, req *http.Request)
return err
}
case "beat":
+ if !s.enabledDeprecated {
+ return utils.HTTPError(nil, http.StatusGone)
+ }
if reader, err = s.handleBeatReader(w, req); err != nil {
return err
}
diff --git a/api/subscriptions/subscriptions_test.go b/api/subscriptions/subscriptions_test.go
index 0c0bffe3a..8cfb55f7f 100644
--- a/api/subscriptions/subscriptions_test.go
+++ b/api/subscriptions/subscriptions_test.go
@@ -36,7 +36,7 @@ var ts *httptest.Server
var blocks []*block.Block
func TestSubscriptions(t *testing.T) {
- initSubscriptionsServer(t)
+ initSubscriptionsServer(t, true)
defer ts.Close()
for name, tt := range map[string]func(*testing.T){
@@ -51,6 +51,17 @@ func TestSubscriptions(t *testing.T) {
}
}
+func TestDeprecatedSubscriptions(t *testing.T) {
+ initSubscriptionsServer(t, false)
+ defer ts.Close()
+
+ u := url.URL{Scheme: "ws", Host: strings.TrimPrefix(ts.URL, "http://"), Path: "/subscriptions/beat"}
+
+ _, resp, err := websocket.DefaultDialer.Dial(u.String(), nil)
+ assert.Error(t, err)
+ assert.Equal(t, http.StatusGone, resp.StatusCode)
+}
+
func testHandleSubjectWithBlock(t *testing.T) {
genesisBlock := blocks[0]
queryArg := fmt.Sprintf("pos=%s", genesisBlock.Header().ID().String())
@@ -216,7 +227,7 @@ func TestParseAddress(t *testing.T) {
assert.Equal(t, expectedAddr, *result)
}
-func initSubscriptionsServer(t *testing.T) {
+func initSubscriptionsServer(t *testing.T, enabledDeprecated bool) {
thorChain, err := testchain.NewIntegrationTestChain()
require.NoError(t, err)
@@ -263,7 +274,7 @@ func initSubscriptionsServer(t *testing.T) {
require.NoError(t, err)
router := mux.NewRouter()
- New(thorChain.Repo(), []string{}, 5, txPool).
+ New(thorChain.Repo(), []string{}, 5, txPool, enabledDeprecated).
Mount(router, "/subscriptions")
ts = httptest.NewServer(router)
}
@@ -319,7 +330,7 @@ func TestSubscriptionsBacktrace(t *testing.T) {
require.NoError(t, err)
router := mux.NewRouter()
- New(thorChain.Repo(), []string{}, 5, txPool).Mount(router, "/subscriptions")
+ New(thorChain.Repo(), []string{}, 5, txPool, true).Mount(router, "/subscriptions")
ts = httptest.NewServer(router)
defer ts.Close()
diff --git a/cmd/thor/flags.go b/cmd/thor/flags.go
index 4b18f22ad..2ce97516e 100644
--- a/cmd/thor/flags.go
+++ b/cmd/thor/flags.go
@@ -69,6 +69,10 @@ var (
Value: 1000,
Usage: "limit the number of logs returned by /logs API",
}
+ apiEnableDeprecatedFlag = cli.BoolFlag{
+ Name: "api-enable-deprecated",
+ Usage: "enable deprecated API endpoints (POST /accounts/{address}, POST /accounts, WS /subscriptions/beat",
+ }
enableAPILogsFlag = cli.BoolFlag{
Name: "enable-api-logs",
Usage: "enables API requests logging",
diff --git a/cmd/thor/main.go b/cmd/thor/main.go
index 4b934bc13..7ddccc08c 100644
--- a/cmd/thor/main.go
+++ b/cmd/thor/main.go
@@ -11,7 +11,6 @@ import (
"io"
"os"
"path/filepath"
- "strings"
"time"
"github.com/ethereum/go-ethereum/accounts/keystore"
@@ -80,6 +79,7 @@ func main() {
apiCallGasLimitFlag,
apiBacktraceLimitFlag,
apiAllowCustomTracerFlag,
+ apiEnableDeprecatedFlag,
enableAPILogsFlag,
apiLogsLimitFlag,
verbosityFlag,
@@ -115,6 +115,7 @@ func main() {
apiCallGasLimitFlag,
apiBacktraceLimitFlag,
apiAllowCustomTracerFlag,
+ apiEnableDeprecatedFlag,
enableAPILogsFlag,
apiLogsLimitFlag,
onDemandFlag,
@@ -255,17 +256,7 @@ func defaultAction(ctx *cli.Context) error {
bftEngine,
p2pCommunicator.Communicator(),
forkConfig,
- ctx.String(apiCorsFlag.Name),
- uint32(ctx.Uint64(apiBacktraceLimitFlag.Name)),
- ctx.Uint64(apiCallGasLimitFlag.Name),
- ctx.Bool(pprofFlag.Name),
- skipLogs,
- ctx.Bool(apiAllowCustomTracerFlag.Name),
- ctx.Bool(enableAPILogsFlag.Name),
- ctx.Bool(enableMetricsFlag.Name),
- ctx.Uint64(apiLogsLimitFlag.Name),
- parseTracerList(strings.TrimSpace(ctx.String(allowedTracersFlag.Name))),
- false,
+ makeAPIConfig(ctx, false),
)
defer func() { log.Info("closing API..."); apiCloser() }()
@@ -399,6 +390,7 @@ func soloAction(ctx *cli.Context) error {
defer func() { log.Info("closing tx pool..."); txPool.Close() }()
bftEngine := solo.NewBFTEngine(repo)
+
apiHandler, apiCloser := api.New(
repo,
state.NewStater(mainDB),
@@ -407,17 +399,7 @@ func soloAction(ctx *cli.Context) error {
bftEngine,
&solo.Communicator{},
forkConfig,
- ctx.String(apiCorsFlag.Name),
- uint32(ctx.Uint64(apiBacktraceLimitFlag.Name)),
- ctx.Uint64(apiCallGasLimitFlag.Name),
- ctx.Bool(pprofFlag.Name),
- skipLogs,
- ctx.Bool(apiAllowCustomTracerFlag.Name),
- ctx.Bool(enableAPILogsFlag.Name),
- ctx.Bool(enableMetricsFlag.Name),
- ctx.Uint64(apiLogsLimitFlag.Name),
- parseTracerList(strings.TrimSpace(ctx.String(allowedTracersFlag.Name))),
- true,
+ makeAPIConfig(ctx, false),
)
defer func() { log.Info("closing API..."); apiCloser() }()
diff --git a/cmd/thor/utils.go b/cmd/thor/utils.go
index 5c6799354..6877a45ee 100644
--- a/cmd/thor/utils.go
+++ b/cmd/thor/utils.go
@@ -37,6 +37,7 @@ import (
"github.com/mattn/go-isatty"
"github.com/mattn/go-tty"
"github.com/pkg/errors"
+ "github.com/vechain/thor/v2/api"
"github.com/vechain/thor/v2/api/doc"
"github.com/vechain/thor/v2/chain"
"github.com/vechain/thor/v2/cmd/thor/node"
@@ -274,6 +275,23 @@ func parseGenesisFile(filePath string) (*genesis.Genesis, thor.ForkConfig, error
return customGen, forkConfig, nil
}
+func makeAPIConfig(ctx *cli.Context, soloMode bool) api.Config {
+ return api.Config{
+ AllowedOrigins: ctx.String(apiCorsFlag.Name),
+ BacktraceLimit: uint32(ctx.Uint64(apiBacktraceLimitFlag.Name)),
+ CallGasLimit: ctx.Uint64(apiCallGasLimitFlag.Name),
+ PprofOn: ctx.Bool(pprofFlag.Name),
+ SkipLogs: ctx.Bool(skipLogsFlag.Name),
+ AllowCustomTracer: ctx.Bool(apiAllowCustomTracerFlag.Name),
+ EnableReqLogger: ctx.Bool(enableAPILogsFlag.Name),
+ EnableMetrics: ctx.Bool(enableMetricsFlag.Name),
+ LogsLimit: ctx.Uint64(apiLogsLimitFlag.Name),
+ AllowedTracers: parseTracerList(strings.TrimSpace(ctx.String(allowedTracersFlag.Name))),
+ EnableDeprecated: ctx.Bool(apiEnableDeprecatedFlag.Name),
+ SoloMode: soloMode,
+ }
+}
+
func makeConfigDir(ctx *cli.Context) (string, error) {
dir := ctx.String(configDirFlag.Name)
if dir == "" {
diff --git a/thorclient/api_test.go b/thorclient/api_test.go
index e6a0e43be..e8ae49a8a 100644
--- a/thorclient/api_test.go
+++ b/thorclient/api_test.go
@@ -50,7 +50,7 @@ func initAPIServer(t *testing.T) (*testchain.Chain, *httptest.Server) {
router := mux.NewRouter()
- accounts.New(thorChain.Repo(), thorChain.Stater(), uint64(gasLimit), thor.NoFork, thorChain.Engine()).
+ accounts.New(thorChain.Repo(), thorChain.Stater(), uint64(gasLimit), thor.NoFork, thorChain.Engine(), true).
Mount(router, "/accounts")
blocks.New(thorChain.Repo(), thorChain.Engine()).Mount(router, "/blocks")
From cebfc39f8bc9e8b97e1e20d79ab1cdca4e1551ff Mon Sep 17 00:00:00 2001
From: Darren Kelly <107671032+darrenvechain@users.noreply.github.com>
Date: Wed, 4 Dec 2024 16:44:52 +0000
Subject: [PATCH 08/15] fix: solo start flags (#906)
---
cmd/thor/main.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/cmd/thor/main.go b/cmd/thor/main.go
index 7ddccc08c..15531e091 100644
--- a/cmd/thor/main.go
+++ b/cmd/thor/main.go
@@ -399,7 +399,7 @@ func soloAction(ctx *cli.Context) error {
bftEngine,
&solo.Communicator{},
forkConfig,
- makeAPIConfig(ctx, false),
+ makeAPIConfig(ctx, true),
)
defer func() { log.Info("closing API..."); apiCloser() }()
From 1ea83b2cf3a26a17c865ea38023cfb25a2e52058 Mon Sep 17 00:00:00 2001
From: Darren Kelly <107671032+darrenvechain@users.noreply.github.com>
Date: Fri, 6 Dec 2024 10:22:27 +0000
Subject: [PATCH 09/15] chore: make thorclient configurable + fix type error
(#908)
* chore: make thorclient configurable
* fix: subscriptions block type
* fix: compile errors
* fix: remove test with lint error
---
thorclient/httpclient/client.go | 6 +++++-
thorclient/thorclient.go | 10 +++++++++-
thorclient/wsclient/client.go | 5 ++---
thorclient/wsclient/client_test.go | 14 ++++++--------
4 files changed, 22 insertions(+), 13 deletions(-)
diff --git a/thorclient/httpclient/client.go b/thorclient/httpclient/client.go
index 8f88783f5..ce05bf17f 100644
--- a/thorclient/httpclient/client.go
+++ b/thorclient/httpclient/client.go
@@ -33,9 +33,13 @@ type Client struct {
// New creates a new Client with the provided URL.
func New(url string) *Client {
+ return NewWithHTTP(url, http.DefaultClient)
+}
+
+func NewWithHTTP(url string, c *http.Client) *Client {
return &Client{
url: url,
- c: &http.Client{},
+ c: c,
}
}
diff --git a/thorclient/thorclient.go b/thorclient/thorclient.go
index 8458a0ae4..0b7939f51 100644
--- a/thorclient/thorclient.go
+++ b/thorclient/thorclient.go
@@ -11,6 +11,7 @@ package thorclient
import (
"fmt"
+ "net/http"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/common/math"
@@ -44,6 +45,13 @@ func New(url string) *Client {
}
}
+// NewWithHTTP creates a new Client using the provided HTTP URL and HTTP client.
+func NewWithHTTP(url string, c *http.Client) *Client {
+ return &Client{
+ httpConn: httpclient.NewWithHTTP(url, c),
+ }
+}
+
// NewWithWS creates a new Client using the provided HTTP and WebSocket URLs.
// Returns an error if the WebSocket connection fails.
func NewWithWS(url string) (*Client, error) {
@@ -202,7 +210,7 @@ func (c *Client) ChainTag() (byte, error) {
}
// SubscribeBlocks subscribes to block updates over WebSocket.
-func (c *Client) SubscribeBlocks(pos string) (*common.Subscription[*blocks.JSONCollapsedBlock], error) {
+func (c *Client) SubscribeBlocks(pos string) (*common.Subscription[*subscriptions.BlockMessage], error) {
if c.wsConn == nil {
return nil, fmt.Errorf("not a websocket typed client")
}
diff --git a/thorclient/wsclient/client.go b/thorclient/wsclient/client.go
index 057d5aa48..9eb1519ab 100644
--- a/thorclient/wsclient/client.go
+++ b/thorclient/wsclient/client.go
@@ -16,7 +16,6 @@ import (
"github.com/vechain/thor/v2/thor"
"github.com/gorilla/websocket"
- "github.com/vechain/thor/v2/api/blocks"
"github.com/vechain/thor/v2/api/subscriptions"
"github.com/vechain/thor/v2/thorclient/common"
)
@@ -89,7 +88,7 @@ func (c *Client) SubscribeEvents(pos string, filter *subscriptions.EventFilter)
// SubscribeBlocks subscribes to block updates based on the provided query.
// It returns a Subscription that streams block messages or an error if the connection fails.
-func (c *Client) SubscribeBlocks(pos string) (*common.Subscription[*blocks.JSONCollapsedBlock], error) {
+func (c *Client) SubscribeBlocks(pos string) (*common.Subscription[*subscriptions.BlockMessage], error) {
queryValues := &url.Values{}
queryValues.Add("pos", pos)
conn, err := c.connect("/subscriptions/block", queryValues)
@@ -97,7 +96,7 @@ func (c *Client) SubscribeBlocks(pos string) (*common.Subscription[*blocks.JSONC
return nil, fmt.Errorf("unable to connect - %w", err)
}
- return subscribe[blocks.JSONCollapsedBlock](conn), nil
+ return subscribe[subscriptions.BlockMessage](conn), nil
}
// SubscribeTransfers subscribes to transfer events based on the provided query.
diff --git a/thorclient/wsclient/client_test.go b/thorclient/wsclient/client_test.go
index 483ae7233..19dd1b395 100644
--- a/thorclient/wsclient/client_test.go
+++ b/thorclient/wsclient/client_test.go
@@ -13,13 +13,11 @@ import (
"testing"
"time"
- "github.com/vechain/thor/v2/test/datagen"
- "github.com/vechain/thor/v2/thor"
-
"github.com/gorilla/websocket"
"github.com/stretchr/testify/assert"
- "github.com/vechain/thor/v2/api/blocks"
"github.com/vechain/thor/v2/api/subscriptions"
+ "github.com/vechain/thor/v2/test/datagen"
+ "github.com/vechain/thor/v2/thor"
"github.com/vechain/thor/v2/thorclient/common"
)
@@ -50,7 +48,7 @@ func TestClient_SubscribeEvents(t *testing.T) {
func TestClient_SubscribeBlocks(t *testing.T) {
pos := "best"
- expectedBlock := &blocks.JSONCollapsedBlock{}
+ expectedBlock := &subscriptions.BlockMessage{}
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, "/subscriptions/block", r.URL.Path)
@@ -288,7 +286,7 @@ func TestClient_SubscribeBlocks_ServerError(t *testing.T) {
func TestClient_SubscribeBlocks_ServerShutdown(t *testing.T) {
pos := "best"
- expectedBlock := &blocks.JSONCollapsedBlock{}
+ expectedBlock := &subscriptions.BlockMessage{}
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, "/subscriptions/block", r.URL.Path)
@@ -325,7 +323,7 @@ func TestClient_SubscribeBlocks_ServerShutdown(t *testing.T) {
func TestClient_SubscribeBlocks_ClientShutdown(t *testing.T) {
pos := "best"
- expectedBlock := &blocks.JSONCollapsedBlock{}
+ expectedBlock := &subscriptions.BlockMessage{}
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, "/subscriptions/block", r.URL.Path)
@@ -377,7 +375,7 @@ func TestClient_SubscribeBlocks_ClientShutdown(t *testing.T) {
func TestClient_SubscribeBlocks_ClientShutdown_LongBlocks(t *testing.T) {
pos := "best"
- expectedBlock := &blocks.JSONCollapsedBlock{}
+ expectedBlock := &subscriptions.BlockMessage{}
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, "/subscriptions/block", r.URL.Path)
From 3aa2935248ee709f3e0230e2679613c14c0f837a Mon Sep 17 00:00:00 2001
From: YeahNotSewerSide <47860375+YeahNotSewerSide@users.noreply.github.com>
Date: Fri, 6 Dec 2024 12:31:28 +0200
Subject: [PATCH 10/15] add 'raw' query parameter to the blocks (#899)
* add 'raw' query parameter to the blocks
* summary -> summary.Header
Co-authored-by: libotony
* change variable name
* make expanded and raw mutually exclusive
* add unit tests
* fix linting
---------
Co-authored-by: libotony
---
api/blocks/blocks.go | 29 ++++++++++++++++++---
api/blocks/blocks_test.go | 55 +++++++++++++++++++++++++++++++++++++++
api/blocks/types.go | 4 +++
api/doc/thor.yaml | 13 +++++++++
api/utils/http.go | 13 +++++++++
5 files changed, 110 insertions(+), 4 deletions(-)
diff --git a/api/blocks/blocks.go b/api/blocks/blocks.go
index ff86e02e6..a8e072a02 100644
--- a/api/blocks/blocks.go
+++ b/api/blocks/blocks.go
@@ -6,8 +6,11 @@
package blocks
import (
+ "encoding/hex"
+ "fmt"
"net/http"
+ "github.com/ethereum/go-ethereum/rlp"
"github.com/gorilla/mux"
"github.com/pkg/errors"
"github.com/vechain/thor/v2/api/utils"
@@ -34,9 +37,17 @@ func (b *Blocks) handleGetBlock(w http.ResponseWriter, req *http.Request) error
if err != nil {
return utils.BadRequest(errors.WithMessage(err, "revision"))
}
- expanded := req.URL.Query().Get("expanded")
- if expanded != "" && expanded != "false" && expanded != "true" {
- return utils.BadRequest(errors.WithMessage(errors.New("should be boolean"), "expanded"))
+ raw, err := utils.StringToBoolean(req.URL.Query().Get("raw"), false)
+ if err != nil {
+ return utils.BadRequest(errors.WithMessage(err, "raw"))
+ }
+ expanded, err := utils.StringToBoolean(req.URL.Query().Get("expanded"), false)
+ if err != nil {
+ return utils.BadRequest(errors.WithMessage(err, "expanded"))
+ }
+
+ if raw && expanded {
+ return utils.BadRequest(errors.WithMessage(errors.New("Raw and Expanded are mutually exclusive"), "raw&expanded"))
}
summary, err := utils.GetSummary(revision, b.repo, b.bft)
@@ -47,6 +58,16 @@ func (b *Blocks) handleGetBlock(w http.ResponseWriter, req *http.Request) error
return err
}
+ if raw {
+ rlpEncoded, err := rlp.EncodeToBytes(summary.Header)
+ if err != nil {
+ return err
+ }
+ return utils.WriteJSON(w, &JSONRawBlockSummary{
+ fmt.Sprintf("0x%s", hex.EncodeToString(rlpEncoded)),
+ })
+ }
+
isTrunk, err := b.isTrunk(summary.Header.ID(), summary.Header.Number())
if err != nil {
return err
@@ -61,7 +82,7 @@ func (b *Blocks) handleGetBlock(w http.ResponseWriter, req *http.Request) error
}
jSummary := buildJSONBlockSummary(summary, isTrunk, isFinalized)
- if expanded == "true" {
+ if expanded {
txs, err := b.repo.GetBlockTransactions(summary.Header.ID())
if err != nil {
return err
diff --git a/api/blocks/blocks_test.go b/api/blocks/blocks_test.go
index dcb6c4e94..8c0439e59 100644
--- a/api/blocks/blocks_test.go
+++ b/api/blocks/blocks_test.go
@@ -6,6 +6,7 @@
package blocks_test
import (
+ "encoding/hex"
"encoding/json"
"math"
"math/big"
@@ -15,6 +16,7 @@ import (
"strings"
"testing"
+ "github.com/ethereum/go-ethereum/rlp"
"github.com/gorilla/mux"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -55,6 +57,8 @@ func TestBlock(t *testing.T) {
"testGetFinalizedBlock": testGetFinalizedBlock,
"testGetJustifiedBlock": testGetJustifiedBlock,
"testGetBlockWithRevisionNumberTooHigh": testGetBlockWithRevisionNumberTooHigh,
+ "testMutuallyExclusiveQueries": testMutuallyExclusiveQueries,
+ "testGetRawBlock": testGetRawBlock,
} {
t.Run(name, tt)
}
@@ -67,6 +71,22 @@ func testBadQueryParams(t *testing.T) {
assert.Equal(t, http.StatusBadRequest, statusCode)
assert.Equal(t, "expanded: should be boolean", strings.TrimSpace(string(res)))
+
+ badQueryParams = "?raw=1"
+ res, statusCode, err = tclient.RawHTTPClient().RawHTTPGet("/blocks/best" + badQueryParams)
+ require.NoError(t, err)
+
+ assert.Equal(t, http.StatusBadRequest, statusCode)
+ assert.Equal(t, "raw: should be boolean", strings.TrimSpace(string(res)))
+}
+
+func testMutuallyExclusiveQueries(t *testing.T) {
+ badQueryParams := "?expanded=true&raw=true"
+ res, statusCode, err := tclient.RawHTTPClient().RawHTTPGet("/blocks/best" + badQueryParams)
+ require.NoError(t, err)
+
+ assert.Equal(t, http.StatusBadRequest, statusCode)
+ assert.Equal(t, "raw&expanded: Raw and Expanded are mutually exclusive", strings.TrimSpace(string(res)))
}
func testGetBestBlock(t *testing.T) {
@@ -80,6 +100,41 @@ func testGetBestBlock(t *testing.T) {
assert.Equal(t, http.StatusOK, statusCode)
}
+func testGetRawBlock(t *testing.T) {
+ res, statusCode, err := tclient.RawHTTPClient().RawHTTPGet("/blocks/best?raw=true")
+ require.NoError(t, err)
+ rawBlock := new(blocks.JSONRawBlockSummary)
+ if err := json.Unmarshal(res, &rawBlock); err != nil {
+ t.Fatal(err)
+ }
+
+ blockBytes, err := hex.DecodeString(rawBlock.Raw[2:len(rawBlock.Raw)])
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ header := block.Header{}
+ err = rlp.DecodeBytes(blockBytes, &header)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ expHeader := blk.Header()
+ assert.Equal(t, expHeader.Number(), header.Number(), "Number should be equal")
+ assert.Equal(t, expHeader.ID(), header.ID(), "Hash should be equal")
+ assert.Equal(t, expHeader.ParentID(), header.ParentID(), "ParentID should be equal")
+ assert.Equal(t, expHeader.Timestamp(), header.Timestamp(), "Timestamp should be equal")
+ assert.Equal(t, expHeader.TotalScore(), header.TotalScore(), "TotalScore should be equal")
+ assert.Equal(t, expHeader.GasLimit(), header.GasLimit(), "GasLimit should be equal")
+ assert.Equal(t, expHeader.GasUsed(), header.GasUsed(), "GasUsed should be equal")
+ assert.Equal(t, expHeader.Beneficiary(), header.Beneficiary(), "Beneficiary should be equal")
+ assert.Equal(t, expHeader.TxsRoot(), header.TxsRoot(), "TxsRoot should be equal")
+ assert.Equal(t, expHeader.StateRoot(), header.StateRoot(), "StateRoot should be equal")
+ assert.Equal(t, expHeader.ReceiptsRoot(), header.ReceiptsRoot(), "ReceiptsRoot should be equal")
+
+ assert.Equal(t, http.StatusOK, statusCode)
+}
+
func testGetBlockByHeight(t *testing.T) {
res, statusCode, err := tclient.RawHTTPClient().RawHTTPGet("/blocks/1")
require.NoError(t, err)
diff --git a/api/blocks/types.go b/api/blocks/types.go
index 38261b2e5..989b63041 100644
--- a/api/blocks/types.go
+++ b/api/blocks/types.go
@@ -33,6 +33,10 @@ type JSONBlockSummary struct {
IsFinalized bool `json:"isFinalized"`
}
+type JSONRawBlockSummary struct {
+ Raw string `json:"raw"`
+}
+
type JSONCollapsedBlock struct {
*JSONBlockSummary
Transactions []thor.Bytes32 `json:"transactions"`
diff --git a/api/doc/thor.yaml b/api/doc/thor.yaml
index 732a5f1a3..dcf0ae6b9 100644
--- a/api/doc/thor.yaml
+++ b/api/doc/thor.yaml
@@ -292,6 +292,7 @@ paths:
parameters:
- $ref: '#/components/parameters/RevisionInPath'
- $ref: '#/components/parameters/ExpandedInQuery'
+ - $ref: '#/components/parameters/RawBlockInQuery'
tags:
- Blocks
summary: Retrieve a block
@@ -2353,6 +2354,18 @@ components:
type: boolean
example: false
+ RawBlockInQuery:
+ name: raw
+ in: query
+ required: false
+ description: |
+ Whether the block should be returned in RLP encoding or not.
+ - `true` returns `block` as an RLP encoded object
+ - `false` returns `block` as a structured JSON object
+ schema:
+ type: boolean
+ example: false
+
PendingInQuery:
name: pending
in: query
diff --git a/api/utils/http.go b/api/utils/http.go
index 652c3e408..2235797de 100644
--- a/api/utils/http.go
+++ b/api/utils/http.go
@@ -9,6 +9,8 @@ import (
"encoding/json"
"io"
"net/http"
+
+ "github.com/pkg/errors"
)
type httpError struct {
@@ -36,6 +38,17 @@ func BadRequest(cause error) error {
}
}
+func StringToBoolean(boolStr string, defaultVal bool) (bool, error) {
+ if boolStr == "" {
+ return defaultVal, nil
+ } else if boolStr == "false" {
+ return false, nil
+ } else if boolStr == "true" {
+ return true, nil
+ }
+ return false, errors.New("should be boolean")
+}
+
// Forbidden convenience method to create http forbidden error.
func Forbidden(cause error) error {
return &httpError{
From 7579db4891fcb98ae4da6fb85a0fd278ed603d39 Mon Sep 17 00:00:00 2001
From: Pedro Gomes
Date: Mon, 9 Dec 2024 10:55:34 +0000
Subject: [PATCH 11/15] Adding Health endpoint (#836)
* Adding Health endpoint
* pr comments + 503 if not healthy
* refactored admin server and api + health endpoint tests
* fix health condition
* fix admin routing
* added comments + changed from ChainSync to ChainBootstrapStatus
* Adding healthcheck for solo mode
* adding solo + tests
* fix log_level handler funcs
* refactor health package + add p2p count
* remove solo methods
* moving health service to api pkg
* added defaults + api health query
* pr comments
* pr comments
* pr comments
* Update cmd/thor/main.go
---
api/admin.go | 62 --------------
api/admin/admin.go | 28 +++++++
api/admin/health/health.go | 84 +++++++++++++++++++
api/admin/health/health_api.go | 68 +++++++++++++++
api/admin/health/health_api_test.go | 59 +++++++++++++
api/admin/health/health_test.go | 71 ++++++++++++++++
api/admin/loglevel/log_level.go | 74 ++++++++++++++++
.../loglevel/log_level_test.go} | 10 ++-
api/admin/loglevel/types.go | 14 ++++
api/admin_server.go | 36 +++-----
api/node/node_test.go | 3 +-
cmd/thor/main.go | 63 +++++++-------
comm/communicator.go | 5 +-
13 files changed, 456 insertions(+), 121 deletions(-)
delete mode 100644 api/admin.go
create mode 100644 api/admin/admin.go
create mode 100644 api/admin/health/health.go
create mode 100644 api/admin/health/health_api.go
create mode 100644 api/admin/health/health_api_test.go
create mode 100644 api/admin/health/health_test.go
create mode 100644 api/admin/loglevel/log_level.go
rename api/{admin_test.go => admin/loglevel/log_level_test.go} (93%)
create mode 100644 api/admin/loglevel/types.go
diff --git a/api/admin.go b/api/admin.go
deleted file mode 100644
index afd299cfa..000000000
--- a/api/admin.go
+++ /dev/null
@@ -1,62 +0,0 @@
-// Copyright (c) 2024 The VeChainThor developers
-
-// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying
-// file LICENSE or
-
-package api
-
-import (
- "log/slog"
- "net/http"
-
- "github.com/pkg/errors"
- "github.com/vechain/thor/v2/api/utils"
- "github.com/vechain/thor/v2/log"
-)
-
-type logLevelRequest struct {
- Level string `json:"level"`
-}
-
-type logLevelResponse struct {
- CurrentLevel string `json:"currentLevel"`
-}
-
-func getLogLevelHandler(logLevel *slog.LevelVar) utils.HandlerFunc {
- return func(w http.ResponseWriter, r *http.Request) error {
- return utils.WriteJSON(w, logLevelResponse{
- CurrentLevel: logLevel.Level().String(),
- })
- }
-}
-
-func postLogLevelHandler(logLevel *slog.LevelVar) utils.HandlerFunc {
- return func(w http.ResponseWriter, r *http.Request) error {
- var req logLevelRequest
-
- if err := utils.ParseJSON(r.Body, &req); err != nil {
- return utils.BadRequest(errors.WithMessage(err, "Invalid request body"))
- }
-
- switch req.Level {
- case "debug":
- logLevel.Set(log.LevelDebug)
- case "info":
- logLevel.Set(log.LevelInfo)
- case "warn":
- logLevel.Set(log.LevelWarn)
- case "error":
- logLevel.Set(log.LevelError)
- case "trace":
- logLevel.Set(log.LevelTrace)
- case "crit":
- logLevel.Set(log.LevelCrit)
- default:
- return utils.BadRequest(errors.New("Invalid verbosity level"))
- }
-
- return utils.WriteJSON(w, logLevelResponse{
- CurrentLevel: logLevel.Level().String(),
- })
- }
-}
diff --git a/api/admin/admin.go b/api/admin/admin.go
new file mode 100644
index 000000000..1e16415f8
--- /dev/null
+++ b/api/admin/admin.go
@@ -0,0 +1,28 @@
+// Copyright (c) 2024 The VeChainThor developers
+
+// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying
+// file LICENSE or
+
+package admin
+
+import (
+ "log/slog"
+ "net/http"
+
+ "github.com/gorilla/handlers"
+ "github.com/gorilla/mux"
+ healthAPI "github.com/vechain/thor/v2/api/admin/health"
+ "github.com/vechain/thor/v2/api/admin/loglevel"
+)
+
+func New(logLevel *slog.LevelVar, health *healthAPI.Health) http.HandlerFunc {
+ router := mux.NewRouter()
+ subRouter := router.PathPrefix("/admin").Subrouter()
+
+ loglevel.New(logLevel).Mount(subRouter, "/loglevel")
+ healthAPI.NewAPI(health).Mount(subRouter, "/health")
+
+ handler := handlers.CompressHandler(router)
+
+ return handler.ServeHTTP
+}
diff --git a/api/admin/health/health.go b/api/admin/health/health.go
new file mode 100644
index 000000000..41522e32d
--- /dev/null
+++ b/api/admin/health/health.go
@@ -0,0 +1,84 @@
+// Copyright (c) 2024 The VeChainThor developers
+
+// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying
+// file LICENSE or
+
+package health
+
+import (
+ "time"
+
+ "github.com/vechain/thor/v2/chain"
+ "github.com/vechain/thor/v2/comm"
+ "github.com/vechain/thor/v2/thor"
+)
+
+type BlockIngestion struct {
+ ID *thor.Bytes32 `json:"id"`
+ Timestamp *time.Time `json:"timestamp"`
+}
+
+type Status struct {
+ Healthy bool `json:"healthy"`
+ BestBlockTime *time.Time `json:"bestBlockTime"`
+ PeerCount int `json:"peerCount"`
+ IsNetworkProgressing bool `json:"isNetworkProgressing"`
+}
+
+type Health struct {
+ repo *chain.Repository
+ p2p *comm.Communicator
+}
+
+const (
+ defaultBlockTolerance = time.Duration(2*thor.BlockInterval) * time.Second // 2 blocks tolerance
+ defaultMinPeerCount = 2
+)
+
+func New(repo *chain.Repository, p2p *comm.Communicator) *Health {
+ return &Health{
+ repo: repo,
+ p2p: p2p,
+ }
+}
+
+// isNetworkProgressing checks if the network is producing new blocks within the allowed interval.
+func (h *Health) isNetworkProgressing(now time.Time, bestBlockTimestamp time.Time, blockTolerance time.Duration) bool {
+ return now.Sub(bestBlockTimestamp) <= blockTolerance
+}
+
+// isNodeConnectedP2P checks if the node is connected to peers
+func (h *Health) isNodeConnectedP2P(peerCount int, minPeerCount int) bool {
+ return peerCount >= minPeerCount
+}
+
+func (h *Health) Status(blockTolerance time.Duration, minPeerCount int) (*Status, error) {
+ // Fetch the best block details
+ bestBlock := h.repo.BestBlockSummary()
+ bestBlockTimestamp := time.Unix(int64(bestBlock.Header.Timestamp()), 0)
+
+ // Fetch the current connected peers
+ var connectedPeerCount int
+ if h.p2p == nil {
+ connectedPeerCount = minPeerCount // ignore peers in solo mode
+ } else {
+ connectedPeerCount = h.p2p.PeerCount()
+ }
+
+ now := time.Now()
+
+ // Perform the checks
+ networkProgressing := h.isNetworkProgressing(now, bestBlockTimestamp, blockTolerance)
+ nodeConnected := h.isNodeConnectedP2P(connectedPeerCount, minPeerCount)
+
+ // Calculate overall health status
+ healthy := networkProgressing && nodeConnected
+
+ // Return the current status
+ return &Status{
+ Healthy: healthy,
+ BestBlockTime: &bestBlockTimestamp,
+ IsNetworkProgressing: networkProgressing,
+ PeerCount: connectedPeerCount,
+ }, nil
+}
diff --git a/api/admin/health/health_api.go b/api/admin/health/health_api.go
new file mode 100644
index 000000000..3bad13f07
--- /dev/null
+++ b/api/admin/health/health_api.go
@@ -0,0 +1,68 @@
+// Copyright (c) 2024 The VeChainThor developers
+
+// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying
+// file LICENSE or
+
+package health
+
+import (
+ "net/http"
+ "strconv"
+ "time"
+
+ "github.com/gorilla/mux"
+ "github.com/vechain/thor/v2/api/utils"
+)
+
+type API struct {
+ healthStatus *Health
+}
+
+func NewAPI(healthStatus *Health) *API {
+ return &API{
+ healthStatus: healthStatus,
+ }
+}
+
+func (h *API) handleGetHealth(w http.ResponseWriter, r *http.Request) error {
+ // Parse query parameters
+ query := r.URL.Query()
+
+ // Default to constants if query parameters are not provided
+ blockTolerance := defaultBlockTolerance
+ minPeerCount := defaultMinPeerCount
+
+ // Override with query parameters if they exist
+ if queryBlockTolerance := query.Get("blockTolerance"); queryBlockTolerance != "" {
+ if parsed, err := time.ParseDuration(queryBlockTolerance); err == nil {
+ blockTolerance = parsed
+ }
+ }
+
+ if queryMinPeerCount := query.Get("minPeerCount"); queryMinPeerCount != "" {
+ if parsed, err := strconv.Atoi(queryMinPeerCount); err == nil {
+ minPeerCount = parsed
+ }
+ }
+
+ acc, err := h.healthStatus.Status(blockTolerance, minPeerCount)
+ if err != nil {
+ return err
+ }
+
+ if !acc.Healthy {
+ w.WriteHeader(http.StatusServiceUnavailable) // Set the status to 503
+ } else {
+ w.WriteHeader(http.StatusOK) // Set the status to 200
+ }
+ return utils.WriteJSON(w, acc)
+}
+
+func (h *API) Mount(root *mux.Router, pathPrefix string) {
+ sub := root.PathPrefix(pathPrefix).Subrouter()
+
+ sub.Path("").
+ Methods(http.MethodGet).
+ Name("health").
+ HandlerFunc(utils.WrapHandlerFunc(h.handleGetHealth))
+}
diff --git a/api/admin/health/health_api_test.go b/api/admin/health/health_api_test.go
new file mode 100644
index 000000000..e50af0398
--- /dev/null
+++ b/api/admin/health/health_api_test.go
@@ -0,0 +1,59 @@
+// Copyright (c) 2024 The VeChainThor developers
+
+// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying
+// file LICENSE or
+
+package health
+
+import (
+ "encoding/json"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "github.com/gorilla/mux"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "github.com/vechain/thor/v2/comm"
+ "github.com/vechain/thor/v2/test/testchain"
+ "github.com/vechain/thor/v2/txpool"
+)
+
+var ts *httptest.Server
+
+func TestHealth(t *testing.T) {
+ initAPIServer(t)
+
+ var healthStatus Status
+ respBody, statusCode := httpGet(t, ts.URL+"/health")
+ require.NoError(t, json.Unmarshal(respBody, &healthStatus))
+ assert.False(t, healthStatus.Healthy)
+ assert.Equal(t, http.StatusServiceUnavailable, statusCode)
+}
+
+func initAPIServer(t *testing.T) {
+ thorChain, err := testchain.NewIntegrationTestChain()
+ require.NoError(t, err)
+
+ router := mux.NewRouter()
+ NewAPI(
+ New(thorChain.Repo(), comm.New(thorChain.Repo(), txpool.New(thorChain.Repo(), nil, txpool.Options{}))),
+ ).Mount(router, "/health")
+
+ ts = httptest.NewServer(router)
+}
+
+func httpGet(t *testing.T, url string) ([]byte, int) {
+ res, err := http.Get(url) //#nosec G107
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer res.Body.Close()
+
+ r, err := io.ReadAll(res.Body)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return r, res.StatusCode
+}
diff --git a/api/admin/health/health_test.go b/api/admin/health/health_test.go
new file mode 100644
index 000000000..60f9a3dcd
--- /dev/null
+++ b/api/admin/health/health_test.go
@@ -0,0 +1,71 @@
+// Copyright (c) 2024 The VeChainThor developers
+
+// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying
+// file LICENSE or
+
+package health
+
+import (
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestHealth_isNetworkProgressing(t *testing.T) {
+ h := &Health{}
+
+ now := time.Now()
+
+ tests := []struct {
+ name string
+ bestBlockTimestamp time.Time
+ expectedProgressing bool
+ }{
+ {
+ name: "Progressing - block within timeBetweenBlocks",
+ bestBlockTimestamp: now.Add(-5 * time.Second),
+ expectedProgressing: true,
+ },
+ {
+ name: "Not Progressing - block outside timeBetweenBlocks",
+ bestBlockTimestamp: now.Add(-25 * time.Second),
+ expectedProgressing: false,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ isProgressing := h.isNetworkProgressing(now, tt.bestBlockTimestamp, defaultBlockTolerance)
+ assert.Equal(t, tt.expectedProgressing, isProgressing, "isNetworkProgressing result mismatch")
+ })
+ }
+}
+
+func TestHealth_isNodeConnectedP2P(t *testing.T) {
+ h := &Health{}
+
+ tests := []struct {
+ name string
+ peerCount int
+ expectedConnected bool
+ }{
+ {
+ name: "Connected - more than one peer",
+ peerCount: 3,
+ expectedConnected: true,
+ },
+ {
+ name: "Not Connected - one or fewer peers",
+ peerCount: 1,
+ expectedConnected: false,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ isConnected := h.isNodeConnectedP2P(tt.peerCount, defaultMinPeerCount)
+ assert.Equal(t, tt.expectedConnected, isConnected, "isNodeConnectedP2P result mismatch")
+ })
+ }
+}
diff --git a/api/admin/loglevel/log_level.go b/api/admin/loglevel/log_level.go
new file mode 100644
index 000000000..d3c339ce2
--- /dev/null
+++ b/api/admin/loglevel/log_level.go
@@ -0,0 +1,74 @@
+// Copyright (c) 2024 The VeChainThor developers
+
+// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying
+// file LICENSE or
+
+package loglevel
+
+import (
+ "log/slog"
+ "net/http"
+
+ "github.com/gorilla/mux"
+ "github.com/pkg/errors"
+ "github.com/vechain/thor/v2/api/utils"
+ "github.com/vechain/thor/v2/log"
+)
+
+type LogLevel struct {
+ logLevel *slog.LevelVar
+}
+
+func New(logLevel *slog.LevelVar) *LogLevel {
+ return &LogLevel{
+ logLevel: logLevel,
+ }
+}
+
+func (l *LogLevel) Mount(root *mux.Router, pathPrefix string) {
+ sub := root.PathPrefix(pathPrefix).Subrouter()
+ sub.Path("").
+ Methods(http.MethodGet).
+ Name("get-log-level").
+ HandlerFunc(utils.WrapHandlerFunc(l.getLogLevelHandler))
+
+ sub.Path("").
+ Methods(http.MethodPost).
+ Name("post-log-level").
+ HandlerFunc(utils.WrapHandlerFunc(l.postLogLevelHandler))
+}
+
+func (l *LogLevel) getLogLevelHandler(w http.ResponseWriter, _ *http.Request) error {
+ return utils.WriteJSON(w, Response{
+ CurrentLevel: l.logLevel.Level().String(),
+ })
+}
+
+func (l *LogLevel) postLogLevelHandler(w http.ResponseWriter, r *http.Request) error {
+ var req Request
+
+ if err := utils.ParseJSON(r.Body, &req); err != nil {
+ return utils.BadRequest(errors.WithMessage(err, "Invalid request body"))
+ }
+
+ switch req.Level {
+ case "debug":
+ l.logLevel.Set(log.LevelDebug)
+ case "info":
+ l.logLevel.Set(log.LevelInfo)
+ case "warn":
+ l.logLevel.Set(log.LevelWarn)
+ case "error":
+ l.logLevel.Set(log.LevelError)
+ case "trace":
+ l.logLevel.Set(log.LevelTrace)
+ case "crit":
+ l.logLevel.Set(log.LevelCrit)
+ default:
+ return utils.BadRequest(errors.New("Invalid verbosity level"))
+ }
+
+ return utils.WriteJSON(w, Response{
+ CurrentLevel: l.logLevel.Level().String(),
+ })
+}
diff --git a/api/admin_test.go b/api/admin/loglevel/log_level_test.go
similarity index 93%
rename from api/admin_test.go
rename to api/admin/loglevel/log_level_test.go
index be2847cbf..3d1a8a960 100644
--- a/api/admin_test.go
+++ b/api/admin/loglevel/log_level_test.go
@@ -3,7 +3,7 @@
// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying
// file LICENSE or
-package api
+package loglevel
import (
"bytes"
@@ -14,6 +14,7 @@ import (
"strings"
"testing"
+ "github.com/gorilla/mux"
"github.com/stretchr/testify/assert"
)
@@ -76,15 +77,16 @@ func TestLogLevelHandler(t *testing.T) {
}
rr := httptest.NewRecorder()
- handler := http.HandlerFunc(HTTPHandler(&logLevel).ServeHTTP)
- handler.ServeHTTP(rr, req)
+ router := mux.NewRouter()
+ New(&logLevel).Mount(router, "/admin/loglevel")
+ router.ServeHTTP(rr, req)
if status := rr.Code; status != tt.expectedStatus {
t.Errorf("handler returned wrong status code: got %v want %v", status, tt.expectedStatus)
}
if tt.expectedLevel != "" {
- var response logLevelResponse
+ var response Response
if err := json.NewDecoder(rr.Body).Decode(&response); err != nil {
t.Fatalf("could not decode response: %v", err)
}
diff --git a/api/admin/loglevel/types.go b/api/admin/loglevel/types.go
new file mode 100644
index 000000000..ce57187b1
--- /dev/null
+++ b/api/admin/loglevel/types.go
@@ -0,0 +1,14 @@
+// Copyright (c) 2024 The VeChainThor developers
+
+// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying
+// file LICENSE or
+
+package loglevel
+
+type Request struct {
+ Level string `json:"level"`
+}
+
+type Response struct {
+ CurrentLevel string `json:"currentLevel"`
+}
diff --git a/api/admin_server.go b/api/admin_server.go
index 26054e908..f2315aeb1 100644
--- a/api/admin_server.go
+++ b/api/admin_server.go
@@ -11,40 +11,28 @@ import (
"net/http"
"time"
- "github.com/gorilla/handlers"
- "github.com/gorilla/mux"
"github.com/pkg/errors"
- "github.com/vechain/thor/v2/api/utils"
+ "github.com/vechain/thor/v2/api/admin"
+ "github.com/vechain/thor/v2/api/admin/health"
+ "github.com/vechain/thor/v2/chain"
"github.com/vechain/thor/v2/co"
+ "github.com/vechain/thor/v2/comm"
)
-func HTTPHandler(logLevel *slog.LevelVar) http.Handler {
- router := mux.NewRouter()
- sub := router.PathPrefix("/admin").Subrouter()
- sub.Path("/loglevel").
- Methods(http.MethodGet).
- Name("get-log-level").
- HandlerFunc(utils.WrapHandlerFunc(getLogLevelHandler(logLevel)))
-
- sub.Path("/loglevel").
- Methods(http.MethodPost).
- Name("post-log-level").
- HandlerFunc(utils.WrapHandlerFunc(postLogLevelHandler(logLevel)))
-
- return handlers.CompressHandler(router)
-}
-
-func StartAdminServer(addr string, logLevel *slog.LevelVar) (string, func(), error) {
+func StartAdminServer(
+ addr string,
+ logLevel *slog.LevelVar,
+ repo *chain.Repository,
+ p2p *comm.Communicator,
+) (string, func(), error) {
listener, err := net.Listen("tcp", addr)
if err != nil {
return "", nil, errors.Wrapf(err, "listen admin API addr [%v]", addr)
}
- router := mux.NewRouter()
- router.PathPrefix("/admin").Handler(HTTPHandler(logLevel))
- handler := handlers.CompressHandler(router)
+ adminHandler := admin.New(logLevel, health.New(repo, p2p))
- srv := &http.Server{Handler: handler, ReadHeaderTimeout: time.Second, ReadTimeout: 5 * time.Second}
+ srv := &http.Server{Handler: adminHandler, ReadHeaderTimeout: time.Second, ReadTimeout: 5 * time.Second}
var goes co.Goes
goes.Go(func() {
srv.Serve(listener)
diff --git a/api/node/node_test.go b/api/node/node_test.go
index 3dd2e96ee..873ad29ad 100644
--- a/api/node/node_test.go
+++ b/api/node/node_test.go
@@ -40,7 +40,8 @@ func initCommServer(t *testing.T) {
Limit: 10000,
LimitPerAccount: 16,
MaxLifetime: 10 * time.Minute,
- }))
+ }),
+ )
router := mux.NewRouter()
node.New(communicator).Mount(router, "/node")
diff --git a/cmd/thor/main.go b/cmd/thor/main.go
index 15531e091..ba19b0c11 100644
--- a/cmd/thor/main.go
+++ b/cmd/thor/main.go
@@ -180,16 +180,6 @@ func defaultAction(ctx *cli.Context) error {
defer func() { log.Info("stopping metrics server..."); closeFunc() }()
}
- adminURL := ""
- if ctx.Bool(enableAdminFlag.Name) {
- url, closeFunc, err := api.StartAdminServer(ctx.String(adminAddrFlag.Name), logLevel)
- if err != nil {
- return fmt.Errorf("unable to start admin server - %w", err)
- }
- adminURL = url
- defer func() { log.Info("stopping admin server..."); closeFunc() }()
- }
-
gene, forkConfig, err := selectGenesis(ctx)
if err != nil {
return err
@@ -243,6 +233,21 @@ func defaultAction(ctx *cli.Context) error {
return err
}
+ adminURL := ""
+ if ctx.Bool(enableAdminFlag.Name) {
+ url, closeFunc, err := api.StartAdminServer(
+ ctx.String(adminAddrFlag.Name),
+ logLevel,
+ repo,
+ p2pCommunicator.Communicator(),
+ )
+ if err != nil {
+ return fmt.Errorf("unable to start admin server - %w", err)
+ }
+ adminURL = url
+ defer func() { log.Info("stopping admin server..."); closeFunc() }()
+ }
+
bftEngine, err := bft.NewEngine(repo, mainDB, forkConfig, master.Address())
if err != nil {
return errors.Wrap(err, "init bft engine")
@@ -287,7 +292,8 @@ func defaultAction(ctx *cli.Context) error {
p2pCommunicator.Communicator(),
ctx.Uint64(targetGasLimitFlag.Name),
skipLogs,
- forkConfig).Run(exitSignal)
+ forkConfig,
+ ).Run(exitSignal)
}
func soloAction(ctx *cli.Context) error {
@@ -301,6 +307,12 @@ func soloAction(ctx *cli.Context) error {
logLevel := initLogger(lvl, ctx.Bool(jsonLogsFlag.Name))
+ onDemandBlockProduction := ctx.Bool(onDemandFlag.Name)
+ blockProductionInterval := ctx.Uint64(blockInterval.Name)
+ if blockProductionInterval == 0 {
+ return errors.New("block-interval cannot be zero")
+ }
+
// enable metrics as soon as possible
metricsURL := ""
if ctx.Bool(enableMetricsFlag.Name) {
@@ -313,16 +325,6 @@ func soloAction(ctx *cli.Context) error {
defer func() { log.Info("stopping metrics server..."); closeFunc() }()
}
- adminURL := ""
- if ctx.Bool(enableAdminFlag.Name) {
- url, closeFunc, err := api.StartAdminServer(ctx.String(adminAddrFlag.Name), logLevel)
- if err != nil {
- return fmt.Errorf("unable to start admin server - %w", err)
- }
- adminURL = url
- defer func() { log.Info("stopping admin server..."); closeFunc() }()
- }
-
var (
gene *genesis.Genesis
forkConfig thor.ForkConfig
@@ -367,6 +369,16 @@ func soloAction(ctx *cli.Context) error {
return err
}
+ adminURL := ""
+ if ctx.Bool(enableAdminFlag.Name) {
+ url, closeFunc, err := api.StartAdminServer(ctx.String(adminAddrFlag.Name), logLevel, repo, nil)
+ if err != nil {
+ return fmt.Errorf("unable to start admin server - %w", err)
+ }
+ adminURL = url
+ defer func() { log.Info("stopping admin server..."); closeFunc() }()
+ }
+
printStartupMessage1(gene, repo, nil, instanceDir, forkConfig)
skipLogs := ctx.Bool(skipLogsFlag.Name)
@@ -412,11 +424,6 @@ func soloAction(ctx *cli.Context) error {
srvCloser()
}()
- blockInterval := ctx.Uint64(blockInterval.Name)
- if blockInterval == 0 {
- return errors.New("block-interval cannot be zero")
- }
-
printStartupMessage2(gene, apiURL, "", metricsURL, adminURL)
optimizer := optimizer.New(mainDB, repo, !ctx.Bool(disablePrunerFlag.Name))
@@ -427,9 +434,9 @@ func soloAction(ctx *cli.Context) error {
logDB,
txPool,
ctx.Uint64(gasLimitFlag.Name),
- ctx.Bool(onDemandFlag.Name),
+ onDemandBlockProduction,
skipLogs,
- blockInterval,
+ blockProductionInterval,
forkConfig).Run(exitSignal)
}
diff --git a/comm/communicator.go b/comm/communicator.go
index 9d0a5a530..48419779a 100644
--- a/comm/communicator.go
+++ b/comm/communicator.go
@@ -72,7 +72,7 @@ func (c *Communicator) Sync(ctx context.Context, handler HandleBlockStream) {
delay := initSyncInterval
syncCount := 0
- shouldSynced := func() bool {
+ isSynced := func() bool {
bestBlockTime := c.repo.BestBlockSummary().Header.Timestamp()
now := uint64(time.Now().Unix())
if bestBlockTime+thor.BlockInterval >= now {
@@ -115,9 +115,10 @@ func (c *Communicator) Sync(ctx context.Context, handler HandleBlockStream) {
}
syncCount++
- if shouldSynced() {
+ if isSynced() {
delay = syncInterval
c.onceSynced.Do(func() {
+ // once off - after a bootstrap the syncedCh trigger the peers.syncTxs
close(c.syncedCh)
})
}
From de248a60128e278d9a8209ac0982f339483a4d46 Mon Sep 17 00:00:00 2001
From: Darren Kelly <107671032+darrenvechain@users.noreply.github.com>
Date: Mon, 9 Dec 2024 11:25:30 +0000
Subject: [PATCH 12/15] Darren/admin api log toggler (#877)
* Adding Health endpoint
* pr comments + 503 if not healthy
* refactored admin server and api + health endpoint tests
* fix health condition
* fix admin routing
* added comments + changed from ChainSync to ChainBootstrapStatus
* Adding healthcheck for solo mode
* adding solo + tests
* fix log_level handler funcs
* feat(admin): toggle api logs via admin API
* feat(admin): add license headers
* refactor health package + add p2p count
* remove solo methods
* moving health service to api pkg
* added defaults + api health query
* pr comments
* pr comments
---------
Co-authored-by: otherview
---
api/admin/admin.go | 8 ++-
api/admin/apilogs/api_logs.go | 70 +++++++++++++++++++++++
api/admin/apilogs/api_logs_test.go | 91 ++++++++++++++++++++++++++++++
api/admin/loglevel/log_level.go | 2 +
api/admin_server.go | 4 +-
api/api.go | 7 +--
api/request_logger.go | 7 ++-
api/request_logger_test.go | 5 +-
cmd/thor/main.go | 23 +++++++-
cmd/thor/utils.go | 5 +-
10 files changed, 208 insertions(+), 14 deletions(-)
create mode 100644 api/admin/apilogs/api_logs.go
create mode 100644 api/admin/apilogs/api_logs_test.go
diff --git a/api/admin/admin.go b/api/admin/admin.go
index 1e16415f8..9b819c875 100644
--- a/api/admin/admin.go
+++ b/api/admin/admin.go
@@ -8,19 +8,23 @@ package admin
import (
"log/slog"
"net/http"
+ "sync/atomic"
"github.com/gorilla/handlers"
"github.com/gorilla/mux"
- healthAPI "github.com/vechain/thor/v2/api/admin/health"
+ "github.com/vechain/thor/v2/api/admin/apilogs"
"github.com/vechain/thor/v2/api/admin/loglevel"
+
+ healthAPI "github.com/vechain/thor/v2/api/admin/health"
)
-func New(logLevel *slog.LevelVar, health *healthAPI.Health) http.HandlerFunc {
+func New(logLevel *slog.LevelVar, health *healthAPI.Health, apiLogsToggle *atomic.Bool) http.HandlerFunc {
router := mux.NewRouter()
subRouter := router.PathPrefix("/admin").Subrouter()
loglevel.New(logLevel).Mount(subRouter, "/loglevel")
healthAPI.NewAPI(health).Mount(subRouter, "/health")
+ apilogs.New(apiLogsToggle).Mount(subRouter, "/apilogs")
handler := handlers.CompressHandler(router)
diff --git a/api/admin/apilogs/api_logs.go b/api/admin/apilogs/api_logs.go
new file mode 100644
index 000000000..0f815d579
--- /dev/null
+++ b/api/admin/apilogs/api_logs.go
@@ -0,0 +1,70 @@
+// Copyright (c) 2024 The VeChainThor developers
+//
+// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying
+// file LICENSE or
+
+package apilogs
+
+import (
+ "net/http"
+ "sync"
+ "sync/atomic"
+
+ "github.com/gorilla/mux"
+ "github.com/vechain/thor/v2/api/utils"
+ "github.com/vechain/thor/v2/log"
+)
+
+type APILogs struct {
+ enabled *atomic.Bool
+ mu sync.Mutex
+}
+
+type Status struct {
+ Enabled bool `json:"enabled"`
+}
+
+func New(enabled *atomic.Bool) *APILogs {
+ return &APILogs{
+ enabled: enabled,
+ }
+}
+
+func (a *APILogs) Mount(root *mux.Router, pathPrefix string) {
+ sub := root.PathPrefix(pathPrefix).Subrouter()
+ sub.Path("").
+ Methods(http.MethodGet).
+ Name("get-api-logs-enabled").
+ HandlerFunc(utils.WrapHandlerFunc(a.areAPILogsEnabled))
+
+ sub.Path("").
+ Methods(http.MethodPost).
+ Name("post-api-logs-enabled").
+ HandlerFunc(utils.WrapHandlerFunc(a.setAPILogsEnabled))
+}
+
+func (a *APILogs) areAPILogsEnabled(w http.ResponseWriter, _ *http.Request) error {
+ a.mu.Lock()
+ defer a.mu.Unlock()
+
+ return utils.WriteJSON(w, Status{
+ Enabled: a.enabled.Load(),
+ })
+}
+
+func (a *APILogs) setAPILogsEnabled(w http.ResponseWriter, r *http.Request) error {
+ a.mu.Lock()
+ defer a.mu.Unlock()
+
+ var req Status
+ if err := utils.ParseJSON(r.Body, &req); err != nil {
+ return utils.BadRequest(err)
+ }
+ a.enabled.Store(req.Enabled)
+
+ log.Info("api logs updated", "pkg", "apilogs", "enabled", req.Enabled)
+
+ return utils.WriteJSON(w, Status{
+ Enabled: a.enabled.Load(),
+ })
+}
diff --git a/api/admin/apilogs/api_logs_test.go b/api/admin/apilogs/api_logs_test.go
new file mode 100644
index 000000000..95cf2c6ac
--- /dev/null
+++ b/api/admin/apilogs/api_logs_test.go
@@ -0,0 +1,91 @@
+// Copyright (c) 2024 The VeChainThor developers
+//
+// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying
+// file LICENSE or
+
+package apilogs
+
+import (
+ "bytes"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "sync/atomic"
+ "testing"
+
+ "github.com/gorilla/mux"
+ "github.com/stretchr/testify/assert"
+)
+
+type TestCase struct {
+ name string
+ method string
+ expectedHTTP int
+ startValue bool
+ expectedEndValue bool
+ requestBody bool
+}
+
+func marshalBody(tt TestCase, t *testing.T) []byte {
+ var reqBody []byte
+ var err error
+ if tt.method == "POST" {
+ reqBody, err = json.Marshal(Status{Enabled: tt.requestBody})
+ if err != nil {
+ t.Fatalf("could not marshal request body: %v", err)
+ }
+ }
+ return reqBody
+}
+
+func TestLogLevelHandler(t *testing.T) {
+ tests := []TestCase{
+ {
+ name: "Valid POST input - set logs to enabled",
+ method: "POST",
+ expectedHTTP: http.StatusOK,
+ startValue: false,
+ requestBody: true,
+ expectedEndValue: true,
+ },
+ {
+ name: "Valid POST input - set logs to disabled",
+ method: "POST",
+ expectedHTTP: http.StatusOK,
+ startValue: true,
+ requestBody: false,
+ expectedEndValue: false,
+ },
+ {
+ name: "GET request - get current level INFO",
+ method: "GET",
+ expectedHTTP: http.StatusOK,
+ startValue: true,
+ expectedEndValue: true,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ logLevel := atomic.Bool{}
+ logLevel.Store(tt.startValue)
+
+ reqBodyBytes := marshalBody(tt, t)
+
+ req, err := http.NewRequest(tt.method, "/admin/apilogs", bytes.NewBuffer(reqBodyBytes))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ rr := httptest.NewRecorder()
+ router := mux.NewRouter()
+ New(&logLevel).Mount(router, "/admin/apilogs")
+ router.ServeHTTP(rr, req)
+
+ assert.Equal(t, tt.expectedHTTP, rr.Code)
+ responseBody := Status{}
+ assert.NoError(t, json.Unmarshal(rr.Body.Bytes(), &responseBody))
+ assert.Equal(t, tt.expectedEndValue, responseBody.Enabled)
+ })
+ }
+}
diff --git a/api/admin/loglevel/log_level.go b/api/admin/loglevel/log_level.go
index d3c339ce2..c91702d2d 100644
--- a/api/admin/loglevel/log_level.go
+++ b/api/admin/loglevel/log_level.go
@@ -68,6 +68,8 @@ func (l *LogLevel) postLogLevelHandler(w http.ResponseWriter, r *http.Request) e
return utils.BadRequest(errors.New("Invalid verbosity level"))
}
+ log.Info("log level changed", "pkg", "loglevel", "level", l.logLevel.Level().String())
+
return utils.WriteJSON(w, Response{
CurrentLevel: l.logLevel.Level().String(),
})
diff --git a/api/admin_server.go b/api/admin_server.go
index f2315aeb1..dca428b36 100644
--- a/api/admin_server.go
+++ b/api/admin_server.go
@@ -9,6 +9,7 @@ import (
"log/slog"
"net"
"net/http"
+ "sync/atomic"
"time"
"github.com/pkg/errors"
@@ -24,13 +25,14 @@ func StartAdminServer(
logLevel *slog.LevelVar,
repo *chain.Repository,
p2p *comm.Communicator,
+ apiLogs *atomic.Bool,
) (string, func(), error) {
listener, err := net.Listen("tcp", addr)
if err != nil {
return "", nil, errors.Wrapf(err, "listen admin API addr [%v]", addr)
}
- adminHandler := admin.New(logLevel, health.New(repo, p2p))
+ adminHandler := admin.New(logLevel, health.New(repo, p2p), apiLogs)
srv := &http.Server{Handler: adminHandler, ReadHeaderTimeout: time.Second, ReadTimeout: 5 * time.Second}
var goes co.Goes
diff --git a/api/api.go b/api/api.go
index 0385929ec..c57e2a957 100644
--- a/api/api.go
+++ b/api/api.go
@@ -9,6 +9,7 @@ import (
"net/http"
"net/http/pprof"
"strings"
+ "sync/atomic"
"github.com/gorilla/handlers"
"github.com/gorilla/mux"
@@ -39,7 +40,7 @@ type Config struct {
PprofOn bool
SkipLogs bool
AllowCustomTracer bool
- EnableReqLogger bool
+ EnableReqLogger *atomic.Bool
EnableMetrics bool
LogsLimit uint64
AllowedTracers []string
@@ -115,9 +116,7 @@ func New(
handlers.ExposedHeaders([]string{"x-genesis-id", "x-thorest-ver"}),
)(handler)
- if config.EnableReqLogger {
- handler = RequestLoggerHandler(handler, logger)
- }
+ handler = RequestLoggerHandler(handler, logger, config.EnableReqLogger)
return handler.ServeHTTP, subs.Close // subscriptions handles hijacked conns, which need to be closed
}
diff --git a/api/request_logger.go b/api/request_logger.go
index 3d48a2d36..451059814 100644
--- a/api/request_logger.go
+++ b/api/request_logger.go
@@ -9,14 +9,19 @@ import (
"bytes"
"io"
"net/http"
+ "sync/atomic"
"time"
"github.com/vechain/thor/v2/log"
)
// RequestLoggerHandler returns a http handler to ensure requests are syphoned into the writer
-func RequestLoggerHandler(handler http.Handler, logger log.Logger) http.Handler {
+func RequestLoggerHandler(handler http.Handler, logger log.Logger, enabled *atomic.Bool) http.Handler {
fn := func(w http.ResponseWriter, r *http.Request) {
+ if !enabled.Load() {
+ handler.ServeHTTP(w, r)
+ return
+ }
// Read and log the body (note: this can only be done once)
// Ensure you don't disrupt the request body for handlers that need to read it
var bodyBytes []byte
diff --git a/api/request_logger_test.go b/api/request_logger_test.go
index 6b8ddcd91..3368e6fc8 100644
--- a/api/request_logger_test.go
+++ b/api/request_logger_test.go
@@ -10,6 +10,7 @@ import (
"net/http"
"net/http/httptest"
"strings"
+ "sync/atomic"
"testing"
"github.com/stretchr/testify/assert"
@@ -59,6 +60,8 @@ func (m *mockLogger) GetLoggedData() []interface{} {
func TestRequestLoggerHandler(t *testing.T) {
mockLog := &mockLogger{}
+ enabled := atomic.Bool{}
+ enabled.Store(true)
// Define a test handler to wrap
testHandler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
@@ -67,7 +70,7 @@ func TestRequestLoggerHandler(t *testing.T) {
})
// Create the RequestLoggerHandler
- loggerHandler := RequestLoggerHandler(testHandler, mockLog)
+ loggerHandler := RequestLoggerHandler(testHandler, mockLog, &enabled)
// Create a test HTTP request
reqBody := "test body"
diff --git a/cmd/thor/main.go b/cmd/thor/main.go
index ba19b0c11..2cb638f2a 100644
--- a/cmd/thor/main.go
+++ b/cmd/thor/main.go
@@ -11,6 +11,7 @@ import (
"io"
"os"
"path/filepath"
+ "sync/atomic"
"time"
"github.com/ethereum/go-ethereum/accounts/keystore"
@@ -234,12 +235,15 @@ func defaultAction(ctx *cli.Context) error {
}
adminURL := ""
+ logAPIRequests := &atomic.Bool{}
+ logAPIRequests.Store(ctx.Bool(enableAPILogsFlag.Name))
if ctx.Bool(enableAdminFlag.Name) {
url, closeFunc, err := api.StartAdminServer(
ctx.String(adminAddrFlag.Name),
logLevel,
repo,
p2pCommunicator.Communicator(),
+ logAPIRequests,
)
if err != nil {
return fmt.Errorf("unable to start admin server - %w", err)
@@ -261,7 +265,7 @@ func defaultAction(ctx *cli.Context) error {
bftEngine,
p2pCommunicator.Communicator(),
forkConfig,
- makeAPIConfig(ctx, false),
+ makeAPIConfig(ctx, logAPIRequests, false),
)
defer func() { log.Info("closing API..."); apiCloser() }()
@@ -370,8 +374,16 @@ func soloAction(ctx *cli.Context) error {
}
adminURL := ""
+ logAPIRequests := &atomic.Bool{}
+ logAPIRequests.Store(ctx.Bool(enableAPILogsFlag.Name))
if ctx.Bool(enableAdminFlag.Name) {
- url, closeFunc, err := api.StartAdminServer(ctx.String(adminAddrFlag.Name), logLevel, repo, nil)
+ url, closeFunc, err := api.StartAdminServer(
+ ctx.String(adminAddrFlag.Name),
+ logLevel,
+ repo,
+ nil,
+ logAPIRequests,
+ )
if err != nil {
return fmt.Errorf("unable to start admin server - %w", err)
}
@@ -411,7 +423,7 @@ func soloAction(ctx *cli.Context) error {
bftEngine,
&solo.Communicator{},
forkConfig,
- makeAPIConfig(ctx, true),
+ makeAPIConfig(ctx, logAPIRequests, true),
)
defer func() { log.Info("closing API..."); apiCloser() }()
@@ -424,6 +436,11 @@ func soloAction(ctx *cli.Context) error {
srvCloser()
}()
+ blockInterval := ctx.Uint64(blockInterval.Name)
+ if blockInterval == 0 {
+ return errors.New("block-interval cannot be zero")
+ }
+
printStartupMessage2(gene, apiURL, "", metricsURL, adminURL)
optimizer := optimizer.New(mainDB, repo, !ctx.Bool(disablePrunerFlag.Name))
diff --git a/cmd/thor/utils.go b/cmd/thor/utils.go
index 6877a45ee..59c5ec284 100644
--- a/cmd/thor/utils.go
+++ b/cmd/thor/utils.go
@@ -23,6 +23,7 @@ import (
"runtime"
"runtime/debug"
"strings"
+ "sync/atomic"
"syscall"
"time"
@@ -275,7 +276,7 @@ func parseGenesisFile(filePath string) (*genesis.Genesis, thor.ForkConfig, error
return customGen, forkConfig, nil
}
-func makeAPIConfig(ctx *cli.Context, soloMode bool) api.Config {
+func makeAPIConfig(ctx *cli.Context, logAPIRequests *atomic.Bool, soloMode bool) api.Config {
return api.Config{
AllowedOrigins: ctx.String(apiCorsFlag.Name),
BacktraceLimit: uint32(ctx.Uint64(apiBacktraceLimitFlag.Name)),
@@ -283,7 +284,7 @@ func makeAPIConfig(ctx *cli.Context, soloMode bool) api.Config {
PprofOn: ctx.Bool(pprofFlag.Name),
SkipLogs: ctx.Bool(skipLogsFlag.Name),
AllowCustomTracer: ctx.Bool(apiAllowCustomTracerFlag.Name),
- EnableReqLogger: ctx.Bool(enableAPILogsFlag.Name),
+ EnableReqLogger: logAPIRequests,
EnableMetrics: ctx.Bool(enableMetricsFlag.Name),
LogsLimit: ctx.Uint64(apiLogsLimitFlag.Name),
AllowedTracers: parseTracerList(strings.TrimSpace(ctx.String(allowedTracersFlag.Name))),
From d9a11a8590a91556c0459724527b11c6d189b290 Mon Sep 17 00:00:00 2001
From: Darren Kelly <107671032+darrenvechain@users.noreply.github.com>
Date: Mon, 9 Dec 2024 11:33:50 +0000
Subject: [PATCH 13/15] Darren/chore/backport metrics (#909)
* chore(muxdb): backport muxdb cache metrics
* chore(muxdb): backport muxdb cache metrics
* chore(metrics): backport disk IO
* chore(metrics): fix lint
* chore(chain): add repo cache metrics
* fix(chain): fix cache return value
* refactor(chain): cache hit miss
---
chain/metric.go | 12 +++++++
chain/repository.go | 24 +++++++++----
metrics/noop.go | 4 ++-
metrics/prometheus.go | 65 ++++++++++++++++++++++++++++++++++
muxdb/internal/trie/cache.go | 20 ++++++-----
muxdb/internal/trie/metrics.go | 12 +++++++
6 files changed, 121 insertions(+), 16 deletions(-)
create mode 100644 chain/metric.go
create mode 100644 muxdb/internal/trie/metrics.go
diff --git a/chain/metric.go b/chain/metric.go
new file mode 100644
index 000000000..8c9a764d4
--- /dev/null
+++ b/chain/metric.go
@@ -0,0 +1,12 @@
+// Copyright (c) 2024 The VeChainThor developers
+//
+// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying
+// file LICENSE or
+
+package chain
+
+import "github.com/vechain/thor/v2/metrics"
+
+var (
+ metricCacheHitMiss = metrics.LazyLoadCounterVec("repo_cache_hit_miss_count", []string{"type", "event"})
+)
diff --git a/chain/repository.go b/chain/repository.go
index 2460d6a3c..81b1f6af4 100644
--- a/chain/repository.go
+++ b/chain/repository.go
@@ -322,23 +322,29 @@ func (r *Repository) GetMaxBlockNum() (uint32, error) {
// GetBlockSummary get block summary by block id.
func (r *Repository) GetBlockSummary(id thor.Bytes32) (summary *BlockSummary, err error) {
- var cached interface{}
- if cached, err = r.caches.summaries.GetOrLoad(id, func() (interface{}, error) {
+ var blk interface{}
+ result := "hit"
+ if blk, err = r.caches.summaries.GetOrLoad(id, func() (interface{}, error) {
+ result = "miss"
return loadBlockSummary(r.data, id)
}); err != nil {
return
}
- return cached.(*BlockSummary), nil
+ metricCacheHitMiss().AddWithLabel(1, map[string]string{"type": "blocks", "event": result})
+ return blk.(*BlockSummary), nil
}
func (r *Repository) getTransaction(key txKey) (*tx.Transaction, error) {
- cached, err := r.caches.txs.GetOrLoad(key, func() (interface{}, error) {
+ result := "hit"
+ trx, err := r.caches.txs.GetOrLoad(key, func() (interface{}, error) {
+ result = "miss"
return loadTransaction(r.data, key)
})
if err != nil {
return nil, err
}
- return cached.(*tx.Transaction), nil
+ metricCacheHitMiss().AddWithLabel(1, map[string]string{"type": "transaction", "event": result})
+ return trx.(*tx.Transaction), nil
}
// GetBlockTransactions get all transactions of the block for given block id.
@@ -377,13 +383,17 @@ func (r *Repository) GetBlock(id thor.Bytes32) (*block.Block, error) {
}
func (r *Repository) getReceipt(key txKey) (*tx.Receipt, error) {
- cached, err := r.caches.receipts.GetOrLoad(key, func() (interface{}, error) {
+ result := "hit"
+ receipt, err := r.caches.receipts.GetOrLoad(key, func() (interface{}, error) {
+ result = "miss"
return loadReceipt(r.data, key)
})
if err != nil {
return nil, err
}
- return cached.(*tx.Receipt), nil
+ metricCacheHitMiss().AddWithLabel(1, map[string]string{"type": "receipt", "event": result})
+
+ return receipt.(*tx.Receipt), nil
}
// GetBlockReceipts get all tx receipts of the block for given block id.
diff --git a/metrics/noop.go b/metrics/noop.go
index 6eb909ff9..a9e24ab2c 100644
--- a/metrics/noop.go
+++ b/metrics/noop.go
@@ -5,7 +5,9 @@
package metrics
-import "net/http"
+import (
+ "net/http"
+)
// noopMetrics implements a no operations metrics service
type noopMetrics struct{}
diff --git a/metrics/prometheus.go b/metrics/prometheus.go
index 50745752c..15447f6dc 100644
--- a/metrics/prometheus.go
+++ b/metrics/prometheus.go
@@ -6,8 +6,15 @@
package metrics
import (
+ "bufio"
+ "fmt"
"net/http"
+ "os"
+ "runtime"
+ "strconv"
+ "strings"
"sync"
+ "time"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -24,6 +31,8 @@ func InitializePrometheusMetrics() {
// don't allow for reset
if _, ok := metrics.(*prometheusMetrics); !ok {
metrics = newPrometheusMetrics()
+ // collection disk io metrics every 5 seconds
+ go metrics.(*prometheusMetrics).collectDiskIO(5 * time.Second)
}
}
@@ -123,6 +132,62 @@ func (o *prometheusMetrics) GetOrCreateGaugeVecMeter(name string, labels []strin
return meter
}
+func getIOLineValue(line string) int64 {
+ fields := strings.Fields(line)
+ if len(fields) != 2 {
+ logger.Warn("this io file line is malformed", "err", line)
+ return 0
+ }
+ value, err := strconv.ParseInt(fields[1], 10, 64)
+ if err != nil {
+ logger.Warn("unable to parse int", "err", err)
+ return 0
+ }
+
+ return value
+}
+
+func getDiskIOData() (int64, int64, error) {
+ pid := os.Getpid()
+ ioFilePath := fmt.Sprintf("/proc/%d/io", pid)
+ file, err := os.Open(ioFilePath)
+ if err != nil {
+ return 0, 0, err
+ }
+
+ // Parse the file line by line
+ scanner := bufio.NewScanner(file)
+ var reads, writes int64
+ for scanner.Scan() {
+ line := scanner.Text()
+ if strings.HasPrefix(line, "syscr") {
+ reads = getIOLineValue(line)
+ } else if strings.HasPrefix(line, "syscw") {
+ writes = getIOLineValue(line)
+ }
+ }
+
+ return reads, writes, nil
+}
+
+func (o *prometheusMetrics) collectDiskIO(refresh time.Duration) {
+ if runtime.GOOS != "linux" {
+ return
+ }
+ for {
+ reads, writes, err := getDiskIOData()
+ if err == nil {
+ readsMeter := o.GetOrCreateGaugeMeter("disk_reads")
+ readsMeter.Set(reads)
+
+ writesMeter := o.GetOrCreateGaugeMeter("disk_writes")
+ writesMeter.Set(writes)
+ }
+
+ time.Sleep(refresh)
+ }
+}
+
func (o *prometheusMetrics) newHistogramMeter(name string, buckets []int64) HistogramMeter {
var floatBuckets []float64
for _, bucket := range buckets {
diff --git a/muxdb/internal/trie/cache.go b/muxdb/internal/trie/cache.go
index cc7bca300..d7d78aaae 100644
--- a/muxdb/internal/trie/cache.go
+++ b/muxdb/internal/trie/cache.go
@@ -45,12 +45,16 @@ func (c *Cache) log() {
last := atomic.SwapInt64(&c.lastLogTime, now)
if now-last > int64(time.Second*20) {
- log1, ok1 := c.nodeStats.ShouldLog("node cache stats")
- log2, ok2 := c.rootStats.ShouldLog("root cache stats")
-
- if ok1 || ok2 {
- log1()
- log2()
+ logNode, hitNode, missNode, okNode := c.nodeStats.shouldLog("node cache stats")
+ logRoot, hitRoot, missRoot, okRoot := c.rootStats.shouldLog("root cache stats")
+
+ if okNode || okRoot {
+ logNode()
+ metricCacheHitMissGaugeVec().SetWithLabel(hitNode, map[string]string{"type": "node", "event": "hit"})
+ metricCacheHitMissGaugeVec().SetWithLabel(missNode, map[string]string{"type": "node", "event": "miss"})
+ logRoot()
+ metricCacheHitMissGaugeVec().SetWithLabel(hitRoot, map[string]string{"type": "root", "event": "hit"})
+ metricCacheHitMissGaugeVec().SetWithLabel(missRoot, map[string]string{"type": "root", "event": "miss"})
}
} else {
atomic.CompareAndSwapInt64(&c.lastLogTime, now, last)
@@ -189,7 +193,7 @@ type cacheStats struct {
func (cs *cacheStats) Hit() int64 { return atomic.AddInt64(&cs.hit, 1) }
func (cs *cacheStats) Miss() int64 { return atomic.AddInt64(&cs.miss, 1) }
-func (cs *cacheStats) ShouldLog(msg string) (func(), bool) {
+func (cs *cacheStats) shouldLog(msg string) (func(), int64, int64, bool) {
hit := atomic.LoadInt64(&cs.hit)
miss := atomic.LoadInt64(&cs.miss)
lookups := hit + miss
@@ -209,5 +213,5 @@ func (cs *cacheStats) ShouldLog(msg string) (func(), bool) {
"hitrate", str,
)
atomic.StoreInt32(&cs.flag, flag)
- }, atomic.LoadInt32(&cs.flag) != flag
+ }, hit, miss, atomic.LoadInt32(&cs.flag) != flag
}
diff --git a/muxdb/internal/trie/metrics.go b/muxdb/internal/trie/metrics.go
new file mode 100644
index 000000000..6db862df9
--- /dev/null
+++ b/muxdb/internal/trie/metrics.go
@@ -0,0 +1,12 @@
+// Copyright (c) 2024 The VeChainThor developers
+//
+// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying
+// file LICENSE or
+
+package trie
+
+import (
+ "github.com/vechain/thor/v2/metrics"
+)
+
+var metricCacheHitMissGaugeVec = metrics.LazyLoadGaugeVec("cache_hit_miss_count", []string{"type", "event"})
From b0a3d7362e30834b525f07b316b6644d8e8199ae Mon Sep 17 00:00:00 2001
From: Darren Kelly <107671032+darrenvechain@users.noreply.github.com>
Date: Mon, 9 Dec 2024 12:49:42 +0000
Subject: [PATCH 14/15] chore(thor): update version (#912)
* chore(thor): update version
* chore(openapi): version
---
api/doc/thor.yaml | 2 +-
cmd/thor/VERSION | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/api/doc/thor.yaml b/api/doc/thor.yaml
index dcf0ae6b9..d8f6c45e8 100644
--- a/api/doc/thor.yaml
+++ b/api/doc/thor.yaml
@@ -12,7 +12,7 @@ info:
license:
name: LGPL 3.0
url: https://www.gnu.org/licenses/lgpl-3.0.en.html
- version: 2.1.4
+ version: 2.1.5
servers:
- url: /
description: Current Node
diff --git a/cmd/thor/VERSION b/cmd/thor/VERSION
index 7d2ed7c70..cd57a8b95 100644
--- a/cmd/thor/VERSION
+++ b/cmd/thor/VERSION
@@ -1 +1 @@
-2.1.4
+2.1.5
From c74bbf0159b08c048181beac82412e3af02af084 Mon Sep 17 00:00:00 2001
From: Delweng
Date: Mon, 9 Dec 2024 21:48:13 +0800
Subject: [PATCH 15/15] feat(api/debug): support debug trace without blockId
(#905)
* api/debug: support debug with txhash
Signed-off-by: jsvisa
api/debug: blockId should use tx's instead
Signed-off-by: jsvisa
fix tests
Signed-off-by: jsvisa
* debug: add test
Signed-off-by: jsvisa
* improve parseTarget
Signed-off-by: jsvisa
* update doc
Signed-off-by: jsvisa
* fix tests
Signed-off-by: jsvisa
---------
Signed-off-by: jsvisa
Co-authored-by: tony
---
api/debug/debug.go | 133 +++++++++++++++++++++++++---------------
api/debug/debug_test.go | 30 +++++++--
api/doc/thor.yaml | 9 +--
3 files changed, 116 insertions(+), 56 deletions(-)
diff --git a/api/debug/debug.go b/api/debug/debug.go
index 5ff54f1dc..497518982 100644
--- a/api/debug/debug.go
+++ b/api/debug/debug.go
@@ -75,22 +75,8 @@ func New(
}
}
-func (d *Debug) prepareClauseEnv(ctx context.Context, blockID thor.Bytes32, txIndex uint64, clauseIndex uint32) (*runtime.Runtime, *runtime.TransactionExecutor, thor.Bytes32, error) {
- block, err := d.repo.GetBlock(blockID)
- if err != nil {
- if d.repo.IsNotFound(err) {
- return nil, nil, thor.Bytes32{}, utils.Forbidden(errors.New("block not found"))
- }
- return nil, nil, thor.Bytes32{}, err
- }
- txs := block.Transactions()
- if txIndex >= uint64(len(txs)) {
- return nil, nil, thor.Bytes32{}, utils.Forbidden(errors.New("tx index out of range"))
- }
- txID := txs[txIndex].ID()
- if clauseIndex >= uint32(len(txs[txIndex].Clauses())) {
- return nil, nil, thor.Bytes32{}, utils.Forbidden(errors.New("clause index out of range"))
- }
+// prepareClauseEnv prepares the runtime environment for the specified clause.
+func (d *Debug) prepareClauseEnv(ctx context.Context, block *block.Block, txID thor.Bytes32, clauseIndex uint32) (*runtime.Runtime, *runtime.TransactionExecutor, thor.Bytes32, error) {
rt, err := consensus.New(
d.repo,
d.stater,
@@ -99,17 +85,29 @@ func (d *Debug) prepareClauseEnv(ctx context.Context, blockID thor.Bytes32, txIn
if err != nil {
return nil, nil, thor.Bytes32{}, err
}
- for i, tx := range txs {
- if uint64(i) > txIndex {
- break
+
+ var found bool
+ txs := block.Transactions()
+ for _, tx := range txs {
+ if txID == tx.ID() {
+ found = true
+ if clauseIndex >= uint32(len(tx.Clauses())) {
+ return nil, nil, thor.Bytes32{}, utils.Forbidden(errors.New("clause index out of range"))
+ }
}
+ }
+ if !found {
+ return nil, nil, thor.Bytes32{}, utils.Forbidden(errors.New("transaction not found"))
+ }
+
+ for _, tx := range block.Transactions() {
txExec, err := rt.PrepareTransaction(tx)
if err != nil {
return nil, nil, thor.Bytes32{}, err
}
clauseCounter := uint32(0)
for txExec.HasNextClause() {
- if txIndex == uint64(i) && clauseIndex == clauseCounter {
+ if tx.ID() == txID && clauseIndex == clauseCounter {
return rt, txExec, txID, nil
}
exec, _ := txExec.PrepareNext()
@@ -127,18 +125,27 @@ func (d *Debug) prepareClauseEnv(ctx context.Context, blockID thor.Bytes32, txIn
default:
}
}
+
+ // no env created, that means tx was reverted at an early clause
return nil, nil, thor.Bytes32{}, utils.Forbidden(errors.New("early reverted"))
}
// trace an existed clause
-func (d *Debug) traceClause(ctx context.Context, tracer tracers.Tracer, blockID thor.Bytes32, txIndex uint64, clauseIndex uint32) (interface{}, error) {
- rt, txExec, txID, err := d.prepareClauseEnv(ctx, blockID, txIndex, clauseIndex)
+func (d *Debug) traceClause(ctx context.Context, tracer tracers.Tracer, block *block.Block, txID thor.Bytes32, clauseIndex uint32) (interface{}, error) {
+ rt, txExec, txID, err := d.prepareClauseEnv(ctx, block, txID, clauseIndex)
if err != nil {
return nil, err
}
+ var txIndex uint64 = math.MaxUint64
+ for i, tx := range block.Transactions() {
+ if tx.ID() == txID {
+ txIndex = uint64(i)
+ break
+ }
+ }
tracer.SetContext(&tracers.Context{
- BlockID: blockID,
+ BlockID: block.Header().ID(),
BlockTime: rt.Context().Time,
TxID: txID,
TxIndex: txIndex,
@@ -178,11 +185,11 @@ func (d *Debug) handleTraceClause(w http.ResponseWriter, req *http.Request) erro
return utils.Forbidden(err)
}
- blockID, txIndex, clauseIndex, err := d.parseTarget(opt.Target)
+ block, txID, clauseIndex, err := d.parseTarget(opt.Target)
if err != nil {
return err
}
- res, err := d.traceClause(req.Context(), tracer, blockID, txIndex, clauseIndex)
+ res, err := d.traceClause(req.Context(), tracer, block, txID, clauseIndex)
if err != nil {
return err
}
@@ -291,8 +298,8 @@ func (d *Debug) traceCall(ctx context.Context, tracer tracers.Tracer, header *bl
return tracer.GetResult()
}
-func (d *Debug) debugStorage(ctx context.Context, contractAddress thor.Address, blockID thor.Bytes32, txIndex uint64, clauseIndex uint32, keyStart []byte, maxResult int) (*StorageRangeResult, error) {
- rt, _, _, err := d.prepareClauseEnv(ctx, blockID, txIndex, clauseIndex)
+func (d *Debug) debugStorage(ctx context.Context, contractAddress thor.Address, block *block.Block, txID thor.Bytes32, clauseIndex uint32, keyStart []byte, maxResult int) (*StorageRangeResult, error) {
+ rt, _, _, err := d.prepareClauseEnv(ctx, block, txID, clauseIndex)
if err != nil {
return nil, err
}
@@ -357,41 +364,71 @@ func (d *Debug) handleDebugStorage(w http.ResponseWriter, req *http.Request) err
return utils.WriteJSON(w, res)
}
-func (d *Debug) parseTarget(target string) (blockID thor.Bytes32, txIndex uint64, clauseIndex uint32, err error) {
+func (d *Debug) parseTarget(target string) (block *block.Block, txID thor.Bytes32, clauseIndex uint32, err error) {
+ // target can be `${blockID}/${txID|txIndex}/${clauseIndex}` or `${txID}/${clauseIndex}`
parts := strings.Split(target, "/")
- if len(parts) != 3 {
- return thor.Bytes32{}, 0, 0, utils.BadRequest(errors.New("target:" + target + " unsupported"))
+ if len(parts) != 3 && len(parts) != 2 {
+ return nil, thor.Bytes32{}, 0, utils.BadRequest(errors.New("target:" + target + " unsupported"))
}
- blockID, err = thor.ParseBytes32(parts[0])
- if err != nil {
- return thor.Bytes32{}, 0, 0, utils.BadRequest(errors.WithMessage(err, "target[0]"))
- }
- if len(parts[1]) == 64 || len(parts[1]) == 66 {
- txID, err := thor.ParseBytes32(parts[1])
+
+ if len(parts) == 2 {
+ txID, err = thor.ParseBytes32(parts[0])
if err != nil {
- return thor.Bytes32{}, 0, 0, utils.BadRequest(errors.WithMessage(err, "target[1]"))
+ return nil, thor.Bytes32{}, 0, utils.BadRequest(errors.WithMessage(err, "target([0]"))
}
-
- txMeta, err := d.repo.NewChain(blockID).GetTransactionMeta(txID)
+ txMeta, err := d.repo.NewBestChain().GetTransactionMeta(txID)
if err != nil {
if d.repo.IsNotFound(err) {
- return thor.Bytes32{}, 0, 0, utils.Forbidden(errors.New("transaction not found"))
+ return nil, thor.Bytes32{}, 0, utils.Forbidden(errors.New("transaction not found"))
}
- return thor.Bytes32{}, 0, 0, err
+ return nil, thor.Bytes32{}, 0, err
+ }
+ block, err = d.repo.GetBlock(txMeta.BlockID)
+ if err != nil {
+ return nil, thor.Bytes32{}, 0, err
}
- txIndex = txMeta.Index
} else {
- i, err := strconv.ParseUint(parts[1], 0, 0)
+ blockID, err := thor.ParseBytes32(parts[0])
+ if err != nil {
+ return nil, thor.Bytes32{}, 0, utils.BadRequest(errors.WithMessage(err, "target[0]"))
+ }
+ block, err = d.repo.GetBlock(blockID)
if err != nil {
- return thor.Bytes32{}, 0, 0, utils.BadRequest(errors.WithMessage(err, "target[1]"))
+ return nil, thor.Bytes32{}, 0, err
+ }
+ if len(parts[1]) == 64 || len(parts[1]) == 66 {
+ txID, err = thor.ParseBytes32(parts[1])
+ if err != nil {
+ return nil, thor.Bytes32{}, 0, utils.BadRequest(errors.WithMessage(err, "target[1]"))
+ }
+
+ var found bool
+ for _, tx := range block.Transactions() {
+ if tx.ID() == txID {
+ found = true
+ break
+ }
+ }
+ if !found {
+ return nil, thor.Bytes32{}, 0, utils.Forbidden(errors.New("transaction not found"))
+ }
+ } else {
+ i, err := strconv.ParseUint(parts[1], 0, 0)
+ if err != nil {
+ return nil, thor.Bytes32{}, 0, utils.BadRequest(errors.WithMessage(err, "target[1]"))
+ }
+ if i >= uint64(len(block.Transactions())) {
+ return nil, thor.Bytes32{}, 0, utils.Forbidden(errors.New("tx index out of range"))
+ }
+ txID = block.Transactions()[i].ID()
}
- txIndex = i
}
- i, err := strconv.ParseUint(parts[2], 0, 0)
+
+ i, err := strconv.ParseUint(parts[len(parts)-1], 0, 0)
if err != nil {
- return thor.Bytes32{}, 0, 0, utils.BadRequest(errors.WithMessage(err, "target[2]"))
+ return nil, thor.Bytes32{}, 0, utils.BadRequest(errors.WithMessage(err, fmt.Sprintf("target[%d]", len(parts)-1)))
} else if i > math.MaxUint32 {
- return thor.Bytes32{}, 0, 0, utils.BadRequest(errors.New("invalid target[2]"))
+ return nil, thor.Bytes32{}, 0, utils.BadRequest(fmt.Errorf("invalid target[%d]", len(parts)-1))
}
clauseIndex = uint32(i)
return
diff --git a/api/debug/debug_test.go b/api/debug/debug_test.go
index 1275a9030..d56718143 100644
--- a/api/debug/debug_test.go
+++ b/api/debug/debug_test.go
@@ -6,7 +6,6 @@
package debug
import (
- "context"
"encoding/json"
"fmt"
"math/big"
@@ -62,6 +61,7 @@ func TestDebug(t *testing.T) {
"testTraceClauseWithClauseIndexOutOfBound": testTraceClauseWithClauseIndexOutOfBound,
"testTraceClauseWithCustomTracer": testTraceClauseWithCustomTracer,
"testTraceClause": testTraceClause,
+ "testTraceClauseWithoutBlockID": testTraceClauseWithoutBlockID,
} {
t.Run(name, tt)
}
@@ -176,9 +176,11 @@ func testTraceClauseWithBadBlockID(t *testing.T) {
}
func testTraceClauseWithNonExistingBlockID(t *testing.T) {
- _, _, _, err := debug.prepareClauseEnv(context.Background(), datagen.RandomHash(), 1, 1)
-
- assert.Error(t, err)
+ traceClauseOption := &TraceClauseOption{
+ Name: "structLogger",
+ Target: fmt.Sprintf("%s/x/x", datagen.RandomHash()),
+ }
+ httpPostAndCheckResponseStatus(t, "/debug/tracers", traceClauseOption, 500)
}
func testTraceClauseWithBadTxID(t *testing.T) {
@@ -265,6 +267,26 @@ func testTraceClause(t *testing.T) {
assert.Equal(t, expectedExecutionResult, parsedExecutionRes)
}
+func testTraceClauseWithoutBlockID(t *testing.T) {
+ traceClauseOption := &TraceClauseOption{
+ Name: "structLogger",
+ Target: fmt.Sprintf("%s/1", transaction.ID()),
+ }
+ expectedExecutionResult := &logger.ExecutionResult{
+ Gas: 0,
+ Failed: false,
+ ReturnValue: "",
+ StructLogs: make([]logger.StructLogRes, 0),
+ }
+ res := httpPostAndCheckResponseStatus(t, "/debug/tracers", traceClauseOption, 200)
+
+ var parsedExecutionRes *logger.ExecutionResult
+ if err := json.Unmarshal([]byte(res), &parsedExecutionRes); err != nil {
+ t.Fatal(err)
+ }
+ assert.Equal(t, expectedExecutionResult, parsedExecutionRes)
+}
+
func testTraceClauseWithTxIndexOutOfBound(t *testing.T) {
traceClauseOption := &TraceClauseOption{
Name: "structLogger",
diff --git a/api/doc/thor.yaml b/api/doc/thor.yaml
index d8f6c45e8..702645411 100644
--- a/api/doc/thor.yaml
+++ b/api/doc/thor.yaml
@@ -2104,11 +2104,12 @@ components:
The unified path of the target to be traced. Currently, only the clause is supported.
Format:
- `blockID/(txIndex|txId)/clauseIndex`
+ `blockID/(txIndex|txId)/clauseIndex` or `txID/clauseIndex`
+
example: '0x010709463c1f0c9aa66a31182fb36d1977d99bfb6526bae0564a0eac4006c31a/0/0'
nullable: false
- pattern: '^0x[0-9a-fA-F]{64}\/(0x[0-9a-fA-F]{64}|\d+)\/[0-9]+$'
+ pattern: '^0x[0-9a-fA-F]{64}(\/(0x[0-9a-fA-F]{64}|\d+))?\/[0-9]+$'
example:
target: '0x010709463c1f0c9aa66a31182fb36d1977d99bfb6526bae0564a0eac4006c31a/0/0'
@@ -2174,9 +2175,9 @@ components:
The unified path of the transaction clause.
Format:
- `blockID/(txIndex|txId)/clauseIndex`
+ `blockID/(txIndex|txId)/clauseIndex` or `txID/clauseIndex`
nullable: false
- pattern: '^0x[0-9a-fA-F]{64}\/(0x[0-9a-fA-F]{64}|\d+)\/[0-9]+$'
+ pattern: '^0x[0-9a-fA-F]{64}(\/(0x[0-9a-fA-F]{64}|\d+))?\/[0-9]+$'
StorageRange:
type: object
+