From b1a07d520e91c1587b2d15c1214e235b2edb7aed Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Wed, 21 Aug 2024 12:04:26 +0200 Subject: [PATCH 1/5] indexer: drop concept of transaction id, hash is now the primary key * api: remove endpoint /chain/transactions/reference/index/{index} indexer migration: * 0013_recreate_table_transactions.sql --- api/chain.go | 38 ------------- vochain/indexer/bench_test.go | 4 +- vochain/indexer/db/db.go | 10 ---- vochain/indexer/db/models.go | 1 - vochain/indexer/db/transactions.sql.go | 33 ++--------- vochain/indexer/indexer_test.go | 5 +- vochain/indexer/indexertypes/types.go | 10 ---- .../0013_recreate_table_transactions.sql | 56 +++++++++++++++++++ vochain/indexer/queries/transactions.sql | 7 +-- vochain/indexer/transaction.go | 13 ----- 10 files changed, 65 insertions(+), 112 deletions(-) create mode 100644 vochain/indexer/migrations/0013_recreate_table_transactions.sql diff --git a/api/chain.go b/api/chain.go index 8c081bd89..9c6c494f2 100644 --- a/api/chain.go +++ b/api/chain.go @@ -107,14 +107,6 @@ func (a *API) enableChainHandlers() error { ); err != nil { return err } - if err := a.Endpoint.RegisterMethod( - "/chain/transactions/reference/index/{index}", - "GET", - apirest.MethodAccessTypePublic, - a.chainTxRefByIndexHandler, - ); err != nil { - return err - } if err := a.Endpoint.RegisterMethod( "/chain/blocks/{height}/transactions/page/{page}", "GET", @@ -717,36 +709,6 @@ func (a *API) chainTxHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) er return ctx.Send(data, apirest.HTTPstatusOK) } -// chainTxRefByIndexHandler -// -// @Summary Transaction by index -// @Description Get transaction by its index. This is not transaction reference (hash), and neither the block height and block index. The transaction index is an incremental counter for each transaction. You could use the transaction `block` and `index` to retrieve full info using [transaction by block and index](transaction-by-block-index). -// @Tags Chain -// @Accept json -// @Produce json -// @Param index path int true "Index of the transaction" -// @Success 200 {object} indexertypes.Transaction -// @Success 204 "See [errors](vocdoni-api#errors) section" -// @Router /chain/transactions/reference/index/{index} [get] -func (a *API) chainTxRefByIndexHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { - index, err := strconv.ParseUint(ctx.URLParam("index"), 10, 64) - if err != nil { - return err - } - ref, err := a.indexer.GetTransaction(index) - if err != nil { - if errors.Is(err, indexer.ErrTransactionNotFound) { - return ErrTransactionNotFound - } - return ErrVochainGetTxFailed.WithErr(err) - } - data, err := json.Marshal(ref) - if err != nil { - return err - } - return ctx.Send(data, apirest.HTTPstatusOK) -} - // chainTxListHandler // // @Summary List transactions diff --git a/vochain/indexer/bench_test.go b/vochain/indexer/bench_test.go index 631e83205..9e8cb1210 100644 --- a/vochain/indexer/bench_test.go +++ b/vochain/indexer/bench_test.go @@ -147,10 +147,10 @@ func BenchmarkFetchTx(b *testing.B) { startTime := time.Now() for j := 0; j < numTxs; j++ { - _, err = idx.GetTransaction(uint64((i * numTxs) + j + 1)) + _, err = idx.GetTxReferenceByBlockHeightAndBlockIndex(int64(i), int64(j)) qt.Assert(b, err, qt.IsNil) } - log.Infof("fetched %d transactions (out of %d total) by index, took %s", + log.Infof("fetched %d transactions (out of %d total) by height+index, took %s", numTxs, (i+1)*numTxs, time.Since(startTime)) startTime = time.Now() for j := 0; j < numTxs; j++ { diff --git a/vochain/indexer/db/db.go b/vochain/indexer/db/db.go index 3695e8b73..dd22f9d0f 100644 --- a/vochain/indexer/db/db.go +++ b/vochain/indexer/db/db.go @@ -81,9 +81,6 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.getTokenTransferStmt, err = db.PrepareContext(ctx, getTokenTransfer); err != nil { return nil, fmt.Errorf("error preparing query GetTokenTransfer: %w", err) } - if q.getTransactionStmt, err = db.PrepareContext(ctx, getTransaction); err != nil { - return nil, fmt.Errorf("error preparing query GetTransaction: %w", err) - } if q.getTransactionByHashStmt, err = db.PrepareContext(ctx, getTransactionByHash); err != nil { return nil, fmt.Errorf("error preparing query GetTransactionByHash: %w", err) } @@ -232,11 +229,6 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing getTokenTransferStmt: %w", cerr) } } - if q.getTransactionStmt != nil { - if cerr := q.getTransactionStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing getTransactionStmt: %w", cerr) - } - } if q.getTransactionByHashStmt != nil { if cerr := q.getTransactionByHashStmt.Close(); cerr != nil { err = fmt.Errorf("error closing getTransactionByHashStmt: %w", cerr) @@ -375,7 +367,6 @@ type Queries struct { getProcessIDsByFinalResultsStmt *sql.Stmt getProcessStatusStmt *sql.Stmt getTokenTransferStmt *sql.Stmt - getTransactionStmt *sql.Stmt getTransactionByHashStmt *sql.Stmt getTxReferenceByBlockHeightAndBlockIndexStmt *sql.Stmt getVoteStmt *sql.Stmt @@ -417,7 +408,6 @@ func (q *Queries) WithTx(tx *sql.Tx) *Queries { getProcessIDsByFinalResultsStmt: q.getProcessIDsByFinalResultsStmt, getProcessStatusStmt: q.getProcessStatusStmt, getTokenTransferStmt: q.getTokenTransferStmt, - getTransactionStmt: q.getTransactionStmt, getTransactionByHashStmt: q.getTransactionByHashStmt, getTxReferenceByBlockHeightAndBlockIndexStmt: q.getTxReferenceByBlockHeightAndBlockIndexStmt, getVoteStmt: q.getVoteStmt, diff --git a/vochain/indexer/db/models.go b/vochain/indexer/db/models.go index 08a6e0e2b..b64fc72c8 100644 --- a/vochain/indexer/db/models.go +++ b/vochain/indexer/db/models.go @@ -57,7 +57,6 @@ type TokenTransfer struct { } type Transaction struct { - ID int64 Hash types.Hash BlockHeight int64 BlockIndex int64 diff --git a/vochain/indexer/db/transactions.sql.go b/vochain/indexer/db/transactions.sql.go index 2d06f135a..a477ebad0 100644 --- a/vochain/indexer/db/transactions.sql.go +++ b/vochain/indexer/db/transactions.sql.go @@ -47,27 +47,8 @@ func (q *Queries) CreateTransaction(ctx context.Context, arg CreateTransactionPa ) } -const getTransaction = `-- name: GetTransaction :one -SELECT id, hash, block_height, block_index, type FROM transactions -WHERE id = ? -LIMIT 1 -` - -func (q *Queries) GetTransaction(ctx context.Context, id int64) (Transaction, error) { - row := q.queryRow(ctx, q.getTransactionStmt, getTransaction, id) - var i Transaction - err := row.Scan( - &i.ID, - &i.Hash, - &i.BlockHeight, - &i.BlockIndex, - &i.Type, - ) - return i, err -} - const getTransactionByHash = `-- name: GetTransactionByHash :one -SELECT id, hash, block_height, block_index, type FROM transactions +SELECT hash, block_height, block_index, type FROM transactions WHERE hash = ? LIMIT 1 ` @@ -76,7 +57,6 @@ func (q *Queries) GetTransactionByHash(ctx context.Context, hash types.Hash) (Tr row := q.queryRow(ctx, q.getTransactionByHashStmt, getTransactionByHash, hash) var i Transaction err := row.Scan( - &i.ID, &i.Hash, &i.BlockHeight, &i.BlockIndex, @@ -86,7 +66,7 @@ func (q *Queries) GetTransactionByHash(ctx context.Context, hash types.Hash) (Tr } const getTxReferenceByBlockHeightAndBlockIndex = `-- name: GetTxReferenceByBlockHeightAndBlockIndex :one -SELECT id, hash, block_height, block_index, type FROM transactions +SELECT hash, block_height, block_index, type FROM transactions WHERE block_height = ? AND block_index = ? LIMIT 1 ` @@ -100,7 +80,6 @@ func (q *Queries) GetTxReferenceByBlockHeightAndBlockIndex(ctx context.Context, row := q.queryRow(ctx, q.getTxReferenceByBlockHeightAndBlockIndexStmt, getTxReferenceByBlockHeightAndBlockIndex, arg.BlockHeight, arg.BlockIndex) var i Transaction err := row.Scan( - &i.ID, &i.Hash, &i.BlockHeight, &i.BlockIndex, @@ -111,16 +90,16 @@ func (q *Queries) GetTxReferenceByBlockHeightAndBlockIndex(ctx context.Context, const searchTransactions = `-- name: SearchTransactions :many WITH results AS ( - SELECT id, hash, block_height, block_index, type + SELECT hash, block_height, block_index, type FROM transactions WHERE ( (?3 = 0 OR block_height = ?3) AND (?4 = '' OR LOWER(type) = LOWER(?4)) ) ) -SELECT id, hash, block_height, block_index, type, COUNT(*) OVER() AS total_count +SELECT hash, block_height, block_index, type, COUNT(*) OVER() AS total_count FROM results -ORDER BY id DESC +ORDER BY block_height DESC, block_index DESC LIMIT ?2 OFFSET ?1 ` @@ -133,7 +112,6 @@ type SearchTransactionsParams struct { } type SearchTransactionsRow struct { - ID int64 Hash []byte BlockHeight int64 BlockIndex int64 @@ -156,7 +134,6 @@ func (q *Queries) SearchTransactions(ctx context.Context, arg SearchTransactions for rows.Next() { var i SearchTransactionsRow if err := rows.Scan( - &i.ID, &i.Hash, &i.BlockHeight, &i.BlockIndex, diff --git a/vochain/indexer/indexer_test.go b/vochain/indexer/indexer_test.go index fad19d248..76c306910 100644 --- a/vochain/indexer/indexer_test.go +++ b/vochain/indexer/indexer_test.go @@ -1404,7 +1404,7 @@ func TestTxIndexer(t *testing.T) { for i := 0; i < totalBlocks; i++ { for j := 0; j < txsPerBlock; j++ { - ref, err := idx.GetTransaction(uint64(i*txsPerBlock + j + 1)) + ref, err := idx.GetTxReferenceByBlockHeightAndBlockIndex(int64(i), int64(j)) qt.Assert(t, err, qt.IsNil) qt.Assert(t, ref.BlockHeight, qt.Equals, uint32(i)) qt.Assert(t, ref.TxBlockIndex, qt.Equals, int32(j)) @@ -1422,8 +1422,6 @@ func TestTxIndexer(t *testing.T) { txs, _, err := idx.SearchTransactions(15, 0, 0, "") qt.Assert(t, err, qt.IsNil) for i, tx := range txs { - // Index is between 1 and totalCount. - qt.Assert(t, tx.Index, qt.Equals, uint64(totalTxs-i)) // BlockIndex and TxBlockIndex start at 0, so subtract 1. qt.Assert(t, tx.BlockHeight, qt.Equals, uint32(totalTxs-i-1)/txsPerBlock) qt.Assert(t, tx.TxBlockIndex, qt.Equals, int32(totalTxs-i-1)%txsPerBlock) @@ -1433,7 +1431,6 @@ func TestTxIndexer(t *testing.T) { txs, _, err = idx.SearchTransactions(1, 5, 0, "") qt.Assert(t, err, qt.IsNil) qt.Assert(t, txs, qt.HasLen, 1) - qt.Assert(t, txs[0].Index, qt.Equals, uint64(95)) } func TestCensusUpdate(t *testing.T) { diff --git a/vochain/indexer/indexertypes/types.go b/vochain/indexer/indexertypes/types.go index d0d2b29aa..b083b2e95 100644 --- a/vochain/indexer/indexertypes/types.go +++ b/vochain/indexer/indexertypes/types.go @@ -176,17 +176,8 @@ type TxPackage struct { Signature types.HexBytes `json:"signature"` } -// TxMetadata contains tx information for the TransactionList api -type TxMetadata struct { - Type string `json:"type"` - BlockHeight uint32 `json:"blockHeight,omitempty"` - Index int32 `json:"index"` - Hash types.HexBytes `json:"hash"` -} - // Transaction holds the db reference for a single transaction type Transaction struct { - Index uint64 `json:"transactionNumber" format:"int64" example:"944"` Hash types.HexBytes `json:"transactionHash" swaggertype:"string" example:"75e8f822f5dd13973ac5158d600f0a2a5fea4bfefce9712ab5195bf17884cfad"` BlockHeight uint32 `json:"blockHeight" format:"int32" example:"64924"` TxBlockIndex int32 `json:"transactionIndex" format:"int32" example:"0"` @@ -195,7 +186,6 @@ type Transaction struct { func TransactionFromDB(dbtx *indexerdb.Transaction) *Transaction { return &Transaction{ - Index: uint64(dbtx.ID), Hash: dbtx.Hash, BlockHeight: uint32(dbtx.BlockHeight), TxBlockIndex: int32(dbtx.BlockIndex), diff --git a/vochain/indexer/migrations/0013_recreate_table_transactions.sql b/vochain/indexer/migrations/0013_recreate_table_transactions.sql new file mode 100644 index 000000000..bf091d7c9 --- /dev/null +++ b/vochain/indexer/migrations/0013_recreate_table_transactions.sql @@ -0,0 +1,56 @@ +-- +goose Up +PRAGMA foreign_keys = OFF; + +-- Create a new table with hash as primary key +CREATE TABLE transactions_new ( + hash BLOB NOT NULL PRIMARY KEY, + block_height INTEGER NOT NULL, + block_index INTEGER NOT NULL, + type TEXT NOT NULL +); + +-- Copy data from the old table to the new table +INSERT INTO transactions_new (hash, block_height, block_index, type) +SELECT hash, block_height, block_index, type +FROM transactions; + +-- Drop the old table +DROP TABLE transactions; + +-- Rename the new table to the old table name +ALTER TABLE transactions_new RENAME TO transactions; + +-- Recreate necessary indexes +CREATE INDEX transactions_block_height_index +ON transactions(block_height, block_index); + +PRAGMA foreign_keys = ON; + +-- +goose Down +PRAGMA foreign_keys = OFF; + +-- Recreate the old table structure +CREATE TABLE transactions ( + id INTEGER NOT NULL PRIMARY KEY, + hash BLOB NOT NULL, + block_height INTEGER NOT NULL, + block_index INTEGER NOT NULL, + type TEXT NOT NULL +); + +-- Copy data back from the new table to the old table +INSERT INTO transactions (hash, block_height, block_index, type) +SELECT hash, block_height, block_index, type +FROM transactions_new; + +-- Drop the new table +DROP TABLE transactions_new; + +-- Recreate the old indexes +CREATE INDEX transactions_hash +ON transactions(hash); + +CREATE INDEX transactions_block_height_index +ON transactions(block_height, block_index); + +PRAGMA foreign_keys = ON; diff --git a/vochain/indexer/queries/transactions.sql b/vochain/indexer/queries/transactions.sql index 0e625a197..eb8b2b617 100644 --- a/vochain/indexer/queries/transactions.sql +++ b/vochain/indexer/queries/transactions.sql @@ -5,11 +5,6 @@ INSERT INTO transactions ( ?, ?, ?, ? ); --- name: GetTransaction :one -SELECT * FROM transactions -WHERE id = ? -LIMIT 1; - -- name: GetTransactionByHash :one SELECT * FROM transactions WHERE hash = ? @@ -34,6 +29,6 @@ WITH results AS ( ) SELECT *, COUNT(*) OVER() AS total_count FROM results -ORDER BY id DESC +ORDER BY block_height DESC, block_index DESC LIMIT sqlc.arg(limit) OFFSET sqlc.arg(offset); diff --git a/vochain/indexer/transaction.go b/vochain/indexer/transaction.go index 4b8c9fe21..16e8cad81 100644 --- a/vochain/indexer/transaction.go +++ b/vochain/indexer/transaction.go @@ -22,18 +22,6 @@ func (idx *Indexer) CountTotalTransactions() (uint64, error) { return uint64(count), err } -// GetTransaction fetches the txReference for the given tx height -func (idx *Indexer) GetTransaction(id uint64) (*indexertypes.Transaction, error) { - sqlTxRef, err := idx.readOnlyQuery.GetTransaction(context.TODO(), int64(id)) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return nil, ErrTransactionNotFound - } - return nil, fmt.Errorf("tx with id %d not found: %v", id, err) - } - return indexertypes.TransactionFromDB(&sqlTxRef), nil -} - // GetTxReferenceByBlockHeightAndBlockIndex fetches the txReference for the given tx height and block tx index func (idx *Indexer) GetTxReferenceByBlockHeightAndBlockIndex(blockHeight, blockIndex int64) (*indexertypes.Transaction, error) { sqlTxRef, err := idx.readOnlyQuery.GetTxReferenceByBlockHeightAndBlockIndex(context.TODO(), indexerdb.GetTxReferenceByBlockHeightAndBlockIndexParams{ @@ -83,7 +71,6 @@ func (idx *Indexer) SearchTransactions(limit, offset int, blockHeight uint64, tx list := []*indexertypes.Transaction{} for _, row := range results { list = append(list, &indexertypes.Transaction{ - Index: uint64(row.ID), Hash: row.Hash, BlockHeight: uint32(row.BlockHeight), TxBlockIndex: int32(row.BlockIndex), From a3574d51578fd3828fdb3c79f0b7cfc7825efa71 Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Mon, 2 Sep 2024 11:14:56 +0200 Subject: [PATCH 2/5] api: rename /chain/transactions fields renames: * transactionHash -> hash * blockHeight -> height * transactionIndex -> index * transactionType -> type affected endpoints: /chain/blocks/{height}/transactions/page/{page} /chain/transactions/page/{page} /chain/transactions /chain/transactions/{height}/{index} /chain/transactions/reference/{hash} --- vochain/indexer/indexertypes/types.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vochain/indexer/indexertypes/types.go b/vochain/indexer/indexertypes/types.go index b083b2e95..bc0e56485 100644 --- a/vochain/indexer/indexertypes/types.go +++ b/vochain/indexer/indexertypes/types.go @@ -178,10 +178,10 @@ type TxPackage struct { // Transaction holds the db reference for a single transaction type Transaction struct { - Hash types.HexBytes `json:"transactionHash" swaggertype:"string" example:"75e8f822f5dd13973ac5158d600f0a2a5fea4bfefce9712ab5195bf17884cfad"` - BlockHeight uint32 `json:"blockHeight" format:"int32" example:"64924"` - TxBlockIndex int32 `json:"transactionIndex" format:"int32" example:"0"` - TxType string `json:"transactionType" enums:"vote,newProcess,admin,setProcess,registerKey,mintTokens,sendTokens,setTransactionCosts,setAccount,collectFaucet,setKeykeeper" example:"Vote"` + Hash types.HexBytes `json:"hash" swaggertype:"string" example:"75e8f822f5dd13973ac5158d600f0a2a5fea4bfefce9712ab5195bf17884cfad"` + BlockHeight uint32 `json:"height" format:"int32" example:"64924"` + TxBlockIndex int32 `json:"index" format:"int32" example:"0"` + TxType string `json:"type" enums:"vote,newProcess,admin,setProcess,registerKey,mintTokens,sendTokens,setTransactionCosts,setAccount,collectFaucet,setKeykeeper" example:"Vote"` } func TransactionFromDB(dbtx *indexerdb.Transaction) *Transaction { From a59d942b93c9bcea9d7a59aab496ab1fd096e6c5 Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Mon, 10 Jun 2024 11:18:11 +0200 Subject: [PATCH 3/5] indexer: CreateBlock during Commit instead of OnBeginBlock since the CreateBlock now fetches the whole block, we need app.NodeClient initialized in all nodes (including seeds), else seeds panic on first Commit --- vochain/app.go | 4 ---- vochain/appsetup.go | 3 --- vochain/indexer/block.go | 17 ----------------- vochain/indexer/indexer.go | 11 +++++++++++ vochain/keykeeper/keykeeper.go | 3 --- .../offchaindatahandler/offchaindatahandler.go | 1 - vochain/state/eventlistener.go | 15 --------------- vochain/state/state_test.go | 1 - 8 files changed, 11 insertions(+), 44 deletions(-) diff --git a/vochain/app.go b/vochain/app.go index 7e0107537..f6691be61 100644 --- a/vochain/app.go +++ b/vochain/app.go @@ -290,10 +290,6 @@ func (app *BaseApplication) beginBlock(t time.Time, height uint32) { app.State.SetHeight(height) go app.State.CachePurge(height) - app.State.OnBeginBlock(vstate.BeginBlock{ - Height: int64(height), - Time: t, - }) } // endBlock is called at the end of every block. diff --git a/vochain/appsetup.go b/vochain/appsetup.go index fb52f66ed..6726b8b21 100644 --- a/vochain/appsetup.go +++ b/vochain/appsetup.go @@ -25,9 +25,6 @@ func (app *BaseApplication) SetNode(vochaincfg *config.VochainCfg) error { if app.Node, err = newTendermint(app, vochaincfg); err != nil { return fmt.Errorf("could not set tendermint node service: %s", err) } - if vochaincfg.IsSeedNode { - return nil - } // Note that cometcli.New logs any error rather than returning it. app.NodeClient = cometcli.New(app.Node) return nil diff --git a/vochain/indexer/block.go b/vochain/indexer/block.go index e40a5fdfb..8c9d76a22 100644 --- a/vochain/indexer/block.go +++ b/vochain/indexer/block.go @@ -6,28 +6,11 @@ import ( "errors" "fmt" "time" - - "go.vocdoni.io/dvote/log" - indexerdb "go.vocdoni.io/dvote/vochain/indexer/db" - "go.vocdoni.io/dvote/vochain/state" ) // ErrBlockNotFound is returned if the block is not found in the indexer database. var ErrBlockNotFound = fmt.Errorf("block not found") -func (idx *Indexer) OnBeginBlock(bb state.BeginBlock) { - idx.blockMu.Lock() - defer idx.blockMu.Unlock() - queries := idx.blockTxQueries() - if _, err := queries.CreateBlock(context.TODO(), indexerdb.CreateBlockParams{ - Height: bb.Height, - Time: bb.Time, - DataHash: nonNullBytes(bb.DataHash), - }); err != nil { - log.Errorw(err, "cannot index new block") - } -} - // BlockTimestamp returns the timestamp of the block at the given height func (idx *Indexer) BlockTimestamp(height int64) (time.Time, error) { block, err := idx.readOnlyQuery.GetBlock(context.TODO(), height) diff --git a/vochain/indexer/indexer.go b/vochain/indexer/indexer.go index 43b59323e..769b76720 100644 --- a/vochain/indexer/indexer.go +++ b/vochain/indexer/indexer.go @@ -414,6 +414,17 @@ func (idx *Indexer) Commit(height uint32) error { queries := idx.blockTxQueries() ctx := context.TODO() + // index the new block + bb := idx.App.GetBlockByHeight(int64(height)) + if _, err := queries.CreateBlock(context.TODO(), indexerdb.CreateBlockParams{ + Height: bb.Height, + Time: bb.Time, + DataHash: nonNullBytes(bb.DataHash), + // TODO: ProposerAddress, Hash, AppHash, ChainID, LastBlockHash?, ValidatorSignatures? + }); err != nil { + log.Errorw(err, "cannot index new block") + } + for _, pidStr := range updateProcs { pid := types.ProcessID(pidStr) if err := idx.updateProcess(ctx, queries, pid); err != nil { diff --git a/vochain/keykeeper/keykeeper.go b/vochain/keykeeper/keykeeper.go index 7f46f654d..4a4425cd1 100644 --- a/vochain/keykeeper/keykeeper.go +++ b/vochain/keykeeper/keykeeper.go @@ -268,9 +268,6 @@ func (*KeyKeeper) OnVote(_ *state.Vote, _ int32) {} // OnNewTx is not used by the KeyKeeper func (*KeyKeeper) OnNewTx(_ *vochaintx.Tx, _ uint32, _ int32) {} -// OnBeginBlock is not used by the KeyKeeper -func (*KeyKeeper) OnBeginBlock(_ state.BeginBlock) {} - // OnCensusUpdate is not used by the KeyKeeper func (*KeyKeeper) OnCensusUpdate(_, _ []byte, _ string, _ uint64) {} diff --git a/vochain/offchaindatahandler/offchaindatahandler.go b/vochain/offchaindatahandler/offchaindatahandler.go index d6e97bd9b..4bc3be6b4 100644 --- a/vochain/offchaindatahandler/offchaindatahandler.go +++ b/vochain/offchaindatahandler/offchaindatahandler.go @@ -166,7 +166,6 @@ func (d *OffChainDataHandler) OnSetAccount(_ []byte, account *state.Account) { func (*OffChainDataHandler) OnCancel(_ []byte, _ int32) {} func (*OffChainDataHandler) OnVote(_ *state.Vote, _ int32) {} func (*OffChainDataHandler) OnNewTx(_ *vochaintx.Tx, _ uint32, _ int32) {} -func (*OffChainDataHandler) OnBeginBlock(state.BeginBlock) {} func (*OffChainDataHandler) OnProcessKeys(_ []byte, _ string, _ int32) {} func (*OffChainDataHandler) OnRevealKeys(_ []byte, _ string, _ int32) {} func (*OffChainDataHandler) OnProcessStatusChange(_ []byte, _ models.ProcessStatus, _ int32) {} diff --git a/vochain/state/eventlistener.go b/vochain/state/eventlistener.go index 1c2d05281..7b599702b 100644 --- a/vochain/state/eventlistener.go +++ b/vochain/state/eventlistener.go @@ -1,8 +1,6 @@ package state import ( - "time" - "go.vocdoni.io/dvote/vochain/transaction/vochaintx" "go.vocdoni.io/proto/build/go/models" ) @@ -32,7 +30,6 @@ type EventListener interface { OnSpendTokens(addr []byte, txType models.TxType, cost uint64, reference string) OnCensusUpdate(pid, censusRoot []byte, censusURI string, censusSize uint64) Commit(height uint32) (err error) - OnBeginBlock(BeginBlock) Rollback() } @@ -46,15 +43,3 @@ func (v *State) AddEventListener(l EventListener) { func (v *State) CleanEventListeners() { v.eventListeners = nil } - -type BeginBlock struct { - Height int64 - Time time.Time - DataHash []byte -} - -func (v *State) OnBeginBlock(bb BeginBlock) { - for _, l := range v.eventListeners { - l.OnBeginBlock(bb) - } -} diff --git a/vochain/state/state_test.go b/vochain/state/state_test.go index e2b76c685..b36dc7722 100644 --- a/vochain/state/state_test.go +++ b/vochain/state/state_test.go @@ -182,7 +182,6 @@ type Listener struct { func (*Listener) OnVote(_ *Vote, _ int32) {} func (*Listener) OnNewTx(_ *vochaintx.Tx, _ uint32, _ int32) {} -func (*Listener) OnBeginBlock(BeginBlock) {} func (*Listener) OnProcess(_ *models.Process, _ int32) {} func (*Listener) OnProcessStatusChange(_ []byte, _ models.ProcessStatus, _ int32) {} func (*Listener) OnProcessDurationChange(_ []byte, _ uint32, _ int32) {} From b34b5bb661c5f5c393c7f857177d742a90582cae Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Mon, 10 Jun 2024 11:18:11 +0200 Subject: [PATCH 4/5] indexer: index more details about blocks and transactions blocks: * indexerdb: add block details chain_id, proposer_address and last_block_hash * indexerdb: rename method GetBlock -> GetBlockByHeight * indexerdb: make CreateBlock an UPSERT * indexer: new method BlockByHeight (and indexerdb.GetBlockByHash) * indexer: new method SearchBlocks (and indexerdb.SearchBlocks) transactions: * indexerdb: add raw_tx, subtype, signature and signer in transactions table * indexerdb: make CreateTransaction an UPSERT * indexer: new methods CountTransactionsByHeight and SearchTransactions * indexer: rename GetTransaction* methods -> GetTxMetadata* indexer migrations: * 0014_alter_columns_table_blocks.sql * 0015_alter_columns_table_transactions.sql indexer: reindex blocks and transactions after migrations indexer: new method ReindexBlocks * vochaintx: add tx.TxSubtype method --- api/api_types.go | 4 +- api/chain.go | 4 +- vochain/indexer/bench_test.go | 13 +- vochain/indexer/block.go | 76 ++++++- vochain/indexer/db/blocks.sql.go | 147 ++++++++++++- vochain/indexer/db/db.go | 198 ++++++++++-------- vochain/indexer/db/models.go | 13 +- vochain/indexer/db/transactions.sql.go | 101 ++++++--- vochain/indexer/indexer.go | 121 ++++++++++- vochain/indexer/indexer_test.go | 9 +- vochain/indexer/indexertypes/block.go | 26 +++ vochain/indexer/indexertypes/types.go | 39 +++- .../0014_alter_columns_table_blocks.sql | 13 ++ .../0015_alter_columns_table_transactions.sql | 11 + vochain/indexer/queries/blocks.sql | 43 +++- vochain/indexer/queries/transactions.sql | 42 ++-- vochain/indexer/transaction.go | 76 +++++-- vochain/transaction/vochaintx/vochaintx.go | 26 +++ 18 files changed, 773 insertions(+), 189 deletions(-) create mode 100644 vochain/indexer/migrations/0014_alter_columns_table_blocks.sql create mode 100644 vochain/indexer/migrations/0015_alter_columns_table_transactions.sql diff --git a/api/api_types.go b/api/api_types.go index efb390503..b1169db8d 100644 --- a/api/api_types.go +++ b/api/api_types.go @@ -275,8 +275,8 @@ type TransactionReference struct { // TransactionsList is used to return a paginated list to the client type TransactionsList struct { - Transactions []*indexertypes.Transaction `json:"transactions"` - Pagination *Pagination `json:"pagination"` + Transactions []*indexertypes.TransactionMetadata `json:"transactions"` + Pagination *Pagination `json:"pagination"` } // FeesList is used to return a paginated list to the client diff --git a/api/chain.go b/api/chain.go index 9c6c494f2..1d84ef095 100644 --- a/api/chain.go +++ b/api/chain.go @@ -646,7 +646,7 @@ func (a *API) chainTxRefByHashHandler(_ *apirest.APIdata, ctx *httprouter.HTTPCo if err != nil { return err } - ref, err := a.indexer.GetTxHashReference(hash) + ref, err := a.indexer.GetTxMetadataByHash(hash) if err != nil { if errors.Is(err, indexer.ErrTransactionNotFound) { return ErrTransactionNotFound @@ -690,7 +690,7 @@ func (a *API) chainTxHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) er return ErrVochainGetTxFailed.WithErr(err) } - ref, err := a.indexer.GetTxReferenceByBlockHeightAndBlockIndex(height, index) + ref, err := a.indexer.GetTransactionByHeightAndIndex(height, index) if err != nil { if errors.Is(err, indexer.ErrTransactionNotFound) { return ErrTransactionNotFound diff --git a/vochain/indexer/bench_test.go b/vochain/indexer/bench_test.go index 9e8cb1210..7e289857a 100644 --- a/vochain/indexer/bench_test.go +++ b/vochain/indexer/bench_test.go @@ -85,6 +85,7 @@ func BenchmarkIndexer(b *testing.B) { tx := &vochaintx.Tx{ TxID: rnd.Random32(), TxModelType: "vote", + Tx: &models.Tx{Payload: &models.Tx_Vote{}}, } idx.OnNewTx(tx, height, txBlockIndex) curTxs = append(curTxs, tx) @@ -112,7 +113,7 @@ func BenchmarkIndexer(b *testing.B) { qt.Check(b, bytes.Equal(voteRef.Meta.TxHash, tx.TxID[:]), qt.IsTrue) } - txRef, err := idx.GetTxHashReference(tx.TxID[:]) + txRef, err := idx.GetTxMetadataByHash(tx.TxID[:]) qt.Check(b, err, qt.IsNil) if err == nil { qt.Check(b, txRef.BlockHeight, qt.Equals, vote.Height) @@ -138,7 +139,11 @@ func BenchmarkFetchTx(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { for j := 0; j < numTxs; j++ { - idx.OnNewTx(&vochaintx.Tx{TxID: util.Random32()}, uint32(i), int32(j)) + idx.OnNewTx(&vochaintx.Tx{ + TxID: util.Random32(), + TxModelType: "vote", + Tx: &models.Tx{Payload: &models.Tx_Vote{}}, + }, uint32(i), int32(j)) } err := idx.Commit(uint32(i)) qt.Assert(b, err, qt.IsNil) @@ -147,14 +152,14 @@ func BenchmarkFetchTx(b *testing.B) { startTime := time.Now() for j := 0; j < numTxs; j++ { - _, err = idx.GetTxReferenceByBlockHeightAndBlockIndex(int64(i), int64(j)) + _, err = idx.GetTransactionByHeightAndIndex(int64(i), int64(j)) qt.Assert(b, err, qt.IsNil) } log.Infof("fetched %d transactions (out of %d total) by height+index, took %s", numTxs, (i+1)*numTxs, time.Since(startTime)) startTime = time.Now() for j := 0; j < numTxs; j++ { - _, err = idx.GetTxHashReference([]byte(fmt.Sprintf("hash%d%d", i, j))) + _, err = idx.GetTxMetadataByHash([]byte(fmt.Sprintf("hash%d%d", i, j))) qt.Assert(b, err, qt.IsNil) } log.Infof("fetched %d transactions (out of %d total) by hash, took %s", diff --git a/vochain/indexer/block.go b/vochain/indexer/block.go index 8c9d76a22..c6021e530 100644 --- a/vochain/indexer/block.go +++ b/vochain/indexer/block.go @@ -6,6 +6,9 @@ import ( "errors" "fmt" "time" + + indexerdb "go.vocdoni.io/dvote/vochain/indexer/db" + "go.vocdoni.io/dvote/vochain/indexer/indexertypes" ) // ErrBlockNotFound is returned if the block is not found in the indexer database. @@ -13,12 +16,77 @@ var ErrBlockNotFound = fmt.Errorf("block not found") // BlockTimestamp returns the timestamp of the block at the given height func (idx *Indexer) BlockTimestamp(height int64) (time.Time, error) { - block, err := idx.readOnlyQuery.GetBlock(context.TODO(), height) + block, err := idx.BlockByHeight(height) if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return time.Time{}, ErrBlockNotFound - } return time.Time{}, err } return block.Time, nil } + +// BlockByHeight returns the available information of the block at the given height +func (idx *Indexer) BlockByHeight(height int64) (*indexertypes.Block, error) { + block, err := idx.readOnlyQuery.GetBlockByHeight(context.TODO(), height) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrBlockNotFound + } + return nil, err + } + return indexertypes.BlockFromDB(&block), nil +} + +// BlockByHash returns the available information of the block with the given hash +func (idx *Indexer) BlockByHash(hash []byte) (*indexertypes.Block, error) { + block, err := idx.readOnlyQuery.GetBlockByHash(context.TODO(), hash) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrBlockNotFound + } + return nil, err + } + return indexertypes.BlockFromDB(&block), nil +} + +// BlockList returns the list of blocks indexed. +// chainID, hash, proposerAddress are optional, if declared as zero-value will be ignored. +// The first one returned is the newest, so they are in descending order. +func (idx *Indexer) BlockList(limit, offset int, chainID, hash, proposerAddress string) ([]*indexertypes.Block, uint64, error) { + if offset < 0 { + return nil, 0, fmt.Errorf("invalid value: offset cannot be %d", offset) + } + if limit <= 0 { + return nil, 0, fmt.Errorf("invalid value: limit cannot be %d", limit) + } + results, err := idx.readOnlyQuery.SearchBlocks(context.TODO(), indexerdb.SearchBlocksParams{ + Limit: int64(limit), + Offset: int64(offset), + ChainID: chainID, + HashSubstr: hash, + ProposerAddress: proposerAddress, + }) + if err != nil { + return nil, 0, err + } + list := []*indexertypes.Block{} + for _, row := range results { + list = append(list, indexertypes.BlockFromDBRow(&row)) + } + if len(results) == 0 { + return list, 0, nil + } + return list, uint64(results[0].TotalCount), nil +} + +// CountBlocks returns how many blocks are indexed. +func (idx *Indexer) CountBlocks() (uint64, error) { + results, err := idx.readOnlyQuery.SearchBlocks(context.TODO(), indexerdb.SearchBlocksParams{ + Limit: 1, + }) + if err != nil { + return 0, err + } + if len(results) == 0 { + return 0, nil + } + return uint64(results[0].TotalCount), nil +} diff --git a/vochain/indexer/db/blocks.sql.go b/vochain/indexer/db/blocks.sql.go index ffa3fa896..669f59602 100644 --- a/vochain/indexer/db/blocks.sql.go +++ b/vochain/indexer/db/blocks.sql.go @@ -13,31 +13,156 @@ import ( const createBlock = `-- name: CreateBlock :execresult INSERT INTO blocks( - height, time, data_hash + chain_id, height, time, hash, proposer_address, last_block_hash ) VALUES ( - ?, ?, ? + ?, ?, ?, ?, ?, ? ) +ON CONFLICT(height) DO UPDATE +SET chain_id = excluded.chain_id, + time = excluded.time, + hash = excluded.hash, + proposer_address = excluded.proposer_address, + last_block_hash = excluded.last_block_hash ` type CreateBlockParams struct { - Height int64 - Time time.Time - DataHash []byte + ChainID string + Height int64 + Time time.Time + Hash []byte + ProposerAddress []byte + LastBlockHash []byte } func (q *Queries) CreateBlock(ctx context.Context, arg CreateBlockParams) (sql.Result, error) { - return q.exec(ctx, q.createBlockStmt, createBlock, arg.Height, arg.Time, arg.DataHash) + return q.exec(ctx, q.createBlockStmt, createBlock, + arg.ChainID, + arg.Height, + arg.Time, + arg.Hash, + arg.ProposerAddress, + arg.LastBlockHash, + ) } -const getBlock = `-- name: GetBlock :one -SELECT height, time, data_hash FROM blocks +const getBlockByHash = `-- name: GetBlockByHash :one +SELECT height, time, chain_id, hash, proposer_address, last_block_hash FROM blocks +WHERE hash = ? +LIMIT 1 +` + +func (q *Queries) GetBlockByHash(ctx context.Context, hash []byte) (Block, error) { + row := q.queryRow(ctx, q.getBlockByHashStmt, getBlockByHash, hash) + var i Block + err := row.Scan( + &i.Height, + &i.Time, + &i.ChainID, + &i.Hash, + &i.ProposerAddress, + &i.LastBlockHash, + ) + return i, err +} + +const getBlockByHeight = `-- name: GetBlockByHeight :one +SELECT height, time, chain_id, hash, proposer_address, last_block_hash FROM blocks WHERE height = ? LIMIT 1 ` -func (q *Queries) GetBlock(ctx context.Context, height int64) (Block, error) { - row := q.queryRow(ctx, q.getBlockStmt, getBlock, height) +func (q *Queries) GetBlockByHeight(ctx context.Context, height int64) (Block, error) { + row := q.queryRow(ctx, q.getBlockByHeightStmt, getBlockByHeight, height) var i Block - err := row.Scan(&i.Height, &i.Time, &i.DataHash) + err := row.Scan( + &i.Height, + &i.Time, + &i.ChainID, + &i.Hash, + &i.ProposerAddress, + &i.LastBlockHash, + ) return i, err } + +const searchBlocks = `-- name: SearchBlocks :many +SELECT + b.height, b.time, b.chain_id, b.hash, b.proposer_address, b.last_block_hash, + COUNT(t.block_index) AS tx_count, + COUNT(*) OVER() AS total_count +FROM blocks AS b +LEFT JOIN transactions AS t + ON b.height = t.block_height +WHERE ( + (?1 = '' OR b.chain_id = ?1) + AND LENGTH(?2) <= 64 -- if passed arg is longer, then just abort the query + AND ( + ?2 = '' + OR (LENGTH(?2) = 64 AND LOWER(HEX(b.hash)) = LOWER(?2)) + OR (LENGTH(?2) < 64 AND INSTR(LOWER(HEX(b.hash)), LOWER(?2)) > 0) + -- TODO: consider keeping an hash_hex column for faster searches + ) + AND (?3 = '' OR LOWER(HEX(b.proposer_address)) = LOWER(?3)) +) +GROUP BY b.height +ORDER BY b.height DESC +LIMIT ?5 +OFFSET ?4 +` + +type SearchBlocksParams struct { + ChainID interface{} + HashSubstr interface{} + ProposerAddress interface{} + Offset int64 + Limit int64 +} + +type SearchBlocksRow struct { + Height int64 + Time time.Time + ChainID string + Hash []byte + ProposerAddress []byte + LastBlockHash []byte + TxCount int64 + TotalCount int64 +} + +func (q *Queries) SearchBlocks(ctx context.Context, arg SearchBlocksParams) ([]SearchBlocksRow, error) { + rows, err := q.query(ctx, q.searchBlocksStmt, searchBlocks, + arg.ChainID, + arg.HashSubstr, + arg.ProposerAddress, + arg.Offset, + arg.Limit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + var items []SearchBlocksRow + for rows.Next() { + var i SearchBlocksRow + if err := rows.Scan( + &i.Height, + &i.Time, + &i.ChainID, + &i.Hash, + &i.ProposerAddress, + &i.LastBlockHash, + &i.TxCount, + &i.TotalCount, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/vochain/indexer/db/db.go b/vochain/indexer/db/db.go index dd22f9d0f..47bf88d46 100644 --- a/vochain/indexer/db/db.go +++ b/vochain/indexer/db/db.go @@ -36,6 +36,9 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.countTransactionsStmt, err = db.PrepareContext(ctx, countTransactions); err != nil { return nil, fmt.Errorf("error preparing query CountTransactions: %w", err) } + if q.countTransactionsByHeightStmt, err = db.PrepareContext(ctx, countTransactionsByHeight); err != nil { + return nil, fmt.Errorf("error preparing query CountTransactionsByHeight: %w", err) + } if q.countVotesStmt, err = db.PrepareContext(ctx, countVotes); err != nil { return nil, fmt.Errorf("error preparing query CountVotes: %w", err) } @@ -60,8 +63,11 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.createVoteStmt, err = db.PrepareContext(ctx, createVote); err != nil { return nil, fmt.Errorf("error preparing query CreateVote: %w", err) } - if q.getBlockStmt, err = db.PrepareContext(ctx, getBlock); err != nil { - return nil, fmt.Errorf("error preparing query GetBlock: %w", err) + if q.getBlockByHashStmt, err = db.PrepareContext(ctx, getBlockByHash); err != nil { + return nil, fmt.Errorf("error preparing query GetBlockByHash: %w", err) + } + if q.getBlockByHeightStmt, err = db.PrepareContext(ctx, getBlockByHeight); err != nil { + return nil, fmt.Errorf("error preparing query GetBlockByHeight: %w", err) } if q.getEntityCountStmt, err = db.PrepareContext(ctx, getEntityCount); err != nil { return nil, fmt.Errorf("error preparing query GetEntityCount: %w", err) @@ -84,8 +90,8 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.getTransactionByHashStmt, err = db.PrepareContext(ctx, getTransactionByHash); err != nil { return nil, fmt.Errorf("error preparing query GetTransactionByHash: %w", err) } - if q.getTxReferenceByBlockHeightAndBlockIndexStmt, err = db.PrepareContext(ctx, getTxReferenceByBlockHeightAndBlockIndex); err != nil { - return nil, fmt.Errorf("error preparing query GetTxReferenceByBlockHeightAndBlockIndex: %w", err) + if q.getTransactionByHeightAndIndexStmt, err = db.PrepareContext(ctx, getTransactionByHeightAndIndex); err != nil { + return nil, fmt.Errorf("error preparing query GetTransactionByHeightAndIndex: %w", err) } if q.getVoteStmt, err = db.PrepareContext(ctx, getVote); err != nil { return nil, fmt.Errorf("error preparing query GetVote: %w", err) @@ -93,6 +99,9 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.searchAccountsStmt, err = db.PrepareContext(ctx, searchAccounts); err != nil { return nil, fmt.Errorf("error preparing query SearchAccounts: %w", err) } + if q.searchBlocksStmt, err = db.PrepareContext(ctx, searchBlocks); err != nil { + return nil, fmt.Errorf("error preparing query SearchBlocks: %w", err) + } if q.searchEntitiesStmt, err = db.PrepareContext(ctx, searchEntities); err != nil { return nil, fmt.Errorf("error preparing query SearchEntities: %w", err) } @@ -154,6 +163,11 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing countTransactionsStmt: %w", cerr) } } + if q.countTransactionsByHeightStmt != nil { + if cerr := q.countTransactionsByHeightStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing countTransactionsByHeightStmt: %w", cerr) + } + } if q.countVotesStmt != nil { if cerr := q.countVotesStmt.Close(); cerr != nil { err = fmt.Errorf("error closing countVotesStmt: %w", cerr) @@ -194,9 +208,14 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing createVoteStmt: %w", cerr) } } - if q.getBlockStmt != nil { - if cerr := q.getBlockStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing getBlockStmt: %w", cerr) + if q.getBlockByHashStmt != nil { + if cerr := q.getBlockByHashStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getBlockByHashStmt: %w", cerr) + } + } + if q.getBlockByHeightStmt != nil { + if cerr := q.getBlockByHeightStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getBlockByHeightStmt: %w", cerr) } } if q.getEntityCountStmt != nil { @@ -234,9 +253,9 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing getTransactionByHashStmt: %w", cerr) } } - if q.getTxReferenceByBlockHeightAndBlockIndexStmt != nil { - if cerr := q.getTxReferenceByBlockHeightAndBlockIndexStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing getTxReferenceByBlockHeightAndBlockIndexStmt: %w", cerr) + if q.getTransactionByHeightAndIndexStmt != nil { + if cerr := q.getTransactionByHeightAndIndexStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getTransactionByHeightAndIndexStmt: %w", cerr) } } if q.getVoteStmt != nil { @@ -249,6 +268,11 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing searchAccountsStmt: %w", cerr) } } + if q.searchBlocksStmt != nil { + if cerr := q.searchBlocksStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing searchBlocksStmt: %w", cerr) + } + } if q.searchEntitiesStmt != nil { if cerr := q.searchEntitiesStmt.Close(); cerr != nil { err = fmt.Errorf("error closing searchEntitiesStmt: %w", cerr) @@ -346,83 +370,89 @@ func (q *Queries) queryRow(ctx context.Context, stmt *sql.Stmt, query string, ar } type Queries struct { - db DBTX - tx *sql.Tx - computeProcessVoteCountStmt *sql.Stmt - countAccountsStmt *sql.Stmt - countTokenTransfersByAccountStmt *sql.Stmt - countTransactionsStmt *sql.Stmt - countVotesStmt *sql.Stmt - createAccountStmt *sql.Stmt - createBlockStmt *sql.Stmt - createProcessStmt *sql.Stmt - createTokenFeeStmt *sql.Stmt - createTokenTransferStmt *sql.Stmt - createTransactionStmt *sql.Stmt - createVoteStmt *sql.Stmt - getBlockStmt *sql.Stmt - getEntityCountStmt *sql.Stmt - getProcessStmt *sql.Stmt - getProcessCountStmt *sql.Stmt - getProcessIDsByFinalResultsStmt *sql.Stmt - getProcessStatusStmt *sql.Stmt - getTokenTransferStmt *sql.Stmt - getTransactionByHashStmt *sql.Stmt - getTxReferenceByBlockHeightAndBlockIndexStmt *sql.Stmt - getVoteStmt *sql.Stmt - searchAccountsStmt *sql.Stmt - searchEntitiesStmt *sql.Stmt - searchProcessesStmt *sql.Stmt - searchTokenFeesStmt *sql.Stmt - searchTokenTransfersStmt *sql.Stmt - searchTransactionsStmt *sql.Stmt - searchVotesStmt *sql.Stmt - setProcessResultsCancelledStmt *sql.Stmt - setProcessResultsReadyStmt *sql.Stmt - updateProcessEndDateStmt *sql.Stmt - updateProcessFromStateStmt *sql.Stmt - updateProcessResultByIDStmt *sql.Stmt - updateProcessResultsStmt *sql.Stmt + db DBTX + tx *sql.Tx + computeProcessVoteCountStmt *sql.Stmt + countAccountsStmt *sql.Stmt + countTokenTransfersByAccountStmt *sql.Stmt + countTransactionsStmt *sql.Stmt + countTransactionsByHeightStmt *sql.Stmt + countVotesStmt *sql.Stmt + createAccountStmt *sql.Stmt + createBlockStmt *sql.Stmt + createProcessStmt *sql.Stmt + createTokenFeeStmt *sql.Stmt + createTokenTransferStmt *sql.Stmt + createTransactionStmt *sql.Stmt + createVoteStmt *sql.Stmt + getBlockByHashStmt *sql.Stmt + getBlockByHeightStmt *sql.Stmt + getEntityCountStmt *sql.Stmt + getProcessStmt *sql.Stmt + getProcessCountStmt *sql.Stmt + getProcessIDsByFinalResultsStmt *sql.Stmt + getProcessStatusStmt *sql.Stmt + getTokenTransferStmt *sql.Stmt + getTransactionByHashStmt *sql.Stmt + getTransactionByHeightAndIndexStmt *sql.Stmt + getVoteStmt *sql.Stmt + searchAccountsStmt *sql.Stmt + searchBlocksStmt *sql.Stmt + searchEntitiesStmt *sql.Stmt + searchProcessesStmt *sql.Stmt + searchTokenFeesStmt *sql.Stmt + searchTokenTransfersStmt *sql.Stmt + searchTransactionsStmt *sql.Stmt + searchVotesStmt *sql.Stmt + setProcessResultsCancelledStmt *sql.Stmt + setProcessResultsReadyStmt *sql.Stmt + updateProcessEndDateStmt *sql.Stmt + updateProcessFromStateStmt *sql.Stmt + updateProcessResultByIDStmt *sql.Stmt + updateProcessResultsStmt *sql.Stmt } func (q *Queries) WithTx(tx *sql.Tx) *Queries { return &Queries{ - db: tx, - tx: tx, - computeProcessVoteCountStmt: q.computeProcessVoteCountStmt, - countAccountsStmt: q.countAccountsStmt, - countTokenTransfersByAccountStmt: q.countTokenTransfersByAccountStmt, - countTransactionsStmt: q.countTransactionsStmt, - countVotesStmt: q.countVotesStmt, - createAccountStmt: q.createAccountStmt, - createBlockStmt: q.createBlockStmt, - createProcessStmt: q.createProcessStmt, - createTokenFeeStmt: q.createTokenFeeStmt, - createTokenTransferStmt: q.createTokenTransferStmt, - createTransactionStmt: q.createTransactionStmt, - createVoteStmt: q.createVoteStmt, - getBlockStmt: q.getBlockStmt, - getEntityCountStmt: q.getEntityCountStmt, - getProcessStmt: q.getProcessStmt, - getProcessCountStmt: q.getProcessCountStmt, - getProcessIDsByFinalResultsStmt: q.getProcessIDsByFinalResultsStmt, - getProcessStatusStmt: q.getProcessStatusStmt, - getTokenTransferStmt: q.getTokenTransferStmt, - getTransactionByHashStmt: q.getTransactionByHashStmt, - getTxReferenceByBlockHeightAndBlockIndexStmt: q.getTxReferenceByBlockHeightAndBlockIndexStmt, - getVoteStmt: q.getVoteStmt, - searchAccountsStmt: q.searchAccountsStmt, - searchEntitiesStmt: q.searchEntitiesStmt, - searchProcessesStmt: q.searchProcessesStmt, - searchTokenFeesStmt: q.searchTokenFeesStmt, - searchTokenTransfersStmt: q.searchTokenTransfersStmt, - searchTransactionsStmt: q.searchTransactionsStmt, - searchVotesStmt: q.searchVotesStmt, - setProcessResultsCancelledStmt: q.setProcessResultsCancelledStmt, - setProcessResultsReadyStmt: q.setProcessResultsReadyStmt, - updateProcessEndDateStmt: q.updateProcessEndDateStmt, - updateProcessFromStateStmt: q.updateProcessFromStateStmt, - updateProcessResultByIDStmt: q.updateProcessResultByIDStmt, - updateProcessResultsStmt: q.updateProcessResultsStmt, + db: tx, + tx: tx, + computeProcessVoteCountStmt: q.computeProcessVoteCountStmt, + countAccountsStmt: q.countAccountsStmt, + countTokenTransfersByAccountStmt: q.countTokenTransfersByAccountStmt, + countTransactionsStmt: q.countTransactionsStmt, + countTransactionsByHeightStmt: q.countTransactionsByHeightStmt, + countVotesStmt: q.countVotesStmt, + createAccountStmt: q.createAccountStmt, + createBlockStmt: q.createBlockStmt, + createProcessStmt: q.createProcessStmt, + createTokenFeeStmt: q.createTokenFeeStmt, + createTokenTransferStmt: q.createTokenTransferStmt, + createTransactionStmt: q.createTransactionStmt, + createVoteStmt: q.createVoteStmt, + getBlockByHashStmt: q.getBlockByHashStmt, + getBlockByHeightStmt: q.getBlockByHeightStmt, + getEntityCountStmt: q.getEntityCountStmt, + getProcessStmt: q.getProcessStmt, + getProcessCountStmt: q.getProcessCountStmt, + getProcessIDsByFinalResultsStmt: q.getProcessIDsByFinalResultsStmt, + getProcessStatusStmt: q.getProcessStatusStmt, + getTokenTransferStmt: q.getTokenTransferStmt, + getTransactionByHashStmt: q.getTransactionByHashStmt, + getTransactionByHeightAndIndexStmt: q.getTransactionByHeightAndIndexStmt, + getVoteStmt: q.getVoteStmt, + searchAccountsStmt: q.searchAccountsStmt, + searchBlocksStmt: q.searchBlocksStmt, + searchEntitiesStmt: q.searchEntitiesStmt, + searchProcessesStmt: q.searchProcessesStmt, + searchTokenFeesStmt: q.searchTokenFeesStmt, + searchTokenTransfersStmt: q.searchTokenTransfersStmt, + searchTransactionsStmt: q.searchTransactionsStmt, + searchVotesStmt: q.searchVotesStmt, + setProcessResultsCancelledStmt: q.setProcessResultsCancelledStmt, + setProcessResultsReadyStmt: q.setProcessResultsReadyStmt, + updateProcessEndDateStmt: q.updateProcessEndDateStmt, + updateProcessFromStateStmt: q.updateProcessFromStateStmt, + updateProcessResultByIDStmt: q.updateProcessResultByIDStmt, + updateProcessResultsStmt: q.updateProcessResultsStmt, } } diff --git a/vochain/indexer/db/models.go b/vochain/indexer/db/models.go index b64fc72c8..d69facf2f 100644 --- a/vochain/indexer/db/models.go +++ b/vochain/indexer/db/models.go @@ -11,9 +11,12 @@ import ( ) type Block struct { - Height int64 - Time time.Time - DataHash []byte + Height int64 + Time time.Time + ChainID string + Hash []byte + ProposerAddress []byte + LastBlockHash []byte } type Process struct { @@ -61,4 +64,8 @@ type Transaction struct { BlockHeight int64 BlockIndex int64 Type string + Subtype string + RawTx []byte + Signature []byte + Signer []byte } diff --git a/vochain/indexer/db/transactions.sql.go b/vochain/indexer/db/transactions.sql.go index a477ebad0..7fabaa528 100644 --- a/vochain/indexer/db/transactions.sql.go +++ b/vochain/indexer/db/transactions.sql.go @@ -23,12 +23,32 @@ func (q *Queries) CountTransactions(ctx context.Context) (int64, error) { return count, err } +const countTransactionsByHeight = `-- name: CountTransactionsByHeight :one +SELECT COUNT(*) FROM transactions +WHERE block_height = ? +` + +func (q *Queries) CountTransactionsByHeight(ctx context.Context, blockHeight int64) (int64, error) { + row := q.queryRow(ctx, q.countTransactionsByHeightStmt, countTransactionsByHeight, blockHeight) + var count int64 + err := row.Scan(&count) + return count, err +} + const createTransaction = `-- name: CreateTransaction :execresult INSERT INTO transactions ( - hash, block_height, block_index, type + hash, block_height, block_index, type, subtype, raw_tx, signature, signer ) VALUES ( - ?, ?, ?, ? + ?, ?, ?, ?, ?, ?, ?, ? ) +ON CONFLICT(hash) DO UPDATE +SET block_height = excluded.block_height, + block_index = excluded.block_index, + type = excluded.type, + subtype = excluded.subtype, + raw_tx = excluded.raw_tx, + signature = excluded.signature, + signer = excluded.signer ` type CreateTransactionParams struct { @@ -36,6 +56,10 @@ type CreateTransactionParams struct { BlockHeight int64 BlockIndex int64 Type string + Subtype string + RawTx []byte + Signature []byte + Signer []byte } func (q *Queries) CreateTransaction(ctx context.Context, arg CreateTransactionParams) (sql.Result, error) { @@ -44,11 +68,15 @@ func (q *Queries) CreateTransaction(ctx context.Context, arg CreateTransactionPa arg.BlockHeight, arg.BlockIndex, arg.Type, + arg.Subtype, + arg.RawTx, + arg.Signature, + arg.Signer, ) } const getTransactionByHash = `-- name: GetTransactionByHash :one -SELECT hash, block_height, block_index, type FROM transactions +SELECT hash, block_height, block_index, type, subtype, raw_tx, signature, signer FROM transactions WHERE hash = ? LIMIT 1 ` @@ -61,70 +89,91 @@ func (q *Queries) GetTransactionByHash(ctx context.Context, hash types.Hash) (Tr &i.BlockHeight, &i.BlockIndex, &i.Type, + &i.Subtype, + &i.RawTx, + &i.Signature, + &i.Signer, ) return i, err } -const getTxReferenceByBlockHeightAndBlockIndex = `-- name: GetTxReferenceByBlockHeightAndBlockIndex :one -SELECT hash, block_height, block_index, type FROM transactions +const getTransactionByHeightAndIndex = `-- name: GetTransactionByHeightAndIndex :one +SELECT hash, block_height, block_index, type, subtype, raw_tx, signature, signer FROM transactions WHERE block_height = ? AND block_index = ? LIMIT 1 ` -type GetTxReferenceByBlockHeightAndBlockIndexParams struct { +type GetTransactionByHeightAndIndexParams struct { BlockHeight int64 BlockIndex int64 } -func (q *Queries) GetTxReferenceByBlockHeightAndBlockIndex(ctx context.Context, arg GetTxReferenceByBlockHeightAndBlockIndexParams) (Transaction, error) { - row := q.queryRow(ctx, q.getTxReferenceByBlockHeightAndBlockIndexStmt, getTxReferenceByBlockHeightAndBlockIndex, arg.BlockHeight, arg.BlockIndex) +func (q *Queries) GetTransactionByHeightAndIndex(ctx context.Context, arg GetTransactionByHeightAndIndexParams) (Transaction, error) { + row := q.queryRow(ctx, q.getTransactionByHeightAndIndexStmt, getTransactionByHeightAndIndex, arg.BlockHeight, arg.BlockIndex) var i Transaction err := row.Scan( &i.Hash, &i.BlockHeight, &i.BlockIndex, &i.Type, + &i.Subtype, + &i.RawTx, + &i.Signature, + &i.Signer, ) return i, err } const searchTransactions = `-- name: SearchTransactions :many -WITH results AS ( - SELECT hash, block_height, block_index, type - FROM transactions - WHERE ( - (?3 = 0 OR block_height = ?3) - AND (?4 = '' OR LOWER(type) = LOWER(?4)) +SELECT hash, block_height, block_index, type, subtype, raw_tx, signature, signer, COUNT(*) OVER() AS total_count +FROM transactions +WHERE + (?1 = 0 OR block_height = ?1) + AND (?2 = '' OR LOWER(type) = LOWER(?2)) + AND (?3 = '' OR LOWER(subtype) = LOWER(?3)) + AND (?4 = '' OR LOWER(HEX(signer)) = LOWER(?4)) + AND ( + ?5 = '' + OR (LENGTH(?5) = 64 AND LOWER(HEX(hash)) = LOWER(?5)) + OR (LENGTH(?5) < 64 AND INSTR(LOWER(HEX(hash)), LOWER(?5)) > 0) + -- TODO: consider keeping an hash_hex column for faster searches ) -) -SELECT hash, block_height, block_index, type, COUNT(*) OVER() AS total_count -FROM results ORDER BY block_height DESC, block_index DESC -LIMIT ?2 -OFFSET ?1 +LIMIT ?7 +OFFSET ?6 ` type SearchTransactionsParams struct { - Offset int64 - Limit int64 BlockHeight interface{} TxType interface{} + TxSubtype interface{} + TxSigner interface{} + HashSubstr interface{} + Offset int64 + Limit int64 } type SearchTransactionsRow struct { - Hash []byte + Hash types.Hash BlockHeight int64 BlockIndex int64 Type string + Subtype string + RawTx []byte + Signature []byte + Signer []byte TotalCount int64 } func (q *Queries) SearchTransactions(ctx context.Context, arg SearchTransactionsParams) ([]SearchTransactionsRow, error) { rows, err := q.query(ctx, q.searchTransactionsStmt, searchTransactions, - arg.Offset, - arg.Limit, arg.BlockHeight, arg.TxType, + arg.TxSubtype, + arg.TxSigner, + arg.HashSubstr, + arg.Offset, + arg.Limit, ) if err != nil { return nil, err @@ -138,6 +187,10 @@ func (q *Queries) SearchTransactions(ctx context.Context, arg SearchTransactions &i.BlockHeight, &i.BlockIndex, &i.Type, + &i.Subtype, + &i.RawTx, + &i.Signature, + &i.Signer, &i.TotalCount, ); err != nil { return nil, err diff --git a/vochain/indexer/indexer.go b/vochain/indexer/indexer.go index 769b76720..b764c8812 100644 --- a/vochain/indexer/indexer.go +++ b/vochain/indexer/indexer.go @@ -1,6 +1,7 @@ package indexer import ( + "bytes" "context" "database/sql" "embed" @@ -172,6 +173,12 @@ func (idx *Indexer) startDB() error { } goose.SetLogger(log.GooseLogger()) goose.SetBaseFS(embedMigrations) + + if gooseMigrationsPending(idx.readWriteDB, "migrations") { + log.Info("indexer db needs migration, scheduling a reindex after sync") + go idx.ReindexBlocks(false) + } + if err := goose.Up(idx.readWriteDB, "migrations"); err != nil { return fmt.Errorf("goose up: %w", err) } @@ -249,6 +256,27 @@ func (idx *Indexer) RestoreBackup(path string) error { return nil } +func gooseMigrationsPending(db *sql.DB, dir string) bool { + // Get the latest applied migration version + currentVersion, err := goose.GetDBVersion(db) + if err != nil { + log.Errorf("failed to get current database version: %v", err) + return false + } + + // Collect migrations after the current version + migrations, err := goose.CollectMigrations(dir, currentVersion, goose.MaxVersion) + if err != nil { + if errors.Is(err, goose.ErrNoMigrationFiles) { + return false + } + log.Errorf("failed to collect migrations: %v", err) + return false + } + + return len(migrations) > 0 +} + // SaveBackup backs up the database to a file on disk. // Note that writes to the database may be blocked until the backup finishes, // and an error may occur if a file at path already exists. @@ -402,6 +430,80 @@ func (idx *Indexer) AfterSyncBootstrap(inTest bool) { log.Infof("live results recovery computation finished, took %s", time.Since(startTime)) } +// ReindexBlocks reindexes all blocks found in blockstore +func (idx *Indexer) ReindexBlocks(inTest bool) { + if !inTest { + <-idx.App.WaitUntilSynced() + } + + // Note that holding blockMu means new votes aren't added until the reindex finishes. + idx.blockMu.Lock() + defer idx.blockMu.Unlock() + + if idx.App.Node == nil || idx.App.Node.BlockStore() == nil { + return + } + + idxBlockCount, err := idx.CountBlocks() + if err != nil { + log.Warnf("indexer CountBlocks returned error: %s", err) + } + log.Infow("start reindexing", + "blockStoreBase", idx.App.Node.BlockStore().Base(), + "blockStoreHeight", idx.App.Node.BlockStore().Height(), + "indexerBlockCount", idxBlockCount, + ) + queries := idx.blockTxQueries() + for height := idx.App.Node.BlockStore().Base(); height <= idx.App.Node.BlockStore().Height(); height++ { + if b := idx.App.GetBlockByHeight(int64(height)); b != nil { + // Blocks + func() { + idxBlock, err := idx.readOnlyQuery.GetBlockByHeight(context.TODO(), b.Height) + if err == nil && idxBlock.Time != b.Time { + log.Errorf("while reindexing blocks, block %d timestamp in db (%s) differs from blockstore (%s), leaving untouched", height, idxBlock.Time, b.Time) + return + } + if _, err := queries.CreateBlock(context.TODO(), indexerdb.CreateBlockParams{ + ChainID: b.ChainID, + Height: b.Height, + Time: b.Time, + Hash: nonNullBytes(b.Hash()), + ProposerAddress: nonNullBytes(b.ProposerAddress), + LastBlockHash: nonNullBytes(b.LastBlockID.Hash), + }); err != nil { + log.Errorw(err, "cannot index new block") + } + }() + + // Transactions + func() { + for index, tx := range b.Data.Txs { + idxTx, err := idx.readOnlyQuery.GetTransactionByHeightAndIndex(context.TODO(), indexerdb.GetTransactionByHeightAndIndexParams{ + BlockHeight: b.Height, + BlockIndex: int64(index), + }) + if err == nil && !bytes.Equal(idxTx.Hash, tx.Hash()) { + log.Errorf("while reindexing txs, tx %d/%d hash in db (%x) differs from blockstore (%x), leaving untouched", b.Height, index, idxTx.Hash, tx.Hash()) + return + } + vtx := new(vochaintx.Tx) + if err := vtx.Unmarshal(tx, b.ChainID); err != nil { + log.Errorw(err, fmt.Sprintf("cannot unmarshal tx %d/%d", b.Height, index)) + continue + } + idx.indexTx(vtx, uint32(b.Height), int32(index)) + } + }() + } + } + + log.Infow("finished reindexing", + "blockStoreBase", idx.App.Node.BlockStore().Base(), + "blockStoreHeight", idx.App.Node.BlockStore().Height(), + "indexerBlockCount", idxBlockCount, + ) +} + // Commit is called by the APP when a block is confirmed and included into the chain func (idx *Indexer) Commit(height uint32) error { idx.blockMu.Lock() @@ -415,14 +517,17 @@ func (idx *Indexer) Commit(height uint32) error { ctx := context.TODO() // index the new block - bb := idx.App.GetBlockByHeight(int64(height)) - if _, err := queries.CreateBlock(context.TODO(), indexerdb.CreateBlockParams{ - Height: bb.Height, - Time: bb.Time, - DataHash: nonNullBytes(bb.DataHash), - // TODO: ProposerAddress, Hash, AppHash, ChainID, LastBlockHash?, ValidatorSignatures? - }); err != nil { - log.Errorw(err, "cannot index new block") + if b := idx.App.GetBlockByHeight(int64(height)); b != nil { + if _, err := queries.CreateBlock(context.TODO(), indexerdb.CreateBlockParams{ + ChainID: b.ChainID, + Height: b.Height, + Time: b.Time, + Hash: nonNullBytes(b.Hash()), + ProposerAddress: nonNullBytes(b.ProposerAddress), + LastBlockHash: nonNullBytes(b.LastBlockID.Hash), + }); err != nil { + log.Errorw(err, "cannot index new block") + } } for _, pidStr := range updateProcs { diff --git a/vochain/indexer/indexer_test.go b/vochain/indexer/indexer_test.go index 76c306910..f3dc00ed3 100644 --- a/vochain/indexer/indexer_test.go +++ b/vochain/indexer/indexer_test.go @@ -1392,6 +1392,7 @@ func TestTxIndexer(t *testing.T) { idx.OnNewTx(&vochaintx.Tx{ TxID: getTxID(i, j), TxModelType: "setAccount", + Tx: &models.Tx{Payload: &models.Tx_SetAccount{}}, }, uint32(i), int32(j)) } } @@ -1404,7 +1405,7 @@ func TestTxIndexer(t *testing.T) { for i := 0; i < totalBlocks; i++ { for j := 0; j < txsPerBlock; j++ { - ref, err := idx.GetTxReferenceByBlockHeightAndBlockIndex(int64(i), int64(j)) + ref, err := idx.GetTransactionByHeightAndIndex(int64(i), int64(j)) qt.Assert(t, err, qt.IsNil) qt.Assert(t, ref.BlockHeight, qt.Equals, uint32(i)) qt.Assert(t, ref.TxBlockIndex, qt.Equals, int32(j)) @@ -1412,14 +1413,14 @@ func TestTxIndexer(t *testing.T) { h := make([]byte, 32) id := getTxID(i, j) copy(h, id[:]) - hashRef, err := idx.GetTxHashReference(h) + hashRef, err := idx.GetTxMetadataByHash(h) qt.Assert(t, err, qt.IsNil) qt.Assert(t, hashRef.BlockHeight, qt.Equals, uint32(i)) qt.Assert(t, hashRef.TxBlockIndex, qt.Equals, int32(j)) } } - txs, _, err := idx.SearchTransactions(15, 0, 0, "") + txs, _, err := idx.SearchTransactions(15, 0, 0, "", "", "", "") qt.Assert(t, err, qt.IsNil) for i, tx := range txs { // BlockIndex and TxBlockIndex start at 0, so subtract 1. @@ -1428,7 +1429,7 @@ func TestTxIndexer(t *testing.T) { qt.Assert(t, tx.TxType, qt.Equals, "setAccount") } - txs, _, err = idx.SearchTransactions(1, 5, 0, "") + txs, _, err = idx.SearchTransactions(1, 5, 0, "", "", "", "") qt.Assert(t, err, qt.IsNil) qt.Assert(t, txs, qt.HasLen, 1) } diff --git a/vochain/indexer/indexertypes/block.go b/vochain/indexer/indexertypes/block.go index 4954dbc22..4b711bdbc 100644 --- a/vochain/indexer/indexertypes/block.go +++ b/vochain/indexer/indexertypes/block.go @@ -4,6 +4,7 @@ import ( "time" "go.vocdoni.io/dvote/types" + indexerdb "go.vocdoni.io/dvote/vochain/indexer/db" ) // Block represents a block handled by the Vochain. @@ -18,3 +19,28 @@ type Block struct { LastBlockHash types.HexBytes `json:"lastBlockHash"` TxCount int64 `json:"txCount"` } + +// BlockFromDB converts the indexerdb.Block into a Block +func BlockFromDB(dbblock *indexerdb.Block) *Block { + return &Block{ + ChainID: dbblock.ChainID, + Height: dbblock.Height, + Time: dbblock.Time, + Hash: nonEmptyBytes(dbblock.Hash), + ProposerAddress: nonEmptyBytes(dbblock.ProposerAddress), + LastBlockHash: nonEmptyBytes(dbblock.LastBlockHash), + } +} + +// BlockFromDBRow converts the indexerdb.SearchBlocksRow into a Block +func BlockFromDBRow(row *indexerdb.SearchBlocksRow) *Block { + return &Block{ + ChainID: row.ChainID, + Height: row.Height, + Time: row.Time, + Hash: nonEmptyBytes(row.Hash), + ProposerAddress: nonEmptyBytes(row.ProposerAddress), + LastBlockHash: nonEmptyBytes(row.LastBlockHash), + TxCount: row.TxCount, + } +} diff --git a/vochain/indexer/indexertypes/types.go b/vochain/indexer/indexertypes/types.go index bc0e56485..ad5b29748 100644 --- a/vochain/indexer/indexertypes/types.go +++ b/vochain/indexer/indexertypes/types.go @@ -176,20 +176,51 @@ type TxPackage struct { Signature types.HexBytes `json:"signature"` } -// Transaction holds the db reference for a single transaction -type Transaction struct { +// TransactionMetadata contains tx information for the TransactionList api +type TransactionMetadata struct { Hash types.HexBytes `json:"hash" swaggertype:"string" example:"75e8f822f5dd13973ac5158d600f0a2a5fea4bfefce9712ab5195bf17884cfad"` BlockHeight uint32 `json:"height" format:"int32" example:"64924"` TxBlockIndex int32 `json:"index" format:"int32" example:"0"` TxType string `json:"type" enums:"vote,newProcess,admin,setProcess,registerKey,mintTokens,sendTokens,setTransactionCosts,setAccount,collectFaucet,setKeykeeper" example:"Vote"` + TxSubtype string `json:"subtype" example:"set_process_census"` + Signer types.HexBytes `json:"signer" swaggertype:"string" example:"0e45513942cf95330fc5e9020851b8bdd9b9c9df"` } -func TransactionFromDB(dbtx *indexerdb.Transaction) *Transaction { - return &Transaction{ +func TransactionMetadataFromDB(dbtx *indexerdb.Transaction) *TransactionMetadata { + return &TransactionMetadata{ + Hash: dbtx.Hash, + BlockHeight: uint32(dbtx.BlockHeight), + TxBlockIndex: int32(dbtx.BlockIndex), + TxType: dbtx.Type, + TxSubtype: dbtx.Subtype, + Signer: dbtx.Signer, + } +} + +func TransactionMetadataFromDBRow(dbtx *indexerdb.SearchTransactionsRow) *TransactionMetadata { + return &TransactionMetadata{ Hash: dbtx.Hash, BlockHeight: uint32(dbtx.BlockHeight), TxBlockIndex: int32(dbtx.BlockIndex), TxType: dbtx.Type, + TxSubtype: dbtx.Subtype, + Signer: dbtx.Signer, + } +} + +// Transaction holds a single transaction +type Transaction struct { + *TransactionMetadata + RawTx types.HexBytes `json:"-"` + Signature types.HexBytes `json:"-"` +} + +// TransactionFromDB converts an indexerdb.Transaction into a Transaction +func TransactionFromDB(dbtx *indexerdb.Transaction) *Transaction { + return &Transaction{ + TransactionMetadata: TransactionMetadataFromDB(dbtx), + RawTx: dbtx.RawTx, + Signature: dbtx.Signature, } } diff --git a/vochain/indexer/migrations/0014_alter_columns_table_blocks.sql b/vochain/indexer/migrations/0014_alter_columns_table_blocks.sql new file mode 100644 index 000000000..c83c32c06 --- /dev/null +++ b/vochain/indexer/migrations/0014_alter_columns_table_blocks.sql @@ -0,0 +1,13 @@ +-- +goose Up +ALTER TABLE blocks DROP COLUMN data_hash; +ALTER TABLE blocks ADD COLUMN chain_id TEXT NOT NULL DEFAULT ''; +ALTER TABLE blocks ADD COLUMN hash BLOB NOT NULL DEFAULT x''; +ALTER TABLE blocks ADD COLUMN proposer_address BLOB NOT NULL DEFAULT x''; +ALTER TABLE blocks ADD COLUMN last_block_hash BLOB NOT NULL DEFAULT x''; + +-- +goose Down +ALTER TABLE blocks ADD COLUMN data_hash BLOB NOT NULL; +ALTER TABLE blocks DROP COLUMN chain_id; +ALTER TABLE blocks DROP COLUMN hash; +ALTER TABLE blocks DROP COLUMN proposer_address; +ALTER TABLE blocks DROP COLUMN last_block_hash; diff --git a/vochain/indexer/migrations/0015_alter_columns_table_transactions.sql b/vochain/indexer/migrations/0015_alter_columns_table_transactions.sql new file mode 100644 index 000000000..81dcb9dec --- /dev/null +++ b/vochain/indexer/migrations/0015_alter_columns_table_transactions.sql @@ -0,0 +1,11 @@ +-- +goose Up +ALTER TABLE transactions ADD COLUMN subtype TEXT NOT NULL DEFAULT ''; +ALTER TABLE transactions ADD COLUMN raw_tx BLOB NOT NULL DEFAULT x''; +ALTER TABLE transactions ADD COLUMN signature BLOB NOT NULL DEFAULT x''; +ALTER TABLE transactions ADD COLUMN signer BLOB NOT NULL DEFAULT x''; + +-- +goose Down +ALTER TABLE transactions DROP COLUMN signer; +ALTER TABLE transactions DROP COLUMN signature; +ALTER TABLE transactions DROP COLUMN raw_tx; +ALTER TABLE transactions DROP COLUMN subtype; diff --git a/vochain/indexer/queries/blocks.sql b/vochain/indexer/queries/blocks.sql index 577e875b5..d15332cdf 100644 --- a/vochain/indexer/queries/blocks.sql +++ b/vochain/indexer/queries/blocks.sql @@ -1,11 +1,46 @@ -- name: CreateBlock :execresult INSERT INTO blocks( - height, time, data_hash + chain_id, height, time, hash, proposer_address, last_block_hash ) VALUES ( - ?, ?, ? -); + ?, ?, ?, ?, ?, ? +) +ON CONFLICT(height) DO UPDATE +SET chain_id = excluded.chain_id, + time = excluded.time, + hash = excluded.hash, + proposer_address = excluded.proposer_address, + last_block_hash = excluded.last_block_hash; --- name: GetBlock :one +-- name: GetBlockByHeight :one SELECT * FROM blocks WHERE height = ? LIMIT 1; + +-- name: GetBlockByHash :one +SELECT * FROM blocks +WHERE hash = ? +LIMIT 1; + +-- name: SearchBlocks :many +SELECT + b.*, + COUNT(t.block_index) AS tx_count, + COUNT(*) OVER() AS total_count +FROM blocks AS b +LEFT JOIN transactions AS t + ON b.height = t.block_height +WHERE ( + (sqlc.arg(chain_id) = '' OR b.chain_id = sqlc.arg(chain_id)) + AND LENGTH(sqlc.arg(hash_substr)) <= 64 -- if passed arg is longer, then just abort the query + AND ( + sqlc.arg(hash_substr) = '' + OR (LENGTH(sqlc.arg(hash_substr)) = 64 AND LOWER(HEX(b.hash)) = LOWER(sqlc.arg(hash_substr))) + OR (LENGTH(sqlc.arg(hash_substr)) < 64 AND INSTR(LOWER(HEX(b.hash)), LOWER(sqlc.arg(hash_substr))) > 0) + -- TODO: consider keeping an hash_hex column for faster searches + ) + AND (sqlc.arg(proposer_address) = '' OR LOWER(HEX(b.proposer_address)) = LOWER(sqlc.arg(proposer_address))) +) +GROUP BY b.height +ORDER BY b.height DESC +LIMIT sqlc.arg(limit) +OFFSET sqlc.arg(offset); diff --git a/vochain/indexer/queries/transactions.sql b/vochain/indexer/queries/transactions.sql index eb8b2b617..c9c152482 100644 --- a/vochain/indexer/queries/transactions.sql +++ b/vochain/indexer/queries/transactions.sql @@ -1,9 +1,18 @@ -- name: CreateTransaction :execresult INSERT INTO transactions ( - hash, block_height, block_index, type + hash, block_height, block_index, type, subtype, raw_tx, signature, signer ) VALUES ( - ?, ?, ?, ? -); + ?, ?, ?, ?, ?, ?, ?, ? +) +ON CONFLICT(hash) DO UPDATE +SET block_height = excluded.block_height, + block_index = excluded.block_index, + type = excluded.type, + subtype = excluded.subtype, + raw_tx = excluded.raw_tx, + signature = excluded.signature, + signer = excluded.signer; + -- name: GetTransactionByHash :one SELECT * FROM transactions @@ -13,22 +22,29 @@ LIMIT 1; -- name: CountTransactions :one SELECT COUNT(*) FROM transactions; --- name: GetTxReferenceByBlockHeightAndBlockIndex :one +-- name: CountTransactionsByHeight :one +SELECT COUNT(*) FROM transactions +WHERE block_height = ?; + +-- name: GetTransactionByHeightAndIndex :one SELECT * FROM transactions WHERE block_height = ? AND block_index = ? LIMIT 1; -- name: SearchTransactions :many -WITH results AS ( - SELECT * - FROM transactions - WHERE ( - (sqlc.arg(block_height) = 0 OR block_height = sqlc.arg(block_height)) - AND (sqlc.arg(tx_type) = '' OR LOWER(type) = LOWER(sqlc.arg(tx_type))) - ) -) SELECT *, COUNT(*) OVER() AS total_count -FROM results +FROM transactions +WHERE + (sqlc.arg(block_height) = 0 OR block_height = sqlc.arg(block_height)) + AND (sqlc.arg(tx_type) = '' OR LOWER(type) = LOWER(sqlc.arg(tx_type))) + AND (sqlc.arg(tx_subtype) = '' OR LOWER(subtype) = LOWER(sqlc.arg(tx_subtype))) + AND (sqlc.arg(tx_signer) = '' OR LOWER(HEX(signer)) = LOWER(sqlc.arg(tx_signer))) + AND ( + sqlc.arg(hash_substr) = '' + OR (LENGTH(sqlc.arg(hash_substr)) = 64 AND LOWER(HEX(hash)) = LOWER(sqlc.arg(hash_substr))) + OR (LENGTH(sqlc.arg(hash_substr)) < 64 AND INSTR(LOWER(HEX(hash)), LOWER(sqlc.arg(hash_substr))) > 0) + -- TODO: consider keeping an hash_hex column for faster searches + ) ORDER BY block_height DESC, block_index DESC LIMIT sqlc.arg(limit) OFFSET sqlc.arg(offset); diff --git a/vochain/indexer/transaction.go b/vochain/indexer/transaction.go index 16e8cad81..31beb4465 100644 --- a/vochain/indexer/transaction.go +++ b/vochain/indexer/transaction.go @@ -5,12 +5,15 @@ import ( "database/sql" "errors" "fmt" + "strings" + "go.vocdoni.io/dvote/crypto/ethereum" "go.vocdoni.io/dvote/log" "go.vocdoni.io/dvote/types" indexerdb "go.vocdoni.io/dvote/vochain/indexer/db" "go.vocdoni.io/dvote/vochain/indexer/indexertypes" "go.vocdoni.io/dvote/vochain/transaction/vochaintx" + "google.golang.org/protobuf/proto" ) // ErrTransactionNotFound is returned if the transaction is not found. @@ -22,37 +25,43 @@ func (idx *Indexer) CountTotalTransactions() (uint64, error) { return uint64(count), err } -// GetTxReferenceByBlockHeightAndBlockIndex fetches the txReference for the given tx height and block tx index -func (idx *Indexer) GetTxReferenceByBlockHeightAndBlockIndex(blockHeight, blockIndex int64) (*indexertypes.Transaction, error) { - sqlTxRef, err := idx.readOnlyQuery.GetTxReferenceByBlockHeightAndBlockIndex(context.TODO(), indexerdb.GetTxReferenceByBlockHeightAndBlockIndexParams{ - BlockHeight: blockHeight, - BlockIndex: blockIndex, - }) +// CountTransactionsByHeight returns the number of transactions indexed for a given height +func (idx *Indexer) CountTransactionsByHeight(height int64) (int64, error) { + return idx.readOnlyQuery.CountTransactionsByHeight(context.TODO(), height) +} + +// GetTxMetadataByHash fetches the tx metadata for the given tx hash +func (idx *Indexer) GetTxMetadataByHash(hash types.HexBytes) (*indexertypes.TransactionMetadata, error) { + sqlTxRef, err := idx.readOnlyQuery.GetTransactionByHash(context.TODO(), hash) if err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, ErrTransactionNotFound } - return nil, fmt.Errorf("tx at block %d and index %d not found: %v", blockHeight, blockIndex, err) + return nil, fmt.Errorf("tx hash %x not found: %v", hash, err) } - return indexertypes.TransactionFromDB(&sqlTxRef), nil + return indexertypes.TransactionMetadataFromDB(&sqlTxRef), nil } -// GetTxHashReference fetches the txReference for the given tx hash -func (idx *Indexer) GetTxHashReference(hash types.HexBytes) (*indexertypes.Transaction, error) { - sqlTxRef, err := idx.readOnlyQuery.GetTransactionByHash(context.TODO(), hash) +// GetTransactionByHeightAndIndex fetches the full tx for the given tx height and block tx index +func (idx *Indexer) GetTransactionByHeightAndIndex(blockHeight, blockIndex int64) (*indexertypes.Transaction, error) { + sqlTxRef, err := idx.readOnlyQuery.GetTransactionByHeightAndIndex(context.TODO(), indexerdb.GetTransactionByHeightAndIndexParams{ + BlockHeight: blockHeight, + BlockIndex: blockIndex, + }) if err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, ErrTransactionNotFound } - return nil, fmt.Errorf("tx hash %x not found: %v", hash, err) + return nil, fmt.Errorf("tx at block %d and index %d not found: %v", blockHeight, blockIndex, err) } return indexertypes.TransactionFromDB(&sqlTxRef), nil } // SearchTransactions returns the list of transactions indexed. -// height and txType are optional, if declared as zero-value will be ignored. +// blockHeight, hash, txType, txSubtype and txSigner are optional, if declared as zero-value will be ignored. +// hash matches substrings. // The first one returned is the newest, so they are in descending order. -func (idx *Indexer) SearchTransactions(limit, offset int, blockHeight uint64, txType string) ([]*indexertypes.Transaction, uint64, error) { +func (idx *Indexer) SearchTransactions(limit, offset int, blockHeight uint64, txHash, txType, txSubtype, txSigner string) ([]*indexertypes.TransactionMetadata, uint64, error) { if offset < 0 { return nil, 0, fmt.Errorf("invalid value: offset cannot be %d", offset) } @@ -62,20 +71,18 @@ func (idx *Indexer) SearchTransactions(limit, offset int, blockHeight uint64, tx results, err := idx.readOnlyQuery.SearchTransactions(context.TODO(), indexerdb.SearchTransactionsParams{ Limit: int64(limit), Offset: int64(offset), + HashSubstr: txHash, BlockHeight: blockHeight, TxType: txType, + TxSubtype: txSubtype, + TxSigner: txSigner, }) if err != nil { return nil, 0, err } - list := []*indexertypes.Transaction{} + list := []*indexertypes.TransactionMetadata{} for _, row := range results { - list = append(list, &indexertypes.Transaction{ - Hash: row.Hash, - BlockHeight: uint32(row.BlockHeight), - TxBlockIndex: int32(row.BlockIndex), - TxType: row.Type, - }) + list = append(list, indexertypes.TransactionMetadataFromDBRow(&row)) } if len(results) == 0 { return list, 0, nil @@ -86,13 +93,38 @@ func (idx *Indexer) SearchTransactions(limit, offset int, blockHeight uint64, tx func (idx *Indexer) OnNewTx(tx *vochaintx.Tx, blockHeight uint32, txIndex int32) { idx.blockMu.Lock() defer idx.blockMu.Unlock() + + idx.indexTx(tx, blockHeight, txIndex) +} + +func (idx *Indexer) indexTx(tx *vochaintx.Tx, blockHeight uint32, txIndex int32) { + rawtx, err := proto.Marshal(tx.Tx) + if err != nil { + log.Errorw(err, "indexer cannot marshal transaction") + return + } + + signer := []byte{} + if len(tx.Signature) > 0 { // not all txs are signed, for example zk ones + addr, err := ethereum.AddrFromSignature(tx.SignedBody, tx.Signature) + if err != nil { + log.Errorw(err, "indexer cannot recover signer from signature") + return + } + signer = addr.Bytes() + } + queries := idx.blockTxQueries() if _, err := queries.CreateTransaction(context.TODO(), indexerdb.CreateTransactionParams{ Hash: tx.TxID[:], BlockHeight: int64(blockHeight), BlockIndex: int64(txIndex), Type: tx.TxModelType, + Subtype: strings.ToLower(tx.TxSubtype()), + RawTx: rawtx, + Signature: nonNullBytes(tx.Signature), + Signer: nonNullBytes(signer), }); err != nil { - log.Errorw(err, "cannot index new transaction") + log.Errorw(err, "cannot index transaction") } } diff --git a/vochain/transaction/vochaintx/vochaintx.go b/vochain/transaction/vochaintx/vochaintx.go index f413e5e3b..66bc950fe 100644 --- a/vochain/transaction/vochaintx/vochaintx.go +++ b/vochain/transaction/vochaintx/vochaintx.go @@ -50,6 +50,32 @@ func (tx *Tx) Unmarshal(content []byte, chainID string) error { return nil } +// TxSubtype returns the content of the "txtype" field inside the tx.Tx. +// +// The function determines the type of the transaction using Protocol Buffers reflection. +// If the field doesn't exist, it returns the empty string "". +func (tx *Tx) TxSubtype() string { + txReflectDescriptor := tx.Tx.ProtoReflect().Descriptor().Oneofs().Get(0) + if txReflectDescriptor == nil { + return "" + } + whichOneTxModelType := tx.Tx.ProtoReflect().WhichOneof(txReflectDescriptor) + if whichOneTxModelType == nil { + return "" + } + // Get the value of the selected field in the oneof + fieldValue := tx.Tx.ProtoReflect().Get(whichOneTxModelType) + // Now, fieldValue is a protoreflect.Value, retrieve the txtype field + txtypeFieldDescriptor := fieldValue.Message().Descriptor().Fields().ByName("txtype") + if txtypeFieldDescriptor == nil { + return "" + } + // Get the integer value of txtype as protoreflect.EnumNumber + enumNumber := fieldValue.Message().Get(txtypeFieldDescriptor).Enum() + // Convert the EnumNumber to a string using the EnumType descriptor + return string(txtypeFieldDescriptor.Enum().Values().ByNumber(enumNumber).Name()) +} + // TxKey computes the checksum of the tx func TxKey(tx []byte) [32]byte { return comettypes.Tx(tx).Key() From 1a47b20fd49257b12f0aeee0ac8b9049d93cb790 Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Mon, 8 Jul 2024 14:13:19 +0200 Subject: [PATCH 5/5] api: fetch blocks and transactions from indexer rather than app BlockStore these endpoints now fetch blocks from indexer, return just `header` and include `txCount` * /chain/blocks * /chain/blocks/{height} * /chain/blocks/hash/{hash} this new endpoint fetches the full tx from indexer, and includes `subtype` and `signer` fields * /chain/transactions/{hash} this legacy endpoint is now marked as deprecated * /chain/transactions/{height}/{index} this endpoint now accepts more filter params (subtype, signer) * /chain/transactions refactor: * api: rename chainBlockHandler -> chainBlockByHeightHandler --- api/api.go | 2 + api/api_types.go | 18 ++-- api/chain.go | 186 ++++++++++++++++++++------------- test/api_test.go | 35 +++++++ vochain/indexer/transaction.go | 12 +++ 5 files changed, 174 insertions(+), 79 deletions(-) diff --git a/api/api.go b/api/api.go index 33fcba409..5fa84931c 100644 --- a/api/api.go +++ b/api/api.go @@ -77,6 +77,8 @@ const ( ParamHeight = "height" ParamReference = "reference" ParamType = "type" + ParamSubtype = "subtype" + ParamSigner = "signer" ParamAccountIdFrom = "accountIdFrom" ParamAccountIdTo = "accountIdTo" ParamStartDateAfter = "startDateAfter" diff --git a/api/api_types.go b/api/api_types.go index b1169db8d..26b13508e 100644 --- a/api/api_types.go +++ b/api/api_types.go @@ -50,8 +50,11 @@ type AccountParams struct { // TransactionParams allows the client to filter transactions type TransactionParams struct { PaginationParams - Height uint64 `json:"height,omitempty"` - Type string `json:"type,omitempty"` + Hash string `json:"hash,omitempty"` + Height uint64 `json:"height,omitempty"` + Type string `json:"type,omitempty"` + Subtype string `json:"subtype,omitempty"` + Signer string `json:"signer,omitempty"` } // BlockParams allows the client to filter blocks @@ -292,9 +295,9 @@ type TransfersList struct { } type GenericTransactionWithInfo struct { - TxContent json.RawMessage `json:"tx"` - TxInfo indexertypes.Transaction `json:"txInfo"` - Signature types.HexBytes `json:"signature"` + TxContent json.RawMessage `json:"tx"` + TxInfo *indexertypes.Transaction `json:"txInfo"` + Signature types.HexBytes `json:"signature"` } type ChainInfo struct { @@ -444,8 +447,9 @@ func CensusTypeToOrigin(ctype CensusTypeDescription) (models.CensusOrigin, []byt } type Block struct { - comettypes.Block `json:",inline"` - Hash types.HexBytes `json:"hash" ` + comettypes.Header `json:"header"` + Hash types.HexBytes `json:"hash" ` + TxCount int64 `json:"txCount"` } // BlockList is used to return a paginated list to the client diff --git a/api/chain.go b/api/chain.go index 1d84ef095..931b4bf21 100644 --- a/api/chain.go +++ b/api/chain.go @@ -13,12 +13,9 @@ import ( "go.vocdoni.io/dvote/crypto/zk/circuit" "go.vocdoni.io/dvote/httprouter" "go.vocdoni.io/dvote/httprouter/apirest" - "go.vocdoni.io/dvote/types" "go.vocdoni.io/dvote/util" - "go.vocdoni.io/dvote/vochain" "go.vocdoni.io/dvote/vochain/genesis" "go.vocdoni.io/dvote/vochain/indexer" - "go.vocdoni.io/dvote/vochain/indexer/indexertypes" "go.vocdoni.io/dvote/vochain/state" ) @@ -119,7 +116,7 @@ func (a *API) enableChainHandlers() error { "/chain/transactions/{height}/{index}", "GET", apirest.MethodAccessTypePublic, - a.chainTxHandler, + a.chainTxByHeightAndIndexHandler, ); err != nil { return err } @@ -139,6 +136,14 @@ func (a *API) enableChainHandlers() error { ); err != nil { return err } + if err := a.Endpoint.RegisterMethod( + "/chain/transactions/{hash}", + "GET", + apirest.MethodAccessTypePublic, + a.chainTxByHashHandler, + ); err != nil { + return err + } if err := a.Endpoint.RegisterMethod( "/chain/transactions/page/{page}", "GET", @@ -159,7 +164,7 @@ func (a *API) enableChainHandlers() error { "/chain/blocks/{height}", "GET", apirest.MethodAccessTypePublic, - a.chainBlockHandler, + a.chainBlockByHeightHandler, ); err != nil { return err } @@ -642,7 +647,7 @@ func (a *API) chainTxCostHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext // @Success 204 "See [errors](vocdoni-api#errors) section" // @Router /chain/transactions/reference/{hash} [get] func (a *API) chainTxRefByHashHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { - hash, err := hex.DecodeString(util.TrimHex(ctx.URLParam("hash"))) + hash, err := hex.DecodeString(util.TrimHex(ctx.URLParam(ParamHash))) if err != nil { return err } @@ -661,10 +666,12 @@ func (a *API) chainTxRefByHashHandler(_ *apirest.APIdata, ctx *httprouter.HTTPCo return ctx.Send(data, apirest.HTTPstatusOK) } -// chainTxHandler +// chainTxByHeightAndIndexHandler // // @Summary Transaction by block height and index // @Description Get transaction full information by block height and index. It returns JSON transaction protobuf encoded. Depending of transaction type will return different types of objects. Current transaction types can be found calling `/chain/transactions/cost` +// @Deprecated +// @Description (deprecated, in favor of /chain/transactions/{hash}) // @Tags Chain // @Accept json // @Produce json @@ -673,7 +680,7 @@ func (a *API) chainTxRefByHashHandler(_ *apirest.APIdata, ctx *httprouter.HTTPCo // @Success 200 {object} GenericTransactionWithInfo // @Success 204 "See [errors](vocdoni-api#errors) section" // @Router /chain/transactions/{height}/{index} [get] -func (a *API) chainTxHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { +func (a *API) chainTxByHeightAndIndexHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { height, err := strconv.ParseInt(ctx.URLParam(ParamHeight), 10, 64) if err != nil { return err @@ -682,15 +689,42 @@ func (a *API) chainTxHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) er if err != nil { return err } - stx, err := a.vocapp.GetTx(uint32(height), int32(index)) + itx, err := a.indexer.GetTransactionByHeightAndIndex(height, index) if err != nil { - if errors.Is(err, vochain.ErrTransactionNotFound) { + if errors.Is(err, indexer.ErrTransactionNotFound) { return ErrTransactionNotFound } return ErrVochainGetTxFailed.WithErr(err) } + tx := &GenericTransactionWithInfo{ + TxContent: protoTxAsJSON(itx.RawTx), + Signature: itx.Signature, + TxInfo: itx, + } + data, err := json.Marshal(tx) + if err != nil { + return err + } + return ctx.Send(data, apirest.HTTPstatusOK) +} - ref, err := a.indexer.GetTransactionByHeightAndIndex(height, index) +// chainTxByHashHandler +// +// @Summary Transaction by hash +// @Description Get transaction full information by hash. It returns JSON transaction protobuf encoded. Depending of transaction type will return different types of objects. Current transaction types can be found calling `/chain/transactions/cost` +// @Tags Chain +// @Accept json +// @Produce json +// @Param hash path string true "Transaction hash" +// @Success 200 {object} GenericTransactionWithInfo +// @Success 204 "See [errors](vocdoni-api#errors) section" +// @Router /chain/transactions/{hash} [get] +func (a *API) chainTxByHashHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { + hash, err := hex.DecodeString(util.TrimHex(ctx.URLParam(ParamHash))) + if err != nil { + return ErrCantParseHexString.WithErr(err) + } + itx, err := a.indexer.GetTransactionByHash(hash) if err != nil { if errors.Is(err, indexer.ErrTransactionNotFound) { return ErrTransactionNotFound @@ -698,9 +732,9 @@ func (a *API) chainTxHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) er return ErrVochainGetTxFailed.WithErr(err) } tx := &GenericTransactionWithInfo{ - TxContent: protoTxAsJSON(stx.Tx), - Signature: stx.Signature, - TxInfo: *ref, + TxContent: protoTxAsJSON(itx.RawTx), + Signature: itx.Signature, + TxInfo: itx, } data, err := json.Marshal(tx) if err != nil { @@ -712,22 +746,28 @@ func (a *API) chainTxHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) er // chainTxListHandler // // @Summary List transactions -// @Description To get full transaction information use [/chain/transaction/{blockHeight}/{txIndex}](transaction-by-block-index).\nWhere transactionIndex is the index of the transaction on the containing block. +// @Description To get full transaction information use [/chain/transaction/{hash}](transaction-by-hash). // @Tags Chain // @Accept json // @Produce json // @Param page query number false "Page" // @Param limit query number false "Items per page" +// @Param hash query string false "Tx hash" // @Param height query number false "Block height" // @Param type query string false "Tx type" -// @Success 200 {object} TransactionsList "List of transactions references" +// @Param subtype query string false "Tx subtype" +// @Param signer query string false "Tx signer" +// @Success 200 {object} TransactionsList "List of transactions (metadata only)" // @Router /chain/transactions [get] func (a *API) chainTxListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { params, err := parseTransactionParams( ctx.QueryParam(ParamPage), ctx.QueryParam(ParamLimit), + ctx.QueryParam(ParamHash), ctx.QueryParam(ParamHeight), ctx.QueryParam(ParamType), + ctx.QueryParam(ParamSubtype), + ctx.QueryParam(ParamSigner), ) if err != nil { return err @@ -743,7 +783,7 @@ func (a *API) chainTxListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext // chainTxListByPageHandler // -// @Summary List transactions +// @Summary List transactions (legacy) // @Description To get full transaction information use [/chain/transaction/{blockHeight}/{txIndex}](transaction-by-block-index).\nWhere transactionIndex is the index of the transaction on the containing block. // @Deprecated // @Description (deprecated, in favor of /chain/transactions?page=xxx) @@ -751,7 +791,7 @@ func (a *API) chainTxListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext // @Accept json // @Produce json // @Param page path number true "Page" -// @Success 200 {object} TransactionsList "List of transactions references" +// @Success 200 {object} TransactionsList "List of transactions (metadata only)" // @Router /chain/transactions/page/{page} [get] func (a *API) chainTxListByPageHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { params, err := parseTransactionParams( @@ -759,6 +799,9 @@ func (a *API) chainTxListByPageHandler(_ *apirest.APIdata, ctx *httprouter.HTTPC "", "", "", + "", + "", + "", ) if err != nil { return err @@ -795,6 +838,9 @@ func (a *API) chainTxListByHeightAndPageHandler(_ *apirest.APIdata, ctx *httprou "", ctx.URLParam(ParamHeight), "", + "", + "", + "", ) if err != nil { return err @@ -820,7 +866,10 @@ func (a *API) transactionList(params *TransactionParams) (*TransactionsList, err params.Limit, params.Page*params.Limit, params.Height, + params.Hash, params.Type, + params.Subtype, + params.Signer, ) if err != nil { return nil, ErrIndexerQueryFailed.WithErr(err) @@ -873,7 +922,7 @@ func (a *API) chainValidatorsHandler(_ *apirest.APIdata, ctx *httprouter.HTTPCon return ctx.Send(data, apirest.HTTPstatusOK) } -// chainBlockHandler +// chainBlockByHeightHandler // // @Summary Get block (by height) // @Description Returns the full block information at the given height @@ -883,23 +932,34 @@ func (a *API) chainValidatorsHandler(_ *apirest.APIdata, ctx *httprouter.HTTPCon // @Param height path int true "Block height" // @Success 200 {object} api.Block // @Router /chain/blocks/{height} [get] -func (a *API) chainBlockHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { +func (a *API) chainBlockByHeightHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { height, err := strconv.ParseUint(ctx.URLParam(ParamHeight), 10, 64) if err != nil { return err } - tmblock := a.vocapp.GetBlockByHeight(int64(height)) - if tmblock == nil { - return ErrBlockNotFound + idxblock, err := a.indexer.BlockByHeight(int64(height)) + if err != nil { + if errors.Is(err, indexer.ErrBlockNotFound) { + return ErrBlockNotFound + } + return ErrBlockNotFound.WithErr(err) + } + txcount, err := a.indexer.CountTransactionsByHeight(int64(height)) + if err != nil { + return ErrIndexerQueryFailed.WithErr(err) } block := &Block{ - Block: comettypes.Block{ - Header: tmblock.Header, - Data: tmblock.Data, - Evidence: tmblock.Evidence, - LastCommit: tmblock.LastCommit, + Header: comettypes.Header{ + ChainID: idxblock.ChainID, + Height: idxblock.Height, + Time: idxblock.Time, + ProposerAddress: []byte(idxblock.ProposerAddress), + LastBlockID: comettypes.BlockID{ + Hash: []byte(idxblock.LastBlockHash), + }, }, - Hash: types.HexBytes(tmblock.Hash()), + Hash: idxblock.Hash, + TxCount: txcount, } data, err := json.Marshal(block) if err != nil { @@ -923,18 +983,29 @@ func (a *API) chainBlockByHashHandler(_ *apirest.APIdata, ctx *httprouter.HTTPCo if err != nil { return err } - tmblock := a.vocapp.GetBlockByHash(hash) - if tmblock == nil { - return ErrBlockNotFound + idxblock, err := a.indexer.BlockByHash(hash) + if err != nil { + if errors.Is(err, indexer.ErrBlockNotFound) { + return ErrBlockNotFound + } + return ErrBlockNotFound.WithErr(err) + } + txcount, err := a.indexer.CountTransactionsByHeight(idxblock.Height) + if err != nil { + return ErrIndexerQueryFailed.WithErr(err) } block := &Block{ - Block: comettypes.Block{ - Header: tmblock.Header, - Data: tmblock.Data, - Evidence: tmblock.Evidence, - LastCommit: tmblock.LastCommit, + Header: comettypes.Header{ + ChainID: idxblock.ChainID, + Height: idxblock.Height, + Time: idxblock.Time, + ProposerAddress: []byte(idxblock.ProposerAddress), + LastBlockID: comettypes.BlockID{ + Hash: []byte(idxblock.LastBlockHash), + }, }, - Hash: types.HexBytes(tmblock.Hash()), + Hash: idxblock.Hash, + TxCount: txcount, } data, err := json.Marshal(block) if err != nil { @@ -977,39 +1048,7 @@ func (a *API) chainBlockListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPCont // // Errors returned are always of type APIerror. func (a *API) sendBlockList(ctx *httprouter.HTTPContext, params *BlockParams) error { - // TODO: replace this by a.indexer.BlockList when it's available - blockList := func(limit, offset int, _, _, _ string) ([]*indexertypes.Block, uint64, error) { - if offset < 0 { - return nil, 0, fmt.Errorf("invalid value: offset cannot be %d", offset) - } - if limit <= 0 { - return nil, 0, fmt.Errorf("invalid value: limit cannot be %d", limit) - } - height := a.vocapp.Height() - total := uint64(height) - uint64(a.vocapp.Node.BlockStore().Base()) - start := height - uint32(params.Page*params.Limit) - end := start - uint32(params.Limit) - list := []*indexertypes.Block{} - for h := start; h > end; h-- { - tmblock := a.vocapp.GetBlockByHeight(int64(h)) - if tmblock == nil { - break - } - list = append(list, &indexertypes.Block{ - ChainID: tmblock.ChainID, - Height: tmblock.Height, - Time: tmblock.Time, - Hash: types.HexBytes(tmblock.Hash()), - ProposerAddress: tmblock.ProposerAddress.Bytes(), - LastBlockHash: tmblock.LastBlockID.Hash.Bytes(), - TxCount: int64(len(tmblock.Txs)), - }) - } - - return list, uint64(total), nil - } - - blocks, total, err := blockList( + blocks, total, err := a.indexer.BlockList( params.Limit, params.Page*params.Limit, params.ChainID, @@ -1363,7 +1402,7 @@ func parseTransfersParams(paramPage, paramLimit, paramAccountId, paramAccountIdF } // parseTransactionParams returns an TransactionParams filled with the passed params -func parseTransactionParams(paramPage, paramLimit, paramHeight, paramType string) (*TransactionParams, error) { +func parseTransactionParams(paramPage, paramLimit, paramHash, paramHeight, paramType, paramSubtype, paramSigner string) (*TransactionParams, error) { pagination, err := parsePaginationParams(paramPage, paramLimit) if err != nil { return nil, err @@ -1376,8 +1415,11 @@ func parseTransactionParams(paramPage, paramLimit, paramHeight, paramType string return &TransactionParams{ PaginationParams: pagination, + Hash: util.TrimHex(paramHash), Height: uint64(height), Type: paramType, + Subtype: paramSubtype, + Signer: util.TrimHex(paramSigner), }, nil } diff --git a/test/api_test.go b/test/api_test.go index 80e4f3699..267895636 100644 --- a/test/api_test.go +++ b/test/api_test.go @@ -461,6 +461,41 @@ func TestAPIAccountTokentxs(t *testing.T) { qt.Assert(t, gotAcct1.Balance, qt.Equals, initBalance+amountAcc2toAcct1-amountAcc1toAcct2-uint64(txBasePrice)) } +func TestAPIBlocks(t *testing.T) { + server := testcommon.APIserver{} + server.Start(t, + api.ChainHandler, + api.CensusHandler, + api.VoteHandler, + api.AccountHandler, + api.ElectionHandler, + api.WalletHandler, + ) + token1 := uuid.New() + c := testutil.NewTestHTTPclient(t, server.ListenAddr, &token1) + + // Block 1 + server.VochainAPP.AdvanceTestBlock() + waitUntilHeight(t, c, 1) + + // create a new account + initBalance := uint64(80) + _ = createAccount(t, c, server, initBalance) + + // Block 2 + server.VochainAPP.AdvanceTestBlock() + waitUntilHeight(t, c, 2) + + // check the txCount + resp, code := c.Request("GET", nil, "chain", "blocks", "1") + qt.Assert(t, code, qt.Equals, 200, qt.Commentf("response: %s", resp)) + + block := api.Block{} + err := json.Unmarshal(resp, &block) + qt.Assert(t, err, qt.IsNil) + qt.Assert(t, block.TxCount, qt.Equals, int64(1)) +} + func runAPIElectionCostWithParams(t *testing.T, electionParams electionprice.ElectionParameters, startBlock uint32, initialBalance, diff --git a/vochain/indexer/transaction.go b/vochain/indexer/transaction.go index 31beb4465..41a6a1818 100644 --- a/vochain/indexer/transaction.go +++ b/vochain/indexer/transaction.go @@ -42,6 +42,18 @@ func (idx *Indexer) GetTxMetadataByHash(hash types.HexBytes) (*indexertypes.Tran return indexertypes.TransactionMetadataFromDB(&sqlTxRef), nil } +// GetTransactionByHash fetches the full tx for the given tx hash +func (idx *Indexer) GetTransactionByHash(hash types.HexBytes) (*indexertypes.Transaction, error) { + sqlTxRef, err := idx.readOnlyQuery.GetTransactionByHash(context.TODO(), hash) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrTransactionNotFound + } + return nil, fmt.Errorf("tx hash %x not found: %v", hash, err) + } + return indexertypes.TransactionFromDB(&sqlTxRef), nil +} + // GetTransactionByHeightAndIndex fetches the full tx for the given tx height and block tx index func (idx *Indexer) GetTransactionByHeightAndIndex(blockHeight, blockIndex int64) (*indexertypes.Transaction, error) { sqlTxRef, err := idx.readOnlyQuery.GetTransactionByHeightAndIndex(context.TODO(), indexerdb.GetTransactionByHeightAndIndexParams{