diff --git a/.github/workflows/test-e2e.yaml b/.github/workflows/test-e2e.yaml index e0b3eaf31..7ac4cc0e8 100644 --- a/.github/workflows/test-e2e.yaml +++ b/.github/workflows/test-e2e.yaml @@ -43,8 +43,8 @@ jobs: uses: actions/checkout@v4 with: repository: vechain/thor-e2e-tests - # https://github.com/vechain/thor-e2e-tests/tree/2cb22d804bb3cdf075917dbece42a182d42d7486 - ref: 2cb22d804bb3cdf075917dbece42a182d42d7486 + # https://github.com/vechain/thor-e2e-tests/tree/00bd3f1b949b05da94e82686e0089a11a136c34c + ref: 00bd3f1b949b05da94e82686e0089a11a136c34c - name: Download artifact uses: actions/download-artifact@v4 diff --git a/api/api.go b/api/api.go index 5bab4af09..631019b77 100644 --- a/api/api.go +++ b/api/api.go @@ -49,6 +49,7 @@ func New( allowCustomTracer bool, enableReqLogger bool, enableMetrics bool, + logsLimit uint64, ) (http.HandlerFunc, func()) { origins := strings.Split(strings.TrimSpace(allowedOrigins), ",") for i, o := range origins { @@ -72,9 +73,9 @@ func New( Mount(router, "/accounts") if !skipLogs { - events.New(repo, logDB). + events.New(repo, logDB, logsLimit). Mount(router, "/logs/event") - transfers.New(repo, logDB). + transfers.New(repo, logDB, logsLimit). Mount(router, "/logs/transfer") } blocks.New(repo, bft). diff --git a/api/events/events.go b/api/events/events.go index ee3bd800c..0d598d499 100644 --- a/api/events/events.go +++ b/api/events/events.go @@ -17,14 +17,16 @@ import ( ) type Events struct { - repo *chain.Repository - db *logdb.LogDB + repo *chain.Repository + db *logdb.LogDB + limit uint64 } -func New(repo *chain.Repository, db *logdb.LogDB) *Events { +func New(repo *chain.Repository, db *logdb.LogDB, logsLimit uint64) *Events { return &Events{ repo, db, + logsLimit, } } @@ -51,6 +53,16 @@ func (e *Events) handleFilter(w http.ResponseWriter, req *http.Request) error { if err := utils.ParseJSON(req.Body, &filter); err != nil { return utils.BadRequest(errors.WithMessage(err, "body")) } + if filter.Options != nil && filter.Options.Limit > e.limit { + return utils.Forbidden(errors.New("options.limit exceeds the maximum allowed value")) + } + if filter.Options == nil { + filter.Options = &logdb.Options{ + Offset: 0, + Limit: e.limit, + } + } + fes, err := e.filter(req.Context(), &filter) if err != nil { return err diff --git a/api/events/events_test.go b/api/events/events_test.go index 0920225d9..7cf9246e5 100644 --- a/api/events/events_test.go +++ b/api/events/events_test.go @@ -11,6 +11,7 @@ import ( "io" "net/http" "net/http/httptest" + "strings" "testing" "github.com/gorilla/mux" @@ -26,6 +27,8 @@ import ( "github.com/vechain/thor/v2/tx" ) +const defaultLogLimit uint64 = 1000 + var ts *httptest.Server var ( @@ -35,7 +38,7 @@ var ( func TestEmptyEvents(t *testing.T) { db := createDb(t) - initEventServer(t, db) + initEventServer(t, db, defaultLogLimit) defer ts.Close() testEventsBadRequest(t) @@ -44,7 +47,7 @@ func TestEmptyEvents(t *testing.T) { func TestEvents(t *testing.T) { db := createDb(t) - initEventServer(t, db) + initEventServer(t, db, defaultLogLimit) defer ts.Close() blocksToInsert := 5 @@ -53,6 +56,38 @@ func TestEvents(t *testing.T) { testEventWithBlocks(t, blocksToInsert) } +func TestOption(t *testing.T) { + db := createDb(t) + initEventServer(t, db, 5) + defer ts.Close() + insertBlocks(t, db, 10) + + filter := events.EventFilter{ + CriteriaSet: make([]*events.EventCriteria, 0), + Range: nil, + Options: &logdb.Options{Limit: 6}, + Order: logdb.DESC, + } + + res, statusCode := httpPost(t, ts.URL+"/events", filter) + assert.Equal(t, "options.limit exceeds the maximum allowed value", strings.Trim(string(res), "\n")) + assert.Equal(t, http.StatusForbidden, statusCode) + + filter.Options.Limit = 5 + _, statusCode = httpPost(t, ts.URL+"/events", filter) + assert.Equal(t, http.StatusOK, statusCode) + + // with nil options, should use default limit + filter.Options = nil + res, statusCode = httpPost(t, ts.URL+"/events", filter) + var tLogs []*events.FilteredEvent + if err := json.Unmarshal(res, &tLogs); err != nil { + t.Fatal(err) + } + assert.Equal(t, http.StatusOK, statusCode) + assert.Equal(t, 5, len(tLogs)) +} + // Test functions func testEventsBadRequest(t *testing.T) { badBody := []byte{0x00, 0x01, 0x02} @@ -128,7 +163,7 @@ func testEventWithBlocks(t *testing.T, expectedBlocks int) { } // Init functions -func initEventServer(t *testing.T, logDb *logdb.LogDB) { +func initEventServer(t *testing.T, logDb *logdb.LogDB, limit uint64) { router := mux.NewRouter() muxDb := muxdb.NewMem() @@ -142,7 +177,7 @@ func initEventServer(t *testing.T, logDb *logdb.LogDB) { repo, _ := chain.NewRepository(muxDb, b) - events.New(repo, logDb).Mount(router, "/events") + events.New(repo, logDb, limit).Mount(router, "/events") ts = httptest.NewServer(router) } diff --git a/api/transfers/transfers.go b/api/transfers/transfers.go index bb64f74a2..215490e77 100644 --- a/api/transfers/transfers.go +++ b/api/transfers/transfers.go @@ -18,14 +18,16 @@ import ( ) type Transfers struct { - repo *chain.Repository - db *logdb.LogDB + repo *chain.Repository + db *logdb.LogDB + limit uint64 } -func New(repo *chain.Repository, db *logdb.LogDB) *Transfers { +func New(repo *chain.Repository, db *logdb.LogDB, logsLimit uint64) *Transfers { return &Transfers{ repo, db, + logsLimit, } } @@ -57,6 +59,16 @@ func (t *Transfers) handleFilterTransferLogs(w http.ResponseWriter, req *http.Re if err := utils.ParseJSON(req.Body, &filter); err != nil { return utils.BadRequest(errors.WithMessage(err, "body")) } + if filter.Options != nil && filter.Options.Limit > t.limit { + return utils.Forbidden(errors.New("options.limit exceeds the maximum allowed value")) + } + if filter.Options == nil { + filter.Options = &logdb.Options{ + Offset: 0, + Limit: t.limit, + } + } + tLogs, err := t.filter(req.Context(), &filter) if err != nil { return err diff --git a/api/transfers/transfers_test.go b/api/transfers/transfers_test.go index 8c940b90b..56db63ee1 100644 --- a/api/transfers/transfers_test.go +++ b/api/transfers/transfers_test.go @@ -13,10 +13,12 @@ import ( "math/big" "net/http" "net/http/httptest" + "strings" "testing" "github.com/gorilla/mux" "github.com/stretchr/testify/assert" + "github.com/vechain/thor/v2/api/events" "github.com/vechain/thor/v2/api/transfers" "github.com/vechain/thor/v2/block" "github.com/vechain/thor/v2/chain" @@ -28,11 +30,13 @@ import ( "github.com/vechain/thor/v2/tx" ) +const defaultLogLimit uint64 = 1000 + var ts *httptest.Server func TestEmptyTransfers(t *testing.T) { db := createDb(t) - initTransferServer(t, db) + initTransferServer(t, db, defaultLogLimit) defer ts.Close() testTransferBadRequest(t) @@ -41,7 +45,7 @@ func TestEmptyTransfers(t *testing.T) { func TestTransfers(t *testing.T) { db := createDb(t) - initTransferServer(t, db) + initTransferServer(t, db, defaultLogLimit) defer ts.Close() blocksToInsert := 5 @@ -50,6 +54,38 @@ func TestTransfers(t *testing.T) { testTransferWithBlocks(t, blocksToInsert) } +func TestOption(t *testing.T) { + db := createDb(t) + initTransferServer(t, db, 5) + defer ts.Close() + insertBlocks(t, db, 10) + + filter := transfers.TransferFilter{ + CriteriaSet: make([]*logdb.TransferCriteria, 0), + Range: nil, + Options: &logdb.Options{Limit: 6}, + Order: logdb.DESC, + } + + res, statusCode := httpPost(t, ts.URL+"/transfers", filter) + assert.Equal(t, "options.limit exceeds the maximum allowed value", strings.Trim(string(res), "\n")) + assert.Equal(t, http.StatusForbidden, statusCode) + + filter.Options.Limit = 5 + _, statusCode = httpPost(t, ts.URL+"/transfers", filter) + assert.Equal(t, http.StatusOK, statusCode) + + // with nil options, should use default limit + filter.Options = nil + res, statusCode = httpPost(t, ts.URL+"/transfers", filter) + var tLogs []*events.FilteredEvent + if err := json.Unmarshal(res, &tLogs); err != nil { + t.Fatal(err) + } + assert.Equal(t, http.StatusOK, statusCode) + assert.Equal(t, 5, len(tLogs)) +} + // Test functions func testTransferBadRequest(t *testing.T) { badBody := []byte{0x00, 0x01, 0x02} @@ -119,7 +155,7 @@ func insertBlocks(t *testing.T, db *logdb.LogDB, n int) { } } -func initTransferServer(t *testing.T, logDb *logdb.LogDB) { +func initTransferServer(t *testing.T, logDb *logdb.LogDB, limit uint64) { router := mux.NewRouter() muxDb := muxdb.NewMem() @@ -133,7 +169,7 @@ func initTransferServer(t *testing.T, logDb *logdb.LogDB) { repo, _ := chain.NewRepository(muxDb, b) - transfers.New(repo, logDb).Mount(router, "/transfers") + transfers.New(repo, logDb, limit).Mount(router, "/transfers") ts = httptest.NewServer(router) } diff --git a/cmd/thor/flags.go b/cmd/thor/flags.go index 645a6f44e..821680936 100644 --- a/cmd/thor/flags.go +++ b/cmd/thor/flags.go @@ -59,6 +59,11 @@ var ( Name: "api-allow-custom-tracer", Usage: "allow custom JS tracer to be used tracer API", } + apiLogsLimitFlag = cli.IntFlag{ + Name: "api-logs-limit", + Value: 1000, + Usage: "limit the number of logs returned by /logs API", + } enableAPILogsFlag = cli.BoolFlag{ Name: "enable-api-logs", Usage: "enables API requests logging", diff --git a/cmd/thor/main.go b/cmd/thor/main.go index 00cb44977..4fff3e737 100644 --- a/cmd/thor/main.go +++ b/cmd/thor/main.go @@ -80,6 +80,7 @@ func main() { apiBacktraceLimitFlag, apiAllowCustomTracerFlag, enableAPILogsFlag, + apiLogsLimitFlag, verbosityFlag, maxPeersFlag, p2pPortFlag, @@ -109,6 +110,7 @@ func main() { apiBacktraceLimitFlag, apiAllowCustomTracerFlag, enableAPILogsFlag, + apiLogsLimitFlag, onDemandFlag, blockInterval, persistFlag, @@ -234,6 +236,7 @@ func defaultAction(ctx *cli.Context) error { ctx.Bool(apiAllowCustomTracerFlag.Name), ctx.Bool(enableAPILogsFlag.Name), ctx.Bool(enableMetricsFlag.Name), + uint64(ctx.Int(apiLogsLimitFlag.Name)), ) defer func() { log.Info("closing API..."); apiCloser() }() @@ -363,6 +366,7 @@ func soloAction(ctx *cli.Context) error { ctx.Bool(apiAllowCustomTracerFlag.Name), ctx.Bool(enableAPILogsFlag.Name), ctx.Bool(enableMetricsFlag.Name), + uint64(ctx.Int(apiLogsLimitFlag.Name)), ) defer func() { log.Info("closing API..."); apiCloser() }() diff --git a/docs/usage.md b/docs/usage.md index ca995ce8c..552bc62b9 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -169,6 +169,7 @@ bin/thor -h | `--api-backtrace-limit` | Limit the distance between 'position' and best block for subscriptions APIs (default: 1000) | | `--api-allow-custom-tracer` | Allow custom JS tracer to be used for the tracer API | | `--enable-api-logs` | Enables API requests logging | +| `--api-logs-limit` | Limit the number of logs returned by /logs API (default: 1000) | | `--verbosity` | Log verbosity (0-9) (default: 3) | | `--max-peers` | Maximum number of P2P network peers (P2P network disabled if set to 0) (default: 25) | | `--p2p-port` | P2P network listening port (default: 11235) | diff --git a/logdb/logdb.go b/logdb/logdb.go index f465efd6e..5d0bf744b 100644 --- a/logdb/logdb.go +++ b/logdb/logdb.go @@ -19,8 +19,7 @@ import ( ) const ( - refIDQuery = "(SELECT id FROM ref WHERE data=?)" - limitThreshold = 1000 + refIDQuery = "(SELECT id FROM ref WHERE data=?)" ) type LogDB struct { @@ -108,14 +107,8 @@ FROM (%v) e LEFT JOIN ref r7 ON e.topic3 = r7.id LEFT JOIN ref r8 ON e.topic4 = r8.id` - if filter == nil { // default query filtering - filter = &EventFilter{ - Options: &Options{ - Offset: 0, - Limit: limitThreshold, - }, - Order: "desc", - } + if filter == nil { + return db.queryEvents(ctx, fmt.Sprintf(query, "event")) } var ( @@ -153,17 +146,10 @@ FROM (%v) e } // if there is limit option, set order inside subquery - subQuery += " LIMIT ?, ?" // all queries are bounded to a max of 1000 results - if filter.Options != nil && filter.Options.Limit > 1000 { - // offset could have been specified - filter.Options.Limit = limitThreshold - } else if filter.Options == nil { - filter.Options = &Options{ - Offset: 0, - Limit: limitThreshold, - } + if filter.Options != nil { + subQuery += " LIMIT ?, ?" + args = append(args, filter.Options.Offset, filter.Options.Limit) } - args = append(args, filter.Options.Offset, filter.Options.Limit) subQuery = "SELECT e.* FROM (" + subQuery + ") s LEFT JOIN event e ON s.seq = e.seq" @@ -179,14 +165,8 @@ FROM (%v) t LEFT JOIN ref r3 ON t.sender = r3.id LEFT JOIN ref r4 ON t.recipient = r4.id` - if filter == nil { // default query filtering - filter = &TransferFilter{ - Options: &Options{ - Offset: 0, - Limit: limitThreshold, - }, - Order: "desc", - } + if filter == nil { + return db.queryTransfers(ctx, fmt.Sprintf(query, "transfer")) } var ( @@ -223,17 +203,10 @@ FROM (%v) t } // if there is limit option, set order inside subquery - subQuery += " LIMIT ?, ?" - if filter.Options != nil && filter.Options.Limit > limitThreshold { - // offset could have been specified - filter.Options.Limit = limitThreshold - } else if filter.Options == nil { - filter.Options = &Options{ - Offset: 0, - Limit: limitThreshold, - } + if filter.Options != nil { + subQuery += " LIMIT ?, ?" + args = append(args, filter.Options.Offset, filter.Options.Limit) } - args = append(args, filter.Options.Offset, filter.Options.Limit) subQuery = "SELECT e.* FROM (" + subQuery + ") s LEFT JOIN transfer e ON s.seq = e.seq" diff --git a/logdb/logdb_test.go b/logdb/logdb_test.go index efa4388cc..684c245cc 100644 --- a/logdb/logdb_test.go +++ b/logdb/logdb_test.go @@ -132,7 +132,7 @@ func TestEvents(t *testing.T) { var allEvents eventLogs var allTransfers transferLogs - for i := 0; i < 2000; i++ { + for i := 0; i < 100; i++ { b = new(block.Builder). ParentID(b.Header().ID()). Transaction(newTx()). @@ -187,14 +187,11 @@ func TestEvents(t *testing.T) { arg *logdb.EventFilter want eventLogs }{ - {"query all events", &logdb.EventFilter{}, allEvents[:1000]}, - {"query all events with nil option", nil, allEvents.Reverse()[:1000]}, - {"query all events asc", &logdb.EventFilter{Order: logdb.ASC}, allEvents[:1000]}, - {"query all events desc", &logdb.EventFilter{Order: logdb.DESC}, allEvents.Reverse()[:1000]}, + {"query all events", &logdb.EventFilter{}, allEvents}, + {"query all events with nil option", nil, allEvents}, + {"query all events asc", &logdb.EventFilter{Order: logdb.ASC}, allEvents}, + {"query all events desc", &logdb.EventFilter{Order: logdb.DESC}, allEvents.Reverse()}, {"query all events limit offset", &logdb.EventFilter{Options: &logdb.Options{Offset: 1, Limit: 10}}, allEvents[1:11]}, - {"query all transfers offset", &logdb.EventFilter{Options: &logdb.Options{Offset: 1500, Limit: 10000}, Order: logdb.ASC}, allEvents[1500:2500]}, - {"query all events outsized limit ", &logdb.EventFilter{Options: &logdb.Options{Limit: 2000}}, allEvents[:1000]}, - {"query all events outsized limit offset", &logdb.EventFilter{Options: &logdb.Options{Offset: 2, Limit: 2000}}, allEvents[2:1002]}, {"query all events range", &logdb.EventFilter{Range: &logdb.Range{From: 10, To: 20}}, allEvents.Filter(func(ev *logdb.Event) bool { return ev.BlockNumber >= 10 && ev.BlockNumber <= 20 })}, {"query events with range and desc", &logdb.EventFilter{Range: &logdb.Range{From: 10, To: 20}, Order: logdb.DESC}, allEvents.Filter(func(ev *logdb.Event) bool { return ev.BlockNumber >= 10 && ev.BlockNumber <= 20 }).Reverse()}, {"query events with limit with desc", &logdb.EventFilter{Order: logdb.DESC, Options: &logdb.Options{Limit: 10}}, allEvents.Reverse()[0:10]}, @@ -221,14 +218,11 @@ func TestEvents(t *testing.T) { arg *logdb.TransferFilter want transferLogs }{ - {"query all transfers", &logdb.TransferFilter{}, allTransfers[:1000]}, - {"query all transfers with nil option", nil, allTransfers.Reverse()[:1000]}, - {"query all transfers asc", &logdb.TransferFilter{Order: logdb.ASC}, allTransfers[:1000]}, - {"query all transfers desc", &logdb.TransferFilter{Order: logdb.DESC}, allTransfers.Reverse()[:1000]}, + {"query all transfers", &logdb.TransferFilter{}, allTransfers}, + {"query all transfers with nil option", nil, allTransfers}, + {"query all transfers asc", &logdb.TransferFilter{Order: logdb.ASC}, allTransfers}, + {"query all transfers desc", &logdb.TransferFilter{Order: logdb.DESC}, allTransfers.Reverse()}, {"query all transfers limit offset", &logdb.TransferFilter{Options: &logdb.Options{Offset: 1, Limit: 10}}, allTransfers[1:11]}, - {"query all transfers offset", &logdb.TransferFilter{Options: &logdb.Options{Offset: 1500, Limit: 10000}, Order: logdb.ASC}, allTransfers[1500:2500]}, - {"query all transfers outsized limit ", &logdb.TransferFilter{Options: &logdb.Options{Limit: 2000}}, allTransfers[:1000]}, - {"query all transfers outsized limit offset", &logdb.TransferFilter{Options: &logdb.Options{Offset: 2, Limit: 2000}}, allTransfers[2:1002]}, {"query all transfers range", &logdb.TransferFilter{Range: &logdb.Range{From: 10, To: 20}}, allTransfers.Filter(func(tr *logdb.Transfer) bool { return tr.BlockNumber >= 10 && tr.BlockNumber <= 20 })}, {"query transfers with range and desc", &logdb.TransferFilter{Range: &logdb.Range{From: 10, To: 20}, Order: logdb.DESC}, allTransfers.Filter(func(tr *logdb.Transfer) bool { return tr.BlockNumber >= 10 && tr.BlockNumber <= 20 }).Reverse()}, {"query transfers with limit with desc", &logdb.TransferFilter{Order: logdb.DESC, Options: &logdb.Options{Limit: 10}}, allTransfers.Reverse()[0:10]},