add api-logs-limit flag

.github/workflows/test-e2e.yaml

@@ -43,8 +43,8 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: vechain/thor-e2e-tests
-          # https://github.com/vechain/thor-e2e-tests/tree/2cb22d804bb3cdf075917dbece42a182d42d7486
-          ref: 2cb22d804bb3cdf075917dbece42a182d42d7486
+          # https://github.com/vechain/thor-e2e-tests/tree/00bd3f1b949b05da94e82686e0089a11a136c34c
+          ref: 00bd3f1b949b05da94e82686e0089a11a136c34c
 
       - name: Download artifact
         uses: actions/download-artifact@v4

api/api.go

@@ -49,6 +49,7 @@ func New(
 	allowCustomTracer bool,
 	enableReqLogger bool,
 	enableMetrics bool,
+	logsLimit uint64,
 ) (http.HandlerFunc, func()) {
 	origins := strings.Split(strings.TrimSpace(allowedOrigins), ",")
 	for i, o := range origins {
@@ -72,9 +73,9 @@ func New(
 		Mount(router, "/accounts")
 
 	if !skipLogs {
-		events.New(repo, logDB).
+		events.New(repo, logDB, logsLimit).
 			Mount(router, "/logs/event")
-		transfers.New(repo, logDB).
+		transfers.New(repo, logDB, logsLimit).
 			Mount(router, "/logs/transfer")
 	}
 	blocks.New(repo, bft).

api/events/events.go

@@ -17,14 +17,16 @@ import (
 )
 
 type Events struct {
-	repo *chain.Repository
-	db   *logdb.LogDB
+	repo  *chain.Repository
+	db    *logdb.LogDB
+	limit uint64
 }
 
-func New(repo *chain.Repository, db *logdb.LogDB) *Events {
+func New(repo *chain.Repository, db *logdb.LogDB, logsLimit uint64) *Events {
 	return &Events{
 		repo,
 		db,
+		logsLimit,
 	}
 }
 
@@ -51,6 +53,16 @@ func (e *Events) handleFilter(w http.ResponseWriter, req *http.Request) error {
 	if err := utils.ParseJSON(req.Body, &filter); err != nil {
 		return utils.BadRequest(errors.WithMessage(err, "body"))
 	}
+	if filter.Options != nil && filter.Options.Limit > e.limit {
+		return utils.Forbidden(errors.New("options.limit exceeds the maximum allowed value"))
+	}
+	if filter.Options == nil {
+		filter.Options = &logdb.Options{
+			Offset: 0,
+			Limit:  e.limit,
+		}
+	}
+
 	fes, err := e.filter(req.Context(), &filter)
 	if err != nil {
 		return err

api/events/events_test.go

@@ -11,6 +11,7 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"testing"
 
 	"github.com/gorilla/mux"
@@ -26,6 +27,8 @@ import (
 	"github.com/vechain/thor/v2/tx"
 )
 
+const defaultLogLimit uint64 = 1000
+
 var ts *httptest.Server
 
 var (
@@ -35,7 +38,7 @@ var (
 
 func TestEmptyEvents(t *testing.T) {
 	db := createDb(t)
-	initEventServer(t, db)
+	initEventServer(t, db, defaultLogLimit)
 	defer ts.Close()
 
 	testEventsBadRequest(t)
@@ -44,7 +47,7 @@ func TestEmptyEvents(t *testing.T) {
 
 func TestEvents(t *testing.T) {
 	db := createDb(t)
-	initEventServer(t, db)
+	initEventServer(t, db, defaultLogLimit)
 	defer ts.Close()
 
 	blocksToInsert := 5
@@ -53,6 +56,38 @@ func TestEvents(t *testing.T) {
 	testEventWithBlocks(t, blocksToInsert)
 }
 
+func TestOption(t *testing.T) {
+	db := createDb(t)
+	initEventServer(t, db, 5)
+	defer ts.Close()
+	insertBlocks(t, db, 10)
+
+	filter := events.EventFilter{
+		CriteriaSet: make([]*events.EventCriteria, 0),
+		Range:       nil,
+		Options:     &logdb.Options{Limit: 6},
+		Order:       logdb.DESC,
+	}
+
+	res, statusCode := httpPost(t, ts.URL+"/events", filter)
+	assert.Equal(t, "options.limit exceeds the maximum allowed value", strings.Trim(string(res), "\n"))
+	assert.Equal(t, http.StatusForbidden, statusCode)
+
+	filter.Options.Limit = 5
+	_, statusCode = httpPost(t, ts.URL+"/events", filter)
+	assert.Equal(t, http.StatusOK, statusCode)
+
+	// with nil options, should use default limit
+	filter.Options = nil
+	res, statusCode = httpPost(t, ts.URL+"/events", filter)
+	var tLogs []*events.FilteredEvent
+	if err := json.Unmarshal(res, &tLogs); err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, http.StatusOK, statusCode)
+	assert.Equal(t, 5, len(tLogs))
+}
+
 // Test functions
 func testEventsBadRequest(t *testing.T) {
 	badBody := []byte{0x00, 0x01, 0x02}
@@ -128,7 +163,7 @@ func testEventWithBlocks(t *testing.T, expectedBlocks int) {
 }
 
 // Init functions
-func initEventServer(t *testing.T, logDb *logdb.LogDB) {
+func initEventServer(t *testing.T, logDb *logdb.LogDB, limit uint64) {
 	router := mux.NewRouter()
 
 	muxDb := muxdb.NewMem()
@@ -142,7 +177,7 @@ func initEventServer(t *testing.T, logDb *logdb.LogDB) {
 
 	repo, _ := chain.NewRepository(muxDb, b)
 
-	events.New(repo, logDb).Mount(router, "/events")
+	events.New(repo, logDb, limit).Mount(router, "/events")
 	ts = httptest.NewServer(router)
 }
 

api/transfers/transfers.go

@@ -18,14 +18,16 @@ import (
 )
 
 type Transfers struct {
-	repo *chain.Repository
-	db   *logdb.LogDB
+	repo  *chain.Repository
+	db    *logdb.LogDB
+	limit uint64
 }
 
-func New(repo *chain.Repository, db *logdb.LogDB) *Transfers {
+func New(repo *chain.Repository, db *logdb.LogDB, logsLimit uint64) *Transfers {
 	return &Transfers{
 		repo,
 		db,
+		logsLimit,
 	}
 }
 
@@ -57,6 +59,16 @@ func (t *Transfers) handleFilterTransferLogs(w http.ResponseWriter, req *http.Re
 	if err := utils.ParseJSON(req.Body, &filter); err != nil {
 		return utils.BadRequest(errors.WithMessage(err, "body"))
 	}
+	if filter.Options != nil && filter.Options.Limit > t.limit {
+		return utils.Forbidden(errors.New("options.limit exceeds the maximum allowed value"))
+	}
+	if filter.Options == nil {
+		filter.Options = &logdb.Options{
+			Offset: 0,
+			Limit:  t.limit,
+		}
+	}
+
 	tLogs, err := t.filter(req.Context(), &filter)
 	if err != nil {
 		return err

api/transfers/transfers_test.go

@@ -13,10 +13,12 @@ import (
 	"math/big"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"testing"
 
 	"github.com/gorilla/mux"
 	"github.com/stretchr/testify/assert"
+	"github.com/vechain/thor/v2/api/events"
 	"github.com/vechain/thor/v2/api/transfers"
 	"github.com/vechain/thor/v2/block"
 	"github.com/vechain/thor/v2/chain"
@@ -28,11 +30,13 @@ import (
 	"github.com/vechain/thor/v2/tx"
 )
 
+const defaultLogLimit uint64 = 1000
+
 var ts *httptest.Server
 
 func TestEmptyTransfers(t *testing.T) {
 	db := createDb(t)
-	initTransferServer(t, db)
+	initTransferServer(t, db, defaultLogLimit)
 	defer ts.Close()
 
 	testTransferBadRequest(t)
@@ -41,7 +45,7 @@ func TestEmptyTransfers(t *testing.T) {
 
 func TestTransfers(t *testing.T) {
 	db := createDb(t)
-	initTransferServer(t, db)
+	initTransferServer(t, db, defaultLogLimit)
 	defer ts.Close()
 
 	blocksToInsert := 5
@@ -50,6 +54,38 @@ func TestTransfers(t *testing.T) {
 	testTransferWithBlocks(t, blocksToInsert)
 }
 
+func TestOption(t *testing.T) {
+	db := createDb(t)
+	initTransferServer(t, db, 5)
+	defer ts.Close()
+	insertBlocks(t, db, 10)
+
+	filter := transfers.TransferFilter{
+		CriteriaSet: make([]*logdb.TransferCriteria, 0),
+		Range:       nil,
+		Options:     &logdb.Options{Limit: 6},
+		Order:       logdb.DESC,
+	}
+
+	res, statusCode := httpPost(t, ts.URL+"/transfers", filter)
+	assert.Equal(t, "options.limit exceeds the maximum allowed value", strings.Trim(string(res), "\n"))
+	assert.Equal(t, http.StatusForbidden, statusCode)
+
+	filter.Options.Limit = 5
+	_, statusCode = httpPost(t, ts.URL+"/transfers", filter)
+	assert.Equal(t, http.StatusOK, statusCode)
+
+	// with nil options, should use default limit
+	filter.Options = nil
+	res, statusCode = httpPost(t, ts.URL+"/transfers", filter)
+	var tLogs []*events.FilteredEvent
+	if err := json.Unmarshal(res, &tLogs); err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, http.StatusOK, statusCode)
+	assert.Equal(t, 5, len(tLogs))
+}
+
 // Test functions
 func testTransferBadRequest(t *testing.T) {
 	badBody := []byte{0x00, 0x01, 0x02}
@@ -119,7 +155,7 @@ func insertBlocks(t *testing.T, db *logdb.LogDB, n int) {
 	}
 }
 
-func initTransferServer(t *testing.T, logDb *logdb.LogDB) {
+func initTransferServer(t *testing.T, logDb *logdb.LogDB, limit uint64) {
 	router := mux.NewRouter()
 
 	muxDb := muxdb.NewMem()
@@ -133,7 +169,7 @@ func initTransferServer(t *testing.T, logDb *logdb.LogDB) {
 
 	repo, _ := chain.NewRepository(muxDb, b)
 
-	transfers.New(repo, logDb).Mount(router, "/transfers")
+	transfers.New(repo, logDb, limit).Mount(router, "/transfers")
 	ts = httptest.NewServer(router)
 }
 

cmd/thor/flags.go

@@ -59,6 +59,11 @@ var (
 		Name:  "api-allow-custom-tracer",
 		Usage: "allow custom JS tracer to be used tracer API",
 	}
+	apiLogsLimitFlag = cli.IntFlag{
+		Name:  "api-logs-limit",
+		Value: 1000,
+		Usage: "limit the number of logs returned by /logs API",
+	}
 	enableAPILogsFlag = cli.BoolFlag{
 		Name:  "enable-api-logs",
 		Usage: "enables API requests logging",

cmd/thor/main.go

@@ -80,6 +80,7 @@ func main() {
 			apiBacktraceLimitFlag,
 			apiAllowCustomTracerFlag,
 			enableAPILogsFlag,
+			apiLogsLimitFlag,
 			verbosityFlag,
 			maxPeersFlag,
 			p2pPortFlag,
@@ -109,6 +110,7 @@ func main() {
 					apiBacktraceLimitFlag,
 					apiAllowCustomTracerFlag,
 					enableAPILogsFlag,
+					apiLogsLimitFlag,
 					onDemandFlag,
 					blockInterval,
 					persistFlag,
@@ -234,6 +236,7 @@ func defaultAction(ctx *cli.Context) error {
 		ctx.Bool(apiAllowCustomTracerFlag.Name),
 		ctx.Bool(enableAPILogsFlag.Name),
 		ctx.Bool(enableMetricsFlag.Name),
+		uint64(ctx.Int(apiLogsLimitFlag.Name)),
 	)
 	defer func() { log.Info("closing API..."); apiCloser() }()
 
@@ -363,6 +366,7 @@ func soloAction(ctx *cli.Context) error {
 		ctx.Bool(apiAllowCustomTracerFlag.Name),
 		ctx.Bool(enableAPILogsFlag.Name),
 		ctx.Bool(enableMetricsFlag.Name),
+		uint64(ctx.Int(apiLogsLimitFlag.Name)),
 	)
 	defer func() { log.Info("closing API..."); apiCloser() }()
 

docs/usage.md

@@ -169,6 +169,7 @@ bin/thor -h
 | `--api-backtrace-limit`     | Limit the distance between 'position' and best block for subscriptions APIs (default: 1000) |
 | `--api-allow-custom-tracer` | Allow custom JS tracer to be used for the tracer API                                        |
 | `--enable-api-logs`         | Enables API requests logging                                                                |
+| `--api-logs-limit`          | Limit the number of logs returned by /logs API (default: 1000)                              |
 | `--verbosity`               | Log verbosity (0-9) (default: 3)                                                            |
 | `--max-peers`               | Maximum number of P2P network peers (P2P network disabled if set to 0) (default: 25)        |
 | `--p2p-port`                | P2P network listening port (default: 11235)                                                 |