fix(mempool): change "mempool is full" log level to debug (backport c…

…ometbft#4123) (cometbft#4145)

it happens in benchmark or production when traffic is high, the log
frequency is very high because it's triggered by p2p tx propagation
message.

Solution:
- change it to debug level
<hr>This is an automatic backport of pull request cometbft#4123 done by
[Mergify](https://mergify.com).

---------

Co-authored-by: yihuang <[email protected]>
Co-authored-by: Andy Nogueira <[email protected]>

.changelog/v0.38.8/improvements/4123-mempool-is-full-log.md

@@ -0,0 +1,2 @@
+- `[log]` Change "mempool is full" log to debug level
+  ([\#4123](https://github.com/cometbft/cometbft/pull/4123))

mempool/clist_mempool.go

@@ -415,7 +415,8 @@ func (mem *CListMempool) resCbFirstTime(
 			if err := mem.isFull(len(tx)); err != nil {
 				// remove from cache (mempool might have a space later)
 				mem.cache.Remove(tx)
-				mem.logger.Error(err.Error())
+				// use debug level to avoid spamming logs when traffic is high
+				mem.logger.Debug(err.Error())
 				mem.metrics.RejectedTxs.Add(1)
 				return
 			}

mempool/reactor.go

@@ -155,10 +155,16 @@ func (memR *Reactor) Receive(e p2p.Envelope) {
 		for _, tx := range protoTxs {
 			ntx := types.Tx(tx)
 			err = memR.mempool.CheckTx(ntx, nil, txInfo)
-			if errors.Is(err, ErrTxInCache) {
-				memR.Logger.Debug("Tx already exists in cache", "tx", ntx.String())
-			} else if err != nil {
-				memR.Logger.Info("Could not check tx", "tx", ntx.String(), "err", err)
+			if err != nil {
+				switch {
+				case errors.Is(err, ErrTxInCache):
+					memR.Logger.Debug("Tx already exists in cache", "tx", ntx.String())
+				case errors.As(err, &ErrMempoolIsFull{}):
+					// using debug level to avoid flooding when traffic is high
+					memR.Logger.Debug(err.Error())
+				default:
+					memR.Logger.Info("Could not check tx", "tx", ntx.String(), "err", err)
+				}
 			}
 		}
 	default: