add witness stage (erigontech#2)

* feat: initial setup for witness stage

* move witness util functions from cmd/stateless

* persist witness

* remove MaxGetProofRewindBlockCount

* fix issues with witness generation

* fix mock sentry

* add flags for witness stage and max limit

* handle batch witness generation and refactor code

* handle mem batch rollback

* delete witness if overlimit

* add debug logs

* parameterize regenerate hash

* make common functions public

* refactor eth_getWitness to consume common functions for witness generation

* add logs

* handle change in witness.limit in between

* fix

* abstract more initialisations into common functions

* load chain config correctly

* fix typecast

* start witness generation from current block instead of past

cmd/utils/flags.go

@@ -940,6 +940,18 @@ var (
 		Usage: "set the cors' allow origins",
 		Value: cli.NewStringSlice(),
 	}
+
+	// Witness stage
+	EnableWitnessGenerationFlag = cli.BoolFlag{
+		Name:  "witness",
+		Usage: "Enables the witness generation stage",
+		Value: true,
+	}
+	MaxWitnessLimitFlag = cli.Uint64Flag{
+		Name:  "witness.limit",
+		Usage: "Maximum number of witness allowed to store at a time",
+		Value: 10_000,
+	}
 )
 
 var MetricFlags = []cli.Flag{&MetricsEnabledFlag, &MetricsHTTPFlag, &MetricsPortFlag}
@@ -1594,6 +1606,11 @@ func setSilkworm(ctx *cli.Context, cfg *ethconfig.Config) {
 	cfg.SilkwormSentry = ctx.Bool(SilkwormSentryFlag.Name)
 }
 
+func setWitness(ctx *cli.Context, cfg *ethconfig.Config) {
+	cfg.EnableWitnessGeneration = ctx.Bool(EnableWitnessGenerationFlag.Name)
+	cfg.MaxWitnessLimit = ctx.Uint64(MaxWitnessLimitFlag.Name)
+}
+
 // CheckExclusive verifies that only a single instance of the provided flags was
 // set by the user. Each flag might optionally be followed by a string type to
 // specialize it further.
@@ -1715,6 +1732,7 @@ func SetEthConfig(ctx *cli.Context, nodeConfig *nodecfg.Config, cfg *ethconfig.C
 		log.Error("Failed to set beacon API", "err", err)
 	}
 	setCaplin(ctx, cfg)
+	setWitness(ctx, cfg)
 
 	cfg.Ethstats = ctx.String(EthStatsURLFlag.Name)
 	cfg.HistoryV3 = ctx.Bool(HistoryV3Flag.Name)

core/state/database.go

@@ -685,7 +685,7 @@ func (tds *TrieDbState) resolveAccountAndStorageTouches(accountTouches common.Ha
 	}
 	if err := tds.t.HookSubTries(subTries, hooks); err != nil {
 		for i, hash := range subTries.Hashes {
-			log.Error("Info for error", "dbPrefix", fmt.Sprintf("%x", dbPrefixes[i]), "fixedbits", fixedbits[i], "hash", hash)
+			log.Error("Info for error", "dbPrefix", fmt.Sprintf("%x", dbPrefixes[i]), "fixedbits", fixedbits[i], "hash", hash, "err", err)
 		}
 		return err
 	}

erigon-lib/kv/tables.go

@@ -232,7 +232,7 @@ Invariants:
 const TrieOfAccounts = "TrieAccount"
 const TrieOfStorage = "TrieStorage"
 const IntermediateTrieHash = "IntermediateTrieHash"
-const Witnesses = "witnesses"
+const Witnesses = "witnesses" // block_num_u64 + "_chunk_" + chunk_num_u64 -> witness
 
 // Mapping [block number] => [Verkle Root]
 const VerkleRoots = "VerkleRoots"

eth/ethconfig/config.go

@@ -230,6 +230,13 @@ type Config struct {
 	//  New DB and Snapshots format of history allows: parallel blocks execution, get state as of given transaction without executing whole block.",
 	HistoryV3 bool
 
+	// EnableWitnessGeneration if true enables the witness generation stage post block execution
+	EnableWitnessGeneration bool
+
+	// MaxWitnessLimit denotes the maximum number of witness allowed to be stored at a time. As new
+	// witnesses are added, the older ones are pruned to maintain the limit.
+	MaxWitnessLimit uint64
+
 	// URL to connect to Heimdall node
 	HeimdallURL string
 

eth/stagedsync/default_stages.go

@@ -20,6 +20,7 @@ func DefaultStages(ctx context.Context,
 	bodies BodiesCfg,
 	senders SendersCfg,
 	exec ExecuteBlockCfg,
+	witness WitnessCfg,
 	hashState HashStateCfg,
 	trieCfg TrieCfg,
 	history HistoryCfg,
@@ -130,6 +131,19 @@ func DefaultStages(ctx context.Context,
 				return PruneExecutionStage(p, tx, exec, ctx, firstCycle)
 			},
 		},
+		{
+			ID:          stages.Witness,
+			Description: "Generate witness of the executed block",
+			Forward: func(firstCycle bool, badBlockUnwind bool, s *StageState, u Unwinder, txc wrap.TxContainer, logger log.Logger) error {
+				return SpawnWitnessStage(s, txc.Tx, witness, ctx, logger)
+			},
+			Unwind: func(firstCycle bool, u *UnwindState, s *StageState, txc wrap.TxContainer, logger log.Logger) error {
+				return UnwindWitnessStage()
+			},
+			Prune: func(firstCycle bool, p *PruneState, tx kv.RwTx, logger log.Logger) error {
+				return PruneWitnessStage()
+			},
+		},
 		{
 			ID:          stages.HashState,
 			Description: "Hash the key in the state",