cannon: Support F_GETFD cmd to fcntl (ethereum-optimism#12050)

* cannon: Support F_GETFD cmd to fcntl.

* cannon: Update fuzz test expectations.

* cannon: Update MIPS.t.sol

* cannon: Introduce a new state version for supporting get_fd.

Switches singlethreaded prestate to use .bin.gz instead of json since it now needs to detect the new state version.

* cannon: Don't override the cannon version.

* Update semver-lock.

* cannon: Update tests to detect old versions but only check writing and parsing for the currently supported versions.

* cannon: Load old version from cannon docker image

* cannon: Improve logging.

* cannon: Restore cannon version arg.

* Fix contrac semvers.

* cannon: Rename singlethreaded-getfd to just singlethreaded-2.

We could just go to using the state version number directly, but particularly the difference between singlethreaded and multithreaded feels useful to keep.

* cannon: Fix comment.

* Update semver again.

.circleci/config.yml

@@ -1058,7 +1058,7 @@ jobs:
           key: cannon-prestate-{{ checksum "./cannon/bin/cannon" }}-{{ checksum "op-program/bin/op-program-client.elf" }}
           name: Save Cannon prestate to cache
           paths:
-            - "op-program/bin/prestate.json"
+            - "op-program/bin/prestate.bin.gz"
             - "op-program/bin/meta.json"
             - "op-program/bin/prestate-proof.json"
       - run:
@@ -1079,7 +1079,7 @@ jobs:
       - persist_to_workspace:
           root: .
           paths:
-            - "op-program/bin/prestate.json"
+            - "op-program/bin/prestate.bin.gz"
             - "op-program/bin/meta.json"
             - "op-program/bin/prestate-proof.json"
 

Makefile

@@ -134,16 +134,16 @@ reproducible-prestate:   ## Builds reproducible-prestate binary
 .PHONY: reproducible-prestate
 
 # Include any files required for the devnet to build and run.
-DEVNET_CANNON_PRESTATE_FILES := op-program/bin/prestate-proof.json op-program/bin/prestate.json op-program/bin/prestate-proof-mt.json op-program/bin/prestate-mt.bin.gz
+DEVNET_CANNON_PRESTATE_FILES := op-program/bin/prestate-proof.json op-program/bin/prestate.bin.gz op-program/bin/prestate-proof-mt.json op-program/bin/prestate-mt.bin.gz
 
 
 $(DEVNET_CANNON_PRESTATE_FILES):
 	make cannon-prestate
 	make cannon-prestate-mt
 
 cannon-prestate: op-program cannon ## Generates prestate using cannon and op-program
-	./cannon/bin/cannon load-elf --type singlethreaded --path op-program/bin/op-program-client.elf --out op-program/bin/prestate.json --meta op-program/bin/meta.json
-	./cannon/bin/cannon run --proof-at '=0'  --stop-at '=1' --input op-program/bin/prestate.json --meta op-program/bin/meta.json --proof-fmt 'op-program/bin/%d.json' --output ""
+	./cannon/bin/cannon load-elf --type singlethreaded-2 --path op-program/bin/op-program-client.elf --out op-program/bin/prestate.bin.gz --meta op-program/bin/meta.json
+	./cannon/bin/cannon run --proof-at '=0'  --stop-at '=1' --input op-program/bin/prestate.bin.gz --meta op-program/bin/meta.json --proof-fmt 'op-program/bin/%d.json' --output ""
 	mv op-program/bin/0.json op-program/bin/prestate-proof.json
 .PHONY: cannon-prestate
 

cannon/.gitignore

@@ -7,9 +7,6 @@ venv
 *.log
 testdata/example/bin
 contracts/out
-state.json
-*.json
-*.json.gz
 *.pprof
 *.out
 bin

cannon/Makefile

@@ -19,7 +19,7 @@ cannon-impl:
 	env GO111MODULE=on GOOS=$(TARGETOS) GOARCH=$(TARGETARCH) go build -v $(LDFLAGS) -o ./bin/cannon-impl .
 
 cannon-embeds: cannon-impl
-	@cp bin/cannon-impl ./multicannon/embeds/cannon-0
+	@cp bin/cannon-impl ./multicannon/embeds/cannon-2
 	@cp bin/cannon-impl ./multicannon/embeds/cannon-1
 
 cannon: cannon-embeds

cannon/cmd/load_elf.go

@@ -69,7 +69,7 @@ func LoadELF(ctx *cli.Context) error {
 		return err
 	}
 	switch ver {
-	case versions.VersionSingleThreaded:
+	case versions.VersionSingleThreaded2:
 		createInitialState = func(f *elf.File) (mipsevm.FPVMState, error) {
 			return program.LoadELF(f, singlethreaded.CreateInitialState)
 		}

cannon/cmd/run.go

@@ -373,6 +373,7 @@ func Run(ctx *cli.Context) error {
 	if err != nil {
 		return fmt.Errorf("failed to load state: %w", err)
 	}
+	l.Info("Loaded input state", "version", state.Version)
 	vm := state.CreateVM(l, po, outLog, errLog, meta)
 	debugProgram := ctx.Bool(RunDebugFlag.Name)
 	if debugProgram {

cannon/mipsevm/exec/mips_syscalls.go

@@ -286,7 +286,15 @@ func HandleSysFcntl(a0, a1 uint32) (v0, v1 uint32) {
 	// args: a0 = fd, a1 = cmd
 	v1 = uint32(0)
 
-	if a1 == 3 { // F_GETFL: get file descriptor flags
+	if a1 == 1 { // F_GETFD: get file descriptor flags
+		switch a0 {
+		case FdStdin, FdStdout, FdStderr, FdPreimageRead, FdHintRead, FdPreimageWrite, FdHintWrite:
+			v0 = 0 // No flags set
+		default:
+			v0 = 0xFFffFFff
+			v1 = MipsEBADF
+		}
+	} else if a1 == 3 { // F_GETFL: get file status flags
 		switch a0 {
 		case FdStdin, FdPreimageRead, FdHintRead:
 			v0 = 0 // O_RDONLY

cannon/mipsevm/multithreaded/state.go

@@ -97,7 +97,6 @@ func CreateInitialState(pc, heapStart uint32) *State {
 }
 
 func (s *State) CreateVM(logger log.Logger, po mipsevm.PreimageOracle, stdOut, stdErr io.Writer, meta mipsevm.Metadata) mipsevm.FPVM {
-	logger.Info("Using cannon multithreaded VM")
 	return NewInstrumentedState(s, po, stdOut, stdErr, logger, meta)
 }
 

cannon/mipsevm/singlethreaded/state.go

@@ -69,7 +69,6 @@ func CreateInitialState(pc, heapStart uint32) *State {
 }
 
 func (s *State) CreateVM(logger log.Logger, po mipsevm.PreimageOracle, stdOut, stdErr io.Writer, meta mipsevm.Metadata) mipsevm.FPVM {
-	logger.Info("Using cannon VM")
 	return NewInstrumentedState(s, po, stdOut, stdErr, meta)
 }
 

cannon/mipsevm/tests/fuzz_evm_common_test.go

@@ -150,7 +150,17 @@ func FuzzStateSyscallFcntl(f *testing.F) {
 				expected.Step += 1
 				expected.PC = state.GetCpu().NextPC
 				expected.NextPC = state.GetCpu().NextPC + 4
-				if cmd == 3 {
+				if cmd == 1 {
+					switch fd {
+					case exec.FdStdin, exec.FdStdout, exec.FdStderr,
+						exec.FdPreimageRead, exec.FdHintRead, exec.FdPreimageWrite, exec.FdHintWrite:
+						expected.Registers[2] = 0
+						expected.Registers[7] = 0
+					default:
+						expected.Registers[2] = 0xFF_FF_FF_FF
+						expected.Registers[7] = exec.MipsEBADF
+					}
+				} else if cmd == 3 {
 					switch fd {
 					case exec.FdStdin, exec.FdPreimageRead, exec.FdHintRead:
 						expected.Registers[2] = 0

cannon/mipsevm/versions/detect.go

@@ -27,7 +27,7 @@ func DetectVersion(path string) (StateVersion, error) {
 	}
 
 	switch ver {
-	case VersionSingleThreaded, VersionMultiThreaded:
+	case VersionSingleThreaded, VersionMultiThreaded, VersionSingleThreaded2:
 		return ver, nil
 	default:
 		return 0, fmt.Errorf("%w: %d", ErrUnknownVersion, ver)

cannon/mipsevm/versions/detect_test.go

@@ -1,8 +1,10 @@
 package versions
 
 import (
+	"embed"
 	"os"
 	"path/filepath"
+	"strconv"
 	"testing"
 
 	"github.com/ethereum-optimism/optimism/cannon/mipsevm/multithreaded"
@@ -11,23 +13,46 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+const statesPath = "testdata/states"
+
+//go:embed testdata/states
+var historicStates embed.FS
+
 func TestDetectVersion(t *testing.T) {
-	t.Run("SingleThreadedJSON", func(t *testing.T) {
-		state, err := NewFromState(singlethreaded.CreateEmptyState())
+	testDetection := func(t *testing.T, version StateVersion, ext string) {
+		filename := strconv.Itoa(int(version)) + ext
+		dir := t.TempDir()
+		path := filepath.Join(dir, filename)
+		in, err := historicStates.ReadFile(filepath.Join(statesPath, filename))
 		require.NoError(t, err)
-		path := writeToFile(t, "state.json", state)
-		version, err := DetectVersion(path)
+		require.NoError(t, os.WriteFile(path, in, 0o644))
+
+		detectedVersion, err := DetectVersion(path)
 		require.NoError(t, err)
-		require.Equal(t, VersionSingleThreaded, version)
-	})
+		require.Equal(t, version, detectedVersion)
+	}
+	// Iterate all known versions to ensure we have a test case to detect every state version
+	for _, version := range StateVersionTypes {
+		version := version
+		t.Run(version.String(), func(t *testing.T) {
+			testDetection(t, version, ".bin.gz")
+		})
+
+		if version == VersionSingleThreaded {
+			t.Run(version.String()+".json", func(t *testing.T) {
+				testDetection(t, version, ".json")
+			})
+		}
+	}
 
+	// Additionally, check that the latest supported versions write new states in a way that is detected correctly
 	t.Run("SingleThreadedBinary", func(t *testing.T) {
 		state, err := NewFromState(singlethreaded.CreateEmptyState())
 		require.NoError(t, err)
 		path := writeToFile(t, "state.bin.gz", state)
 		version, err := DetectVersion(path)
 		require.NoError(t, err)
-		require.Equal(t, VersionSingleThreaded, version)
+		require.Equal(t, VersionSingleThreaded2, version)
 	})
 
 	t.Run("MultiThreadedBinary", func(t *testing.T) {

cannon/mipsevm/versions/state.go

@@ -19,14 +19,16 @@ const (
 	// VersionSingleThreaded is the version of the Cannon STF found in op-contracts/v1.6.0 - https://github.com/ethereum-optimism/optimism/blob/op-contracts/v1.6.0/packages/contracts-bedrock/src/cannon/MIPS.sol
 	VersionSingleThreaded StateVersion = iota
 	VersionMultiThreaded
+	// VersionSingleThreaded2 is based on VersionSingleThreaded with the addition of support for fcntl(F_GETFD) syscall
+	VersionSingleThreaded2
 )
 
 var (
 	ErrUnknownVersion   = errors.New("unknown version")
 	ErrJsonNotSupported = errors.New("json not supported")
 )
 
-var StateVersionTypes = []StateVersion{VersionSingleThreaded, VersionMultiThreaded}
+var StateVersionTypes = []StateVersion{VersionSingleThreaded, VersionMultiThreaded, VersionSingleThreaded2}
 
 func LoadStateFromFile(path string) (*VersionedState, error) {
 	if !serialize.IsBinaryFile(path) {
@@ -44,7 +46,7 @@ func NewFromState(state mipsevm.FPVMState) (*VersionedState, error) {
 	switch state := state.(type) {
 	case *singlethreaded.State:
 		return &VersionedState{
-			Version:   VersionSingleThreaded,
+			Version:   VersionSingleThreaded2,
 			FPVMState: state,
 		}, nil
 	case *multithreaded.State:
@@ -79,7 +81,7 @@ func (s *VersionedState) Deserialize(in io.Reader) error {
 	}
 
 	switch s.Version {
-	case VersionSingleThreaded:
+	case VersionSingleThreaded2:
 		state := &singlethreaded.State{}
 		if err := state.Deserialize(in); err != nil {
 			return err
@@ -113,6 +115,8 @@ func (s StateVersion) String() string {
 		return "singlethreaded"
 	case VersionMultiThreaded:
 		return "multithreaded"
+	case VersionSingleThreaded2:
+		return "singlethreaded-2"
 	default:
 		return "unknown"
 	}
@@ -124,6 +128,8 @@ func ParseStateVersion(ver string) (StateVersion, error) {
 		return VersionSingleThreaded, nil
 	case "multithreaded":
 		return VersionMultiThreaded, nil
+	case "singlethreaded-2":
+		return VersionSingleThreaded2, nil
 	default:
 		return StateVersion(0), errors.New("unknown state version")
 	}

cannon/mipsevm/versions/state_test.go

@@ -4,18 +4,19 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/ethereum-optimism/optimism/cannon/mipsevm"
 	"github.com/ethereum-optimism/optimism/cannon/mipsevm/multithreaded"
 	"github.com/ethereum-optimism/optimism/cannon/mipsevm/singlethreaded"
 	"github.com/ethereum-optimism/optimism/cannon/serialize"
 	"github.com/stretchr/testify/require"
 )
 
 func TestNewFromState(t *testing.T) {
-	t.Run("singlethreaded", func(t *testing.T) {
+	t.Run("singlethreaded-2", func(t *testing.T) {
 		actual, err := NewFromState(singlethreaded.CreateEmptyState())
 		require.NoError(t, err)
 		require.IsType(t, &singlethreaded.State{}, actual.FPVMState)
-		require.Equal(t, VersionSingleThreaded, actual.Version)
+		require.Equal(t, VersionSingleThreaded2, actual.Version)
 	})
 
 	t.Run("multithreaded", func(t *testing.T) {
@@ -27,16 +28,6 @@ func TestNewFromState(t *testing.T) {
 }
 
 func TestLoadStateFromFile(t *testing.T) {
-	t.Run("SinglethreadedFromJSON", func(t *testing.T) {
-		expected, err := NewFromState(singlethreaded.CreateEmptyState())
-		require.NoError(t, err)
-
-		path := writeToFile(t, "state.json", expected)
-		actual, err := LoadStateFromFile(path)
-		require.NoError(t, err)
-		require.Equal(t, expected, actual)
-	})
-
 	t.Run("SinglethreadedFromBinary", func(t *testing.T) {
 		expected, err := NewFromState(singlethreaded.CreateEmptyState())
 		require.NoError(t, err)
@@ -58,14 +49,26 @@ func TestLoadStateFromFile(t *testing.T) {
 	})
 }
 
-func TestMultithreadedDoesNotSupportJSON(t *testing.T) {
-	state, err := NewFromState(multithreaded.CreateEmptyState())
-	require.NoError(t, err)
+func TestVersionsOtherThanZeroDoNotSupportJSON(t *testing.T) {
+	tests := []struct {
+		version     StateVersion
+		createState func() mipsevm.FPVMState
+	}{
+		{VersionSingleThreaded2, func() mipsevm.FPVMState { return singlethreaded.CreateEmptyState() }},
+		{VersionMultiThreaded, func() mipsevm.FPVMState { return multithreaded.CreateEmptyState() }},
+	}
+	for _, test := range tests {
+		test := test
+		t.Run(test.version.String(), func(t *testing.T) {
+			state, err := NewFromState(test.createState())
+			require.NoError(t, err)
 
-	dir := t.TempDir()
-	path := filepath.Join(dir, "test.json")
-	err = serialize.Write(path, state, 0o644)
-	require.ErrorIs(t, err, ErrJsonNotSupported)
+			dir := t.TempDir()
+			path := filepath.Join(dir, "test.json")
+			err = serialize.Write(path, state, 0o644)
+			require.ErrorIs(t, err, ErrJsonNotSupported)
+		})
+	}
 }
 
 func writeToFile(t *testing.T, filename string, data serialize.Serializable) string {

cannon/mipsevm/versions/testdata/states/0.json

@@ -0,0 +1,48 @@
+{
+  "memory": [],
+  "preimageKey": "0x0000000000000000000000000000000000000000000000000000000000000000",
+  "preimageOffset": 0,
+  "pc": 0,
+  "nextPC": 4,
+  "lo": 0,
+  "hi": 0,
+  "heap": 0,
+  "exit": 0,
+  "exited": false,
+  "step": 0,
+  "registers": [
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0
+  ]
+}
+

cannon/multicannon/exec.go

@@ -21,7 +21,7 @@ const baseDir = "embeds"
 
 func ExecuteCannon(ctx context.Context, args []string, ver versions.StateVersion) error {
 	switch ver {
-	case versions.VersionSingleThreaded, versions.VersionMultiThreaded:
+	case versions.VersionSingleThreaded, versions.VersionSingleThreaded2, versions.VersionMultiThreaded:
 	default:
 		return errors.New("unsupported version")
 	}

op-challenger/README.md

@@ -43,7 +43,7 @@ DISPUTE_GAME_FACTORY=$(jq -r .DisputeGameFactoryProxy .devnet/addresses.json)
   --cannon-l2-genesis .devnet/genesis-l2.json \
   --cannon-bin ./cannon/bin/cannon \
   --cannon-server ./op-program/bin/op-program \
-  --cannon-prestate ./op-program/bin/prestate.json \
+  --cannon-prestate ./op-program/bin/prestate.bin.gz \
   --l2-eth-rpc http://localhost:9545 \
   --mnemonic "test test test test test test test test test test test junk" \
   --hd-path "m/44'/60'/0'/0/8" \