fuzz: adds logging_no_diff target (tetratelabs#2077)

Signed-off-by: Takeshi Yoneda <[email protected]>

internal/integration_test/fuzz/README.md

@@ -14,6 +14,8 @@ Currently, we have the following fuzzing targets:
 - `no_diff`: compares the results from the compiler and interpreter engines, and see if there's a diff in them.
 - `memory_no_diff`: same as `no_diff` except that in addition to the results, it also compares the entire memory buffer between engines to ensure the consistency around memory access.
   Therefore, this takes much longer than `no_diff`.
+- `logging_no_diff`: same as `no_diff` except that in addition to the results, it also compares the entire logging filter result between engines to ensure the consistency around function calls.
+  Therefore, this takes much longer than `no_diff`.
 - `validation`: try compiling maybe-invalid Wasm module binaries. This is to ensure that our validation phase works correctly as well as the engines do not panic during compilation.
 
 

internal/integration_test/fuzz/fuzz/Cargo.toml

@@ -32,3 +32,9 @@ name = "no_diff"
 path = "fuzz_targets/no_diff.rs"
 test = false
 doc = false
+
+[[bin]]
+name = "logging_no_diff"
+path = "fuzz_targets/logging_no_diff.rs"
+test = false
+doc = false

internal/integration_test/fuzz/fuzz/fuzz_targets/logging_no_diff.rs

@@ -0,0 +1,7 @@
+#![no_main]
+use libfuzzer_sys::fuzz_target;
+mod util;
+
+fuzz_target!(|data: &[u8]| {
+    let _ = util::run_nodiff(data, false, true);
+});

internal/integration_test/fuzz/fuzz/fuzz_targets/memory_no_diff.rs

@@ -1,63 +1,7 @@
-//! memory_diff fuzzing test is almost the same as basic.rs
-//! but this also ensure that there's no difference in the memory buffer
-//! state after the execution.
-
 #![no_main]
-use libfuzzer_sys::arbitrary::{Result, Unstructured};
 use libfuzzer_sys::fuzz_target;
-use wasm_smith::SwarmConfig;
-mod wazero_abi;
+mod util;
 
 fuzz_target!(|data: &[u8]| {
-    let _ = run(data);
+    let _ = util::run_nodiff(data, true, false);
 });
-
-fn run(data: &[u8]) -> Result<()> {
-    // Create the random source.
-    let mut u = Unstructured::new(data);
-
-    // Generate the configuration.
-    let mut config: SwarmConfig = u.arbitrary()?;
-
-    // 64-bit memory won't be supported by wazero.
-    config.memory64_enabled = false;
-    // For exactly one memory exists.
-    config.max_memories = 1;
-    config.min_memories = 1;
-    // If we don't set the limit, we will soon reach the OOM and the fuzzing will be killed by OS.
-    config.max_memory_pages = 10;
-    config.memory_max_size_required = true;
-    // Don't test too large tables.
-    config.max_tables = 2;
-    config.max_table_elements = 1_000;
-    config.table_max_size_required = true;
-
-    // max_instructions is set to 100 by default which seems a little bit smaller.
-    config.max_instructions = 5000;
-
-    // Without canonicalization of NaNs, the results cannot be matched among engines.
-    config.canonicalize_nans = true;
-
-    // Export all the things so that we can invoke them.
-    config.export_everything = true;
-
-    // Ensures that at least one function exists.
-    config.min_funcs = 1;
-    config.max_funcs = config.max_funcs.max(1);
-
-    // TODO: enable after threads support in wazevo.
-    config.threads_enabled = false;
-
-    // Generate the random module via wasm-smith.
-    let mut module = wasm_smith::Module::new(config.clone(), &mut u)?;
-    module.ensure_termination(1000);
-    let module_bytes = module.to_bytes();
-
-    // Pass the randomly generated module to the wazero library.
-    unsafe {
-        wazero_abi::require_no_diff(module_bytes.as_ptr(), module_bytes.len(), true);
-    }
-
-    // We always return Ok as inside require_no_diff, we cause panic if the binary is interesting.
-    Ok(())
-}

internal/integration_test/fuzz/fuzz/fuzz_targets/no_diff.rs

@@ -1,61 +1,7 @@
 #![no_main]
-
-use libfuzzer_sys::arbitrary::{Result, Unstructured};
 use libfuzzer_sys::fuzz_target;
-use wasm_smith::SwarmConfig;
-
-mod wazero_abi;
+mod util;
 
 fuzz_target!(|data: &[u8]| {
-    let _ = run(data);
+    let _ = util::run_nodiff(data, false, false);
 });
-
-fn run(data: &[u8]) -> Result<()> {
-    // Create the random source.
-    let mut u = Unstructured::new(data);
-
-    // Generate the configuration.
-    let mut config: SwarmConfig = u.arbitrary()?;
-
-    // 64-bit memory won't be supported by wazero.
-    config.memory64_enabled = false;
-    // Also, multiple memories are not supported.
-    config.max_memories = 1;
-    config.max_imports = 10;
-    // If we don't set the limit, we will soon reach the OOM and the fuzzing will be killed by OS.
-    config.max_memory_pages = 10;
-    config.memory_max_size_required = true;
-    // Don't test too large tables.
-    config.max_tables = 2;
-    config.max_table_elements = 1_000;
-    config.table_max_size_required = true;
-
-    // max_instructions is set to 100 by default which seems a little bit smaller.
-    config.max_instructions = 5000;
-
-    // Without canonicalization of NaNs, the results cannot be matched among engines.
-    config.canonicalize_nans = true;
-
-    // Export all the things so that we can invoke them.
-    config.export_everything = true;
-
-    // Ensures that at least one function exists.
-    config.min_funcs = 1;
-    config.max_funcs = config.max_funcs.max(1);
-
-    // TODO: enable after threads support in wazevo.
-    config.threads_enabled = false;
-
-    // Generate the random module via wasm-smith.
-    let mut module = wasm_smith::Module::new(config.clone(), &mut u)?;
-    module.ensure_termination(1000);
-    let module_bytes = module.to_bytes();
-
-    // Pass the randomly generated module to the wazero library.
-    unsafe {
-        wazero_abi::require_no_diff(module_bytes.as_ptr(), module_bytes.len(), false);
-    }
-
-    // We always return Ok as inside require_no_diff, we cause panic if the binary is interesting.
-    Ok(())
-}

internal/integration_test/fuzz/fuzz/fuzz_targets/wazero_abi.rs → internal/integration_test/fuzz/fuzz/fuzz_targets/util.rs

@@ -2,8 +2,12 @@
 
 extern "C" {
     // require_no_diff is implemented in Go, and accepts the pointer to the binary and its size.
-    #[allow(dead_code)]
-    pub fn require_no_diff(binary_ptr: *const u8, binary_size: usize, check_memory: bool);
+    pub fn require_no_diff(
+        binary_ptr: *const u8,
+        binary_size: usize,
+        check_memory: bool,
+        check_logging: bool,
+    );
 
     // validate is implemented in Go, and accepts the pointer to the binary and its size.
     #[allow(dead_code)]
@@ -12,9 +16,11 @@ extern "C" {
 
 use ctor::ctor;
 use libc::SIGSTKSZ;
+use libfuzzer_sys::arbitrary::Unstructured;
 use nix::libc::{sigaltstack, stack_t};
 use nix::sys::signal::{sigaction, SaFlags, SigAction, SigHandler, SigSet, Signal};
 use std::ptr::null_mut;
+use wasm_smith::SwarmConfig;
 
 #[ctor]
 /// Sets up the separate stack for signal handlers, and sets the SA_ONSTACK flag for signals that are handled by libFuzzer
@@ -75,3 +81,61 @@ fn set_signal_stack() {
         std::mem::forget(stack);
     }
 }
+
+#[allow(dead_code)]
+pub fn run_nodiff(
+    data: &[u8],
+    check_memory: bool,
+    check_logging: bool,
+) -> libfuzzer_sys::arbitrary::Result<()> {
+    // Create the random source.
+    let mut u = Unstructured::new(data);
+
+    // Generate the configuration.
+    let mut config: SwarmConfig = u.arbitrary()?;
+
+    // 64-bit memory won't be supported by wazero.
+    config.memory64_enabled = false;
+    // For exactly one memory exists.
+    config.max_memories = 1;
+    config.min_memories = 1;
+    // If we don't set the limit, we will soon reach the OOM and the fuzzing will be killed by OS.
+    config.max_memory_pages = 10;
+    config.memory_max_size_required = true;
+    // Don't test too large tables.
+    config.max_tables = 2;
+    config.max_table_elements = 1_000;
+    config.table_max_size_required = true;
+
+    // max_instructions is set to 100 by default which seems a little bit smaller.
+    config.max_instructions = 5000;
+
+    // Without canonicalization of NaNs, the results cannot be matched among engines.
+    config.canonicalize_nans = true;
+
+    // Export all the things so that we can invoke them.
+    config.export_everything = true;
+
+    // Ensures that at least one function exists.
+    config.min_funcs = 1;
+    config.max_funcs = config.max_funcs.max(1);
+
+    // TODO: enable after threads support in wazevo.
+    config.threads_enabled = false;
+
+    // Generate the random module via wasm-smith.
+    let mut module = wasm_smith::Module::new(config.clone(), &mut u)?;
+    module.ensure_termination(1000);
+    let module_bytes = module.to_bytes();
+
+    // Pass the randomly generated module to the wazero library.
+    unsafe {
+        require_no_diff(
+            module_bytes.as_ptr(),
+            module_bytes.len(),
+            check_memory,
+            check_logging,
+        );
+    }
+    Ok(())
+}

internal/integration_test/fuzz/fuzz/fuzz_targets/validation.rs

@@ -3,7 +3,7 @@
 use libfuzzer_sys::arbitrary::{Result, Unstructured};
 use libfuzzer_sys::fuzz_target;
 
-mod wazero_abi;
+mod util;
 
 fuzz_target!(|data: &[u8]| {
     let _ = run(data);
@@ -18,7 +18,7 @@ fn run(data: &[u8]) -> Result<()> {
     let module_bytes = module.to_bytes();
 
     unsafe {
-        wazero_abi::validate(module_bytes.as_ptr(), module_bytes.len());
+        util::validate(module_bytes.as_ptr(), module_bytes.len());
     }
 
     // We always return Ok as inside validate, we cause panic if the binary is interesting.

internal/integration_test/fuzz/wazerolib/extern.go

@@ -21,7 +21,7 @@ func main() {}
 // And if there's diff, this also saves the problematic binary and wat into testdata directory.
 //
 //export require_no_diff
-func require_no_diff(binaryPtr uintptr, binarySize int, checkMemory bool) {
+func require_no_diff(binaryPtr uintptr, binarySize int, checkMemory bool, checkLogging bool) {
 	// TODO: use unsafe.Slice after flooring Go 1.20.
 	var wasmBin []byte
 	wasmHdr := (*reflect.SliceHeader)(unsafe.Pointer(&wasmBin))
@@ -37,7 +37,7 @@ func require_no_diff(binaryPtr uintptr, binarySize int, checkMemory bool) {
 		}
 	}()
 
-	requireNoDiff(wasmBin, checkMemory, func(err error) {
+	requireNoDiff(wasmBin, checkMemory, checkLogging, func(err error) {
 		if err != nil {
 			panic(err)
 		}

internal/integration_test/fuzz/wazerolib/nodiff.go

@@ -12,6 +12,7 @@ import (
 	"github.com/tetratelabs/wazero"
 	"github.com/tetratelabs/wazero/api"
 	"github.com/tetratelabs/wazero/experimental"
+	"github.com/tetratelabs/wazero/experimental/logging"
 	"github.com/tetratelabs/wazero/experimental/opt"
 	"github.com/tetratelabs/wazero/internal/testing/binaryencoding"
 	"github.com/tetratelabs/wazero/internal/wasm"
@@ -42,24 +43,37 @@ func extractInternalWasmModuleFromCompiledModule(c wazero.CompiledModule) (*wasm
 }
 
 // requireNoDiff ensures that the behavior is the same between the compiler and the interpreter for any given binary.
-func requireNoDiff(wasmBin []byte, checkMemory bool, requireNoError func(err error)) {
-	// Choose the context to use for function calls.
-	ctx := context.Background()
-
+func requireNoDiff(wasmBin []byte, checkMemory, loggingCheck bool, requireNoError func(err error)) {
 	const features = api.CoreFeaturesV2 | experimental.CoreFeaturesThreads
-	compiler := wazero.NewRuntimeWithConfig(ctx, opt.NewRuntimeConfigOptimizingCompiler().WithCoreFeatures(features))
-	interpreter := wazero.NewRuntimeWithConfig(ctx, wazero.NewRuntimeConfigInterpreter().WithCoreFeatures(features))
-	defer compiler.Close(ctx)
-	defer interpreter.Close(ctx)
+	compiler := wazero.NewRuntimeWithConfig(context.Background(), opt.NewRuntimeConfigOptimizingCompiler().WithCoreFeatures(features))
+	interpreter := wazero.NewRuntimeWithConfig(context.Background(), wazero.NewRuntimeConfigInterpreter().WithCoreFeatures(features))
+	defer compiler.Close(context.Background())
+	defer interpreter.Close(context.Background())
+
+	interpreterCtx, compilerCtx := context.Background(), context.Background()
+	var interPreterLoggingBuf, compilerLoggingBuf bytes.Buffer
+	var errorDuringInvocation bool
+	if loggingCheck {
+		interpreterCtx = context.WithValue(interpreterCtx, experimental.FunctionListenerFactoryKey{}, logging.NewLoggingListenerFactory(&interPreterLoggingBuf))
+		compilerCtx = context.WithValue(compilerCtx, experimental.FunctionListenerFactoryKey{}, logging.NewLoggingListenerFactory(&compilerLoggingBuf))
+		defer func() {
+			if !errorDuringInvocation {
+				if !bytes.Equal(compilerLoggingBuf.Bytes(), interPreterLoggingBuf.Bytes()) {
+					requireNoError(fmt.Errorf("logging mismatch\ncompiler: %s\ninterpreter: %s",
+						compilerLoggingBuf.String(), interPreterLoggingBuf.String()))
+				}
+			}
+		}()
+	}
 
-	compilerCompiled, err := compiler.CompileModule(ctx, wasmBin)
+	compilerCompiled, err := compiler.CompileModule(compilerCtx, wasmBin)
 	if err != nil && strings.Contains(err.Error(), "has an empty module name") {
 		// This is the limitation wazero imposes to allow special-casing of anonymous modules.
 		return
 	}
 	requireNoError(err)
 
-	interpreterCompiled, err := interpreter.CompileModule(ctx, wasmBin)
+	interpreterCompiled, err := interpreter.CompileModule(interpreterCtx, wasmBin)
 	requireNoError(err)
 
 	internalMod, err := extractInternalWasmModuleFromCompiledModule(compilerCompiled)
@@ -71,16 +85,18 @@ func requireNoDiff(wasmBin []byte, checkMemory bool, requireNoError func(err err
 	ensureDummyImports(interpreter, internalMod, requireNoError)
 
 	// Instantiate module.
-	compilerMod, compilerInstErr := compiler.InstantiateModule(ctx, compilerCompiled,
+	compilerMod, compilerInstErr := compiler.InstantiateModule(compilerCtx, compilerCompiled,
 		wazero.NewModuleConfig().WithName(string(internalMod.ID[:])))
-	interpreterMod, interpreterInstErr := interpreter.InstantiateModule(ctx, interpreterCompiled,
+	interpreterMod, interpreterInstErr := interpreter.InstantiateModule(interpreterCtx, interpreterCompiled,
 		wazero.NewModuleConfig().WithName(string(internalMod.ID[:])))
 
 	okToInvoke, err := ensureInstantiationError(compilerInstErr, interpreterInstErr)
 	requireNoError(err)
 
 	if okToInvoke {
-		err = ensureInvocationResultMatch(compilerMod, interpreterMod, interpreterCompiled.ExportedFunctions())
+		err, errorDuringInvocation = ensureInvocationResultMatch(
+			compilerCtx, interpreterCtx,
+			compilerMod, interpreterMod, interpreterCompiled.ExportedFunctions())
 		requireNoError(err)
 
 		compilerMem, _ := compilerMod.Memory().(*wasm.MemoryInstance)
@@ -248,9 +264,10 @@ func ensureDummyImports(r wazero.Runtime, origin *wasm.Module, requireNoError fu
 const valueTypeVector = 0x7b
 
 // ensureInvocationResultMatch invokes all the exported functions from the module, and compare all the results between compiler vs interpreter.
-func ensureInvocationResultMatch(compiledMod, interpreterMod api.Module, exportedFunctions map[string]api.FunctionDefinition) (err error) {
-	ctx := context.Background()
-
+func ensureInvocationResultMatch(
+	compilerCtx, interpreterCtx context.Context, compiledMod, interpreterMod api.Module,
+	exportedFunctions map[string]api.FunctionDefinition,
+) (err error, errorDuringInvocation bool) {
 	// In order to do the deterministic execution, we need to sort the exported functions.
 	var names []string
 	for f := range exportedFunctions {
@@ -275,8 +292,9 @@ outer:
 		intF := interpreterMod.ExportedFunction(name)
 
 		params := getDummyValues(def.ParamTypes())
-		cmpRes, cmpErr := cmpF.Call(ctx, params...)
-		intRes, intErr := intF.Call(ctx, params...)
+		cmpRes, cmpErr := cmpF.Call(compilerCtx, params...)
+		intRes, intErr := intF.Call(interpreterCtx, params...)
+		errorDuringInvocation = errorDuringInvocation || cmpErr != nil || intErr != nil
 		if errMismatch := ensureInvocationError(cmpErr, intErr); errMismatch != nil {
 			panic(fmt.Sprintf("error mismatch on invoking %s: %v", name, errMismatch))
 		}

internal/integration_test/fuzz/wazerolib/nodiff_test.go

@@ -17,7 +17,7 @@ func TestReRunFailedRequireNoDiffCase(t *testing.T) {
 		t.Skip(err)
 	}
 
-	requireNoDiff(wasmBin, true, func(err error) { require.NoError(t, err) })
+	requireNoDiff(wasmBin, true, true, func(err error) { require.NoError(t, err) })
 }
 
 func Test_ensureMutableGlobalsMatch(t *testing.T) {