From 356379b6fbb2fd7f462ab7581cff859625008bf8 Mon Sep 17 00:00:00 2001 From: Takeshi Yoneda Date: Tue, 20 Feb 2024 10:56:54 +0900 Subject: [PATCH] fuzz: adds logging_no_diff target (#2077) Signed-off-by: Takeshi Yoneda --- internal/integration_test/fuzz/README.md | 2 + .../integration_test/fuzz/fuzz/Cargo.toml | 6 ++ .../fuzz/fuzz/fuzz_targets/logging_no_diff.rs | 7 ++ .../fuzz/fuzz/fuzz_targets/memory_no_diff.rs | 60 +--------------- .../fuzz/fuzz/fuzz_targets/no_diff.rs | 58 +--------------- .../fuzz_targets/{wazero_abi.rs => util.rs} | 68 ++++++++++++++++++- .../fuzz/fuzz/fuzz_targets/validation.rs | 4 +- .../integration_test/fuzz/wazerolib/extern.go | 4 +- .../integration_test/fuzz/wazerolib/nodiff.go | 54 ++++++++++----- .../fuzz/wazerolib/nodiff_test.go | 2 +- 10 files changed, 126 insertions(+), 139 deletions(-) create mode 100644 internal/integration_test/fuzz/fuzz/fuzz_targets/logging_no_diff.rs rename internal/integration_test/fuzz/fuzz/fuzz_targets/{wazero_abi.rs => util.rs} (56%) diff --git a/internal/integration_test/fuzz/README.md b/internal/integration_test/fuzz/README.md index fe03073b51..ef28f58b8a 100644 --- a/internal/integration_test/fuzz/README.md +++ b/internal/integration_test/fuzz/README.md @@ -14,6 +14,8 @@ Currently, we have the following fuzzing targets: - `no_diff`: compares the results from the compiler and interpreter engines, and see if there's a diff in them. - `memory_no_diff`: same as `no_diff` except that in addition to the results, it also compares the entire memory buffer between engines to ensure the consistency around memory access. Therefore, this takes much longer than `no_diff`. +- `logging_no_diff`: same as `no_diff` except that in addition to the results, it also compares the entire logging filter result between engines to ensure the consistency around function calls. + Therefore, this takes much longer than `no_diff`. - `validation`: try compiling maybe-invalid Wasm module binaries. This is to ensure that our validation phase works correctly as well as the engines do not panic during compilation. diff --git a/internal/integration_test/fuzz/fuzz/Cargo.toml b/internal/integration_test/fuzz/fuzz/Cargo.toml index ade3d42b11..561814279e 100644 --- a/internal/integration_test/fuzz/fuzz/Cargo.toml +++ b/internal/integration_test/fuzz/fuzz/Cargo.toml @@ -32,3 +32,9 @@ name = "no_diff" path = "fuzz_targets/no_diff.rs" test = false doc = false + +[[bin]] +name = "logging_no_diff" +path = "fuzz_targets/logging_no_diff.rs" +test = false +doc = false diff --git a/internal/integration_test/fuzz/fuzz/fuzz_targets/logging_no_diff.rs b/internal/integration_test/fuzz/fuzz/fuzz_targets/logging_no_diff.rs new file mode 100644 index 0000000000..30054b4187 --- /dev/null +++ b/internal/integration_test/fuzz/fuzz/fuzz_targets/logging_no_diff.rs @@ -0,0 +1,7 @@ +#![no_main] +use libfuzzer_sys::fuzz_target; +mod util; + +fuzz_target!(|data: &[u8]| { + let _ = util::run_nodiff(data, false, true); +}); diff --git a/internal/integration_test/fuzz/fuzz/fuzz_targets/memory_no_diff.rs b/internal/integration_test/fuzz/fuzz/fuzz_targets/memory_no_diff.rs index 35aa4cb732..a64bb314e5 100644 --- a/internal/integration_test/fuzz/fuzz/fuzz_targets/memory_no_diff.rs +++ b/internal/integration_test/fuzz/fuzz/fuzz_targets/memory_no_diff.rs @@ -1,63 +1,7 @@ -//! memory_diff fuzzing test is almost the same as basic.rs -//! but this also ensure that there's no difference in the memory buffer -//! state after the execution. - #![no_main] -use libfuzzer_sys::arbitrary::{Result, Unstructured}; use libfuzzer_sys::fuzz_target; -use wasm_smith::SwarmConfig; -mod wazero_abi; +mod util; fuzz_target!(|data: &[u8]| { - let _ = run(data); + let _ = util::run_nodiff(data, true, false); }); - -fn run(data: &[u8]) -> Result<()> { - // Create the random source. - let mut u = Unstructured::new(data); - - // Generate the configuration. - let mut config: SwarmConfig = u.arbitrary()?; - - // 64-bit memory won't be supported by wazero. - config.memory64_enabled = false; - // For exactly one memory exists. - config.max_memories = 1; - config.min_memories = 1; - // If we don't set the limit, we will soon reach the OOM and the fuzzing will be killed by OS. - config.max_memory_pages = 10; - config.memory_max_size_required = true; - // Don't test too large tables. - config.max_tables = 2; - config.max_table_elements = 1_000; - config.table_max_size_required = true; - - // max_instructions is set to 100 by default which seems a little bit smaller. - config.max_instructions = 5000; - - // Without canonicalization of NaNs, the results cannot be matched among engines. - config.canonicalize_nans = true; - - // Export all the things so that we can invoke them. - config.export_everything = true; - - // Ensures that at least one function exists. - config.min_funcs = 1; - config.max_funcs = config.max_funcs.max(1); - - // TODO: enable after threads support in wazevo. - config.threads_enabled = false; - - // Generate the random module via wasm-smith. - let mut module = wasm_smith::Module::new(config.clone(), &mut u)?; - module.ensure_termination(1000); - let module_bytes = module.to_bytes(); - - // Pass the randomly generated module to the wazero library. - unsafe { - wazero_abi::require_no_diff(module_bytes.as_ptr(), module_bytes.len(), true); - } - - // We always return Ok as inside require_no_diff, we cause panic if the binary is interesting. - Ok(()) -} diff --git a/internal/integration_test/fuzz/fuzz/fuzz_targets/no_diff.rs b/internal/integration_test/fuzz/fuzz/fuzz_targets/no_diff.rs index 0a046b91ac..8f8ab8e991 100644 --- a/internal/integration_test/fuzz/fuzz/fuzz_targets/no_diff.rs +++ b/internal/integration_test/fuzz/fuzz/fuzz_targets/no_diff.rs @@ -1,61 +1,7 @@ #![no_main] - -use libfuzzer_sys::arbitrary::{Result, Unstructured}; use libfuzzer_sys::fuzz_target; -use wasm_smith::SwarmConfig; - -mod wazero_abi; +mod util; fuzz_target!(|data: &[u8]| { - let _ = run(data); + let _ = util::run_nodiff(data, false, false); }); - -fn run(data: &[u8]) -> Result<()> { - // Create the random source. - let mut u = Unstructured::new(data); - - // Generate the configuration. - let mut config: SwarmConfig = u.arbitrary()?; - - // 64-bit memory won't be supported by wazero. - config.memory64_enabled = false; - // Also, multiple memories are not supported. - config.max_memories = 1; - config.max_imports = 10; - // If we don't set the limit, we will soon reach the OOM and the fuzzing will be killed by OS. - config.max_memory_pages = 10; - config.memory_max_size_required = true; - // Don't test too large tables. - config.max_tables = 2; - config.max_table_elements = 1_000; - config.table_max_size_required = true; - - // max_instructions is set to 100 by default which seems a little bit smaller. - config.max_instructions = 5000; - - // Without canonicalization of NaNs, the results cannot be matched among engines. - config.canonicalize_nans = true; - - // Export all the things so that we can invoke them. - config.export_everything = true; - - // Ensures that at least one function exists. - config.min_funcs = 1; - config.max_funcs = config.max_funcs.max(1); - - // TODO: enable after threads support in wazevo. - config.threads_enabled = false; - - // Generate the random module via wasm-smith. - let mut module = wasm_smith::Module::new(config.clone(), &mut u)?; - module.ensure_termination(1000); - let module_bytes = module.to_bytes(); - - // Pass the randomly generated module to the wazero library. - unsafe { - wazero_abi::require_no_diff(module_bytes.as_ptr(), module_bytes.len(), false); - } - - // We always return Ok as inside require_no_diff, we cause panic if the binary is interesting. - Ok(()) -} diff --git a/internal/integration_test/fuzz/fuzz/fuzz_targets/wazero_abi.rs b/internal/integration_test/fuzz/fuzz/fuzz_targets/util.rs similarity index 56% rename from internal/integration_test/fuzz/fuzz/fuzz_targets/wazero_abi.rs rename to internal/integration_test/fuzz/fuzz/fuzz_targets/util.rs index f21f9d13ff..0a74acc5d1 100644 --- a/internal/integration_test/fuzz/fuzz/fuzz_targets/wazero_abi.rs +++ b/internal/integration_test/fuzz/fuzz/fuzz_targets/util.rs @@ -2,8 +2,12 @@ extern "C" { // require_no_diff is implemented in Go, and accepts the pointer to the binary and its size. - #[allow(dead_code)] - pub fn require_no_diff(binary_ptr: *const u8, binary_size: usize, check_memory: bool); + pub fn require_no_diff( + binary_ptr: *const u8, + binary_size: usize, + check_memory: bool, + check_logging: bool, + ); // validate is implemented in Go, and accepts the pointer to the binary and its size. #[allow(dead_code)] @@ -12,9 +16,11 @@ extern "C" { use ctor::ctor; use libc::SIGSTKSZ; +use libfuzzer_sys::arbitrary::Unstructured; use nix::libc::{sigaltstack, stack_t}; use nix::sys::signal::{sigaction, SaFlags, SigAction, SigHandler, SigSet, Signal}; use std::ptr::null_mut; +use wasm_smith::SwarmConfig; #[ctor] /// Sets up the separate stack for signal handlers, and sets the SA_ONSTACK flag for signals that are handled by libFuzzer @@ -75,3 +81,61 @@ fn set_signal_stack() { std::mem::forget(stack); } } + +#[allow(dead_code)] +pub fn run_nodiff( + data: &[u8], + check_memory: bool, + check_logging: bool, +) -> libfuzzer_sys::arbitrary::Result<()> { + // Create the random source. + let mut u = Unstructured::new(data); + + // Generate the configuration. + let mut config: SwarmConfig = u.arbitrary()?; + + // 64-bit memory won't be supported by wazero. + config.memory64_enabled = false; + // For exactly one memory exists. + config.max_memories = 1; + config.min_memories = 1; + // If we don't set the limit, we will soon reach the OOM and the fuzzing will be killed by OS. + config.max_memory_pages = 10; + config.memory_max_size_required = true; + // Don't test too large tables. + config.max_tables = 2; + config.max_table_elements = 1_000; + config.table_max_size_required = true; + + // max_instructions is set to 100 by default which seems a little bit smaller. + config.max_instructions = 5000; + + // Without canonicalization of NaNs, the results cannot be matched among engines. + config.canonicalize_nans = true; + + // Export all the things so that we can invoke them. + config.export_everything = true; + + // Ensures that at least one function exists. + config.min_funcs = 1; + config.max_funcs = config.max_funcs.max(1); + + // TODO: enable after threads support in wazevo. + config.threads_enabled = false; + + // Generate the random module via wasm-smith. + let mut module = wasm_smith::Module::new(config.clone(), &mut u)?; + module.ensure_termination(1000); + let module_bytes = module.to_bytes(); + + // Pass the randomly generated module to the wazero library. + unsafe { + require_no_diff( + module_bytes.as_ptr(), + module_bytes.len(), + check_memory, + check_logging, + ); + } + Ok(()) +} diff --git a/internal/integration_test/fuzz/fuzz/fuzz_targets/validation.rs b/internal/integration_test/fuzz/fuzz/fuzz_targets/validation.rs index 97660eeeb7..43b7654c49 100644 --- a/internal/integration_test/fuzz/fuzz/fuzz_targets/validation.rs +++ b/internal/integration_test/fuzz/fuzz/fuzz_targets/validation.rs @@ -3,7 +3,7 @@ use libfuzzer_sys::arbitrary::{Result, Unstructured}; use libfuzzer_sys::fuzz_target; -mod wazero_abi; +mod util; fuzz_target!(|data: &[u8]| { let _ = run(data); @@ -18,7 +18,7 @@ fn run(data: &[u8]) -> Result<()> { let module_bytes = module.to_bytes(); unsafe { - wazero_abi::validate(module_bytes.as_ptr(), module_bytes.len()); + util::validate(module_bytes.as_ptr(), module_bytes.len()); } // We always return Ok as inside validate, we cause panic if the binary is interesting. diff --git a/internal/integration_test/fuzz/wazerolib/extern.go b/internal/integration_test/fuzz/wazerolib/extern.go index 9f5d9de636..e3a4d06e8f 100644 --- a/internal/integration_test/fuzz/wazerolib/extern.go +++ b/internal/integration_test/fuzz/wazerolib/extern.go @@ -21,7 +21,7 @@ func main() {} // And if there's diff, this also saves the problematic binary and wat into testdata directory. // //export require_no_diff -func require_no_diff(binaryPtr uintptr, binarySize int, checkMemory bool) { +func require_no_diff(binaryPtr uintptr, binarySize int, checkMemory bool, checkLogging bool) { // TODO: use unsafe.Slice after flooring Go 1.20. var wasmBin []byte wasmHdr := (*reflect.SliceHeader)(unsafe.Pointer(&wasmBin)) @@ -37,7 +37,7 @@ func require_no_diff(binaryPtr uintptr, binarySize int, checkMemory bool) { } }() - requireNoDiff(wasmBin, checkMemory, func(err error) { + requireNoDiff(wasmBin, checkMemory, checkLogging, func(err error) { if err != nil { panic(err) } diff --git a/internal/integration_test/fuzz/wazerolib/nodiff.go b/internal/integration_test/fuzz/wazerolib/nodiff.go index 545d97dee0..2afab74a77 100644 --- a/internal/integration_test/fuzz/wazerolib/nodiff.go +++ b/internal/integration_test/fuzz/wazerolib/nodiff.go @@ -12,6 +12,7 @@ import ( "github.com/tetratelabs/wazero" "github.com/tetratelabs/wazero/api" "github.com/tetratelabs/wazero/experimental" + "github.com/tetratelabs/wazero/experimental/logging" "github.com/tetratelabs/wazero/experimental/opt" "github.com/tetratelabs/wazero/internal/testing/binaryencoding" "github.com/tetratelabs/wazero/internal/wasm" @@ -42,24 +43,37 @@ func extractInternalWasmModuleFromCompiledModule(c wazero.CompiledModule) (*wasm } // requireNoDiff ensures that the behavior is the same between the compiler and the interpreter for any given binary. -func requireNoDiff(wasmBin []byte, checkMemory bool, requireNoError func(err error)) { - // Choose the context to use for function calls. - ctx := context.Background() - +func requireNoDiff(wasmBin []byte, checkMemory, loggingCheck bool, requireNoError func(err error)) { const features = api.CoreFeaturesV2 | experimental.CoreFeaturesThreads - compiler := wazero.NewRuntimeWithConfig(ctx, opt.NewRuntimeConfigOptimizingCompiler().WithCoreFeatures(features)) - interpreter := wazero.NewRuntimeWithConfig(ctx, wazero.NewRuntimeConfigInterpreter().WithCoreFeatures(features)) - defer compiler.Close(ctx) - defer interpreter.Close(ctx) + compiler := wazero.NewRuntimeWithConfig(context.Background(), opt.NewRuntimeConfigOptimizingCompiler().WithCoreFeatures(features)) + interpreter := wazero.NewRuntimeWithConfig(context.Background(), wazero.NewRuntimeConfigInterpreter().WithCoreFeatures(features)) + defer compiler.Close(context.Background()) + defer interpreter.Close(context.Background()) + + interpreterCtx, compilerCtx := context.Background(), context.Background() + var interPreterLoggingBuf, compilerLoggingBuf bytes.Buffer + var errorDuringInvocation bool + if loggingCheck { + interpreterCtx = context.WithValue(interpreterCtx, experimental.FunctionListenerFactoryKey{}, logging.NewLoggingListenerFactory(&interPreterLoggingBuf)) + compilerCtx = context.WithValue(compilerCtx, experimental.FunctionListenerFactoryKey{}, logging.NewLoggingListenerFactory(&compilerLoggingBuf)) + defer func() { + if !errorDuringInvocation { + if !bytes.Equal(compilerLoggingBuf.Bytes(), interPreterLoggingBuf.Bytes()) { + requireNoError(fmt.Errorf("logging mismatch\ncompiler: %s\ninterpreter: %s", + compilerLoggingBuf.String(), interPreterLoggingBuf.String())) + } + } + }() + } - compilerCompiled, err := compiler.CompileModule(ctx, wasmBin) + compilerCompiled, err := compiler.CompileModule(compilerCtx, wasmBin) if err != nil && strings.Contains(err.Error(), "has an empty module name") { // This is the limitation wazero imposes to allow special-casing of anonymous modules. return } requireNoError(err) - interpreterCompiled, err := interpreter.CompileModule(ctx, wasmBin) + interpreterCompiled, err := interpreter.CompileModule(interpreterCtx, wasmBin) requireNoError(err) internalMod, err := extractInternalWasmModuleFromCompiledModule(compilerCompiled) @@ -71,16 +85,18 @@ func requireNoDiff(wasmBin []byte, checkMemory bool, requireNoError func(err err ensureDummyImports(interpreter, internalMod, requireNoError) // Instantiate module. - compilerMod, compilerInstErr := compiler.InstantiateModule(ctx, compilerCompiled, + compilerMod, compilerInstErr := compiler.InstantiateModule(compilerCtx, compilerCompiled, wazero.NewModuleConfig().WithName(string(internalMod.ID[:]))) - interpreterMod, interpreterInstErr := interpreter.InstantiateModule(ctx, interpreterCompiled, + interpreterMod, interpreterInstErr := interpreter.InstantiateModule(interpreterCtx, interpreterCompiled, wazero.NewModuleConfig().WithName(string(internalMod.ID[:]))) okToInvoke, err := ensureInstantiationError(compilerInstErr, interpreterInstErr) requireNoError(err) if okToInvoke { - err = ensureInvocationResultMatch(compilerMod, interpreterMod, interpreterCompiled.ExportedFunctions()) + err, errorDuringInvocation = ensureInvocationResultMatch( + compilerCtx, interpreterCtx, + compilerMod, interpreterMod, interpreterCompiled.ExportedFunctions()) requireNoError(err) compilerMem, _ := compilerMod.Memory().(*wasm.MemoryInstance) @@ -248,9 +264,10 @@ func ensureDummyImports(r wazero.Runtime, origin *wasm.Module, requireNoError fu const valueTypeVector = 0x7b // ensureInvocationResultMatch invokes all the exported functions from the module, and compare all the results between compiler vs interpreter. -func ensureInvocationResultMatch(compiledMod, interpreterMod api.Module, exportedFunctions map[string]api.FunctionDefinition) (err error) { - ctx := context.Background() - +func ensureInvocationResultMatch( + compilerCtx, interpreterCtx context.Context, compiledMod, interpreterMod api.Module, + exportedFunctions map[string]api.FunctionDefinition, +) (err error, errorDuringInvocation bool) { // In order to do the deterministic execution, we need to sort the exported functions. var names []string for f := range exportedFunctions { @@ -275,8 +292,9 @@ outer: intF := interpreterMod.ExportedFunction(name) params := getDummyValues(def.ParamTypes()) - cmpRes, cmpErr := cmpF.Call(ctx, params...) - intRes, intErr := intF.Call(ctx, params...) + cmpRes, cmpErr := cmpF.Call(compilerCtx, params...) + intRes, intErr := intF.Call(interpreterCtx, params...) + errorDuringInvocation = errorDuringInvocation || cmpErr != nil || intErr != nil if errMismatch := ensureInvocationError(cmpErr, intErr); errMismatch != nil { panic(fmt.Sprintf("error mismatch on invoking %s: %v", name, errMismatch)) } diff --git a/internal/integration_test/fuzz/wazerolib/nodiff_test.go b/internal/integration_test/fuzz/wazerolib/nodiff_test.go index 7b51f46dde..e232edeee9 100644 --- a/internal/integration_test/fuzz/wazerolib/nodiff_test.go +++ b/internal/integration_test/fuzz/wazerolib/nodiff_test.go @@ -17,7 +17,7 @@ func TestReRunFailedRequireNoDiffCase(t *testing.T) { t.Skip(err) } - requireNoDiff(wasmBin, true, func(err error) { require.NoError(t, err) }) + requireNoDiff(wasmBin, true, true, func(err error) { require.NoError(t, err) }) } func Test_ensureMutableGlobalsMatch(t *testing.T) {