From 33575b82df0188e1049e8a5b389a6b197a5fff97 Mon Sep 17 00:00:00 2001 From: Ryan Liptak Date: Sun, 24 Nov 2024 22:18:07 -0800 Subject: [PATCH] Cleanup fuzzy tests a bit and run them in CI The fuzzy tests rely on rc.exe/cvtres.exe being in the PATH to check that our output is expected, so they are only run on the Windows CI --- .github/workflows/ci.yml | 4 ++ build.zig | 61 +++++++++++++---------- test/fuzzy_accelerators.zig | 2 +- test/fuzzy_ascii_strings.zig | 5 +- test/fuzzy_bitmaps.zig | 5 +- test/fuzzy_code_pages.zig | 5 +- test/fuzzy_common_resource_attributes.zig | 3 +- test/fuzzy_cvtres.zig | 5 +- test/fuzzy_dlginclude.zig | 2 +- test/fuzzy_icons.zig | 5 +- test/fuzzy_name_or_ordinal.zig | 2 +- test/fuzzy_number_expressions.zig | 2 +- test/fuzzy_numbers.zig | 2 +- test/fuzzy_numeric_types.zig | 2 +- test/fuzzy_raw_data.zig | 2 +- test/fuzzy_res.zig | 10 ++-- test/fuzzy_strings.zig | 2 +- test/fuzzy_stringtable.zig | 5 +- 18 files changed, 73 insertions(+), 51 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 140d707..ae67a7f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,6 +47,10 @@ jobs: if: ${{ matrix.os != 'macos-latest' }} run: zig build test -Dtarget=x86-native + - name: Fuzzy Tests + if: ${{ matrix.os == 'windows-latest' }} + run: zig build test_fuzzy + - name: Test big endian (mips) # TODO: Disabled due to segfault when running the compiled resinator # binary with qemu-mips, unsure of the cause but might be in the diff --git a/build.zig b/build.zig index 3c1d7a9..167586b 100644 --- a/build.zig +++ b/build.zig @@ -83,6 +83,13 @@ pub fn build(b: *std.Build) void { test_step.dependOn(&run_compiler_tests.step); test_step.dependOn(run_cli_tests_step); + const test_utils_module = b.createModule(.{ + .root_source_file = b.path("test/utils.zig"), + .imports = &.{ + .{ .name = "resinator", .module = resinator }, + }, + }); + // Tools const cvtres_strip = b.addExecutable(.{ .name = "cvtres-strip", @@ -90,12 +97,7 @@ pub fn build(b: *std.Build) void { .target = target, .optimize = mode, }); - cvtres_strip.root_module.addAnonymousImport("utils", .{ - .root_source_file = b.path("test/utils.zig"), - .imports = &.{ - .{ .name = "resinator", .module = resinator }, - }, - }); + cvtres_strip.root_module.addImport("utils", test_utils_module); const install_cvtres_strip = b.addInstallArtifact(cvtres_strip, .{}); const cvtres_strip_step = b.step("cvtres-strip", "Build and install cvtres-strip tool"); cvtres_strip_step.dependOn(&install_cvtres_strip.step); @@ -105,28 +107,32 @@ pub fn build(b: *std.Build) void { // Fuzzy tests const fuzzy_max_iterations = b.option(u64, "fuzzy-iterations", "The max iterations for fuzzy tests (default: 1000)") orelse 1000; + const fuzzy_debug = b.option(bool, "fuzzy-debug", "When enabled, fuzzy tests will write their inputs to the cache dir (default: false)") orelse false; const test_options = b.addOptions(); test_options.addOption(u64, "max_iterations", fuzzy_max_iterations); + test_options.addOption(bool, "fuzzy_debug", fuzzy_debug); const all_fuzzy_tests_step = b.step("test_fuzzy", "Run all fuzz/property-testing-like tests with a max number of iterations for each"); - _ = addFuzzyTest(b, "numbers", mode, target, resinator, all_fuzzy_tests_step, test_options); - _ = addFuzzyTest(b, "number_expressions", mode, target, resinator, all_fuzzy_tests_step, test_options); - _ = addFuzzyTest(b, "ascii_strings", mode, target, resinator, all_fuzzy_tests_step, test_options); - _ = addFuzzyTest(b, "numeric_types", mode, target, resinator, all_fuzzy_tests_step, test_options); - _ = addFuzzyTest(b, "common_resource_attributes", mode, target, resinator, all_fuzzy_tests_step, test_options); - _ = addFuzzyTest(b, "raw_data", mode, target, resinator, all_fuzzy_tests_step, test_options); - _ = addFuzzyTest(b, "name_or_ordinal", mode, target, resinator, all_fuzzy_tests_step, test_options); - _ = addFuzzyTest(b, "code_pages", mode, target, resinator, all_fuzzy_tests_step, test_options); - _ = addFuzzyTest(b, "icons", mode, target, resinator, all_fuzzy_tests_step, test_options); - _ = addFuzzyTest(b, "bitmaps", mode, target, resinator, all_fuzzy_tests_step, test_options); - _ = addFuzzyTest(b, "stringtable", mode, target, resinator, all_fuzzy_tests_step, test_options); - _ = addFuzzyTest(b, "fonts", mode, target, resinator, all_fuzzy_tests_step, test_options); - _ = addFuzzyTest(b, "dlginclude", mode, target, resinator, all_fuzzy_tests_step, test_options); - _ = addFuzzyTest(b, "strings", mode, target, resinator, all_fuzzy_tests_step, test_options); - _ = addFuzzyTest(b, "accelerators", mode, target, resinator, all_fuzzy_tests_step, test_options); - _ = addFuzzyTest(b, "cvtres", mode, target, resinator, all_fuzzy_tests_step, test_options); - _ = addFuzzyTest(b, "res", mode, target, resinator, all_fuzzy_tests_step, test_options); + _ = addFuzzyTest(b, "numbers", mode, target, resinator, test_utils_module, all_fuzzy_tests_step, test_options); + _ = addFuzzyTest(b, "number_expressions", mode, target, resinator, test_utils_module, all_fuzzy_tests_step, test_options); + _ = addFuzzyTest(b, "ascii_strings", mode, target, resinator, test_utils_module, all_fuzzy_tests_step, test_options); + _ = addFuzzyTest(b, "numeric_types", mode, target, resinator, test_utils_module, all_fuzzy_tests_step, test_options); + _ = addFuzzyTest(b, "common_resource_attributes", mode, target, resinator, test_utils_module, all_fuzzy_tests_step, test_options); + _ = addFuzzyTest(b, "raw_data", mode, target, resinator, test_utils_module, all_fuzzy_tests_step, test_options); + _ = addFuzzyTest(b, "name_or_ordinal", mode, target, resinator, test_utils_module, all_fuzzy_tests_step, test_options); + _ = addFuzzyTest(b, "code_pages", mode, target, resinator, test_utils_module, all_fuzzy_tests_step, test_options); + _ = addFuzzyTest(b, "icons", mode, target, resinator, test_utils_module, all_fuzzy_tests_step, test_options); + _ = addFuzzyTest(b, "stringtable", mode, target, resinator, test_utils_module, all_fuzzy_tests_step, test_options); + _ = addFuzzyTest(b, "strings", mode, target, resinator, test_utils_module, all_fuzzy_tests_step, test_options); + _ = addFuzzyTest(b, "cvtres", mode, target, resinator, test_utils_module, all_fuzzy_tests_step, test_options); + _ = addFuzzyTest(b, "res", mode, target, resinator, test_utils_module, all_fuzzy_tests_step, test_options); + // Exclude these fuzzy tests from the test_fuzzy step since they don't fully work as tests + // and are more geared towards gathering info at the moment. + _ = addFuzzyTest(b, "bitmaps", mode, target, resinator, test_utils_module, null, test_options); + _ = addFuzzyTest(b, "fonts", mode, target, resinator, test_utils_module, null, test_options); + _ = addFuzzyTest(b, "dlginclude", mode, target, resinator, test_utils_module, null, test_options); + _ = addFuzzyTest(b, "accelerators", mode, target, resinator, test_utils_module, null, test_options); _ = addFuzzer(b, "fuzz_rc", &.{}, resinator, target); @@ -327,7 +333,8 @@ fn addFuzzyTest( mode: std.builtin.Mode, target: std.Build.ResolvedTarget, resinator: *std.Build.Module, - all_fuzzy_tests_step: *std.Build.Step, + test_utils_module: *std.Build.Module, + all_fuzzy_tests_step: ?*std.Build.Step, fuzzy_options: *std.Build.Step.Options, ) *std.Build.Step.Compile { var test_step = b.addTest(.{ @@ -336,6 +343,8 @@ fn addFuzzyTest( .optimize = mode, }); test_step.root_module.addImport("resinator", resinator); + // We use an import to avoid pulling in the test cases of the test utils themselves + test_step.root_module.addImport("test_utils", test_utils_module); test_step.root_module.addOptions("fuzzy_options", fuzzy_options); const run_test = b.addRunArtifact(test_step); @@ -343,7 +352,9 @@ fn addFuzzyTest( var test_run_step = b.step("test_fuzzy_" ++ name, "Some fuzz/property-testing-like tests for " ++ name); test_run_step.dependOn(&run_test.step); - all_fuzzy_tests_step.dependOn(test_run_step); + if (all_fuzzy_tests_step) |all_step| { + all_step.dependOn(test_run_step); + } return test_step; } diff --git a/test/fuzzy_accelerators.zig b/test/fuzzy_accelerators.zig index ccf59cc..0bd7049 100644 --- a/test/fuzzy_accelerators.zig +++ b/test/fuzzy_accelerators.zig @@ -1,6 +1,6 @@ const std = @import("std"); const resinator = @import("resinator"); -const utils = @import("utils.zig"); +const utils = @import("test_utils"); const fuzzy_options = @import("fuzzy_options"); const iterations = fuzzy_options.max_iterations; diff --git a/test/fuzzy_ascii_strings.zig b/test/fuzzy_ascii_strings.zig index c764710..291886a 100644 --- a/test/fuzzy_ascii_strings.zig +++ b/test/fuzzy_ascii_strings.zig @@ -1,5 +1,5 @@ const std = @import("std"); -const utils = @import("utils.zig"); +const utils = @import("test_utils"); const fuzzy_options = @import("fuzzy_options"); const iterations = fuzzy_options.max_iterations; @@ -86,7 +86,8 @@ test "fuzz" { const source = source_buffer.items; // write out the source file to disk for debugging - try std.fs.cwd().writeFile(.{ .sub_path = ".zig-cache/tmp/fuzzy_ascii_strings.rc", .data = source }); + if (fuzzy_options.fuzzy_debug) + try std.fs.cwd().writeFile(.{ .sub_path = ".zig-cache/tmp/fuzzy_ascii_strings.rc", .data = source }); try utils.expectSameResOutput(allocator, source, .{ .cwd = tmp.dir, diff --git a/test/fuzzy_bitmaps.zig b/test/fuzzy_bitmaps.zig index c23aff8..99a2a27 100644 --- a/test/fuzzy_bitmaps.zig +++ b/test/fuzzy_bitmaps.zig @@ -1,5 +1,5 @@ const std = @import("std"); -const utils = @import("utils.zig"); +const utils = @import("test_utils"); const fuzzy_options = @import("fuzzy_options"); const iterations = fuzzy_options.max_iterations; const resinator = @import("resinator"); @@ -78,7 +78,8 @@ test "BITMAP fuzz" { try tmp.dir.writeFile(.{ .sub_path = "test.bin", .data = image_buffer.items }); // also write it to the top-level tmp dir for debugging - try std.fs.cwd().writeFile(.{ .sub_path = ".zig-cache/tmp/fuzzy_bitmaps.bin", .data = image_buffer.items }); + if (fuzzy_options.fuzzy_debug) + try std.fs.cwd().writeFile(.{ .sub_path = ".zig-cache/tmp/fuzzy_bitmaps.bin", .data = image_buffer.items }); var diagnostics = resinator.errors.Diagnostics.init(allocator); defer diagnostics.deinit(); diff --git a/test/fuzzy_code_pages.zig b/test/fuzzy_code_pages.zig index 30a8fb6..ac0d382 100644 --- a/test/fuzzy_code_pages.zig +++ b/test/fuzzy_code_pages.zig @@ -1,5 +1,5 @@ const std = @import("std"); -const utils = @import("utils.zig"); +const utils = @import("test_utils"); const fuzzy_options = @import("fuzzy_options"); const iterations = fuzzy_options.max_iterations; @@ -112,7 +112,8 @@ test "fuzz" { const source = source_buffer.items; // write out the source file to disk for debugging - try std.fs.cwd().writeFile(.{ .sub_path = ".zig-cache/tmp/fuzzy_code_pages.rc", .data = source }); + if (fuzzy_options.fuzzy_debug) + try std.fs.cwd().writeFile(.{ .sub_path = ".zig-cache/tmp/fuzzy_code_pages.rc", .data = source }); try utils.expectSameResOutput(allocator, source, .{ .cwd = tmp.dir, diff --git a/test/fuzzy_common_resource_attributes.zig b/test/fuzzy_common_resource_attributes.zig index 1f59685..c6de7d6 100644 --- a/test/fuzzy_common_resource_attributes.zig +++ b/test/fuzzy_common_resource_attributes.zig @@ -1,5 +1,5 @@ const std = @import("std"); -const utils = @import("utils.zig"); +const utils = @import("test_utils"); const common_resource_attributes: []const []const u8 = &.{ "PRELOAD", "LOADONCALL", "FIXED", @@ -41,7 +41,6 @@ test "RCDATA common resource attribute permutations" { // batching large amounts of permutations together we hugely reduce the amount of time it takes this // test to run, since the bottleneck is the creation of each `.rc` and `.res` file. if (is_batch_i) { - std.debug.print("{}\n", .{perm_i}); const source = source_buffer.items; try utils.expectSameResOutput(allocator, source, .{ diff --git a/test/fuzzy_cvtres.zig b/test/fuzzy_cvtres.zig index aa949ad..7df1fba 100644 --- a/test/fuzzy_cvtres.zig +++ b/test/fuzzy_cvtres.zig @@ -1,5 +1,5 @@ const std = @import("std"); -const utils = @import("utils.zig"); +const utils = @import("test_utils"); const fuzzy_options = @import("fuzzy_options"); const iterations = fuzzy_options.max_iterations; const resinator = @import("resinator"); @@ -64,7 +64,8 @@ test "cvtres fuzz" { } // also write it to the top-level tmp dir for debugging - try std.fs.cwd().writeFile(.{ .sub_path = ".zig-cache/tmp/fuzzy_cvtres.res", .data = res_buffer.items }); + if (fuzzy_options.fuzzy_debug) + try std.fs.cwd().writeFile(.{ .sub_path = ".zig-cache/tmp/fuzzy_cvtres.res", .data = res_buffer.items }); const random_target: std.coff.MachineType = switch (rand.uintLessThan(u8, 8)) { 0 => .X64, diff --git a/test/fuzzy_dlginclude.zig b/test/fuzzy_dlginclude.zig index 71d233b..d57bb20 100644 --- a/test/fuzzy_dlginclude.zig +++ b/test/fuzzy_dlginclude.zig @@ -1,6 +1,6 @@ const std = @import("std"); const resinator = @import("resinator"); -const utils = @import("utils.zig"); +const utils = @import("test_utils"); const fuzzy_options = @import("fuzzy_options"); const iterations = fuzzy_options.max_iterations; diff --git a/test/fuzzy_icons.zig b/test/fuzzy_icons.zig index de0737f..375895e 100644 --- a/test/fuzzy_icons.zig +++ b/test/fuzzy_icons.zig @@ -1,5 +1,5 @@ const std = @import("std"); -const utils = @import("utils.zig"); +const utils = @import("test_utils"); const fuzzy_options = @import("fuzzy_options"); const iterations = fuzzy_options.max_iterations; @@ -79,7 +79,8 @@ test "ICON fuzz" { try tmp.dir.writeFile(.{ .sub_path = "test.ico", .data = icon_buffer.items }); // also write it to the top-level tmp dir for debugging - try std.fs.cwd().writeFile(.{ .sub_path = ".zig-cache/tmp/fuzzy_icons.ico", .data = icon_buffer.items }); + if (fuzzy_options.fuzzy_debug) + try std.fs.cwd().writeFile(.{ .sub_path = ".zig-cache/tmp/fuzzy_icons.ico", .data = icon_buffer.items }); try utils.expectSameResOutput(allocator, source, .{ .cwd = tmp.dir, diff --git a/test/fuzzy_name_or_ordinal.zig b/test/fuzzy_name_or_ordinal.zig index 5738503..1bf4a0f 100644 --- a/test/fuzzy_name_or_ordinal.zig +++ b/test/fuzzy_name_or_ordinal.zig @@ -1,5 +1,5 @@ const std = @import("std"); -const utils = @import("utils.zig"); +const utils = @import("test_utils"); const fuzzy_options = @import("fuzzy_options"); const iterations = fuzzy_options.max_iterations; diff --git a/test/fuzzy_number_expressions.zig b/test/fuzzy_number_expressions.zig index 96cc1e8..80f8076 100644 --- a/test/fuzzy_number_expressions.zig +++ b/test/fuzzy_number_expressions.zig @@ -1,5 +1,5 @@ const std = @import("std"); -const utils = @import("utils.zig"); +const utils = @import("test_utils"); const fuzzy_options = @import("fuzzy_options"); const iterations = fuzzy_options.max_iterations; diff --git a/test/fuzzy_numbers.zig b/test/fuzzy_numbers.zig index e2293bb..04e043b 100644 --- a/test/fuzzy_numbers.zig +++ b/test/fuzzy_numbers.zig @@ -1,5 +1,5 @@ const std = @import("std"); -const utils = @import("utils.zig"); +const utils = @import("test_utils"); const fuzzy_options = @import("fuzzy_options"); const iterations = fuzzy_options.max_iterations; diff --git a/test/fuzzy_numeric_types.zig b/test/fuzzy_numeric_types.zig index eb450a6..f99dd70 100644 --- a/test/fuzzy_numeric_types.zig +++ b/test/fuzzy_numeric_types.zig @@ -1,5 +1,5 @@ const std = @import("std"); -const utils = @import("utils.zig"); +const utils = @import("test_utils"); test "raw data" { const allocator = std.testing.allocator; diff --git a/test/fuzzy_raw_data.zig b/test/fuzzy_raw_data.zig index 0652c9f..505d005 100644 --- a/test/fuzzy_raw_data.zig +++ b/test/fuzzy_raw_data.zig @@ -1,5 +1,5 @@ const std = @import("std"); -const utils = @import("utils.zig"); +const utils = @import("test_utils"); test "single char in raw data block" { var source_buf = "1 RCDATA { ? }".*; diff --git a/test/fuzzy_res.zig b/test/fuzzy_res.zig index 9630ef1..9eb44c1 100644 --- a/test/fuzzy_res.zig +++ b/test/fuzzy_res.zig @@ -1,5 +1,5 @@ const std = @import("std"); -const utils = @import("utils.zig"); +const utils = @import("test_utils"); const fuzzy_options = @import("fuzzy_options"); const iterations = fuzzy_options.max_iterations; const resinator = @import("resinator"); @@ -24,12 +24,13 @@ test "res preface fuzz" { res_buffer.clearRetainingCapacity(); switch (rand.boolean()) { - true => try utils.writeRandomValidResource(allocator, rand, res_buffer.writer(), .{}), + true => _ = try utils.writeRandomValidResource(allocator, rand, res_buffer.writer(), .{}), false => try utils.writeRandomPotentiallyInvalidResource(allocator, rand, res_buffer.writer()), } // also write it to the top-level tmp dir for debugging - try std.fs.cwd().writeFile(.{ .sub_path = ".zig-cache/tmp/fuzzy_res_preface.res", .data = res_buffer.items }); + if (fuzzy_options.fuzzy_debug) + try std.fs.cwd().writeFile(.{ .sub_path = ".zig-cache/tmp/fuzzy_res_preface.res", .data = res_buffer.items }); var fbs = std.io.fixedBufferStream(res_buffer.items); var resources = resinator.cvtres.parseRes(allocator, fbs.reader(), .{ @@ -66,7 +67,8 @@ test "res fuzz" { } // also write it to the top-level tmp dir for debugging - try std.fs.cwd().writeFile(.{ .sub_path = ".zig-cache/tmp/fuzzy_res.res", .data = res_buffer.items }); + if (fuzzy_options.fuzzy_debug) + try std.fs.cwd().writeFile(.{ .sub_path = ".zig-cache/tmp/fuzzy_res.res", .data = res_buffer.items }); var fbs = std.io.fixedBufferStream(res_buffer.items); var resources = resinator.cvtres.parseRes(allocator, fbs.reader(), .{ diff --git a/test/fuzzy_strings.zig b/test/fuzzy_strings.zig index 1160491..217a18a 100644 --- a/test/fuzzy_strings.zig +++ b/test/fuzzy_strings.zig @@ -1,6 +1,6 @@ const std = @import("std"); const resinator = @import("resinator"); -const utils = @import("utils.zig"); +const utils = @import("test_utils"); const fuzzy_options = @import("fuzzy_options"); const iterations = fuzzy_options.max_iterations; diff --git a/test/fuzzy_stringtable.zig b/test/fuzzy_stringtable.zig index 8d59d5b..ebe293e 100644 --- a/test/fuzzy_stringtable.zig +++ b/test/fuzzy_stringtable.zig @@ -1,5 +1,5 @@ const std = @import("std"); -const utils = @import("utils.zig"); +const utils = @import("test_utils"); const fuzzy_options = @import("fuzzy_options"); const iterations = fuzzy_options.max_iterations; @@ -30,7 +30,8 @@ test "fuzz" { const source = source_buffer.items; // write out the source file to disk for debugging - try std.fs.cwd().writeFile(.{ .sub_path = ".zig-cache/tmp/fuzzy_stringtable_strings.rc", .data = source }); + if (fuzzy_options.fuzzy_debug) + try std.fs.cwd().writeFile(.{ .sub_path = ".zig-cache/tmp/fuzzy_stringtable_strings.rc", .data = source }); try utils.expectSameResOutput(allocator, source, .{ .cwd = tmp.dir,