Cleanup fuzzy tests a bit and run them in CI

The fuzzy tests rely on rc.exe/cvtres.exe being in the PATH to check that our output is expected, so they are only run on the Windows CI

.github/workflows/ci.yml

@@ -47,6 +47,10 @@ jobs:
         if: ${{ matrix.os != 'macos-latest' }}
         run: zig build test -Dtarget=x86-native
 
+      - name: Fuzzy Tests
+        if: ${{ matrix.os == 'windows-latest' }}
+        run: zig build test_fuzzy
+
       - name: Test big endian (mips)
         # TODO: Disabled due to segfault when running the compiled resinator
         #       binary with qemu-mips, unsure of the cause but might be in the

build.zig

@@ -83,19 +83,21 @@ pub fn build(b: *std.Build) void {
     test_step.dependOn(&run_compiler_tests.step);
     test_step.dependOn(run_cli_tests_step);
 
+    const test_utils_module = b.createModule(.{
+        .root_source_file = b.path("test/utils.zig"),
+        .imports = &.{
+            .{ .name = "resinator", .module = resinator },
+        },
+    });
+
     // Tools
     const cvtres_strip = b.addExecutable(.{
         .name = "cvtres-strip",
         .root_source_file = b.path("tools/cvtres-strip.zig"),
         .target = target,
         .optimize = mode,
     });
-    cvtres_strip.root_module.addAnonymousImport("utils", .{
-        .root_source_file = b.path("test/utils.zig"),
-        .imports = &.{
-            .{ .name = "resinator", .module = resinator },
-        },
-    });
+    cvtres_strip.root_module.addImport("utils", test_utils_module);
     const install_cvtres_strip = b.addInstallArtifact(cvtres_strip, .{});
     const cvtres_strip_step = b.step("cvtres-strip", "Build and install cvtres-strip tool");
     cvtres_strip_step.dependOn(&install_cvtres_strip.step);
@@ -105,28 +107,32 @@ pub fn build(b: *std.Build) void {
 
     // Fuzzy tests
     const fuzzy_max_iterations = b.option(u64, "fuzzy-iterations", "The max iterations for fuzzy tests (default: 1000)") orelse 1000;
+    const fuzzy_debug = b.option(bool, "fuzzy-debug", "When enabled, fuzzy tests will write their inputs to the cache dir (default: false)") orelse false;
 
     const test_options = b.addOptions();
     test_options.addOption(u64, "max_iterations", fuzzy_max_iterations);
+    test_options.addOption(bool, "fuzzy_debug", fuzzy_debug);
 
     const all_fuzzy_tests_step = b.step("test_fuzzy", "Run all fuzz/property-testing-like tests with a max number of iterations for each");
-    _ = addFuzzyTest(b, "numbers", mode, target, resinator, all_fuzzy_tests_step, test_options);
-    _ = addFuzzyTest(b, "number_expressions", mode, target, resinator, all_fuzzy_tests_step, test_options);
-    _ = addFuzzyTest(b, "ascii_strings", mode, target, resinator, all_fuzzy_tests_step, test_options);
-    _ = addFuzzyTest(b, "numeric_types", mode, target, resinator, all_fuzzy_tests_step, test_options);
-    _ = addFuzzyTest(b, "common_resource_attributes", mode, target, resinator, all_fuzzy_tests_step, test_options);
-    _ = addFuzzyTest(b, "raw_data", mode, target, resinator, all_fuzzy_tests_step, test_options);
-    _ = addFuzzyTest(b, "name_or_ordinal", mode, target, resinator, all_fuzzy_tests_step, test_options);
-    _ = addFuzzyTest(b, "code_pages", mode, target, resinator, all_fuzzy_tests_step, test_options);
-    _ = addFuzzyTest(b, "icons", mode, target, resinator, all_fuzzy_tests_step, test_options);
-    _ = addFuzzyTest(b, "bitmaps", mode, target, resinator, all_fuzzy_tests_step, test_options);
-    _ = addFuzzyTest(b, "stringtable", mode, target, resinator, all_fuzzy_tests_step, test_options);
-    _ = addFuzzyTest(b, "fonts", mode, target, resinator, all_fuzzy_tests_step, test_options);
-    _ = addFuzzyTest(b, "dlginclude", mode, target, resinator, all_fuzzy_tests_step, test_options);
-    _ = addFuzzyTest(b, "strings", mode, target, resinator, all_fuzzy_tests_step, test_options);
-    _ = addFuzzyTest(b, "accelerators", mode, target, resinator, all_fuzzy_tests_step, test_options);
-    _ = addFuzzyTest(b, "cvtres", mode, target, resinator, all_fuzzy_tests_step, test_options);
-    _ = addFuzzyTest(b, "res", mode, target, resinator, all_fuzzy_tests_step, test_options);
+    _ = addFuzzyTest(b, "numbers", mode, target, resinator, test_utils_module, all_fuzzy_tests_step, test_options);
+    _ = addFuzzyTest(b, "number_expressions", mode, target, resinator, test_utils_module, all_fuzzy_tests_step, test_options);
+    _ = addFuzzyTest(b, "ascii_strings", mode, target, resinator, test_utils_module, all_fuzzy_tests_step, test_options);
+    _ = addFuzzyTest(b, "numeric_types", mode, target, resinator, test_utils_module, all_fuzzy_tests_step, test_options);
+    _ = addFuzzyTest(b, "common_resource_attributes", mode, target, resinator, test_utils_module, all_fuzzy_tests_step, test_options);
+    _ = addFuzzyTest(b, "raw_data", mode, target, resinator, test_utils_module, all_fuzzy_tests_step, test_options);
+    _ = addFuzzyTest(b, "name_or_ordinal", mode, target, resinator, test_utils_module, all_fuzzy_tests_step, test_options);
+    _ = addFuzzyTest(b, "code_pages", mode, target, resinator, test_utils_module, all_fuzzy_tests_step, test_options);
+    _ = addFuzzyTest(b, "icons", mode, target, resinator, test_utils_module, all_fuzzy_tests_step, test_options);
+    _ = addFuzzyTest(b, "stringtable", mode, target, resinator, test_utils_module, all_fuzzy_tests_step, test_options);
+    _ = addFuzzyTest(b, "strings", mode, target, resinator, test_utils_module, all_fuzzy_tests_step, test_options);
+    _ = addFuzzyTest(b, "cvtres", mode, target, resinator, test_utils_module, all_fuzzy_tests_step, test_options);
+    _ = addFuzzyTest(b, "res", mode, target, resinator, test_utils_module, all_fuzzy_tests_step, test_options);
+    // Exclude these fuzzy tests from the test_fuzzy step since they don't fully work as tests
+    // and are more geared towards gathering info at the moment.
+    _ = addFuzzyTest(b, "bitmaps", mode, target, resinator, test_utils_module, null, test_options);
+    _ = addFuzzyTest(b, "fonts", mode, target, resinator, test_utils_module, null, test_options);
+    _ = addFuzzyTest(b, "dlginclude", mode, target, resinator, test_utils_module, null, test_options);
+    _ = addFuzzyTest(b, "accelerators", mode, target, resinator, test_utils_module, null, test_options);
 
     _ = addFuzzer(b, "fuzz_rc", &.{}, resinator, target);
 
@@ -327,7 +333,8 @@ fn addFuzzyTest(
     mode: std.builtin.Mode,
     target: std.Build.ResolvedTarget,
     resinator: *std.Build.Module,
-    all_fuzzy_tests_step: *std.Build.Step,
+    test_utils_module: *std.Build.Module,
+    all_fuzzy_tests_step: ?*std.Build.Step,
     fuzzy_options: *std.Build.Step.Options,
 ) *std.Build.Step.Compile {
     var test_step = b.addTest(.{
@@ -336,14 +343,18 @@ fn addFuzzyTest(
         .optimize = mode,
     });
     test_step.root_module.addImport("resinator", resinator);
+    // We use an import to avoid pulling in the test cases of the test utils themselves
+    test_step.root_module.addImport("test_utils", test_utils_module);
     test_step.root_module.addOptions("fuzzy_options", fuzzy_options);
 
     const run_test = b.addRunArtifact(test_step);
 
     var test_run_step = b.step("test_fuzzy_" ++ name, "Some fuzz/property-testing-like tests for " ++ name);
     test_run_step.dependOn(&run_test.step);
 
-    all_fuzzy_tests_step.dependOn(test_run_step);
+    if (all_fuzzy_tests_step) |all_step| {
+        all_step.dependOn(test_run_step);
+    }
 
     return test_step;
 }

test/fuzzy_accelerators.zig

@@ -1,6 +1,6 @@
 const std = @import("std");
 const resinator = @import("resinator");
-const utils = @import("utils.zig");
+const utils = @import("test_utils");
 const fuzzy_options = @import("fuzzy_options");
 const iterations = fuzzy_options.max_iterations;
 

test/fuzzy_ascii_strings.zig

@@ -1,5 +1,5 @@
 const std = @import("std");
-const utils = @import("utils.zig");
+const utils = @import("test_utils");
 const fuzzy_options = @import("fuzzy_options");
 const iterations = fuzzy_options.max_iterations;
 
@@ -86,7 +86,8 @@ test "fuzz" {
         const source = source_buffer.items;
 
         // write out the source file to disk for debugging
-        try std.fs.cwd().writeFile(.{ .sub_path = ".zig-cache/tmp/fuzzy_ascii_strings.rc", .data = source });
+        if (fuzzy_options.fuzzy_debug)
+            try std.fs.cwd().writeFile(.{ .sub_path = ".zig-cache/tmp/fuzzy_ascii_strings.rc", .data = source });
 
         try utils.expectSameResOutput(allocator, source, .{
             .cwd = tmp.dir,

test/fuzzy_bitmaps.zig

@@ -1,5 +1,5 @@
 const std = @import("std");
-const utils = @import("utils.zig");
+const utils = @import("test_utils");
 const fuzzy_options = @import("fuzzy_options");
 const iterations = fuzzy_options.max_iterations;
 const resinator = @import("resinator");
@@ -78,7 +78,8 @@ test "BITMAP fuzz" {
         try tmp.dir.writeFile(.{ .sub_path = "test.bin", .data = image_buffer.items });
 
         // also write it to the top-level tmp dir for debugging
-        try std.fs.cwd().writeFile(.{ .sub_path = ".zig-cache/tmp/fuzzy_bitmaps.bin", .data = image_buffer.items });
+        if (fuzzy_options.fuzzy_debug)
+            try std.fs.cwd().writeFile(.{ .sub_path = ".zig-cache/tmp/fuzzy_bitmaps.bin", .data = image_buffer.items });
 
         var diagnostics = resinator.errors.Diagnostics.init(allocator);
         defer diagnostics.deinit();

test/fuzzy_code_pages.zig

@@ -1,5 +1,5 @@
 const std = @import("std");
-const utils = @import("utils.zig");
+const utils = @import("test_utils");
 const fuzzy_options = @import("fuzzy_options");
 const iterations = fuzzy_options.max_iterations;
 
@@ -112,7 +112,8 @@ test "fuzz" {
         const source = source_buffer.items;
 
         // write out the source file to disk for debugging
-        try std.fs.cwd().writeFile(.{ .sub_path = ".zig-cache/tmp/fuzzy_code_pages.rc", .data = source });
+        if (fuzzy_options.fuzzy_debug)
+            try std.fs.cwd().writeFile(.{ .sub_path = ".zig-cache/tmp/fuzzy_code_pages.rc", .data = source });
 
         try utils.expectSameResOutput(allocator, source, .{
             .cwd = tmp.dir,

test/fuzzy_common_resource_attributes.zig

@@ -1,5 +1,5 @@
 const std = @import("std");
-const utils = @import("utils.zig");
+const utils = @import("test_utils");
 
 const common_resource_attributes: []const []const u8 = &.{
     "PRELOAD",  "LOADONCALL",  "FIXED",
@@ -41,7 +41,6 @@ test "RCDATA common resource attribute permutations" {
         // batching large amounts of permutations together we hugely reduce the amount of time it takes this
         // test to run, since the bottleneck is the creation of each `.rc` and `.res` file.
         if (is_batch_i) {
-            std.debug.print("{}\n", .{perm_i});
             const source = source_buffer.items;
 
             try utils.expectSameResOutput(allocator, source, .{

test/fuzzy_cvtres.zig

@@ -1,5 +1,5 @@
 const std = @import("std");
-const utils = @import("utils.zig");
+const utils = @import("test_utils");
 const fuzzy_options = @import("fuzzy_options");
 const iterations = fuzzy_options.max_iterations;
 const resinator = @import("resinator");
@@ -64,7 +64,8 @@ test "cvtres fuzz" {
         }
 
         // also write it to the top-level tmp dir for debugging
-        try std.fs.cwd().writeFile(.{ .sub_path = ".zig-cache/tmp/fuzzy_cvtres.res", .data = res_buffer.items });
+        if (fuzzy_options.fuzzy_debug)
+            try std.fs.cwd().writeFile(.{ .sub_path = ".zig-cache/tmp/fuzzy_cvtres.res", .data = res_buffer.items });
 
         const random_target: std.coff.MachineType = switch (rand.uintLessThan(u8, 8)) {
             0 => .X64,

test/fuzzy_dlginclude.zig

@@ -1,6 +1,6 @@
 const std = @import("std");
 const resinator = @import("resinator");
-const utils = @import("utils.zig");
+const utils = @import("test_utils");
 const fuzzy_options = @import("fuzzy_options");
 const iterations = fuzzy_options.max_iterations;
 

test/fuzzy_icons.zig

@@ -1,5 +1,5 @@
 const std = @import("std");
-const utils = @import("utils.zig");
+const utils = @import("test_utils");
 const fuzzy_options = @import("fuzzy_options");
 const iterations = fuzzy_options.max_iterations;
 
@@ -79,7 +79,8 @@ test "ICON fuzz" {
         try tmp.dir.writeFile(.{ .sub_path = "test.ico", .data = icon_buffer.items });
 
         // also write it to the top-level tmp dir for debugging
-        try std.fs.cwd().writeFile(.{ .sub_path = ".zig-cache/tmp/fuzzy_icons.ico", .data = icon_buffer.items });
+        if (fuzzy_options.fuzzy_debug)
+            try std.fs.cwd().writeFile(.{ .sub_path = ".zig-cache/tmp/fuzzy_icons.ico", .data = icon_buffer.items });
 
         try utils.expectSameResOutput(allocator, source, .{
             .cwd = tmp.dir,

test/fuzzy_name_or_ordinal.zig

@@ -1,5 +1,5 @@
 const std = @import("std");
-const utils = @import("utils.zig");
+const utils = @import("test_utils");
 const fuzzy_options = @import("fuzzy_options");
 const iterations = fuzzy_options.max_iterations;
 

test/fuzzy_number_expressions.zig

@@ -1,5 +1,5 @@
 const std = @import("std");
-const utils = @import("utils.zig");
+const utils = @import("test_utils");
 const fuzzy_options = @import("fuzzy_options");
 const iterations = fuzzy_options.max_iterations;
 

test/fuzzy_numbers.zig

@@ -1,5 +1,5 @@
 const std = @import("std");
-const utils = @import("utils.zig");
+const utils = @import("test_utils");
 const fuzzy_options = @import("fuzzy_options");
 const iterations = fuzzy_options.max_iterations;
 

test/fuzzy_numeric_types.zig

@@ -1,5 +1,5 @@
 const std = @import("std");
-const utils = @import("utils.zig");
+const utils = @import("test_utils");
 
 test "raw data" {
     const allocator = std.testing.allocator;

test/fuzzy_raw_data.zig

@@ -1,5 +1,5 @@
 const std = @import("std");
-const utils = @import("utils.zig");
+const utils = @import("test_utils");
 
 test "single char in raw data block" {
     var source_buf = "1 RCDATA { ? }".*;

test/fuzzy_res.zig

@@ -1,5 +1,5 @@
 const std = @import("std");
-const utils = @import("utils.zig");
+const utils = @import("test_utils");
 const fuzzy_options = @import("fuzzy_options");
 const iterations = fuzzy_options.max_iterations;
 const resinator = @import("resinator");
@@ -24,12 +24,13 @@ test "res preface fuzz" {
         res_buffer.clearRetainingCapacity();
 
         switch (rand.boolean()) {
-            true => try utils.writeRandomValidResource(allocator, rand, res_buffer.writer(), .{}),
+            true => _ = try utils.writeRandomValidResource(allocator, rand, res_buffer.writer(), .{}),
             false => try utils.writeRandomPotentiallyInvalidResource(allocator, rand, res_buffer.writer()),
         }
 
         // also write it to the top-level tmp dir for debugging
-        try std.fs.cwd().writeFile(.{ .sub_path = ".zig-cache/tmp/fuzzy_res_preface.res", .data = res_buffer.items });
+        if (fuzzy_options.fuzzy_debug)
+            try std.fs.cwd().writeFile(.{ .sub_path = ".zig-cache/tmp/fuzzy_res_preface.res", .data = res_buffer.items });
 
         var fbs = std.io.fixedBufferStream(res_buffer.items);
         var resources = resinator.cvtres.parseRes(allocator, fbs.reader(), .{
@@ -66,7 +67,8 @@ test "res fuzz" {
         }
 
         // also write it to the top-level tmp dir for debugging
-        try std.fs.cwd().writeFile(.{ .sub_path = ".zig-cache/tmp/fuzzy_res.res", .data = res_buffer.items });
+        if (fuzzy_options.fuzzy_debug)
+            try std.fs.cwd().writeFile(.{ .sub_path = ".zig-cache/tmp/fuzzy_res.res", .data = res_buffer.items });
 
         var fbs = std.io.fixedBufferStream(res_buffer.items);
         var resources = resinator.cvtres.parseRes(allocator, fbs.reader(), .{

test/fuzzy_strings.zig

@@ -1,6 +1,6 @@
 const std = @import("std");
 const resinator = @import("resinator");
-const utils = @import("utils.zig");
+const utils = @import("test_utils");
 const fuzzy_options = @import("fuzzy_options");
 const iterations = fuzzy_options.max_iterations;
 

test/fuzzy_stringtable.zig

@@ -1,5 +1,5 @@
 const std = @import("std");
-const utils = @import("utils.zig");
+const utils = @import("test_utils");
 const fuzzy_options = @import("fuzzy_options");
 const iterations = fuzzy_options.max_iterations;
 
@@ -30,7 +30,8 @@ test "fuzz" {
         const source = source_buffer.items;
 
         // write out the source file to disk for debugging
-        try std.fs.cwd().writeFile(.{ .sub_path = ".zig-cache/tmp/fuzzy_stringtable_strings.rc", .data = source });
+        if (fuzzy_options.fuzzy_debug)
+            try std.fs.cwd().writeFile(.{ .sub_path = ".zig-cache/tmp/fuzzy_stringtable_strings.rc", .data = source });
 
         try utils.expectSameResOutput(allocator, source, .{
             .cwd = tmp.dir,