From 65c9143e841f38c5714beeaa09482203ab05cc15 Mon Sep 17 00:00:00 2001
From: Thore Husfeldt <thore.husfeldt@gmail.com>
Date: Thu, 1 Feb 2024 17:30:31 +0100
Subject: [PATCH] Only sanity-check if all validators accepted

---
 bin/testcase.py | 58 ++++++++++++++++++++++++++++++++++---------------
 1 file changed, 40 insertions(+), 18 deletions(-)

diff --git a/bin/testcase.py b/bin/testcase.py
index 771557f9..4026f29a 100644
--- a/bin/testcase.py
+++ b/bin/testcase.py
@@ -17,11 +17,12 @@ class Testcase:
     # tested, and `short_path` is the name of the testcase relative to
     # `problem.path / 'data'`.
     """
+
     def __init__(self, base_problem, path: Path, *, short_path=None):
         assert path.suffix == '.in' or path.suffixes == [".in", ".statement"]
 
         self.problem = base_problem
-    
+
         self.in_path = path
         self.ans_path = (
             self.in_path.with_suffix('.ans')
@@ -47,8 +48,13 @@ def __init__(self, base_problem, path: Path, *, short_path=None):
         if self.root == 'bad':
             warn('data/bad is deprecated. Use data/{invalid_inputs,invalid_answers} instead.')
             self.root = 'invalid_inputs' if self.ans_path.is_file() else 'invalid_answers'
-        assert self.root in ['invalid_inputs', 'invalid_answers', 'secret', 'sample', 'test'], self.root # TODO add invalid_outputs
-
+        assert self.root in [
+            'invalid_inputs',
+            'invalid_answers',
+            'secret',
+            'sample',
+            'test',
+        ], self.root  # TODO add invalid_outputs
 
         self.included = False
         if path.is_symlink():
@@ -63,7 +69,9 @@ def __init__(self, base_problem, path: Path, *, short_path=None):
         # Read using the short_path instead of the in_path, because during
         # generate the testcase will live in a temporary directory, where
         # testdata.yaml doesn't exist.
-        self.testdata_yaml = self.problem.get_testdata_yaml(self.problem.path / 'data' / self.short_path)
+        self.testdata_yaml = self.problem.get_testdata_yaml(
+            self.problem.path / 'data' / self.short_path
+        )
 
     def with_suffix(self, ext):
         return self.in_path.with_suffix(ext)
@@ -124,25 +132,27 @@ def validate_format(
         bar,
         constraints=None,
         warn_instead_of_error=False,
-        args=None, # TODO never used?
+        args=None,  # TODO never used?
     ):
 
-
         check_constraints = constraints is not None
         match mode:
             case validate.Mode.INPUT:
-                validators = self.problem.validators(validate.InputValidator, check_constraints=check_constraints)
+                validators = self.problem.validators(
+                    validate.InputValidator, check_constraints=check_constraints
+                )
                 expect_rejection = self.root == 'invalid_inputs'
             case validate.Mode.ANSWER:
-                validators = (self.problem.validators(validate.AnswerValidator, check_constraints=check_constraints) + 
-                              self.problem.validators(validate.OutputValidator, check_constraints=check_constraints)
-                              )
+                validators = self.problem.validators(
+                    validate.AnswerValidator, check_constraints=check_constraints
+                ) + self.problem.validators(
+                    validate.OutputValidator, check_constraints=check_constraints
+                )
                 expect_rejection = self.root == 'invalid_answers'
             case validate.Mode.OUTPUT:
                 raise NotImplementedError
             case _:
                 raise ValueError
-                
 
         validator_accepted = []
         for validator in validators:
@@ -155,7 +165,9 @@ def validate_format(
 
             ret = validator.run(self, constraints=constraints, args=flags)
             if ret.ok not in [True, config.RTV_WA]:
-                warn(f"Expected {validator} to exit with {config.RTV_AC} or {config.RTV_WA}, got {ret.ok}")
+                warn(
+                    f"Expected {validator} to exit with {config.RTV_AC} or {config.RTV_WA}, got {ret.ok}"
+                )
                 ret.ok = config.RTV_WA
                 # TODO could also interpret 0 as RTV_AC
             ok = ret.ok == True
@@ -186,7 +198,10 @@ def validate_format(
                 data = ret.err
 
             bar.part_done(
-                ok or expect_rejection, message, data=data, warn_instead_of_error=warn_instead_of_error
+                ok or expect_rejection,
+                message,
+                data=data,
+                warn_instead_of_error=warn_instead_of_error,
             )
 
             if ok is True:
@@ -216,9 +231,16 @@ def validate_format(
 
             break
 
-        validate.sanity_check(self.in_path if mode == validate.Mode.INPUT else self.ans_path, bar)
-
-        if expect_rejection and all(validator_accepted):
-            bar.error(f"{mode} validation (unexpectedly) succeeded")
+        if all(validator_accepted):
+            if expect_rejection:
+                success = False
+                bar.error(f"{mode} validation (unexpectedly) succeeded")
+            else:
+                success = True
+                validate.sanity_check(
+                    self.in_path if mode == validate.Mode.INPUT else self.ans_path, bar
+                )
+        else:
+            success = not expect_rejection
 
-        return all(validator_accepted) != expect_rejection
+        return success