Skip to content

Commit

Permalink
bt validate learns --invalid
Browse files Browse the repository at this point in the history
  • Loading branch information
@thorehusfeldt
thorehusfeldt committed Feb 7, 2024
1 parent ea268d3 commit a870130
Show file tree
Hide file tree
Showing 5 changed files with 59 additions and 34 deletions.
2 changes: 1 addition & 1 deletion bin/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@
grep -Ev '^(h|jobs|time|verbose)$' | sed "s/^/'/;s/$/',/" | tr '\n' ' ' | sed 's/^/args_list = [/;s/, $/]\n/'
"""
# fmt: off
args_list = ['1', 'add_unlisted', 'all', 'api', 'author', 'check_deterministic', 'clean', 'clean_generated', 'cleanup_generated', 'colors', 'contest', 'contest_id', 'contestname', 'cp', 'cpp_flags', 'default_solution', 'directory', 'error', 'force', 'force_build', 'language', 'no_validators', 'input', 'interaction', 'interactive', 'kattis', 'memory', 'move_to', 'no_bar', 'no_generate', 'no_solutions', 'no_timelimit', 'order', 'order_from_ccs', 'answer', 'password', 'post_freeze', 'problem', 'problemname', 'remove', 'samples', 'skel', 'skip', 'no_solution', 'no_testcase_sanity_checks', 'no_visualizer', 'submissions', 'table', 'testcases', 'timelimit', 'timeout', 'token', 'username', 'validation', 'watch', 'web']
args_list = ['1', 'add_unlisted', 'all', 'answer', 'api', 'author', 'check_deterministic', 'clean', 'clean_generated', 'cleanup_generated', 'colors', 'contest', 'contest_id', 'contestname', 'cp', 'cpp_flags', 'default_solution', 'directory', 'error', 'force', 'force_build', 'language', 'no_validators', 'input', 'interaction', 'interactive', 'invalid', 'kattis', 'memory', 'move_to', 'no_bar', 'no_generate', 'no_solutions', 'no_timelimit', 'order', 'order_from_ccs', 'password', 'post_freeze', 'problem', 'problemname', 'remove', 'samples', 'skel', 'skip', 'no_solution', 'no_testcase_sanity_checks', 'no_visualizer', 'submissions', 'table', 'testcases', 'timelimit', 'timeout', 'token', 'username', 'validation', 'watch', 'web']
# fmt: on


Expand Down
40 changes: 26 additions & 14 deletions bin/problem.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,7 @@ def testcases(
only_sample=False,
statement_samples=False,
include_bad=False,
invalid_outputs_only=False,
mode=None,
copy=False,
):
def maybe_copy(x):
Expand Down Expand Up @@ -218,8 +218,14 @@ def maybe_copy(x):
in_paths.append(t)

in_paths = list(set(in_paths))
elif invalid_outputs_only:
in_paths = list(glob(p.path, 'data/invalid_outputs/**/*.in'))
elif mode is not None:
in_paths = []
for prefix in {
validate.Mode.INPUT: ['secret', 'sample'],
validate.Mode.ANSWER: ['secret', 'sample'],
validate.Mode.INVALID: ['bad', 'invalid_*'],
}[mode]:
in_paths += glob(p.path, f'data/{prefix}/**/*.in')
else:
in_paths = list(glob(p.path, 'data/sample/**/*.in'))
if statement_samples:
Expand All @@ -237,6 +243,8 @@ def maybe_copy(x):
in_paths += list(glob(p.path, 'data/invalid_answers/**/*.in'))
in_paths += list(glob(p.path, 'data/invalid_outputs/**/*.in'))



testcases = []
for f in in_paths:
t = testcase.Testcase(p, f)
Expand All @@ -246,9 +254,9 @@ def maybe_copy(x):
warn(f'Found input file {f} without a .interaction file. Skipping.')
continue
if needans and not t.ans_path.is_file():
if not t.root == 'invalid_inputs':
if t.root != 'invalid_inputs':
warn(f'Found input file {f} without a .ans file. Skipping.')
continue
continue
testcases.append(t)
testcases.sort(key=lambda t: t.name)

Expand Down Expand Up @@ -582,7 +590,7 @@ def validate_data(problem, mode: validate.Mode, constraints: dict | bool | None
"""Validate aspects of the test data files.
Arguments:
mode: validate.Mode.INPUT | validate.Mode.ANSWER | (not implemented) Validate.Mode.OUT_FILE
mode: validate.Mode.INPUT | validate.Mode.ANSWER | Validate.Mode.INVALID
constraints: True | dict | None. True means "do check constraints but discard the result."
False: TODO is this ever used?
Return:
Expand All @@ -600,21 +608,25 @@ def validate_data(problem, mode: validate.Mode, constraints: dict | bool | None
ok = True

# Pre-build the relevant Validators so as to avoid clash with ProgressBar bar below
# Also, pick the relevant testcases
check_constraints = constraints is not None
match mode:
case validate.Mode.INPUT:
problem.validators(validate.InputValidator, check_constraints=check_constraints)
testcases = problem.testcases(mode=mode, include_bad=not check_constraints)
case validate.Mode.ANSWER:
problem.validators(validate.AnswerValidator, check_constraints=check_constraints)
problem.validators(validate.OutputValidator, check_constraints=check_constraints)
case validate.Mode.OUT_FILE:
problem.validators(validate.OutputValidator, check_constraints=check_constraints)

needans = mode != validate.Mode.INPUT
testcases = (problem.testcases(needans=needans, include_bad=not check_constraints)
if mode != validate.Mode.OUT_FILE
else problem.testcases(invalid_outputs_only=True)
)
testcases = problem.testcases(mode=mode, include_bad=not check_constraints)
case validate.Mode.INVALID:
problem.validators(validate.InputValidator)
problem.validators(validate.AnswerValidator)
problem.validators(validate.OutputValidator)
testcases = problem.testcases(mode=mode)
case _:
ValueError(mode)

needans = mode != validate.Mode.INPUT # TODO

if testcases is False:
return True
Expand Down
18 changes: 12 additions & 6 deletions bin/testcase.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,10 @@ def __init__(self, base_problem, path: Path, *, short_path=None):
self.problem.path / 'data' / self.short_path
)


def __repr__(self):
return self.name

def with_suffix(self, ext):
return self.in_path.with_suffix(ext)

Expand Down Expand Up @@ -203,21 +207,23 @@ def validate_format(
validators = self.problem.validators(
InputValidator, check_constraints=check_constraints
)
expect_rejection = self.root == 'invalid_inputs'
expect_rejection = False
case Mode.ANSWER:
validators = self.problem.validators(
AnswerValidator, check_constraints=check_constraints
) + self.problem.validators(OutputValidator, check_constraints=check_constraints)
expect_rejection = self.root == 'invalid_answers'
case Mode.OUT_FILE:
validators = self.problem.validators(OutputValidator)
expect_rejection = self.root == 'invalid_outputs' # TODO: currently alsways True
expect_rejection = False
case Mode.INVALID:
validators = self.problem.validators(InputValidator)[::]
if self.root in ['invalid_answers', 'invalid_outputs']:
validators += self.problem.validators(AnswerValidator) + self.problem.validators(OutputValidator)
expect_rejection = True
case _:
raise ValueError

validator_accepted = []
for validator in validators:
if type(validator) == OutputValidator:
if type(validator) == OutputValidator and self.root.startswith("invalid"):
args = ['case_sensitive', 'space_change_sensitive']
flags = self.testdata_yaml_validator_flags(validator)
if flags is False:
Expand Down
10 changes: 7 additions & 3 deletions bin/tools.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -451,6 +451,9 @@ def build_parser():
input_answer_group.add_argument(
'--answer', action='store_true', help='Only validate answer.'
)
input_answer_group.add_argument(
'--invalid', action='store_true', help='Only check invalid files for validity.'
)

move_or_remove_group = validate_parser.add_mutually_exclusive_group()
move_or_remove_group.add_argument(
Expand Down Expand Up @@ -918,11 +921,12 @@ def run_parsed_arguments(args):
if level == 'problem':
success &= latex.build_problem_pdfs(problem, solutions=True, web=config.args.web)
if action in ['validate', 'all']:
if not (action == 'validate' and config.args.answer):
if not (action == 'validate' and (config.args.input or config.args.answer)):
success &= problem.validate_data(validate.Mode.INVALID)
if not (action == 'validate' and (config.args.answer or config.args.invalid)):
success &= problem.validate_data(validate.Mode.INPUT)
if not (action == 'validate' and config.args.input):
if not (action == 'validate' and (config.args.input or config.args.invalid)):
success &= problem.validate_data(validate.Mode.ANSWER)
success &= problem.validate_data(validate.Mode.OUT_FILE)
if action in ['run', 'all']:
success &= problem.run_submissions()
if action in ['test']:
Expand Down
23 changes: 13 additions & 10 deletions bin/validate.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,13 @@ class Mode(Enum):

INPUT = 1
ANSWER = 2
OUT_FILE = 3
INVALID = 3

def __str__(self):
return {
Mode.INPUT: "input",
Mode.ANSWER: "answer",
Mode.OUT_FILE: "output file",
Mode.INVALID: "invalid files",
}[self]


Expand Down Expand Up @@ -167,8 +167,8 @@ def run(self, testcase, mode=Mode.INPUT, constraints=None, args=None) -> ExecRes
mode:
must be Mode.INPUT
"""
if mode != Mode.INPUT:
raise ValueError("InputValidators only support Mode.INPUT")
if mode == Mode.ANSWER:
raise ValueError("InputValidators do not support Mode.ANSWER")
cwd, constraints_path, arglist = self._run_helper(testcase, constraints, args)

if self.language in Validator.FORMAT_VALIDATOR_LANGUAGES:
Expand Down Expand Up @@ -211,8 +211,8 @@ def run(self, testcase, mode=Mode.ANSWER, constraints=None, args=None):
ExecResult
"""

if mode != Mode.ANSWER:
raise ValueError("AnswerValidators only support Mode.ANSWER")
if mode == Mode.INPUT:
raise ValueError("AnswerValidators do no support Mode.INPUT")

cwd, constraints_path, arglist = self._run_helper(testcase, constraints, args)

Expand Down Expand Up @@ -269,10 +269,13 @@ def run(self, testcase, run=None, mode=None, constraints=None, args=None):
path = run.out_path
else:
match mode:
case Mode.OUT_FILE:
if testcase.out_path is None:
raise ValueError(f"Test case {testcase.name} has no .out file")
path = testcase.out_path.resolve()
case Mode.INVALID:
if testcase.root == 'invalid_answers':
path = testcase.ans_path.resolve()
else:
if testcase.out_path is None:
raise ValueError(f"Test case {testcase.name} has no .out file")
path = testcase.out_path.resolve()
case Mode.ANSWER:
path = testcase.ans_path.resolve()
case Mode.INPUT:
Expand Down

0 comments on commit a870130

Please sign in to comment.