Skip to content

Commit

Permalink
Mark expected failures under crosshair
Browse files Browse the repository at this point in the history
  • Loading branch information
Zac-HD committed Jul 7, 2024
1 parent eabcfcc commit b85bcdb
Show file tree
Hide file tree
Showing 3 changed files with 36 additions and 0 deletions.
30 changes: 30 additions & 0 deletions hypothesis-python/tests/common/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
# obtain one at https://mozilla.org/MPL/2.0/.

import contextlib
import enum
import sys
from io import StringIO
from types import SimpleNamespace
Expand Down Expand Up @@ -228,3 +229,32 @@ def temp_registered(type_, strat_or_factory):
# config option, so *linking against* something built this way can break us.
# Everything is terrible
PYTHON_FTZ = next_down(sys.float_info.min) == 0.0


class Why(enum.Enum):
# Use an enum here so it's easier to find and/or exclude some cases later
other = "other"
flaky_replay = "Inconsistent results from replaying a failing test..."
symbolic_outside_context = (
"CrosshairInternal error from using value outside context"
)
undiscovered = "crosshair doesn't find the failing input"


def xfail_on_crosshair(why: Why, /):
import pytest
try:
import crosshair.util
except ImportError:
return pytest.mark.xf_crosshair

kw = {
"strict": True,
"reason": f"Expected failure due to: {why.value}",
"condition": settings.get_profile(settings._current_profile).backend
== "crosshair",
}
if why is Why.symbolic_outside_context:
kw["raises"] = crosshair.util.CrosshairInternal

return lambda fn: pytest.mark.xf_crosshair(pytest.mark.xfail(**kw)(fn))
1 change: 1 addition & 0 deletions hypothesis-python/tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ def pytest_configure(config):
"markers",
"xp_min_version(api_version): run when greater or equal to api_version",
)
config.addinivalue_line("markers", "xf_crosshair: selection for xfailing symbolics")


def pytest_addoption(parser):
Expand Down
5 changes: 5 additions & 0 deletions hypothesis-python/tests/cover/test_testdecorators.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,15 @@
)

from tests.common.utils import (
Why,
assert_falsifying_output,
capture_out,
fails,
fails_with,
no_shrink,
raises,
skipif_emscripten,
xfail_on_crosshair,
)

# This particular test file is run under both pytest and nose, so it can't
Expand Down Expand Up @@ -304,6 +306,7 @@ def test_has_ascii(x):
assert any(c in ascii_characters for c in x)


@xfail_on_crosshair(Why.symbolic_outside_context)
def test_can_derandomize():
values = []

Expand Down Expand Up @@ -393,6 +396,7 @@ def test_mixed_text(x):
assert set(x).issubset(set("abcdefg"))


@xfail_on_crosshair(Why.other) # runs ~five failing examples
def test_when_set_to_no_simplifies_runs_failing_example_twice():
failing = []

Expand Down Expand Up @@ -478,6 +482,7 @@ def test_empty_lists(xs):
assert xs == []


@xfail_on_crosshair(Why.other) # executes >> 2 tests, but there are only two bools
def test_given_usable_inline_on_lambdas():
xs = []
given(booleans())(lambda x: xs.append(x))()
Expand Down

0 comments on commit b85bcdb

Please sign in to comment.