From 52a2e1418769b605c4512c42e30c99a4f866f719 Mon Sep 17 00:00:00 2001 From: Bot Date: Fri, 12 Jan 2024 19:08:58 +0000 Subject: [PATCH] DOCS : Regenerated docs. --- README.md | 214 ++++++------------ docs/public/approach/index.md | 1 - ...rns.md => separation-of-test-concerns.org} | 36 ++- docs/public/index.md | 214 ++++++------------ docs/public/using/engine/rewrite-given.md | 189 +++++++++++++++- 5 files changed, 331 insertions(+), 323 deletions(-) rename docs/public/approach/{separation-of-test-concerns.md => separation-of-test-concerns.org} (73%) diff --git a/README.md b/README.md index 6aca5328..3756badf 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,8 @@ [![Main branch status](https://github.com/hitchdev/hitchstory/actions/workflows/regression.yml/badge.svg)](https://github.com/hitchdev/hitchstory/actions/workflows/regression.yml) -Type-safe [StrictYAML](https://hitchdev.com/hitchstory/why/strictyaml) integration tests run from pytest. They can: +Type-safe [StrictYAML](https://hitchdev.com/hitchstory/why/strictyaml) python integration testing framework. With this +framework, your tests can: ## Rewrite themselves from program output (command line test example) @@ -12,6 +13,7 @@ Type-safe [StrictYAML](https://hitchdev.com/hitchstory/why/strictyaml) integrati ![Test writing docs](https://hitchdev-videos.netlify.app/rewrite-docs-demo.gif) +The tests can be run on their own or as pytest tests. ## Demo projects with demo tests @@ -25,8 +27,6 @@ Project | Storytests | Python code | Doc template | Autogenerated docs -Minimal example (two files) demonstrating two short YAML tests and the -python code necessary to run them from within a pytest file. # Code Example @@ -36,174 +36,99 @@ python code necessary to run them from within a pytest file. example.story: ```yaml -Log in as James: +Logged in: given: - browser: firefox # test preconditions + website: /login # preconditions steps: - - Enter text: - username: james - password: password - - Click: log in - -See James analytics: - based on: log in as james # test inheritance + - Form filled: + username: AzureDiamond + password: hunter2 + - Clicked: login + + +Email sent: + about: | + The most basic email with no subject, cc or bcc + set. + based on: logged in # inherits from and continues from test above following steps: - - Click: analytics + - Clicked: new email + - Form filled: + to: Cthon98@aol.com + contents: | # long form text + Hey guys, + + I think I got hacked! + - Clicked: send email + - Email was sent ``` -test_hitchstory.py: +engine.py: ```python from hitchstory import BaseEngine, GivenDefinition, GivenProperty from hitchstory import Failure, strings_match -from hitchstory import StoryCollection from strictyaml import Str -from pathlib import Path -from os import getenv class Engine(BaseEngine): - """Interprets and validates the hitchstory stories.""" - given_definition = GivenDefinition( - browser=GivenProperty( - # Available validators: https://hitchdev.com/strictyaml/using/ - Str() - ), + website=GivenProperty(Str()), ) def __init__(self, rewrite=False): self._rewrite = rewrite def set_up(self): - print(f"Using browser {self.given['browser']}") + print(f"Load web page at {self.given['website']}") + + def form_filled(self, **textboxes): + for name, contents in sorted(textboxes.items()): + print(f"Put {contents} in name") - def click(self, name): + def clicked(self, name): print(f"Click on {name}") - - if name == "analytics": - raise Failure(f"button {name} not found") - def enter_text(self, **textboxes): - for name, text in textboxes.items(): - print(f"Enter {text} in {name}") + def failing_step(self): + raise Failure("This was not supposed to happen") - def tear_down(self): - pass - - -collection = StoryCollection( - # All .story files in this file's directory. - Path(__file__).parent.glob("*.story"), - - Engine( - # If REWRITE environment variable is set to yes -> rewrite mode. - rewrite=getenv("REWRITE", "no") == "yes" - ) -) - -#You can embed the stories in tests manually: -#def test_log_in_as_james(): -# collection.named("Log in as james").play() - -#def test_see_james_analytics(): -# collection.named("See James analytics").play() - -# Or autogenerate runnable tests from the YAML stories like so: -# E.g. "Log in as James" -> "def test_login_in_as_james" -collection.with_external_test_runner().ordered_by_name().add_pytests_to( - module=__import__(__name__) # This module -) + def error_message_displayed(self, expected_message): + """Demonstrates steps that can rewrite themselves.""" + actual_message = "error message!" + try: + strings_match(expected_message, actual_message) + except Failure: + if self._rewrite: + self.current_step.rewrite("expected_message").to(actual_message) + else: + raise + + def email_was_sent(self): + print("Check email was sent!") ``` -## Run passing "log in as James" test - -Running test_log_in_as_james runs the "Log in as James" story. - - - -`pytest -s -k test_log_in_as_james` - -Outputs: -``` -============================= test session starts ============================== -platform linux -- Python n.n.n, pytest-n.n.n, pluggy-n.n.n -rootdir: /path/to -collected 2 items / 1 deselected / 1 selected - -test_hitchstory.py Using browser firefox -Enter james in username -Enter password in password -Click on log in -. - -======================= 1 passed, 1 deselected in 0.1s ======================== -``` - - -## Run failing "see James' analytics" test - -Failing tests also have colors and highlighting when run for real. - - - - - -`pytest -k test_see_james_analytics` - -Outputs: -``` -============================= test session starts ============================== -platform linux -- Python n.n.n, pytest-n.n.n, pluggy-n.n.n -rootdir: /path/to -collected 2 items / 1 deselected / 1 selected - -test_hitchstory.py F [100%] - -=================================== FAILURES =================================== -___________________________ test_see_james_analytics ___________________________ - -story = Story('see-james-analytics') - - def hitchstory(story=story): -> story.play() -E hitchstory.exceptions.StoryFailure: RUNNING See James analytics in /path/to/example.story ... FAILED in 0.1 seconds. -E -E based on: log in as james # test inheritance -E following steps: -E - Click: analytics -E -E -E hitchstory.exceptions.Failure -E -E Test failed. -E -E button analytics not found - -/src/hitchstory/story_list.py:51: StoryFailure ------------------------------ Captured stdout call ----------------------------- -Using browser firefox -Enter james in username -Enter password in password -Click on log in -Click on analytics -=========================== short test summary info ============================ -FAILED test_hitchstory.py::test_see_james_analytics - hitchstory.exceptions.StoryFailure: RUNNING See James analytics in /path/to/example.story ... FAILED in 0.1 seconds. - - based on: log in as james # test inheritance - following steps: - - Click: analytics - - -hitchstory.exceptions.Failure - - Test failed. - -button analytics not found -======================= 1 failed, 1 deselected in 0.1s ======================== +```python +>>> from hitchstory import StoryCollection +>>> from pathlib import Path +>>> from engine import Engine +>>> +>>> StoryCollection(Path(".").glob("*.story"), Engine()).named("Email sent").play() +RUNNING Email sent in /path/to/working/example.story ... Load web page at /login +Put hunter2 in name +Put AzureDiamond in name +Click on login +Click on new email +Put Hey guys, + +I think I got hacked! + in name +Put Cthon98@aol.com in name +Click on send email +Check email was sent! +SUCCESS in 0.1 seconds. ``` @@ -231,8 +156,8 @@ It is tested and documented with itself. ## Using HitchStory: With Pytest -If you already have pytest set up and running integration -tests, you can use it with hitchstory: +If you already have pytest set up, you can quickly and easily write +a test using hitchstory that runs alongside your other pytest tests: - [Self rewriting tests with pytest and hitchstory](https://hitchdev.com/hitchstory/using/pytest/rewrite) @@ -298,7 +223,6 @@ Best practices, how the tool was meant to be used, etc. - [Flaky Tests](https://hitchdev.com/hitchstory/approach/flaky-tests) - [The Hermetic End to End Testing Pattern](https://hitchdev.com/hitchstory/approach/hermetic-end-to-end-test) - [ANTIPATTERN - Analysts writing stories for the developer](https://hitchdev.com/hitchstory/approach/human-writable) -- [Separation of Test Concerns](https://hitchdev.com/hitchstory/approach/separation-of-test-concerns) - [Snapshot Test Driven Development (STDD)](https://hitchdev.com/hitchstory/approach/snapshot-test-driven-development-stdd) - [Test Artefact Environment Isolation](https://hitchdev.com/hitchstory/approach/test-artefact-environment-isolation) - [Test concern leakage](https://hitchdev.com/hitchstory/approach/test-concern-leakage) diff --git a/docs/public/approach/index.md b/docs/public/approach/index.md index 640c2f91..f4e8bb6a 100644 --- a/docs/public/approach/index.md +++ b/docs/public/approach/index.md @@ -11,7 +11,6 @@ HitchStory best practices are documented here: - [Flaky Tests](flaky-tests) - [The Hermetic End to End Testing Pattern](hermetic-end-to-end-test) - [ANTIPATTERN - Analysts writing stories for the developer](human-writable) -- [Separation of Test Concerns](separation-of-test-concerns) - [Snapshot Test Driven Development (STDD)](snapshot-test-driven-development-stdd) - [Test Artefact Environment Isolation](test-artefact-environment-isolation) - [Test concern leakage](test-concern-leakage) diff --git a/docs/public/approach/separation-of-test-concerns.md b/docs/public/approach/separation-of-test-concerns.org similarity index 73% rename from docs/public/approach/separation-of-test-concerns.md rename to docs/public/approach/separation-of-test-concerns.org index 07545104..ce1f8303 100644 --- a/docs/public/approach/separation-of-test-concerns.md +++ b/docs/public/approach/separation-of-test-concerns.org @@ -1,11 +1,17 @@ ---- -title: Separation of Test Concerns ---- -# Separation of Test Concerns +* Separation of Test Concerns :wdoc:wpublish: +:PROPERTIES: +:directory: hitchstory/docs/src/approach/ +:END: -Separation of Test Concerns is a form of [separation of concerns](https://en.wikipedia.org/wiki/Separation_of_concerns) where the user story of a test is described declaratively and kept entirely separate from the [turing complete](https://en.m.wikipedia.org/wiki/Turing_completeness) code (the "engine") that executes it. +** #intro -It mirrors the separation of concerns exhibited by web applications adhering to the [MVC](https://en.m.wikipedia.org/wiki/Model–view–controller) pattern ([MTV](https://docs.djangoproject.com/en/2.1/faq/general/#django-appears-to-be-a-mvc-framework-but-you-call-the-controller-the-view-and-the-view-the-template-how-come-you-don-t-use-the-standard-names) in the Django world).With hitchstory, an example of a non-turing complete, declarative user story would look something like this: +Separation of Test Concerns is a form of [[https://en.wikipedia.org/wiki/Separation_of_concerns][separation of concerns]] where the user story of a test is described declaratively and kept entirely separate from the [[https://en.m.wikipedia.org/wiki/Turing_completeness][turing complete]] code (the "engine") that executes it. + +It mirrors the separation of concerns exhibited by web applications adhering to the [[https://en.m.wikipedia.org/wiki/Model–view–controller][MVC]] pattern ([[https://docs.djangoproject.com/en/2.1/faq/general/#django-appears-to-be-a-mvc-framework-but-you-call-the-controller-the-view-and-the-view-the-template-how-come-you-don-t-use-the-standard-names][MTV]] in the Django world). + +** # + +With hitchstory, an example of a non-turing complete, declarative user story would look something like this: ```yaml Logged in as AzureDiamond: @@ -31,8 +37,7 @@ With corresponding engine code looking like this: self.driver.click(name=name) ``` - -## xUnit tests : no separation of concerns +** xUnit tests : no separation of concerns Unlike with hitchstory, xUnit tests do not have a layer to enforce separation test of concerns. @@ -59,10 +64,7 @@ def test_add_user(browser, web_server, init, dbsession): This isnt always the case, however, and it is very frequently the case that the intended behavior is difficult to derive from the test even for a skilled developer. - - - -## No separation: less readable +** No separation: less readable Not all tests have an easily discernable specification. For example: @@ -74,10 +76,7 @@ Not all tests have an easily discernable specification. For example: This isnt a problem with the writer of the test - it is an intrinsic problem with xUnit tests. As they grow, they become more unreadable, and since they largely interact with code APIs the relevance to the overall app may be difficult to discern. - - - -## Cucumber / Gherkin: Separation only between high level description and the rest +** Cucumber / Gherkin: Separation only between high level description and the rest Cucumber (as well as its derivatives) is another framework that enforces a language layer, but instead of enforcing a separation between specification and execution it is shaped around creating a much less useful separation between high level specifications and the rest. @@ -90,11 +89,8 @@ An example of such a high level scenario (drawn from a representative cucumber t And API: I check that POST call status code is 200 ``` -This example exhibits [test concern leakage](../test-concern-leakage). +This example exhibits [[../test-concern-leakage][test concern leakage]]. This provides a limited window into the intended (or actual) behavior of the API as it is a very high level overview of the API's behavior. Key specification details about this story will still exist in this executable specification but they will be buried in the step code that the above translates to. Due to the need to bury key specification details (e.g. the contents of the API call creating the person), the step code will also have to be highly specialized and step code reusability will be inhibited. - - - diff --git a/docs/public/index.md b/docs/public/index.md index 496c1a77..5125ca8c 100644 --- a/docs/public/index.md +++ b/docs/public/index.md @@ -6,7 +6,8 @@ title: HitchStory GitHub Repo starsPyPI - Downloads -Type-safe [StrictYAML](why/strictyaml) integration tests run from pytest. They can: +Type-safe [StrictYAML](why/strictyaml) python integration testing framework. With this +framework, your tests can: ## Rewrite themselves from program output (command line test example) @@ -16,6 +17,7 @@ Type-safe [StrictYAML](why/strictyaml) integration tests run from pytest. They c ![Test writing docs](https://hitchdev-videos.netlify.app/rewrite-docs-demo.gif) +The tests can be run on their own or as pytest tests. ## Demo projects with demo tests @@ -29,8 +31,6 @@ Project | Storytests | Python code | Doc template | Autogenerated docs -Minimal example (two files) demonstrating two short YAML tests and the -python code necessary to run them from within a pytest file. # Code Example @@ -40,174 +40,99 @@ python code necessary to run them from within a pytest file. example.story: ```yaml -Log in as James: +Logged in: given: - browser: firefox # test preconditions + website: /login # preconditions steps: - - Enter text: - username: james - password: password - - Click: log in - -See James analytics: - based on: log in as james # test inheritance + - Form filled: + username: AzureDiamond + password: hunter2 + - Clicked: login + + +Email sent: + about: | + The most basic email with no subject, cc or bcc + set. + based on: logged in # inherits from and continues from test above following steps: - - Click: analytics + - Clicked: new email + - Form filled: + to: Cthon98@aol.com + contents: | # long form text + Hey guys, + + I think I got hacked! + - Clicked: send email + - Email was sent ``` -test_hitchstory.py: +engine.py: ```python from hitchstory import BaseEngine, GivenDefinition, GivenProperty from hitchstory import Failure, strings_match -from hitchstory import StoryCollection from strictyaml import Str -from pathlib import Path -from os import getenv class Engine(BaseEngine): - """Interprets and validates the hitchstory stories.""" - given_definition = GivenDefinition( - browser=GivenProperty( - # Available validators: https://hitchdev.com/strictyaml/using/ - Str() - ), + website=GivenProperty(Str()), ) def __init__(self, rewrite=False): self._rewrite = rewrite def set_up(self): - print(f"Using browser {self.given['browser']}") + print(f"Load web page at {self.given['website']}") + + def form_filled(self, **textboxes): + for name, contents in sorted(textboxes.items()): + print(f"Put {contents} in name") - def click(self, name): + def clicked(self, name): print(f"Click on {name}") - - if name == "analytics": - raise Failure(f"button {name} not found") - def enter_text(self, **textboxes): - for name, text in textboxes.items(): - print(f"Enter {text} in {name}") + def failing_step(self): + raise Failure("This was not supposed to happen") - def tear_down(self): - pass - - -collection = StoryCollection( - # All .story files in this file's directory. - Path(__file__).parent.glob("*.story"), - - Engine( - # If REWRITE environment variable is set to yes -> rewrite mode. - rewrite=getenv("REWRITE", "no") == "yes" - ) -) - -#You can embed the stories in tests manually: -#def test_log_in_as_james(): -# collection.named("Log in as james").play() - -#def test_see_james_analytics(): -# collection.named("See James analytics").play() - -# Or autogenerate runnable tests from the YAML stories like so: -# E.g. "Log in as James" -> "def test_login_in_as_james" -collection.with_external_test_runner().ordered_by_name().add_pytests_to( - module=__import__(__name__) # This module -) + def error_message_displayed(self, expected_message): + """Demonstrates steps that can rewrite themselves.""" + actual_message = "error message!" + try: + strings_match(expected_message, actual_message) + except Failure: + if self._rewrite: + self.current_step.rewrite("expected_message").to(actual_message) + else: + raise + + def email_was_sent(self): + print("Check email was sent!") ``` -## Run passing "log in as James" test - -Running test_log_in_as_james runs the "Log in as James" story. - - - -`pytest -s -k test_log_in_as_james` - -Outputs: -``` -============================= test session starts ============================== -platform linux -- Python n.n.n, pytest-n.n.n, pluggy-n.n.n -rootdir: /path/to -collected 2 items / 1 deselected / 1 selected - -test_hitchstory.py Using browser firefox -Enter james in username -Enter password in password -Click on log in -. - -======================= 1 passed, 1 deselected in 0.1s ======================== -``` - - -## Run failing "see James' analytics" test - -Failing tests also have colors and highlighting when run for real. - - - - - -`pytest -k test_see_james_analytics` - -Outputs: -``` -============================= test session starts ============================== -platform linux -- Python n.n.n, pytest-n.n.n, pluggy-n.n.n -rootdir: /path/to -collected 2 items / 1 deselected / 1 selected - -test_hitchstory.py F [100%] - -=================================== FAILURES =================================== -___________________________ test_see_james_analytics ___________________________ - -story = Story('see-james-analytics') - - def hitchstory(story=story): -> story.play() -E hitchstory.exceptions.StoryFailure: RUNNING See James analytics in /path/to/example.story ... FAILED in 0.1 seconds. -E -E based on: log in as james # test inheritance -E following steps: -E - Click: analytics -E -E -E hitchstory.exceptions.Failure -E -E Test failed. -E -E button analytics not found - -/src/hitchstory/story_list.py:51: StoryFailure ------------------------------ Captured stdout call ----------------------------- -Using browser firefox -Enter james in username -Enter password in password -Click on log in -Click on analytics -=========================== short test summary info ============================ -FAILED test_hitchstory.py::test_see_james_analytics - hitchstory.exceptions.StoryFailure: RUNNING See James analytics in /path/to/example.story ... FAILED in 0.1 seconds. - - based on: log in as james # test inheritance - following steps: - - Click: analytics - - -hitchstory.exceptions.Failure - - Test failed. - -button analytics not found -======================= 1 failed, 1 deselected in 0.1s ======================== +```python +>>> from hitchstory import StoryCollection +>>> from pathlib import Path +>>> from engine import Engine +>>> +>>> StoryCollection(Path(".").glob("*.story"), Engine()).named("Email sent").play() +RUNNING Email sent in /path/to/working/example.story ... Load web page at /login +Put hunter2 in name +Put AzureDiamond in name +Click on login +Click on new email +Put Hey guys, + +I think I got hacked! + in name +Put Cthon98@aol.com in name +Click on send email +Check email was sent! +SUCCESS in 0.1 seconds. ``` @@ -235,8 +160,8 @@ It is tested and documented with itself. ## Using HitchStory: With Pytest -If you already have pytest set up and running integration -tests, you can use it with hitchstory: +If you already have pytest set up, you can quickly and easily write +a test using hitchstory that runs alongside your other pytest tests: - [Self rewriting tests with pytest and hitchstory](using/pytest/rewrite) @@ -302,7 +227,6 @@ Best practices, how the tool was meant to be used, etc. - [Flaky Tests](approach/flaky-tests) - [The Hermetic End to End Testing Pattern](approach/hermetic-end-to-end-test) - [ANTIPATTERN - Analysts writing stories for the developer](approach/human-writable) -- [Separation of Test Concerns](approach/separation-of-test-concerns) - [Snapshot Test Driven Development (STDD)](approach/snapshot-test-driven-development-stdd) - [Test Artefact Environment Isolation](approach/test-artefact-environment-isolation) - [Test concern leakage](approach/test-concern-leakage) diff --git a/docs/public/using/engine/rewrite-given.md b/docs/public/using/engine/rewrite-given.md index b17e4281..88c85859 100644 --- a/docs/public/using/engine/rewrite-given.md +++ b/docs/public/using/engine/rewrite-given.md @@ -4,25 +4,32 @@ title: Story that rewrites given preconditions -These examples show how to build stories that rewrite themselves -from program output (in-test snapshot testing) but that rewrite -the given preconditions. +These examples show how to build stories that rewrite their given +preconditions from program output. -This is useful for changing +This is useful for auto-updating given preconditions when the +outside world changes. For example, if a a REST API service that +is being mocked starts returning different data you can +run the story in rewrite mode to update the mock. + +The command to perform this rewrite is: ``` self.current_step.rewrite("argument").to("new output") ``` +Note that if there is a story inheritance hierarchy then only the +child story's given preconditions will be updated. + # Code Example -example.story: +example1.story: ```yaml -Call API: +Basic: given: mock api: request: | @@ -32,6 +39,46 @@ Call API: steps: - Call API ``` +example2.story: + +```yaml +Overridden response: + based on: basic + given: + mock api: + response: | + {"greeting": "bonjour"} +``` +example3.story: + +```yaml +Overridden request: + based on: basic + given: + mock api: + request: | + {"greeting": "hi there"} +``` +example4.story: + +```yaml +Story with variations: + steps: + - Call API + + variations: + French: + given: + mock api: + response: | + {"greeting": "bonjour"} + + Chinese: + given: + mock api: + request: | + {"greeting": "Ni hao"} +``` engine.py: ```python @@ -51,7 +98,7 @@ class Engine(BaseEngine): def call_api(self): if self._rewrite: - self.given.rewrite("Mock API", "response").to("""{"greeting": "bye"}""") + self.given.rewrite("Mock API", "response").to("""{"greeting": "bye"}\n""") ``` With code: @@ -66,25 +113,31 @@ from engine import Engine +## Simple + + + + + ```python -StoryCollection(Path(".").glob("*.story"), Engine(rewrite=True)).ordered_by_name().play() +StoryCollection(Path(".").glob("*.story"), Engine(rewrite=True)).named("Basic").play() ``` Will output: ``` -RUNNING Call API in /path/to/working/example.story ... SUCCESS in 0.1 seconds. +RUNNING Basic in /path/to/working/example1.story ... SUCCESS in 0.1 seconds. ``` -File example.story should now contain: +File example1.story should now contain: ``` -Call API: +Basic: given: mock api: request: | @@ -92,9 +145,121 @@ Call API: response: | {"greeting": "bye"} steps: - - Call API + - Call API +``` + + +## Overridden response + + + + + + + +```python +StoryCollection(Path(".").glob("*.story"), Engine(rewrite=True)).named("Overridden response").play() + +``` + +Will output: +``` +RUNNING Overridden response in /path/to/working/example2.story ... SUCCESS in 0.1 seconds. +``` + + + + +File example2.story should now contain: + +``` +Overridden response: + based on: basic + given: + mock api: + response: | + {"greeting": "bye"} +``` + + +## Overridden request + + + + + + + +```python +StoryCollection(Path(".").glob("*.story"), Engine(rewrite=True)).named("Overridden request").play() + +``` + +Will output: +``` +RUNNING Overridden request in /path/to/working/example3.story ... SUCCESS in 0.1 seconds. +``` + + + + +File example3.story should now contain: + +``` +Overridden request: + based on: basic + given: + mock api: + request: | + {"greeting": "hi there"} + response: | + {"greeting": "bye"} +``` + + +## Story with variations + + + + + + + +```python +StoryCollection(Path(".").glob("*.story"), Engine(rewrite=True)).named("Story with variations/French").play() + ``` +Will output: +``` +RUNNING Story with variations/French in /path/to/working/example4.story ... SUCCESS in 0.1 seconds. +``` + + + + +File example4.story should now contain: + +``` +Story with variations: + steps: + - Call API + + variations: + French: + given: + mock api: + response: | + {"greeting": "bye"} + + Chinese: + given: + mock api: + request: | + {"greeting": "Ni hao"} +``` + +