From 4e319f3712e7a634b2c4b66c4c3a1b8838bdfbfc Mon Sep 17 00:00:00 2001 From: Tobias Persson Date: Fri, 9 Feb 2024 10:43:21 +0100 Subject: [PATCH] Updated tests to run the ESR more like the real world Also added a few extra checks and testability things to the ESR --- .../src/etos_suite_runner/esr.py | 18 +- .../src/etos_suite_runner/lib/suite.py | 4 + .../etos_suite_runner/test-requirements.txt | 1 + .../tests/library/fake_database.py | 124 ++++++ .../tests/library/handler.py | 368 +++++------------- .../etos_suite_runner/tests/scenario/tercc.py | 11 + .../tests/scenario/test_permutations.py | 223 ++++++++--- .../tests/scenario/test_regular.py | 227 +++++++++-- projects/etos_suite_runner/tox.ini | 1 + 9 files changed, 617 insertions(+), 360 deletions(-) create mode 100644 projects/etos_suite_runner/tests/library/fake_database.py diff --git a/projects/etos_suite_runner/src/etos_suite_runner/esr.py b/projects/etos_suite_runner/src/etos_suite_runner/esr.py index 19f3a55..a083222 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/esr.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/esr.py @@ -102,12 +102,13 @@ def _release_environment(self) -> None: if not status: self.logger.error(message) - def run_suites(self, triggered: EiffelActivityTriggeredEvent) -> None: + def run_suites(self, triggered: EiffelActivityTriggeredEvent) -> list[str]: """Start up a suite runner handling multiple suites that execute within test runners. Will only start the test activity if there's a 'slot' available. :param triggered: Activity triggered. + :return: List of main suite IDs """ context = triggered.meta.event_id self.etos.config.set("context", context) @@ -125,12 +126,15 @@ def run_suites(self, triggered: EiffelActivityTriggeredEvent) -> None: try: self.logger.info("Get test environment.") - threading.Thread(target=self._request_environment, args=(ids,), daemon=True).start() + threading.Thread( + target=self._request_environment, args=(ids.copy(),), daemon=True + ).start() self.etos.events.send_activity_started(triggered, {"CONTEXT": context}) self.logger.info("Starting ESR.") runner.start_suites_and_wait() + return ids except EnvironmentProviderException: self.logger.info("Release test environment.") self._release_environment() @@ -143,8 +147,11 @@ def verify_input() -> None: assert os.getenv("SOURCE_HOST"), "SOURCE_HOST environment variable not provided." assert os.getenv("TERCC"), "TERCC environment variable not provided." - def run(self) -> None: - """Run the ESR main loop.""" + def run(self) -> list[str]: + """Run the ESR main loop. + + :return: List of test suites (main suites) that were started. + """ tercc_id = None try: tercc_id = self.params.tercc.meta.event_id @@ -186,10 +193,11 @@ def run(self) -> None: raise try: - self.run_suites(triggered) + ids = self.run_suites(triggered) self.etos.events.send_activity_finished( triggered, {"conclusion": "SUCCESSFUL"}, {"CONTEXT": context} ) + return ids except Exception as exception: # pylint:disable=broad-except reason = str(exception) self.logger.exception( diff --git a/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py b/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py index 186a986..dd7d924 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py @@ -314,6 +314,10 @@ def start(self) -> None: "Environment received. Starting up a sub suite", extra={"user_log": True} ) sub_suite_definition = self._download_sub_suite(sub_suite_environment) + if sub_suite_definition is None: + raise EnvironmentProviderException( + "URL to sub suite is missing", self.etos.config.get("task_id") + ) sub_suite_definition["id"] = sub_suite_environment["meta"]["id"] sub_suite = SubSuite( self.etos, sub_suite_definition, self.suite["test_suite_started_id"] diff --git a/projects/etos_suite_runner/test-requirements.txt b/projects/etos_suite_runner/test-requirements.txt index 27054f2..820b2fe 100644 --- a/projects/etos_suite_runner/test-requirements.txt +++ b/projects/etos_suite_runner/test-requirements.txt @@ -1,3 +1,4 @@ mock pytest pytest-cov +mongomock diff --git a/projects/etos_suite_runner/tests/library/fake_database.py b/projects/etos_suite_runner/tests/library/fake_database.py new file mode 100644 index 0000000..c1d9756 --- /dev/null +++ b/projects/etos_suite_runner/tests/library/fake_database.py @@ -0,0 +1,124 @@ +# Copyright Axis Communications AB. +# +# For a full list of individual contributors, please see the commit history. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Fake database library helpers.""" +from queue import Queue +from threading import Event, RLock, Timer +from typing import Any, Iterator, Optional + +from etcd3gw.lease import Lease + +# pylint:disable=unused-argument + + +class FakeDatabase: + """A fake database that follows the etcd client.""" + + lock = RLock() + + def __init__(self): + """Initialize fake reader and writer.""" + self.db_dict = {} + self.expire = [] + self.leases = {} + self.watchers = [] + + def __call__(self): + """Database instantiation faker.""" + return self + + def watch(self, path: str, range_end: Optional[str] = None) -> (Event, Iterator[dict]): + """Watch for changes of a path.""" + canceled = Event() + queue = Queue() + + def cancel(): + canceled.set() + queue.put(None) + + def iterator(): + self.watchers.append(queue) + try: + while not canceled.is_set(): + event = queue.get() + if event is None: + canceled.set() + if not canceled.is_set(): + yield event + finally: + self.watchers.remove(queue) + + return iterator(), cancel + + def __event(self, event: dict) -> None: + """Send an event to all watchers.""" + for watcher in self.watchers: + watcher.put_nowait(event) + + def put(self, item: str, value: Any, lease: Optional[Lease] = None) -> None: + """Put an item into database.""" + with self.lock: + self.db_dict.setdefault(item, []) + if lease is not None: + self.leases[item] = lease + timer = Timer(self.expire[lease.id], self.delete, args=(item,)) + timer.daemon = True + timer.start() + self.db_dict[item].append(str(value).encode()) + self.__event({"kv": {"key": item.encode(), "value": str(value).encode()}}) + + def lease(self, ttl=30) -> Lease: + """Create a lease.""" + with self.lock: + self.expire.append(ttl) + # ttl is unused since we do not actually make the post request that the regular + # etcd client does. First argument to `Lease` is the ID that was returned by the + # etcd server. + return Lease(len(self.expire) - 1) + + def get(self, path: str) -> list[bytes]: + """Get an item from database.""" + if isinstance(path, bytes): + path = path.decode() + with self.lock: + return list(reversed(self.db_dict.get(path, []))) + + def get_prefix(self, prefix: str) -> list[tuple[bytes, dict]]: + """Get items based on prefix.""" + if isinstance(prefix, bytes): + prefix = prefix.decode() + paths = [] + with self.lock: + for key, value in self.db_dict.items(): + if key.startswith(prefix): + paths.append((value[-1], {"key": key.encode()})) + return paths + + def delete(self, path: str) -> None: + """Delete a single item.""" + with self.lock: + del self.db_dict[path] + if self.leases.get(path): + self.expire.pop(self.leases.get(path).id) + del self.leases[path] + self.__event({"kv": {"key": path.encode()}, "type": "DELETE"}) + + def delete_prefix(self, prefix: str) -> None: + """Delete items based on prefix.""" + with self.lock: + db_dict = self.db_dict.copy() + for key in db_dict: + if key.startswith(prefix): + self.delete(key) diff --git a/projects/etos_suite_runner/tests/library/handler.py b/projects/etos_suite_runner/tests/library/handler.py index 91c3150..9b4d3eb 100644 --- a/projects/etos_suite_runner/tests/library/handler.py +++ b/projects/etos_suite_runner/tests/library/handler.py @@ -14,15 +14,19 @@ # See the License for the specific language governing permissions and # limitations under the License. """ETOS suite runner request handler.""" -import os import json import logging from http.server import BaseHTTPRequestHandler -from uuid import uuid4 +from urllib.parse import urlparse -from graphql import parse -from etos_lib.lib.debug import Debug +import mongomock import requests +from eiffellib.events import EiffelTestSuiteFinishedEvent, EiffelTestSuiteStartedEvent +from etos_lib.lib.debug import Debug +from graphql import parse + +CLIENT = mongomock.MongoClient() +DB = CLIENT["Database"] class Handler(BaseHTTPRequestHandler): @@ -30,10 +34,8 @@ class Handler(BaseHTTPRequestHandler): logger = logging.getLogger(__name__) requests = [] - sub_suites_and_main_suites = {} - environments = {} - suite_runner_ids = [] - activities = {} + uploads = {} + debug = Debug() def __init__(self, tercc, *args, **kwargs): """Initialize a BaseHTTPRequestHandler. This must be initialized with functools.partial. @@ -46,7 +48,6 @@ def __init__(self, tercc, *args, **kwargs): :param tercc: Test execution recipe collection for a test scenario. :type tercc: dict """ - self.debug = Debug() self.tercc = tercc super().__init__(*args, **kwargs) @@ -54,23 +55,37 @@ def __init__(self, tercc, *args, **kwargs): def reset(cls): """Reset the handler. This has to be done after each test.""" cls.requests.clear() - cls.sub_suites_and_main_suites.clear() - cls.environments.clear() - cls.activities.clear() - cls.suite_runner_ids.clear() + CLIENT.drop_database("Database") + cls.uploads.clear() + + @classmethod + def insert_to_db(cls, event): + """Insert an event to the database. - @property - def main_suites(self): - """Test suites started sent by ESR. + :param event: Event to store into database. + :type event: :obj:`eiffellib.events.eiffel_base_event.EiffelBaseEvent` + """ + collection = DB[event.meta.type] + response = collection.find_one({"_id": event.meta.event_id}) + if not response: + doc = event.json + doc["_id"] = event.meta.event_id + collection.insert_one(doc) + + @classmethod + def get_from_db(cls, collection_name, query): + """Send a query to a database collection. - :return: A list of test suite started stored in the ETOS library debug module. + :param collection_name: The collection to query. + :type collection_name: str + :param query: The query to send. + :type query: dict + :return: a list of events from the database. :rtype: list """ - started = [] - for event in self.debug.events_published: - if event.meta.type == "EiffelTestSuiteStartedEvent": - started.append(event) - return started + for event in cls.debug.events_published: + cls.insert_to_db(event) + return list(DB[collection_name].find(query)) def store_request(self, data): """Store a request for testing purposes. @@ -101,184 +116,20 @@ def get_gql_query(self, request_data): ) raise TypeError("Not a valid GraphQL query") - def artifact_created(self): - """Artifact under test. - - :return: A GraphQL response for an artifact. - :rtype: dict - """ - return { - "data": { - "artifactCreated": { - "edges": [ - { - "node": { - "data": { - "identity": "pkg:etos/suite-runner", - }, - "links": [], - "meta": { - "id": "b44e0d4a-bc88-4c2a-b808-d336448c959e", - "time": 1664263414557, - "type": "EiffelArtifactCreatedEvent", - "version": "3.0.0", - }, - } - } - ] - } - } - } + def to_graphql(self, name, data): + """Convert an Eiffel event dictionary to a GraphQL response. - def environment_defined(self, query): - """Create environment defined events for all expected sub suites. - - :param query: GraphQL query string. - :type query: dict - :return: A GraphQL response for several environment defined. - :rtype: dict - """ - if self.suite_runner_ids and self.main_suites: - activity = None - for activity_id, activity_event in self.activities.items(): - if activity_id == query["links.target"]: - activity = activity_event - if activity is None: - return {"data": {"environmentDefined": {"edges": []}}} - activity_id = activity["meta"]["id"] - if self.environments.get(activity_id): - return {"data": {"environmentDefined": {"edges": self.environments[activity_id]}}} - link_id = None - for link in activity["links"]: - if link["type"] == "CONTEXT": - link_id = link["target"] - break - self.environments.setdefault(activity_id, []) - - for suite in self.main_suites: - if suite.meta.event_id != link_id: - continue - host = os.getenv("ETOS_ENVIRONMENT_PROVIDER") - tercc = None - index = None - for number, batch in enumerate(self.tercc["data"]["batches"]): - if batch["name"] == suite.data.data.get("name"): - index = number - tercc = batch - for subindex, _ in enumerate(tercc["recipes"]): - sub_suite_name = f"{suite.data.data.get('name')}_SubSuite_{subindex+1}" - self.environments[activity_id].append( - { - "node": { - "data": { - "name": sub_suite_name, - "uri": f"{host}/sub_suite/{index}/{subindex}", - }, - "meta": {"id": str(uuid4())}, - } - } - ) - return {"data": {"environmentDefined": {"edges": self.environments[activity_id]}}} - return {"data": {"environmentDefined": {"edges": []}}} - - def test_suite_started(self, query): - """Create a test suite started for sub suites based on ESR suites. - - :param query: GraphQL query string. - :type query: dict - :return: A graphql response with a test suite started for a "started" sub suite. - :rtype: dict - """ - edges = [] - for key, values in self.sub_suites_and_main_suites.items(): - if key == query["links.target"]: - for value in values: - edges.append( - { - "node": { - "data": {"name": value["name"]}, - "meta": {"id": value["id"]}, - } - } - ) - break - return {"data": {"testSuiteStarted": {"edges": edges}}} - - def test_suite_finished(self, query): - """Create a test suite finished event based on sub suites and ESR suites. - - :param query: GraphQL query string. - :type query: dict - :return: A graphql response with a test suite finished for a "started" sub suite. + :param name: Name of the event, in graphql. + :type name: str + :param data: Data to set in graphql response. + :type data: list or None + :return: A graphql-valid dictionary :rtype: dict """ - edges = [] - for key, values in self.sub_suites_and_main_suites.items(): - if key == query["links.target"]: - for value in values: - edges.append( - { - "node": { - "data": {"testSuiteOutcome": {"verdict": "PASSED"}}, - "meta": {"id": value["finished"]}, - } - } - ) - break - return {"data": {"testSuiteFinished": {"edges": edges}}} - - def activity_triggered(self, query): - """Create an activity triggered event. - - :param query: GraphQL query string. - :type query: dict - :return: A graphql response with an activity triggered for a triggered environment provider. - :rtype: dict - """ - suite_id = query["links.target"] - event_id = str(uuid4()) - activity = { - "data": { - "activityTriggered": { - "meta": {"id": event_id}, - "links": [{"type": "CONTEXT", "target": suite_id}], - } - } - } - self.activities[event_id] = activity["data"]["activityTriggered"] - return activity - - def activity_finished(self, query): - """Create an activity finished event. - - :param query: GraphQL query string. - :type query: dict - :return: A graphql response with an activity finished for a "finished" activity. - :rtype: dict - """ - if self.environments: - for activity in self.activities: - if activity == query["links.target"]: - return { - "data": { - "activityFinished": { - "edges": [ - { - "node": { - "meta": {"id": str(uuid4())}, - "data": { - "activityOutcome": { - "conclusion": "SUCCESSFUL", - "description": None, - } - }, - } - } - ] - } - } - } - return {"data": {"activityFinished": {"edges": []}}} + if data: + edges = [{"node": d} for d in data] + return {"data": {name: {"edges": edges}}} + return {"data": {name: {"edges": []}}} def do_graphql(self, query_name, query): # pylint:disable=too-many-return-statements """Handle GraphQL queries to a fake ER. @@ -290,86 +141,71 @@ def do_graphql(self, query_name, query): # pylint:disable=too-many-return-state :return: JSON data mimicking an ER. :rtype: dict """ + data = None + if query_name == "testExecutionRecipeCollectionCreated": + data = self.get_from_db("EiffelTestExecutionRecipeCollectionCreatedEvent", query) if query_name == "artifactCreated": - return self.artifact_created() + data = self.get_from_db("EiffelArtifactCreatedEvent", query) if query_name == "activityTriggered": - return self.activity_triggered(query) + data = self.get_from_db("EiffelActivityTriggeredEvent", query) if query_name == "activityFinished": - return self.activity_finished(query) + data = self.get_from_db("EiffelActivityFinishedEvent", query) + for event in data: + # The GraphQL API changes the outcome fields since they are the same for multiple + # events so we have to correct the data here as well. + event["data"]["activityOutcome"] = event["data"].pop("outcome") if query_name == "environmentDefined": - return self.environment_defined(query) + data = self.get_from_db("EiffelEnvironmentDefinedEvent", query) if query_name == "testSuiteStarted": - return self.test_suite_started(query) + data = self.get_from_db("EiffelTestSuiteStartedEvent", query) if query_name == "testSuiteFinished": - return self.test_suite_finished(query) - return None - - def sub_suite(self): - """Get fake sub suite information mimicking the ETOS environment provider. - - :return: Sub suite definitions that the ESR can act upon. - :rtype: dict + data = self.get_from_db("EiffelTestSuiteFinishedEvent", query) + for event in data: + # The GraphQL API changes the outcome fields since they are the same for multiple + # events so we have to correct the data here as well. + event["data"]["testSuiteOutcome"] = event["data"].pop("outcome") + return self.to_graphql(query_name, data) + + def fake_start_etr(self, request_data): + """Handle the ETR start requests from the ESR. + + :param request_data: Request data from the ESR, with instructions. + :type request_data: bytes """ - subindex = int(self.path.split("/")[-1]) - index = int(self.path.split("/")[-2]) - sub_suite = self.tercc["data"]["batches"][index].copy() - for main_suite in self.main_suites: - if main_suite.data.data.get("name") == sub_suite["name"]: - main_suite_id = main_suite.meta.event_id - - sub_suite["name"] = f"{sub_suite['name']}_SubSuite_{subindex+1}" - host = os.getenv("ETOS_ENVIRONMENT_PROVIDER") - sub_suite["test_suite_started_id"] = main_suite_id - self.sub_suites_and_main_suites.setdefault(main_suite_id, []) - self.sub_suites_and_main_suites[main_suite_id].append( - { - "name": sub_suite["name"], - "id": main_suite_id, - "finished": str(uuid4()), - } + json_request = json.loads(request_data) + environment = DB["EiffelEnvironmentDefinedEvent"].find_one( + {"meta.id": json_request["environment"]["ENVIRONMENT_ID"]}, ) - sub_suite["executor"] = {"request": {"method": "GET", "url": f"{host}/etr"}} - return sub_suite - - def do_environment_provider_post(self, request_data): - """Handle POST requests to a fake environment provider. - - :return: JSON data mimicking the ETOS environment provider. - :rtype: dict - """ - json_data = json.loads(request_data) - if json_data.get("suite_runner_ids") is not None and not self.suite_runner_ids: - self.suite_runner_ids.extend(json_data["suite_runner_ids"].split(",")) - return {"result": "success", "data": {"id": "12345"}} - - def do_environment_provider_get(self): - """Handle GET requests to a fake environment provider. - - :return: JSON data mimicking the ETOS environment provider. - :rtype: dict - """ - if self.path.startswith("/sub_suite"): - return self.sub_suite() - if self.path == "/etr" or "?single_release" in self.path or "?release" in self.path: - # These are not being tested, just return SUCCESS. - return {"status": "SUCCESS"} - if self.path == "/?id=12345": - if self.environments: - return {"status": "SUCCESS"} - return {"status": "PENDING"} - return None + sub_suite = json.loads(self.uploads.get(urlparse(environment["data"]["uri"]).path)) + started = EiffelTestSuiteStartedEvent() + started.data.add("name", environment["data"]["name"]) + started.links.add("CAUSE", sub_suite["test_suite_started_id"]) + started.validate() + finished = EiffelTestSuiteFinishedEvent() + finished.data.add("outcome", {"verdict": "PASSED", "conclusion": "SUCCESSFUL"}) + finished.links.add("TEST_SUITE_EXECUTION", started) + finished.validate() + Debug().events_published.append(started) + Debug().events_published.append(finished) # pylint:disable=invalid-name def do_POST(self): """Handle POST requests.""" self.store_request(self.request) request_data = self.rfile.read(int(self.headers["Content-Length"])) - try: - query_name, query = self.get_gql_query(request_data) - except (TypeError, KeyError): - response = self.do_environment_provider_post(request_data) + if self.path.startswith("/log"): + self.uploads[self.path] = request_data.decode() + response = {} + elif self.path.startswith("/etr"): + self.fake_start_etr(request_data) + response = {} else: - response = self.do_graphql(query_name, query) + try: + query_name, query = self.get_gql_query(request_data) + except (TypeError, KeyError): + response = {} + else: + response = self.do_graphql(query_name, query) self.send_response(requests.codes["ok"]) self.send_header("Content-Type", "application/json; charset=utf-8") @@ -382,11 +218,9 @@ def do_POST(self): def do_GET(self): """Handle GET requests.""" self.store_request(self.request) - response = self.do_environment_provider_get() + response = self.uploads.get(self.path) self.send_response(requests.codes["ok"]) - self.send_header("Content-Type", "application/json; charset=utf-8") + self.send_header("Content-Type", "text/html; charset=utf-8") self.end_headers() - - response_content = json.dumps(response) - self.wfile.write(response_content.encode("utf-8")) + self.wfile.write(response.encode("utf-8")) diff --git a/projects/etos_suite_runner/tests/scenario/tercc.py b/projects/etos_suite_runner/tests/scenario/tercc.py index 6ecce1f..0cf35d1 100644 --- a/projects/etos_suite_runner/tests/scenario/tercc.py +++ b/projects/etos_suite_runner/tests/scenario/tercc.py @@ -333,3 +333,14 @@ }, "links": [{"type": "CAUSE", "target": "349f9bf9-0fc7-4dd4-b641-ac5f1c9ea7aa"}], } + + +ARTIFACT_CREATED = { + "data": {"identity": "pkg:etos-suite-runner/tests"}, + "meta": { + "type": "EiffelArtifactCreatedEvent", + "id": "349f9bf9-0fc7-4dd4-b641-ac5f1c9ea7aa", + "time": 1664260578384, + "version": "3.3.0", + }, +} diff --git a/projects/etos_suite_runner/tests/scenario/test_permutations.py b/projects/etos_suite_runner/tests/scenario/test_permutations.py index 4169df3..23097b6 100644 --- a/projects/etos_suite_runner/tests/scenario/test_permutations.py +++ b/projects/etos_suite_runner/tests/scenario/test_permutations.py @@ -14,20 +14,90 @@ # See the License for the specific language governing permissions and # limitations under the License. """Scenario tests for permutations.""" -import os -import time import json import logging -from unittest import TestCase +import os +import time +from copy import deepcopy from functools import partial +from unittest import TestCase -from etos_lib.lib.debug import Debug +from eiffellib.events import ( + EiffelArtifactCreatedEvent, + EiffelTestExecutionRecipeCollectionCreatedEvent, +) from etos_lib.lib.config import Config +from etos_lib.lib.debug import Debug from etos_suite_runner.esr import ESR -from tests.scenario.tercc import PERMUTATION_TERCC, PERMUTATION_TERCC_SUB_SUITES +from tests.library.fake_database import FakeDatabase from tests.library.fake_server import FakeServer from tests.library.handler import Handler +from tests.scenario.tercc import ARTIFACT_CREATED, PERMUTATION_TERCC, PERMUTATION_TERCC_SUB_SUITES + +IUT_PROVIDER = { + "iut": { + "id": "default", + "list": { + "possible": { + "$expand": { + "value": { + "type": "$identity.type", + "namespace": "$identity.namespace", + "name": "$identity.name", + "version": "$identity.version", + "qualifiers": "$identity.qualifiers", + "subpath": "$identity.subpath", + }, + "to": "$amount", + } + }, + "available": "$this.possible", + }, + } +} + +EXECUTION_SPACE_PROVIDER = { + "execution_space": { + "id": "default", + "list": { + "possible": { + "$expand": { + "value": { + "instructions": "$execution_space_instructions", + "request": { + "url": {"$join": {"strings": ["$dataset.host", "/etr"]}}, + "method": "POST", + "json": "$expand_value.instructions", + }, + }, + "to": "$amount", + } + }, + "available": "$this.possible", + }, + } +} + +LOG_AREA_PROVIDER = { + "log": { + "id": "default", + "list": { + "possible": { + "$expand": { + "value": { + "upload": { + "url": {"$join": {"strings": ["$dataset.host", "/log/", "{name}"]}}, + "method": "POST", + } + }, + "to": "$amount", + } + }, + "available": "$this.possible", + }, + } +} class TestPermutationScenario(TestCase): @@ -41,6 +111,7 @@ def setUp(self): os.environ["ESR_WAIT_FOR_ENVIRONMENT_TIMEOUT"] = "20" os.environ["SUITE_RUNNER"] = "registry.nordix.org/eiffel/etos-suite-runner" os.environ["SOURCE_HOST"] = "localhost" + Config().set("database", FakeDatabase()) def tearDown(self): """Reset all globally stored data for the next test.""" @@ -50,77 +121,114 @@ def tearDown(self): Debug()._Debug__events_published.clear() Debug()._Debug__events_received.clear() - def validate_event_name_order(self, events): - """Validate ESR sent events. + def setup_providers(self): + """Setup providers in the fake ETCD database.""" + Config().get("database").put("/environment/provider/iut/default", json.dumps(IUT_PROVIDER)) + Config().get("database").put( + "/environment/provider/execution-space/default", json.dumps(EXECUTION_SPACE_PROVIDER) + ) + Config().get("database").put( + "/environment/provider/log-area/default", json.dumps(LOG_AREA_PROVIDER) + ) - :raises AssertionError: If events are not correct. + def register_providers(self, testrun_id, host): + """Register providers for a testrun in the fake ETCD database. - :param events: All events sent, in order. - :type events: deque + :param testrun_id: ID to set up providers for. + :type testrun_id: str + :param host: The host to set in the dataset. + :type host: str """ - self.logger.info(events) - event_names_any_order = [ - "EiffelAnnouncementPublishedEvent", - "EiffelAnnouncementPublishedEvent", - "EiffelAnnouncementPublishedEvent", - ] - event_names_in_order = [ - "EiffelActivityTriggeredEvent", - "EiffelEnvironmentDefinedEvent", - "EiffelActivityStartedEvent", - "EiffelTestSuiteStartedEvent", - "EiffelTestSuiteStartedEvent", - "EiffelTestSuiteFinishedEvent", - "EiffelTestSuiteFinishedEvent", - "EiffelActivityFinishedEvent", - ] - for event_name in event_names_in_order: - sent_event = events.popleft().meta.type - while sent_event in event_names_any_order: - event_names_any_order.remove(sent_event) - sent_event = events.popleft().meta.type - self.assertEqual(sent_event, event_name) - self.assertEqual(list(events), []) - self.assertEqual(event_names_any_order, []) + Config().get("database").put( + f"/testrun/{testrun_id}/provider/iut", json.dumps(IUT_PROVIDER) + ) + Config().get("database").put( + f"/testrun/{testrun_id}/provider/log-area", json.dumps(LOG_AREA_PROVIDER) + ) + Config().get("database").put( + f"/testrun/{testrun_id}/provider/execution-space", json.dumps(EXECUTION_SPACE_PROVIDER) + ) + Config().get("database").put( + f"/testrun/{testrun_id}/provider/dataset", json.dumps({"host": host}) + ) + + def publish_initial_events(self, tercc, artc): + """Publish events that are required for the ESR to start. + + :param tercc: The test execution recipe collection created event that triggered the ESR. + :type tercc: dict + :param artc: The IUT artifact that the tercc links to. + :type artc: dict + """ + tercc_event = EiffelTestExecutionRecipeCollectionCreatedEvent() + tercc_event.rebuild(deepcopy(tercc)) + artc_event = EiffelArtifactCreatedEvent() + artc_event.rebuild(deepcopy(artc)) + Debug().events_published.append(artc_event) + Debug().events_published.append(tercc_event) def test_permutation_scenario(self): """Test permutations using 2 suites with 1 sub suite each. Approval criteria: - It shall be possible to execute permutations in ESR without failures. - - The ESR shall send events in the correct order. Test steps: 1. Start up a fake server. 2. Initialize and run ESR. 3. Verify that the ESR executes without errors. - 4. Verify that all events were sent and in the correct order. """ os.environ["TERCC"] = json.dumps(PERMUTATION_TERCC) tercc = json.loads(os.environ["TERCC"]) + testrun_id = tercc["meta"]["id"] + self.setup_providers() + self.publish_initial_events(tercc, ARTIFACT_CREATED) handler = partial(Handler, tercc) end = time.time() + 25 self.logger.info("STEP: Start up a fake server.") with FakeServer(handler) as server: + self.register_providers(testrun_id, server.host) os.environ["ETOS_GRAPHQL_SERVER"] = server.host os.environ["ETOS_ENVIRONMENT_PROVIDER"] = server.host + os.environ["ETOS_API"] = server.host self.logger.info("STEP: Initialize and run ESR.") esr = ESR() try: self.logger.info("STEP: Verify that the ESR executes without errors.") - esr.run() - - self.logger.info("STEP: Verify that all events were sent and in the correct order.") - self.validate_event_name_order(Debug().events_published.copy()) + suite_ids = esr.run() + self.assertEqual( + len(suite_ids), 2, "There shall only be a single test suite started." + ) + for suite_id in suite_ids: + suite_finished = Handler.get_from_db( + "EiffelTestSuiteFinishedEvent", {"links.target": suite_id} + ) + self.assertEqual( + len(suite_finished), 1, "There shall only be a single test suite finished." + ) + outcome = suite_finished[0].get("data", {}).get("outcome", {}) + self.logger.info(outcome) + self.assertDictEqual( + outcome, + { + "conclusion": "SUCCESSFUL", + "verdict": "PASSED", + "description": "All tests passed.", + }, + f"Wrong outcome {outcome!r}, outcome should be successful.", + ) finally: # If the _get_environment_status method in ESR does not time out before the test # finishes there will be loads of tracebacks in the log. Won't fail the test but # the noise is immense. while time.time() <= end: + status = esr.params.get_status() + if status["status"] != "NOT_STARTED": + break time.sleep(1) def test_permutation_scenario_sub_suites(self): @@ -128,37 +236,62 @@ def test_permutation_scenario_sub_suites(self): Approval criteria: - It shall be possible to execute permutations in ESR without failures. - - The ESR shall send events in the correct order. Test steps: 1. Start up a fake server. 2. Initialize and run ESR. 3. Verify that the ESR executes without errors. - 4. Verify that all events were sent and in the correct order. """ os.environ["TERCC"] = json.dumps(PERMUTATION_TERCC_SUB_SUITES) tercc = json.loads(os.environ["TERCC"]) + testrun_id = tercc["meta"]["id"] + self.setup_providers() + self.publish_initial_events(tercc, ARTIFACT_CREATED) handler = partial(Handler, tercc) end = time.time() + 25 self.logger.info("STEP: Start up a fake server.") with FakeServer(handler) as server: + self.register_providers(testrun_id, server.host) os.environ["ETOS_GRAPHQL_SERVER"] = server.host os.environ["ETOS_ENVIRONMENT_PROVIDER"] = server.host + os.environ["ETOS_API"] = server.host self.logger.info("STEP: Initialize and run ESR.") esr = ESR() try: self.logger.info("STEP: Verify that the ESR executes without errors.") - esr.run() + suite_ids = esr.run() - self.logger.info("STEP: Verify that all events were sent and in the correct order.") - self.validate_event_name_order(Debug().events_published.copy()) + self.assertEqual( + len(suite_ids), 2, "There shall only be a single test suite started." + ) + for suite_id in suite_ids: + suite_finished = Handler.get_from_db( + "EiffelTestSuiteFinishedEvent", {"links.target": suite_id} + ) + self.assertEqual( + len(suite_finished), 1, "There shall only be a single test suite finished." + ) + outcome = suite_finished[0].get("data", {}).get("outcome", {}) + self.logger.info(outcome) + self.assertDictEqual( + outcome, + { + "conclusion": "SUCCESSFUL", + "verdict": "PASSED", + "description": "All tests passed.", + }, + f"Wrong outcome {outcome!r}, outcome should be successful.", + ) finally: # If the _get_environment_status method in ESR does not time out before the test # finishes there will be loads of tracebacks in the log. Won't fail the test but # the noise is immense. while time.time() <= end: + status = esr.params.get_status() + if status["status"] != "NOT_STARTED": + break time.sleep(1) diff --git a/projects/etos_suite_runner/tests/scenario/test_regular.py b/projects/etos_suite_runner/tests/scenario/test_regular.py index fd2cee0..e178287 100644 --- a/projects/etos_suite_runner/tests/scenario/test_regular.py +++ b/projects/etos_suite_runner/tests/scenario/test_regular.py @@ -14,21 +14,90 @@ # See the License for the specific language governing permissions and # limitations under the License. """Scenario tests for the most regular cases.""" -import os -import time import json import logging -from unittest import TestCase +import os +import time +from copy import deepcopy from functools import partial +from unittest import TestCase -from etos_lib.lib.debug import Debug +from eiffellib.events import ( + EiffelArtifactCreatedEvent, + EiffelTestExecutionRecipeCollectionCreatedEvent, +) from etos_lib.lib.config import Config +from etos_lib.lib.debug import Debug from etos_suite_runner.esr import ESR - -from tests.scenario.tercc import TERCC, TERCC_SUB_SUITES, TERCC_EMPTY +from tests.library.fake_database import FakeDatabase from tests.library.fake_server import FakeServer from tests.library.handler import Handler +from tests.scenario.tercc import ARTIFACT_CREATED, TERCC, TERCC_EMPTY, TERCC_SUB_SUITES + +IUT_PROVIDER = { + "iut": { + "id": "default", + "list": { + "possible": { + "$expand": { + "value": { + "type": "$identity.type", + "namespace": "$identity.namespace", + "name": "$identity.name", + "version": "$identity.version", + "qualifiers": "$identity.qualifiers", + "subpath": "$identity.subpath", + }, + "to": "$amount", + } + }, + "available": "$this.possible", + }, + } +} + +EXECUTION_SPACE_PROVIDER = { + "execution_space": { + "id": "default", + "list": { + "possible": { + "$expand": { + "value": { + "instructions": "$execution_space_instructions", + "request": { + "url": {"$join": {"strings": ["$dataset.host", "/etr"]}}, + "method": "POST", + "json": "$expand_value.instructions", + }, + }, + "to": "$amount", + } + }, + "available": "$this.possible", + }, + } +} + +LOG_AREA_PROVIDER = { + "log": { + "id": "default", + "list": { + "possible": { + "$expand": { + "value": { + "upload": { + "url": {"$join": {"strings": ["$dataset.host", "/log/", "{name}"]}}, + "method": "POST", + } + }, + "to": "$amount", + } + }, + "available": "$this.possible", + }, + } +} class TestRegularScenario(TestCase): @@ -42,6 +111,7 @@ def setUp(self): os.environ["ESR_WAIT_FOR_ENVIRONMENT_TIMEOUT"] = "20" os.environ["SUITE_RUNNER"] = "registry.nordix.org/eiffel/etos-suite-runner" os.environ["SOURCE_HOST"] = "localhost" + Config().set("database", FakeDatabase()) def tearDown(self): """Reset all globally stored data for the next test.""" @@ -51,72 +121,112 @@ def tearDown(self): Debug()._Debug__events_published.clear() Debug()._Debug__events_received.clear() - def validate_event_name_order(self, events): - """Validate ESR sent events. + def setup_providers(self): + """Setup providers in the fake ETCD database.""" + Config().get("database").put("/environment/provider/iut/default", json.dumps(IUT_PROVIDER)) + Config().get("database").put( + "/environment/provider/execution-space/default", json.dumps(EXECUTION_SPACE_PROVIDER) + ) + Config().get("database").put( + "/environment/provider/log-area/default", json.dumps(LOG_AREA_PROVIDER) + ) + + def register_providers(self, testrun_id, host): + """Register providers for a testrun in the fake ETCD database. + + :param testrun_id: ID to set up providers for. + :type testrun_id: str + :param host: The host to set in the dataset. + :type host: str + """ + Config().get("database").put( + f"/testrun/{testrun_id}/provider/iut", json.dumps(IUT_PROVIDER) + ) + Config().get("database").put( + f"/testrun/{testrun_id}/provider/log-area", json.dumps(LOG_AREA_PROVIDER) + ) + Config().get("database").put( + f"/testrun/{testrun_id}/provider/execution-space", json.dumps(EXECUTION_SPACE_PROVIDER) + ) + Config().get("database").put( + f"/testrun/{testrun_id}/provider/dataset", json.dumps({"host": host}) + ) - :raises AssertionError: If events are not correct. + def publish_initial_events(self, tercc, artc): + """Publish events that are required for the ESR to start. - :param events: All events sent, in order. - :type events: deque + :param tercc: The test execution recipe collection created event that triggered the ESR. + :type tercc: dict + :param artc: The IUT artifact that the tercc links to. + :type artc: dict """ - self.logger.info(events) - event_names_any_order = [ - "EiffelAnnouncementPublishedEvent", - "EiffelAnnouncementPublishedEvent", - ] - event_names_in_order = [ - "EiffelActivityTriggeredEvent", - "EiffelEnvironmentDefinedEvent", - "EiffelActivityStartedEvent", - "EiffelTestSuiteStartedEvent", - "EiffelTestSuiteFinishedEvent", - "EiffelActivityFinishedEvent", - ] - for event_name in event_names_in_order: - sent_event = events.popleft().meta.type - while sent_event in event_names_any_order: - event_names_any_order.remove(sent_event) - sent_event = events.popleft().meta.type - self.assertEqual(sent_event, event_name) - self.assertEqual(list(events), []) + tercc_event = EiffelTestExecutionRecipeCollectionCreatedEvent() + tercc_event.rebuild(deepcopy(tercc)) + artc_event = EiffelArtifactCreatedEvent() + artc_event.rebuild(deepcopy(artc)) + Debug().events_published.append(artc_event) + Debug().events_published.append(tercc_event) def test_full_scenario(self): """Test ESR using 1 suite with 1 sub suite. Approval criteria: - It shall be possible to execute a full scenario in ESR without failures. - - The ESR shall send events in the correct order. Test steps: 1. Start up a fake server. 2. Initialize and run ESR. 3. Verify that the ESR executes without errors. - 4. Verify that all events were sent and in the correct order. """ os.environ["TERCC"] = json.dumps(TERCC) tercc = json.loads(os.environ["TERCC"]) + testrun_id = tercc["meta"]["id"] + self.setup_providers() + self.publish_initial_events(tercc, ARTIFACT_CREATED) handler = partial(Handler, tercc) end = time.time() + 25 self.logger.info("STEP: Start up a fake server.") with FakeServer(handler) as server: + self.register_providers(testrun_id, server.host) os.environ["ETOS_GRAPHQL_SERVER"] = server.host os.environ["ETOS_ENVIRONMENT_PROVIDER"] = server.host + os.environ["ETOS_API"] = server.host self.logger.info("STEP: Initialize and run ESR.") esr = ESR() try: self.logger.info("STEP: Verify that the ESR executes without errors.") - esr.run() - - self.logger.info("STEP: Verify that all events were sent and in the correct order.") - self.validate_event_name_order(Debug().events_published.copy()) + suite_ids = esr.run() + self.assertEqual( + len(suite_ids), 1, "There shall only be a single test suite started." + ) + suite_finished = Handler.get_from_db( + "EiffelTestSuiteFinishedEvent", {"links.target": suite_ids[0]} + ) + self.assertEqual( + len(suite_finished), 1, "There shall only be a single test suite finished." + ) + outcome = suite_finished[0].get("data", {}).get("outcome", {}) + self.logger.info(outcome) + self.assertDictEqual( + outcome, + { + "conclusion": "SUCCESSFUL", + "verdict": "PASSED", + "description": "All tests passed.", + }, + f"Wrong outcome {outcome!r}, outcome should be successful.", + ) finally: # If the _get_environment_status method in ESR does not time out before the test # finishes there will be loads of tracebacks in the log. Won't fail the test but # the noise is immense. while time.time() <= end: + status = esr.params.get_status() + if status["status"] != "NOT_STARTED": + break time.sleep(1) def test_full_scenario_sub_suites(self): @@ -125,38 +235,61 @@ def test_full_scenario_sub_suites(self): Approval criteria: - It shall be possible to execute a full scenario with sub suites in ESR without failures. - - The ESR shall send events in the correct order. Test steps: 1. Start up a fake server. 2. Initialize and run ESR. 3. Verify that the ESR executes without errors. - 4. Verify that all events were sent and in the correct order. """ os.environ["TERCC"] = json.dumps(TERCC_SUB_SUITES) tercc = json.loads(os.environ["TERCC"]) + testrun_id = tercc["meta"]["id"] + self.setup_providers() + self.publish_initial_events(tercc, ARTIFACT_CREATED) handler = partial(Handler, tercc) end = time.time() + 25 self.logger.info("STEP: Start up a fake server.") with FakeServer(handler) as server: + self.register_providers(testrun_id, server.host) os.environ["ETOS_GRAPHQL_SERVER"] = server.host os.environ["ETOS_ENVIRONMENT_PROVIDER"] = server.host + os.environ["ETOS_API"] = server.host self.logger.info("STEP: Initialize and run ESR.") esr = ESR() try: self.logger.info("STEP: Verify that the ESR executes without errors.") - esr.run() - - self.logger.info("STEP: Verify that all events were sent and in the correct order.") - self.validate_event_name_order(Debug().events_published.copy()) + suite_ids = esr.run() + self.assertEqual( + len(suite_ids), 1, "There shall only be a single test suite started." + ) + suite_finished = Handler.get_from_db( + "EiffelTestSuiteFinishedEvent", {"links.target": suite_ids[0]} + ) + self.assertEqual( + len(suite_finished), 1, "There shall only be a single test suite finished." + ) + outcome = suite_finished[0].get("data", {}).get("outcome", {}) + self.logger.info(outcome) + self.assertDictEqual( + outcome, + { + "conclusion": "SUCCESSFUL", + "verdict": "PASSED", + "description": "All tests passed.", + }, + f"Wrong outcome {outcome!r}, outcome should be successful.", + ) finally: # If the _get_environment_status method in ESR does not time out before the test # finishes there will be loads of tracebacks in the log. Won't fail the test but # the noise is immense. while time.time() <= end: + status = esr.params.get_status() + if status["status"] != "NOT_STARTED": + break time.sleep(1) def test_esr_without_recipes(self): @@ -172,13 +305,18 @@ def test_esr_without_recipes(self): """ os.environ["TERCC"] = json.dumps(TERCC_EMPTY) tercc = json.loads(os.environ["TERCC"]) + testrun_id = tercc["meta"]["id"] + self.setup_providers() + self.publish_initial_events(tercc, ARTIFACT_CREATED) handler = partial(Handler, tercc) end = time.time() + 25 self.logger.info("STEP: Start up a fake server.") with FakeServer(handler) as server: + self.register_providers(testrun_id, server.host) os.environ["ETOS_GRAPHQL_SERVER"] = server.host os.environ["ETOS_ENVIRONMENT_PROVIDER"] = server.host + os.environ["ETOS_API"] = server.host self.logger.info("STEP: Initialize and run ESR.") esr = ESR() @@ -205,4 +343,7 @@ def test_esr_without_recipes(self): # finishes there will be loads of tracebacks in the log. Won't fail the test but # the noise is immense. while time.time() <= end: + status = esr.params.get_status() + if status["status"] != "NOT_STARTED": + break time.sleep(1) diff --git a/projects/etos_suite_runner/tox.ini b/projects/etos_suite_runner/tox.ini index 65d3eac..7c08452 100644 --- a/projects/etos_suite_runner/tox.ini +++ b/projects/etos_suite_runner/tox.ini @@ -22,6 +22,7 @@ commands = [testenv:pylint] deps = pylint + -r{toxinidir}/test-requirements.txt commands = pylint src/etos_suite_runner tests