Skip to content

Commit

Permalink
Start and release as environments come in (#26)
Browse files Browse the repository at this point in the history
ESR will now start ETR instances as environments become available
and it will release environments as ETR instances finishes.
There was also some general cleanup made to make it easier to
understand what is going on.
This cleanup was necessary since I could not understand the
resulting code when I just patched in this support.
t-persson authored Aug 26, 2022

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent b627291 commit 5a74a83
Showing 9 changed files with 471 additions and 429 deletions.
4 changes: 1 addition & 3 deletions docs/conf.py
Original file line number Diff line number Diff line change
@@ -14,9 +14,7 @@
import inspect
import shutil

__location__ = os.path.join(
os.getcwd(), os.path.dirname(inspect.getfile(inspect.currentframe()))
)
__location__ = os.path.join(os.getcwd(), os.path.dirname(inspect.getfile(inspect.currentframe())))

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
30 changes: 9 additions & 21 deletions src/etos_suite_runner/__main__.py
Original file line number Diff line number Diff line change
@@ -47,9 +47,7 @@ class ESR: # pylint:disable=too-many-instance-attributes
def __init__(self):
"""Initialize ESR by creating a rabbitmq publisher."""
self.logger = logging.getLogger("ESR")
self.etos = ETOS(
"ETOS Suite Runner", os.getenv("SOURCE_HOST"), "ETOS Suite Runner"
)
self.etos = ETOS("ETOS Suite Runner", os.getenv("SOURCE_HOST"), "ETOS Suite Runner")
signal.signal(signal.SIGTERM, self.graceful_exit)
self.params = ESRParameters(self.etos)
FORMAT_CONFIG.identifier = self.params.tercc.meta.event_id
@@ -109,11 +107,7 @@ def _get_environment_status(self, task_id):
response = None
for generator in wait_generator:
for response in generator:
result = (
response.get("result", {})
if response.get("result") is not None
else {}
)
result = response.get("result", {}) if response.get("result") is not None else {}
self.params.set_status(response.get("status"), result.get("error"))
if response and result:
break
@@ -163,11 +157,12 @@ def run_suites(self, triggered):
:type triggered: :obj:`eiffel.events.EiffelActivityTriggeredEvent`
"""
context = triggered.meta.event_id
self.etos.config.set("context", context)
LOGGER.info("Sending ESR Docker environment event.")
self.etos.events.send_environment_defined(
"ESR Docker", {"CONTEXT": context}, image=os.getenv("SUITE_RUNNER")
)
runner = SuiteRunner(self.params, self.etos, context)
runner = SuiteRunner(self.params, self.etos)

ids = []
for suite in self.params.test_suite:
@@ -197,12 +192,8 @@ def run_suites(self, triggered):
@staticmethod
def verify_input():
"""Verify that the data input to ESR are correct."""
assert os.getenv(
"SUITE_RUNNER"
), "SUITE_RUNNER enviroment variable not provided."
assert os.getenv(
"SOURCE_HOST"
), "SOURCE_HOST environment variable not provided."
assert os.getenv("SUITE_RUNNER"), "SUITE_RUNNER enviroment variable not provided."
assert os.getenv("SOURCE_HOST"), "SOURCE_HOST environment variable not provided."
assert os.getenv("TERCC"), "TERCC environment variable not provided."

def run(self):
@@ -249,9 +240,7 @@ def run(self):
)
except Exception as exception: # pylint:disable=broad-except
reason = str(exception)
self.etos.events.send_activity_canceled(
triggered, {"CONTEXT": context}, reason=reason
)
self.etos.events.send_activity_canceled(triggered, {"CONTEXT": context}, reason=reason)
self.etos.events.send_announcement_published(
"[ESR] Test suite execution failed",
traceback.format_exc(),
@@ -262,9 +251,7 @@ def run(self):

def graceful_exit(self, *_):
"""Attempt to gracefully exit the running job."""
self.logger.info(
"Kill command received - Attempting to shut down all processes."
)
self.logger.info("Kill command received - Attempting to shut down all processes.")
raise Exception("Terminate command received - Shutting down.")


@@ -278,6 +265,7 @@ def main():
termination_log.write(traceback.format_exc())
raise
finally:
esr.etos.publisher.wait_for_unpublished_events()
esr.etos.publisher.stop()
LOGGER.info("ESR Finished Executing.")

89 changes: 89 additions & 0 deletions src/etos_suite_runner/lib/esr_parameters.py
Original file line number Diff line number Diff line change
@@ -16,11 +16,15 @@
"""ESR parameters module."""
import os
import json
import time
import logging
from threading import Lock

from packageurl import PackageURL
from etos_lib.logging.logger import FORMAT_CONFIG
from eiffellib.events import EiffelTestExecutionRecipeCollectionCreatedEvent
from .graphql import request_environment_defined
from .exceptions import EnvironmentProviderException

ARTIFACTS = """
{
@@ -46,13 +50,16 @@ class ESRParameters:

logger = logging.getLogger("ESRParameters")
lock = Lock()
environment_provider_done = False
error = False
__test_suite = None

def __init__(self, etos):
"""ESR parameters instance."""
self.etos = etos
self.issuer = {"name": "ETOS Suite Runner"}
self.environment_status = {"status": "NOT_STARTED", "error": None}
self.__environments = {}

def set_status(self, status, error):
"""Set environment provider status."""
@@ -150,3 +157,85 @@ def product(self):
purl = PackageURL.from_string(identity)
self.etos.config.set("product", purl.name)
return self.etos.config.get("product")

def collect_environments(self):
"""Get environments for all test suites in this ETOS run."""
FORMAT_CONFIG.identifier = self.tercc.meta.event_id
downloaded = []
status = {
"status": "FAILURE",
"error": "Couldn't collect any error information",
}
timeout = time.time() + self.etos.config.get("WAIT_FOR_ENVIRONMENT_TIMEOUT")
while time.time() < timeout:
status = self.environment_status
for environment in request_environment_defined(
self.etos, self.etos.config.get("context")
):
if environment["meta"]["id"] in downloaded:
continue
suite = self._download_sub_suite(environment)
if self.error:
break
downloaded.append(environment["meta"]["id"])
if suite is None: # Not a real sub suite environment defined event.
continue
suite["id"] = environment["meta"]["id"]
with self.lock:
self.__environments.setdefault(suite["test_suite_started_id"], [])
self.__environments[suite["test_suite_started_id"]].append(suite)
if status["status"] == "FAILURE":
break
if status["status"] != "PENDING" and len(downloaded) >= len(self.test_suite):
# We must have found at least one environment for each test suite.
self.environment_provider_done = True
break
time.sleep(5)
if status["status"] == "FAILURE":
with self.lock:
self.error = EnvironmentProviderException(
status["error"], self.etos.config.get("task_id")
)

def _download_sub_suite(self, environment):
"""Download a sub suite from an EnvironmentDefined event.
:param environment: Environment defined event to download from.
:type environment: dict
:return: Downloaded sub suite information.
:rtype: dict
"""
if environment["data"].get("uri") is None:
return None
uri = environment["data"]["uri"]
json_header = {"Accept": "application/json"}
json_response = self.etos.http.wait_for_request(
uri,
headers=json_header,
)
suite = {}
for suite in json_response:
break
else:
self.error = Exception("Could not download sub suite instructions")
return suite

def environments(self, test_suite_started_id):
"""Iterate over all environments correlated to a test suite started.
:param test_suite_started_id: The ID to correlate environments with.
:type test_suite_started_id: str
"""
found = 0
while not self.error:
time.sleep(1)
finished = self.environment_provider_done
with self.lock:
environments = self.__environments.get(test_suite_started_id, []).copy()
for environment in environments:
with self.lock:
self.__environments[test_suite_started_id].remove(environment)
found += 1
yield environment
if finished and found > 0:
break
38 changes: 15 additions & 23 deletions src/etos_suite_runner/lib/graphql.py
Original file line number Diff line number Diff line change
@@ -55,32 +55,26 @@ def request_test_suite_started(etos, main_suite_id):
return None # StopIteration


def request_test_suite_finished(etos, test_suite_ids):
"""Request test suite finished from graphql.
def request_test_suite_finished(etos, test_suite_started_id):
"""Request test suite started from graphql.
:param etos: ETOS client instance.
:type etos: :obj:`etos_lib.etos.Etos`
:param test_suite_ids: list of test suite started IDs of which finished to search for.
:type test_suite_ids: list
:return: Iterator of test suite finished graphql responses.
:param test_suite_started_id: ID of test suite which caused the test suites started
:type test_suite_started_id: str
:return: Iterator of test suite started graphql responses.
:rtype: iterator
"""
or_query = "{'$or': ["
or_query += ", ".join(
[
f"{{'links.type': 'TEST_SUITE_EXECUTION', 'links.target': '{test_suite_id}'}}"
for test_suite_id in test_suite_ids
]
)
or_query += "]}"
for response in request(etos, TEST_SUITE_FINISHED % or_query):
for response in request(etos, TEST_SUITE_FINISHED % test_suite_started_id):
if response:
for _, test_suite_finished in etos.graphql.search_for_nodes(
response, "testSuiteFinished"
):
yield test_suite_finished
return None # StopIteration
return None # StopIteration
try:
_, test_suite_finished = next(
etos.graphql.search_for_nodes(response, "testSuiteFinished")
)
except StopIteration:
return None
return test_suite_finished
return None


def request_environment_defined(etos, activity_id):
@@ -95,9 +89,7 @@ def request_environment_defined(etos, activity_id):
"""
for response in request(etos, ENVIRONMENTS % activity_id):
if response:
for _, environment in etos.graphql.search_for_nodes(
response, "environmentDefined"
):
for _, environment in etos.graphql.search_for_nodes(response, "environmentDefined"):
yield environment
return None # StopIteration
return None # StopIteration
14 changes: 7 additions & 7 deletions src/etos_suite_runner/lib/graphql_queries.py
Original file line number Diff line number Diff line change
@@ -20,35 +20,35 @@
edges {
node {
data {
testSuiteCategories {
type
}
}
meta {
id
name
}
}
}
}
}
"""


TEST_SUITE_FINISHED = """
{
testSuiteFinished(search: "%s") {
testSuiteFinished(search: "{'links.target': '%s', 'links.type': 'TEST_SUITE_EXECUTION'}" last: 1) {
edges {
node {
data {
testSuiteOutcome {
verdict
}
}
meta {
id
}
}
}
}
}
"""


ENVIRONMENTS = """
{
environmentDefined(search:"{'links.type': 'CONTEXT', 'links.target': '%s'}") {
181 changes: 0 additions & 181 deletions src/etos_suite_runner/lib/result_handler.py

This file was deleted.

220 changes: 27 additions & 193 deletions src/etos_suite_runner/lib/runner.py
Original file line number Diff line number Diff line change
@@ -15,19 +15,12 @@
# limitations under the License.
"""ETOS suite runner executor."""
import logging
import time
from threading import Lock, Thread
from threading import Thread
from multiprocessing.pool import ThreadPool

from etos_lib.logging.logger import FORMAT_CONFIG
from eiffellib.events.eiffel_test_suite_started_event import EiffelTestSuiteStartedEvent

from etos_suite_runner.lib.result_handler import ResultHandler
from etos_suite_runner.lib.executor import Executor
from etos_suite_runner.lib.exceptions import EnvironmentProviderException
from etos_suite_runner.lib.graphql import (
request_environment_defined,
)
from .suite import TestSuite


class SuiteRunner: # pylint:disable=too-few-public-methods
@@ -37,25 +30,18 @@ class SuiteRunner: # pylint:disable=too-few-public-methods
Starts ETOS test runner (ETR) and sends out a test suite finished.
"""

lock = Lock()
environment_provider_done = False
error = False
logger = logging.getLogger("ESR - Runner")

def __init__(self, params, etos, context):
def __init__(self, params, etos):
"""Initialize.
:param params: Parameters object for this suite runner.
:type params: :obj:`etos_suite_runner.lib.esr_parameters.ESRParameters`
:param etos: ETOS library object.
:type etos: :obj:`etos_lib.etos.ETOS`
:param context: Context which triggered the runner.
:type context: str
"""
self.params = params
self.etos = etos
self.context = context
self.sub_suites = {}

def _release_environment(self, task_id):
"""Release an environment from the environment provider.
@@ -64,192 +50,40 @@ def _release_environment(self, task_id):
:type task_id: str
"""
wait_generator = self.etos.http.wait_for_request(
self.etos.debug.environment_provider, params={"release": task_id}
self.etos.debug.environment_provider, params={"release": task_id}, timeout=60
)
for response in wait_generator:
if response:
break

def _run_etr(self, environment):
"""Trigger an instance of ETR.
:param environment: Environment which to execute in.
:type environment: dict
"""
executor = Executor(self.etos)
executor.run_tests(environment)

def _environments(self):
"""Get environments for all test suites in this ETOS run."""
FORMAT_CONFIG.identifier = self.params.tercc.meta.event_id
downloaded = []
status = {
"status": "FAILURE",
"error": "Couldn't collect any error information",
}
timeout = time.time() + self.etos.config.get("WAIT_FOR_ENVIRONMENT_TIMEOUT")
while time.time() < timeout:
status = self.params.environment_status
self.logger.info(status)
for environment in request_environment_defined(self.etos, self.context):
if environment["meta"]["id"] in downloaded:
continue
suite = self._download_sub_suite(environment)
if self.error:
break
downloaded.append(environment["meta"]["id"])
if suite is None: # Not a real sub suite environment defined event.
continue
with self.lock:
self.sub_suites.setdefault(suite["test_suite_started_id"], [])
self.sub_suites[suite["test_suite_started_id"]].append(suite)
# We must have found at least one environment for each test suite.
if status["status"] != "PENDING" and len(downloaded) >= len(
self.params.test_suite
):
self.environment_provider_done = True
break
time.sleep(5)
if status["status"] == "FAILURE":
self.error = EnvironmentProviderException(
status["error"], self.etos.config.get("task_id")
)

def _download_sub_suite(self, environment):
"""Download a sub suite from an EnvironmentDefined event.
:param environment: Environment defined event to download from.
:type environment: dict
:return: Downloaded sub suite information.
:rtype: dict
"""
if environment["data"].get("uri") is None:
return None
uri = environment["data"]["uri"]
json_header = {"Accept": "application/json"}
json_response = self.etos.http.wait_for_request(
uri,
headers=json_header,
)
suite = {}
for suite in json_response:
break
else:
self.error = Exception("Could not download sub suite instructions")
return suite

def _sub_suites(self, main_suite_id):
"""Get all sub suites that correlates with ID.
:param main_suite_id: Main suite ID to correlate sub suites to.
:type main_suite_id: str
:return: Each correlated sub suite.
:rtype: Iterator
"""
while not self.error:
downloaded_all = self.environment_provider_done
time.sleep(1)
with self.lock:
sub_suites = self.sub_suites.get(main_suite_id, []).copy()
for sub_suite in sub_suites:
with self.lock:
self.sub_suites[main_suite_id].remove(sub_suite)
yield sub_suite
if downloaded_all:
break

def start_sub_suites(self, suite):
"""Start up all sub suites within a TERCC suite.
:param suite: TERCC suite to start up sub suites from.
:type suite: dict
"""
suite_name = suite.get("name")
self.etos.events.send_announcement_published(
"[ESR] Starting tests.",
"Starting test suites on all checked out IUTs.",
"MINOR",
{"CONTEXT": self.context},
)
self.logger.info("Starting sub suites for %r", suite_name)
started = []
for sub_suite in self._sub_suites(suite["test_suite_started_id"]):
started.append(sub_suite)

self.logger.info("Triggering sub suite %r", sub_suite["name"])
self._run_etr(sub_suite)
self.logger.info("%r Triggered", sub_suite["name"])
time.sleep(1)
self.logger.info("All %d sub suites for %r started", len(started), suite_name)

self.etos.events.send_announcement_published(
"[ESR] Waiting.",
"Waiting for test suites to finish",
"MINOR",
{"CONTEXT": self.context},
)
return started

def start_suite(self, suite):
"""Send test suite events and launch test runners.
:param suite: Test suite to start.
:type suite: dict
"""
FORMAT_CONFIG.identifier = self.params.tercc.meta.event_id
suite_name = suite.get("name")
self.logger.info("Starting %s.", suite_name)

categories = ["Regression test suite"]
if self.params.product:
categories.append(self.params.product)

test_suite_started = EiffelTestSuiteStartedEvent()

# This ID has been stored in Environment so that the ETR know which test suite to link to.
test_suite_started.meta.event_id = suite.get("test_suite_started_id")
data = {"name": suite_name, "categories": categories, "types": ["FUNCTIONAL"]}
links = {"CONTEXT": self.context}
self.etos.events.send(test_suite_started, links, data)

verdict = "INCONCLUSIVE"
conclusion = "INCONCLUSIVE"
description = ""

result_handler = ResultHandler(self.etos, test_suite_started)
try:
started = self.start_sub_suites(suite)
self.logger.info("Wait for test results.")
result_handler.wait_for_test_suite_finished(len(started))
verdict, conclusion, description = result_handler.test_results()
time.sleep(5)
except Exception as exc:
conclusion = "FAILED"
description = str(exc)
raise
finally:
self.etos.events.send_test_suite_finished(
test_suite_started,
{"CONTEXT": self.context},
outcome={
"verdict": verdict,
"conclusion": conclusion,
"description": description,
},
)
# TODO: Add releasing of environment defined IDs when that is supported
self.logger.info("Test suite finished.")

def start_suites_and_wait(self):
"""Get environments and start all test suites."""
Thread(target=self._environments, daemon=True).start()
Thread(target=self.params.collect_environments, daemon=True).start()
try:
test_suites = [
TestSuite(self.etos, self.params, suite) for suite in self.params.test_suite
]
with ThreadPool() as pool:
pool.map(self.start_suite, self.params.test_suite)
if self.error:
raise self.error
pool.map(self.run, test_suites)
if self.params.error:
raise self.params.error
finally:
task_id = self.etos.config.get("task_id")
self.logger.info("Release test environment.")
self.logger.info("Release the full test environment.")
if task_id is not None:
self._release_environment(task_id)

def run(self, test_suite):
"""Run test suite runner.
:param test_suite: Test suite to run.
:type test_suite: :obj:`TestSuite`
"""
FORMAT_CONFIG.identifier = self.params.tercc.meta.event_id
try:
test_suite.start() # send EiffelTestSuiteStartedEvent
# All sub suites finished.
finally:
results = test_suite.results()
test_suite.finish(*results) # send EiffelTestSuiteFinishedEvent
test_suite.release_all()
322 changes: 322 additions & 0 deletions src/etos_suite_runner/lib/suite.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,322 @@
# Copyright 2022 Axis Communications AB.
#
# For a full list of individual contributors, please see the commit history.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Test suite handler."""
import logging
import threading
import time

from etos_lib.logging.logger import FORMAT_CONFIG
from eiffellib.events import EiffelTestSuiteStartedEvent

from .executor import Executor
from .graphql import request_test_suite_finished, request_test_suite_started
from .log_filter import DuplicateFilter


class SubSuite:
"""Handle test results and tracking of a single sub suite."""

released = False

def __init__(self, etos, environment):
"""Initialize a sub suite."""
self.etos = etos
self.environment = environment
self.name = self.environment.get("name")
self.logger = logging.getLogger(f"SubSuite - {self.name}")
self.logger.addFilter(DuplicateFilter(self.logger))
self.test_suite_started = {} # This is set by a different thread.
self.test_suite_finished = {}

@property
def finished(self):
"""Whether or not this sub suite has finished."""
return bool(self.test_suite_finished)

@property
def started(self):
"""Whether or not this sub suite has started."""
return bool(self.test_suite_started)

def request_finished_event(self):
"""Request a test suite finished event for this sub suite."""
# Prevent ER requests if we know we're not even started.
if not self.started:
return
# Prevent ER requests if we know we're already finished.
if not self.test_suite_finished:
self.test_suite_finished = request_test_suite_finished(
self.etos, self.test_suite_started["meta"]["id"]
)

def outcome(self):
"""Outcome of this sub suite.
:return: Test suite outcome from the test suite finished event.
:rtype: dict
"""
if self.finished:
return self.test_suite_finished.get("data", {}).get("testSuiteOutcome", {})
return {}

def start(self, identifier):
"""Start ETR for this sub suite.
:param identifier: An identifier for logs in this sub suite.
:type identifier: str
"""
FORMAT_CONFIG.identifier = identifier
self.logger.info("Triggering ETR.")
executor = Executor(self.etos)
executor.run_tests(self.environment)
self.logger.info("ETR triggered.")
timeout = time.time() + self.etos.debug.default_test_result_timeout
try:
while time.time() < timeout:
time.sleep(1)
if not self.started:
continue
self.logger.info("ETR started.")
self.request_finished_event()
if self.finished:
self.logger.info("ETR finished.")
break
finally:
self.release()

def release(self):
"""Release this sub suite."""
self.logger.info("Releasing environment")
wait_generator = self.etos.http.wait_for_request(
self.etos.debug.environment_provider,
params={"single_release": self.environment["id"]},
timeout=60,
)
for response in wait_generator:
if response:
self.logger.info("Successfully released")
self.released = True
break


class TestSuite:
"""Handle the starting and waiting for test suites in ETOS."""

test_suite_started = None
started = False
lock = threading.Lock()

def __init__(self, etos, params, suite):
"""Initialize a TestSuite instance."""
self.etos = etos
self.params = params
self.suite = suite
self.logger = logging.getLogger(f"TestSuite - {self.suite.get('name')}")
self.logger.addFilter(DuplicateFilter(self.logger))
self.sub_suites = []

@property
def sub_suite_definitions(self):
"""All sub suite definitions from the environment provider.
Each sub suite definition is an environment for the sub suites to execute in.
"""
yield from self.params.environments(self.suite["test_suite_started_id"])

@property
def all_finished(self):
"""Whether or not all sub suites are finished."""
with self.lock:
return all(sub_suite.finished for sub_suite in self.sub_suites)

def _announce(self, header, body):
"""Send an announcement over Eiffel.
:param header: Header of the announcement.
:type header: str
:param body: Body of the announcement.
:type body: str
"""
self.etos.events.send_announcement_published(
f"[ESR] {header}",
body,
"MINOR",
{"CONTEXT": self.etos.config.get("context")},
)

def _send_test_suite_started(self):
"""Send a test suite started event.
:return: Test suite started event.
:rtype: :obj:`eiffellib.events.EiffelTestSuiteStartedEvent`
"""
test_suite_started = EiffelTestSuiteStartedEvent()

categories = ["Regression test suite"]
if self.params.product:
categories.append(self.params.product)

# This ID has been stored in Environment so that the ETR know which test suite to link to.
test_suite_started.meta.event_id = self.suite.get("test_suite_started_id")
data = {"name": self.suite.get("name"), "categories": categories, "types": ["FUNCTIONAL"]}
links = {"CONTEXT": self.etos.config.get("context")}
return self.etos.events.send(test_suite_started, links, data)

def start(self):
"""Send test suite started, trigger and wait for all sub suites to start."""
self._announce("Starting tests", f"Starting up sub suites for '{self.suite.get('name')}'")

self.test_suite_started = self._send_test_suite_started()
self.logger.info("Test suite started %r", self.test_suite_started.meta.event_id)

self.logger.info("Starting sub suites")
threads = []
assigner = None
try:
for sub_suite_definition in self.sub_suite_definitions:
sub_suite = SubSuite(self.etos, sub_suite_definition)
with self.lock:
self.sub_suites.append(sub_suite)
thread = threading.Thread(
target=sub_suite.start, args=(self.params.tercc.meta.event_id,)
)
threads.append(thread)
thread.start()

self.logger.info("Assigning test suite started events to sub suites")
assigner = threading.Thread(target=self._assign_test_suite_started)
assigner.start()

if self.params.error:
self.logger.error("Environment provider error: %r", self.params.error)
self._announce(
"Error",
f"Environment provider failed to provide an environment: '{self.params.error}'"
"\nWill finish already started sub suites\n",
)
return
with self.lock:
number_of_suites = len(self.sub_suites)
self.logger.info("All %d sub suites triggered", number_of_suites)
self.started = True
finally:
if assigner is not None:
assigner.join()
for thread in threads:
thread.join()
self.logger.info("All %d sub suites finished", number_of_suites)

def _assign_test_suite_started(self):
"""Assign test suite started events to all sub suites."""
FORMAT_CONFIG.identifier = self.params.tercc.meta.event_id
timeout = time.time() + self.etos.debug.default_test_result_timeout
self.logger.info("Assigning test suite started to sub suites")
while time.time() < timeout:
time.sleep(1)
suites = []
with self.lock:
sub_suites = self.sub_suites.copy()
if len(sub_suites) == 0 and self.params.error:
self.logger.info("Environment provider error")
return
if len(sub_suites) == 0:
self.logger.info("No sub suites started just yet")
continue
for test_suite_started in request_test_suite_started(
self.etos, self.suite["test_suite_started_id"]
):
self.logger.info("Found test suite started")
suites.append(test_suite_started)
for sub_suite in sub_suites:
if sub_suite.started:
continue
# Using name to match here is safe because we're only searching for
# sub suites that are connected to this test_suite_started ID and the
# "_SubSuite_\d" part of the name is set by ETOS and not humans.
if sub_suite.name == test_suite_started["data"]["name"]:
self.logger.info("Test suite started correlates to %r", sub_suite.name)
sub_suite.test_suite_started = test_suite_started
else:
self.logger.info(
"No correlation for %r", test_suite_started["data"]["name"]
)
if len(suites) == len(sub_suites):
self.logger.info("All %d sub suites started", len(sub_suites))
break

def release_all(self):
"""Release all, unreleased, sub suites."""
self.logger.info("Releasing all sub suite environments")
for sub_suite in self.sub_suites:
if not sub_suite.released:
sub_suite.release()
self.logger.info("All sub suite environments are released")

def finish(self, verdict, conclusion, description):
"""Send test suite finished for this test suite.
:param verdict: Verdict of the execution.
:type verdict: str
:param conclusion: Conclusion taken on the results.
:type conclusion: str
:param description: Description of the verdict and conclusion.
:type description: str
"""
self.etos.events.send_test_suite_finished(
self.test_suite_started,
{"CONTEXT": self.etos.config.get("context")},
outcome={
"verdict": verdict,
"conclusion": conclusion,
"description": description,
},
)
self.logger.info("Test suite finished.")

def results(self):
"""Test results for this execution.
:return: Verdict, conclusion and description.
:rtype: tuple
"""
verdict = "INCONCLUSIVE"
conclusion = "SUCCESSFUL"
description = ""

if not self.started:
verdict = "INCONCLUSIVE"
conclusion = "FAILED"
description = (
f"No sub suites started at all for {self.test_suite_started.meta.event_id}."
)
elif not self.all_finished:
verdict = "INCONCLUSIVE"
conclusion = "FAILED"
description = "Did not receive test results from sub suites."
else:
for sub_suite in self.sub_suites:
if sub_suite.outcome().get("verdict") != "PASSED":
verdict = "FAILED"
description = sub_suite.outcome().get("description")
# If we get this far without exceptions or return statements
# and the verdict is still inconclusive, it would mean that
# that we passed everything.
if verdict == "INCONCLUSIVE":
description = "All tests passed."
verdict = "PASSED"
if not description:
description = "No description received from ESR or ETR."
return verdict, conclusion, description
2 changes: 1 addition & 1 deletion tox.ini
Original file line number Diff line number Diff line change
@@ -16,7 +16,7 @@ envlist = py3,black,pylint,pydocstyle
deps =
black
commands =
black --check --diff .
black --check --diff -l 100 .

[testenv:pylint]
deps =

0 comments on commit 5a74a83

Please sign in to comment.