Test benchmarks

Signed-off-by: Divya Madala <[email protected]>

jenkins/opensearch/benchmark-test-endpoint.jenkinsfile

@@ -121,6 +121,7 @@ parameters {
                         password: PASSWORD,
                         workload: TEST_WORKLOAD,
                         userTag: USER_TAGS.isEmpty() ? "security-enabled:${SECURITY_ENABLED}" : "${USER_TAGS},security-enabled:${SECURITY_ENABLED}",
+                        suffix: "${BUILD_NUMBER}",
                         workloadParams: WORKLOAD_PARAMS,
                         testProcedure: TEST_PROCEDURE,
                         excludeTasks: EXCLUDE_TASKS,

src/test_workflow/benchmark_test/benchmark_test_runner_opensearch.py

@@ -8,8 +8,9 @@
 import logging
 import os
 from typing import Union
-
+import subprocess
 import yaml
+
 from retry.api import retry_call  # type: ignore
 
 from git.git_repository import GitRepository
@@ -42,8 +43,10 @@ def run_tests(self) -> None:
             cluster = BenchmarkTestCluster(self.args)
             cluster.start()
             benchmark_test_suite = BenchmarkTestSuite(cluster.endpoint_with_port, self.security, self.args, cluster.fetch_password())
-            retry_call(benchmark_test_suite.execute, tries=3, delay=60, backoff=2)
-
+            try:
+                retry_call(benchmark_test_suite.execute, tries=3, delay=60, backoff=2)
+            finally:
+                subprocess.check_call(f"docker rm docker-container-{self.args.stack_suffix}", cwd=os.getcwd(), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         else:
             config = yaml.safe_load(self.args.config)
 
@@ -53,4 +56,7 @@ def run_tests(self) -> None:
                     with WorkingDirectory(current_workspace):
                         with BenchmarkCreateCluster.create(self.args, self.test_manifest, config, current_workspace) as test_cluster:
                             benchmark_test_suite = BenchmarkTestSuite(test_cluster.endpoint_with_port, self.security, self.args, test_cluster.fetch_password())
-                            retry_call(benchmark_test_suite.execute, tries=3, delay=60, backoff=2)
+                            try:
+                                benchmark_test_suite.execute()
+                            finally:
+                                subprocess.check_call(f"docker rm docker-container-{self.args.stack_suffix}", cwd=os.getcwd(), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

src/test_workflow/benchmark_test/benchmark_test_suite.py

@@ -29,9 +29,6 @@ class BenchmarkTestSuite:
     Represents a performance test suite. This class runs rally test on the deployed cluster with the provided IP.
     """
 
-    def __exit__(self, exc_type: Any, exc_value: Any, exc_traceback: Any) -> None:
-        self.tmp_dir.__exit__(exc_type, exc_value, exc_traceback)
-
     def __init__(
             self,
             endpoint: Any,
@@ -43,11 +40,9 @@ def __init__(
         self.security = security
         self.args = args
         self.password = password
-        self.tmp_dir = TemporaryDirectory(keep=True)
 
         # Pass the cluster endpoints with -t for multi-cluster use cases(e.g. cross-cluster-replication)
-        self.container_name = "test_container"  # container name
-        self.command = f'docker run --name {self.container_name}'
+        self.command = f'docker run --name docker-container-{self.args.stack_suffix}'
         if self.args.benchmark_config:
             self.command += f" -v {args.benchmark_config}:/opensearch-benchmark/.benchmark/benchmark.ini"
         self.command += f" opensearchproject/opensearch-benchmark:latest execute-test --workload={self.args.workload} " \
@@ -85,23 +80,18 @@ def execute(self) -> None:
         log_info = f"Executing {self.command.replace(self.endpoint, len(self.endpoint) * '*').replace(self.args.username, len(self.args.username) * '*')}"
         logging.info(log_info.replace(self.password, len(self.password) * '*') if self.password else log_info)
         subprocess.check_call(f"{self.command}", cwd=os.getcwd(), shell=True)
-
-        subprocess.check_call(f"docker cp {self.container_name}:opensearch-benchmark/. .", cwd=os.getcwd(), shell=True)
-        file_path = glob.glob(os.path.join(os.getcwd(), "test_executions", "*", "test_execution.json"))
-        logging.info(file_path)
-        self.convert(file_path[0])
-        subprocess.check_call(f"docker rm {self.container_name}", cwd=os.getcwd(), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        with TemporaryDirectory() as work_dir:
+            subprocess.check_call(f"docker cp docker-container-{self.args.stack_suffix}:opensearch-benchmark/. {str(work_dir.path)}", cwd=os.getcwd(), shell=True)
+            file_path = glob.glob(os.path.join(str(work_dir.path), "test_executions", "*", "test_execution.json"))
+            self.convert(file_path[0])
 
     def convert(self, results: str) -> None:
         with open(results) as file:
             data = json.load(file)
         formatted_data = pd.json_normalize(data["results"]["op_metrics"])
         formatted_data.to_csv(os.path.join(os.getcwd(), "test_execution.csv"), index=False)
         df = pd.read_csv(os.path.join(os.getcwd(), "test_execution.csv"))
-        terminal_width = shutil.get_terminal_size().columns
-        width_95_percent = int(1 * terminal_width)
-        pd.set_option('display.width', width_95_percent)
-
+        pd.set_option('display.width', int(1 * shutil.get_terminal_size().columns))
         pd.set_option('display.max_rows', None)
         pd.set_option('display.max_columns', None)
         logging.info(f"\n{df}")