diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 636dc76..5ba1fa1 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -22,7 +22,7 @@ jobs: pull_number: context.issue.number }) const isTitleValid = /^\[#\d+\] /.test(pr.data.title) - const isDescriptionValid = /([Ff]ix(es|ed)?|[Cc]lose(s|d)?|[Rr]esolve(s|d)?|[Pp]art [Oo]f) \(.*\)\[.*\]/.test(pr.data.body) + const isDescriptionValid = /([Ff]ix(es|ed)?|[Cc]lose(s|d)?|[Rr]esolve(s|d)?|[Pp]art [Oo]f) \[.*\]\(.*\)/.test(pr.data.body) if (isTitleValid && isDescriptionValid) { return } diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 87561a2..75ca21f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,6 +4,26 @@ We value your interest in contributing to `PyHPCC.` Thank you +## Project Structure +``` +. +└── pyhpcc/ + ├── .github # contains build, release, test and other gh actions + ├── docs/ # contains files for documentation + ├── examples/ # contains starter examples + ├── src/ + │ ├── pyhpcc/ + │ │ ├── handlers/ # contains thor and roxie handler + │ │ └── models/ # contains classes auth, workunit submit + │ └── tests/ + │ ├── models/ + │ ├── test_files/ # contains resource files needed for testing + │ └── hanlders/ + ├── pyproject.toml # Project config + ├── CONTRIBUTING.md + └── README.md +``` + ## Set up the repository locally. ## Prerequisites Before starting to develop, make sure you install the following software: @@ -25,6 +45,15 @@ To install the dependencies, run the following command, which downloads the depe poetry install ``` +## How to run tests +Since ecl client tools aren't installed in the GitHub runner, some tests are skipped in the github runner. + +Some tests will fail if `ecl client tools` aren't installed. + +``` +pytest run # Run in project root +``` + ## Linting and Formatting PyHPCC uses [Ruff](https://docs.astral.sh/ruff/) as its formatter and linter. @@ -56,8 +85,4 @@ The base branch is the main repo's main branch. - PR name: copy-and-paste the relevant issue name and include the issue number in front in square brackets, e.g. `[#1020] Make bash_runcommand in WorkUnitSubmit class configurable ` - PR description: mention the issue number in this format: Fixes #1020. Doing so will automatically close the related issue once the PR is merged. - Please Ensure that "Allow edits from maintainers" is ticked. -- Please describe the changes you have made in your branch and how they resolve the issue. - - - - +- Please describe the changes you have made in your branch and how they resolve the issue. \ No newline at end of file diff --git a/README.md b/README.md index 48e162c..e572567 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,8 @@ To use PyHPCC, you need these
1. [Python3](https://www.python.org/downloads/) 2. [ECL Client Tools](https://hpccsystems.com/download/): Select your operating systems to download client tools + + Download the latest stable build from releases in GitHub.
``` bash @@ -36,5 +38,4 @@ Contributions to PyHPCC are welcomed and encouraged.
For specific contribution guidelines, please take a look at [CONTRIBUTING.md](CONTRIBUTING.md). -For more information about the package, please refer to the detailed documentation - https://upgraded-bassoon-daa9d010.pages.github.io/build/html/index.html - +For more information about the package, please refer to the detailed documentation - https://upgraded-bassoon-daa9d010.pages.github.io/build/html/index.html \ No newline at end of file diff --git a/src/pyhpcc/command_config.py b/src/pyhpcc/command_config.py index 58b834b..af86287 100644 --- a/src/pyhpcc/command_config.py +++ b/src/pyhpcc/command_config.py @@ -1,7 +1,11 @@ import logging from pyhpcc.config import ( + CLUSTER_OPTION, COMPILE_OPTIONS, + JOB_NAME_OPTION, + LIMIT_OPTION, + OUTPUT_FILE_OPTION, PASSWORD_OPTIONS, PORT_OPTION, RUN_AUTH_OPTIONS, @@ -35,6 +39,9 @@ class CompileConfig(object): create_compile_bash_command: Generate the eclcc command for the given options and input_file + get_option: + Retrieves the compile config option + """ @@ -43,31 +50,35 @@ def __init__(self, options: dict): def validate_options(self): """Validate if the compiler options are supported or not""" - invalid_options = set() + invalid_options = [] for option in self.options: if option not in COMPILE_OPTIONS and not ( option.startswith("-R") or option.startswith("-f") ): - invalid_options.add(option) + invalid_options.append(option) if len(invalid_options) > 0: raise CompileConfigException(str(invalid_options)) def set_output_file(self, output_file): """Set name of output file (default a.out if linking to""" - self.options["-o"] = output_file + self.options[OUTPUT_FILE_OPTION] = output_file def create_compile_bash_command(self, input_file): """Generate the eclcc command for the given options and input_file""" self.validate_options() eclcc_command = "eclcc" for key, value in self.options.items(): - if isinstance(value, bool): + if value is bool: eclcc_command += f" {key}" else: eclcc_command += f" {key} {value}" eclcc_command = f"{eclcc_command} {input_file}" return eclcc_command + def get_option(self, option): + """Get the option available for the option""" + return self.options[option] + class RunConfig(object): """ @@ -106,6 +117,10 @@ class RunConfig(object): set_password: Set password for accessing ecl services + + get_option: + Retrieves the run config option + """ def __init__(self, options: dict): @@ -125,17 +140,17 @@ def validate_options(self): f"Invalid options not supported by pyhpcc {str(invalid_options)}" ) - def create_run_bash_command(self, target_file=""): + def create_run_bash_command(self, target_file): """Generate the ecl command for the given options and target_file""" self.validate_options() ecl_command = "ecl run" params = "" for key, value in self.options.items(): - if isinstance(value, bool): + if value is bool: params += f" {key}" else: params += f" {key} {value}" - ecl_command = f"{ecl_command} {target_file} {params}" + ecl_command = f"{ecl_command} {target_file}{params}" log.info(ecl_command) return ecl_command @@ -152,15 +167,15 @@ def set_auth_params(self, auth: Auth): def set_target(self, target): """Set the target""" - self.options["--target"] = target + self.options[CLUSTER_OPTION] = target def set_job_name(self, job_name): """Specify the job name for the workunit""" - self.options["--name"] = job_name + self.options[JOB_NAME_OPTION] = job_name def set_limit(self, limit): """Sets the result limit for the query""" - self.options["--limit"] = limit + self.options[LIMIT_OPTION] = limit def set_server(self, server): """Set IP of server running ecl services (eclwatch)""" @@ -168,7 +183,7 @@ def set_server(self, server): def set_port(self, port): """Set ECL services port""" - self.options[PORT_OPTION[0]] = port + self.options[PORT_OPTION] = port def set_username(self, username): """Set username for accessing ecl services""" @@ -177,3 +192,7 @@ def set_username(self, username): def set_password(self, password): """Set password for accessing ecl services""" self.options[PASSWORD_OPTIONS[0]] = password + + def get_option(self, option): + """Get the option available for the option""" + return self.options[option] diff --git a/src/pyhpcc/config.py b/src/pyhpcc/config.py index 64f05e2..2e9acc4 100644 --- a/src/pyhpcc/config.py +++ b/src/pyhpcc/config.py @@ -28,29 +28,27 @@ } -platforms = {"hthor", "thor", "roxie"} - DEFAULT_COMPILE_OPTIONS = {"-platform": "thor", "-wu": True, "-E": True} DEFUALT_RUN_OPTIONS = {} -CLUSTER_PARAM = "--target" -JOB_NAME_PARAM = "--name" -LIMIT_PARAM = "--limit" +CLUSTER_OPTION = "--target" +JOB_NAME_OPTION = "--name" +LIMIT_OPTION = "--limit" DEFAULT_LIMIT = 100 USER_OPTIONS = ["-u", "--username"] PASSWORD_OPTIONS = ["-pw", "--password"] SERVER_OPTIONS = ["-s", "--s"] -PORT_OPTION = ["--port"] +PORT_OPTION = "--port" +OUTPUT_FILE_OPTION = "-o" VERBOSE_OPTIONS = [ "-v", "--verbose", ] -RUN_AUTH_OPTIONS = {*USER_OPTIONS, *PASSWORD_OPTIONS, *SERVER_OPTIONS, *PORT_OPTION} +RUN_AUTH_OPTIONS = {*USER_OPTIONS, *PASSWORD_OPTIONS, *SERVER_OPTIONS, PORT_OPTION} COMPILE_OPTIONS = { "-I", "-L", - "-o", "-manifest", "--main", "-syntax", @@ -84,7 +82,8 @@ *VERBOSE_OPTIONS, "-wxxxx", "--version", - CLUSTER_PARAM, + CLUSTER_OPTION, + OUTPUT_FILE_OPTION, } RUN_OPTIONS = { @@ -118,12 +117,12 @@ "--cert", "--key", "--cacert", - *PORT_OPTION, + PORT_OPTION, *USER_OPTIONS, *PASSWORD_OPTIONS, "--wait-connect", "--wait-read", - CLUSTER_PARAM, - JOB_NAME_PARAM, + CLUSTER_OPTION, + JOB_NAME_OPTION, *VERBOSE_OPTIONS, } diff --git a/src/pyhpcc/errors.py b/src/pyhpcc/errors.py index 1c49d16..cf1e0a4 100644 --- a/src/pyhpcc/errors.py +++ b/src/pyhpcc/errors.py @@ -17,19 +17,6 @@ def __init__(self, message: str): super().__init__(self.message) -class TypeError(Error): - """Exception raised for type errors. - - Attributes: - message: - The error message - """ - - def __init__(self, message: str): - self.message = message - super().__init__(self.message) - - class HPCCException(Error): """Exception raised for HPCC errors. diff --git a/src/pyhpcc/handlers/roxie_handler.py b/src/pyhpcc/handlers/roxie_handler.py index c205186..ec73e34 100644 --- a/src/pyhpcc/handlers/roxie_handler.py +++ b/src/pyhpcc/handlers/roxie_handler.py @@ -1,7 +1,7 @@ import logging import pyhpcc.config as conf -from pyhpcc.errors import HPCCAuthenticationError, TypeError +from pyhpcc.errors import HPCCAuthenticationError from pyhpcc.utils import convert_arg_to_utf8_str log = logging.getLogger(__name__) diff --git a/src/pyhpcc/handlers/thor_handler.py b/src/pyhpcc/handlers/thor_handler.py index c09efef..b10bbcd 100644 --- a/src/pyhpcc/handlers/thor_handler.py +++ b/src/pyhpcc/handlers/thor_handler.py @@ -1,7 +1,7 @@ import logging import pyhpcc.config as conf -from pyhpcc.errors import HPCCAuthenticationError, TypeError +from pyhpcc.errors import HPCCAuthenticationError from pyhpcc.utils import convert_arg_to_utf8_str log = logging.getLogger(__name__) @@ -134,10 +134,13 @@ def execute(self): # self.api.cached_result = True # return result - full_url = self.api.auth.get_url() + self.path + "." + self.response_type + full_url = ( + self.api.auth.get_url() + "/" + self.path + "." + self.response_type + ) # Debugging if conf.DEBUG: + print("Came jere") print("full_url: ", full_url) print("self.session.params: ", self.session.params) print("self.session.headers: ", self.session.headers) diff --git a/src/pyhpcc/models/workunit_submit.py b/src/pyhpcc/models/workunit_submit.py index db40317..fa9bc65 100644 --- a/src/pyhpcc/models/workunit_submit.py +++ b/src/pyhpcc/models/workunit_submit.py @@ -2,6 +2,7 @@ import logging import os import subprocess +from collections import Counter import requests @@ -64,12 +65,20 @@ class WorkunitSubmit(object): run_workunit: Legacy function to run the workunit + + configure_run_config: + Creates run config from given options """ - def __init__(self, hpcc: HPCC, cluster1="", cluster2=""): - self.hpcc = hpcc - self.cluster1 = cluster1 - self.cluster2 = cluster2 + def __init__( + self, + hpcc: HPCC, + clusters: tuple, + ): + if len(clusters) == 0: + raise ValueError("Minimum one cluster should be specified") + self.hpcc: HPCC = hpcc + self.clusters: tuple = clusters self.stateid = conf.WORKUNIT_STATE_MAP def write_file(self, query_text, folder, job_name): @@ -128,16 +137,19 @@ def get_bash_command(self, file_name, compile_config: CompileConfig): A generic exception """ try: - if "-o" not in compile_config.options: + if conf.OUTPUT_FILE_OPTION not in compile_config.options: output_file = utils.create_compile_file_name(file_name) compile_config.set_output_file(output_file) + else: + output_file = compile_config.get_option(conf.OUTPUT_FILE_OPTION) + log.info(compile_config.options) bash_command = compile_config.create_compile_bash_command(file_name) log.info(bash_command) return bash_command, output_file except Exception as e: raise HPCCException("Could not get bash command: " + str(e)) - def get_work_load(self): + def get_least_active_cluster(self): """Get the workload for the given two HPCC clusters Parameters @@ -156,23 +168,41 @@ def get_work_load(self): A generic exception """ try: + if len(self.clusters) == 1: + return self.clusters[0] payload = {"SortBy": "Name", "Descending": 1} + return self.get_cluster_from_response(self.hpcc.activity(**payload).json()) + except Exception as e: + raise HPCCException("Could not get workload: " + str(e)) - resp = self.hpcc.activity(**payload).json() - len1 = 0 - len2 = 0 - if "Running" in list(resp["ActivityResponse"].keys()): - workunits = resp["ActivityResponse"]["Running"]["ActiveWorkunit"] - for workunit in workunits: - if workunit["TargetClusterName"] == self.cluster1: - len1 = len1 + 1 - if workunit["TargetClusterName"] == self.cluster2: - len2 = len2 + 1 + def get_cluster_from_response(self, resp): + """Extract the cluster from the Activity API Response - return len1, len2 + Parameters + ---------- + self: + The object pointer + resp: + Activity API response - except Exception as e: - raise HPCCException("Could not get workload: " + str(e)) + Returns + ------- + str + Cluster with least activity + + Raises + ------ + HPCCException: + A generic exception + """ + cluster_activity = Counter(self.clusters) + if "Running" in list(resp["ActivityResponse"].keys()): + workunits = resp["ActivityResponse"]["Running"]["ActiveWorkunit"] + for workunit in workunits: + cluster = workunit["TargetClusterName"] + if cluster in cluster_activity: + cluster_activity[cluster] -= 1 + return cluster_activity.most_common(1)[0][0] def create_file_name(self, query_text, working_folder, job_name): """Create a filename for the ecl file @@ -258,23 +288,7 @@ def bash_run(self, compiled_file, options: dict = None): A generic exception """ try: - # Select the cluster to run the query on - if options is None: - options = conf.DEFUALT_RUN_OPTIONS - run_config = RunConfig(options) - if conf.CLUSTER_PARAM not in run_config.options: - len1, len2 = self.get_work_load() - if len2 > len1: - cluster = self.cluster1 - else: - cluster = self.cluster2 - run_config.set_target(cluster) - if conf.JOB_NAME_PARAM not in run_config.options: - self.job_name = self.job_name.replace(" ", "_") - run_config.set_job_name(self.job_name) - if conf.LIMIT_PARAM not in run_config.options: - run_config.set_limit(conf.DEFAULT_LIMIT) - run_config.set_auth_params(self.hpcc.auth) + run_config = self.configure_run_config(options) bash_command = run_config.create_run_bash_command(compiled_file) process = subprocess.Popen( bash_command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT @@ -287,6 +301,32 @@ def bash_run(self, compiled_file, options: dict = None): except Exception as e: raise HPCCException("Could not run: " + str(e)) + def configure_run_config(self, options: dict) -> RunConfig: + """Creates run config from given options + + Parameters + ---------- + options: + dict of run config options + + Returns + ------- + run_config: + Returns RunConfig object configured with additional options + """ + if options is None: + options = conf.DEFUALT_RUN_OPTIONS + run_config = RunConfig(options) + if conf.CLUSTER_OPTION not in run_config.options: + run_config.set_target(self.get_least_active_cluster()) + if conf.JOB_NAME_OPTION not in run_config.options: + self.job_name = self.job_name.replace(" ", "_") + run_config.set_job_name(self.job_name) + if conf.LIMIT_OPTION not in run_config.options: + run_config.set_limit(conf.DEFAULT_LIMIT) + run_config.set_auth_params(self.hpcc.auth) + return run_config + def compile_workunit(self, wuid, cluster=""): """Legacy function to compile a workunit - use bash_compile instead @@ -298,11 +338,8 @@ def compile_workunit(self, wuid, cluster=""): The HPCC cluster to run the query on """ if cluster == "": - len1, len2 = self.get_work_load() - if len2 > len1: - cluster = self.cluster1 - else: - cluster = self.cluster2 + cluster = self.get_least_active_cluster() + self.hpcc.wu_submit(Wuid=wuid, Cluster=cluster) try: w3 = self.hpcc.wu_wait_compiled(Wuid=wuid) @@ -335,12 +372,7 @@ def create_workunit( The data to pass to the query """ if cluster_orig == "": - len1, len2 = self.get_work_load() - - if len2 > len1: - cluster_orig = self.cluster1 - else: - cluster_orig = self.cluster2 + cluster_orig = self.get_least_active_cluster() if query_text is None: data = {"QueryText": data} kwargs = {"data": data} @@ -417,12 +449,7 @@ def run_workunit(self, wuid, cluster=""): The HPCC cluster to run the query on """ if cluster == "": - len1, len2 = self.get_work_load() - - if len2 > len1: - cluster = self.cluster1 - else: - cluster = self.cluster2 + cluster = self.get_least_active_cluster() try: w4 = self.hpcc.wu_run(Wuid=wuid, Cluster=cluster, Variables=[]) except requests.exceptions.Timeout: diff --git a/src/pyhpcc/utils.py b/src/pyhpcc/utils.py index 003ebb2..05e85d8 100644 --- a/src/pyhpcc/utils.py +++ b/src/pyhpcc/utils.py @@ -8,7 +8,6 @@ from StringIO import StringIO else: from io import StringIO -from pyhpcc.command_config import CompileConfig from pyhpcc.errors import HPCCException """ diff --git a/tests/config.py b/tests/config.py index 35debb1..cc9a214 100644 --- a/tests/config.py +++ b/tests/config.py @@ -13,13 +13,14 @@ class DUMMY_SECRETS: DUMMY_HPCC_PORT = 0 WUID = "" DEBUG = False + ENV = "LOCAL" try: import my_secret except Exception: my_secret = DUMMY_SECRETS - +ENV_VAR = "ENV" ## HPCC Config HPCC_USERNAME = os.environ.get("HPCC_USERNAME") or my_secret.HPCC_USERNAME HPCC_PASSWORD = os.environ.get("HPCC_PASSWORD") or my_secret.HPCC_PASSWORD @@ -31,3 +32,4 @@ class DUMMY_SECRETS: DUMMY_HPCC_HOST = os.environ.get("DUMMY_HPCC_HOST") or my_secret.DUMMY_HPCC_HOST DUMMY_HPCC_PORT = os.environ.get("DUMMY_HPCC_PORT") or my_secret.DUMMY_HPCC_PORT WUID = os.environ.get("WUID") or my_secret.WUID +ENV = os.environ.get(ENV_VAR) or my_secret.ENV diff --git a/tests/models/test_workunit_submit.py b/tests/models/test_workunit_submit.py new file mode 100644 index 0000000..4fbcee3 --- /dev/null +++ b/tests/models/test_workunit_submit.py @@ -0,0 +1,204 @@ +import copy +import os +import subprocess + +import config +import pytest +from pyhpcc.command_config import CompileConfig +from pyhpcc.config import OUTPUT_FILE_OPTION +from pyhpcc.models.auth import Auth +from pyhpcc.models.hpcc import HPCC +from pyhpcc.models.workunit_submit import WorkunitSubmit + +DUMMY_OUTPUT = "dummy_output" + +HPCC_HOST = config.HPCC_HOST +HPCC_PASSWORD = config.HPCC_PASSWORD +HPCC_PORT = config.HPCC_PORT +HPCC_USERNAME = config.HPCC_USERNAME +ENV = "LOCAL" + + +@pytest.fixture +def auth(): + return Auth(HPCC_HOST, HPCC_PORT, HPCC_USERNAME, HPCC_PASSWORD) + + +@pytest.fixture +def hpcc(auth): + return HPCC(auth) + + +@pytest.fixture +def clusters(): + return ("thor", "hthor") + + +@pytest.fixture +def ws(hpcc, clusters): + return WorkunitSubmit(hpcc, clusters) + + +# Test if creation of WorkUnitSubmit raises error if no clusters are provided +def test_work_unit_creation_error(hpcc): + with pytest.raises(ValueError): + WorkunitSubmit(hpcc, ()) + + +# Test WorkUnitSubmit creation with correct parameters +def test_work_unit_creation_noerror(hpcc, clusters): + try: + WorkunitSubmit(hpcc, clusters) + except Exception as error: + pytest.fail( + f"Faced with exception while creating WorkunitSubmit object {str(error)}" + ) + + +# Test if get_bash_command produces correct output with and without output file in CompileConfig +@pytest.mark.parametrize( + "config, input_file, expected_output", + [ + ({}, "a.ecl", ("eclcc -o a.eclxml a.ecl", "a.eclxml")), + ( + {OUTPUT_FILE_OPTION: "hello.eclxml"}, + "a.ecl", + ("eclcc -o hello.eclxml a.ecl", "hello.eclxml"), + ), + ], +) +def test_get_bash_command_output_file(ws, config, input_file, expected_output): + compile_config = CompileConfig(config) + assert expected_output == ws.get_bash_command(input_file, compile_config) + + +class MockProcess: + def communicate(*args, **kwargs): + return (DUMMY_OUTPUT, "dummy_error") + + +# Test WorkunitSubmit bash_compile with mocking compile method +@pytest.mark.parametrize( + "config, input_file, expected_output", + [ + ({}, "a.ecl", (DUMMY_OUTPUT, "a.eclxml")), + ( + {OUTPUT_FILE_OPTION: "hello.eclxml"}, + "a.ecl", + (DUMMY_OUTPUT, "hello.eclxml"), + ), + ], +) +def test_bash_compile(config, input_file, expected_output, monkeypatch, ws): + def dummy_process(*args, **kwargs): + return MockProcess() + + monkeypatch.setattr(subprocess, "Popen", dummy_process) + output = ws.bash_compile(input_file, config) + assert output == expected_output + + +activity_response_skeleton = {"ActivityResponse": {"Running": {"ActiveWorkunit": []}}} + + +def create_tests(): + active_workunits = { + ("thor", "hthor", "thor", "dthor"): "hthor", + ("dthor", "thor"): "hthor", + ("dthor", "hthor", "hthor", "thor"): "thor", + } + inputs = [] + for resp_clusters, answer in active_workunits.items(): + temp = [] + for cluster in resp_clusters: + temp.append({"TargetClusterName": cluster}) + input = copy.deepcopy(activity_response_skeleton) + input["ActivityResponse"]["Running"]["ActiveWorkunit"] = temp + inputs.append((input, answer)) + return inputs + + +# Test if WorkunitSubmit get_cluster_from_response cluster selection is correct from Activity API response +@pytest.mark.parametrize("activity, expected_output", create_tests()) +def test_get_cluster_from_response(activity, expected_output, ws): + output = ws.get_cluster_from_response(activity) + assert output == expected_output + + +# Test if get_least_active_cluster return least active cluster directly returns if only one cluster is configured +def test_get_cluster_when_one_cluster(hpcc): + clusters = ("thor",) + ws = WorkunitSubmit(hpcc, clusters) + ws.get_least_active_cluster() == clusters[0] + + +# Test if file is created with the contents for create_file_name function +@pytest.mark.parametrize( + "content, job_name, expected_file", + [("OUTPUT('HELLO WORLD!');", "Basic Job", "Basic_Job.ecl")], +) +def test_create_file(tmp_path, ws, content, job_name, expected_file): + output = ws.create_file_name(content, tmp_path, job_name) + ecl_file_path = tmp_path / expected_file + assert output == str(ecl_file_path) + assert ecl_file_path.read_text() == content + + +# Test if compilation is working for bash_compile: Runs only in local +@pytest.mark.skipif( + config.ENV != "LOCAL", + reason="ECL Client Tools required. Can't run on github runner", +) +@pytest.mark.parametrize( + "job_name, expected_file, options, content, error_code", + [ + ("Basic Job", "Basic_job.eclxml", {"-E": True}, "OUTPUT('HELLO WORLD!');", -1), + # ("Basic Job", "Basic_job.eclxml", {"-E": True}, "OUTPUT('HELLO WORL", 185), + ("Basic Job", "Basic_job.eclxml", None, "OUTPUT('HELLO WORLD!');", -1), + ], +) +def test_bash_compile_full( + tmp_path, ws, job_name, options, expected_file, content, error_code +): + output_file = ws.create_file_name(content, tmp_path, job_name) + output, error = ws.bash_compile(output_file, options) + assert os.path.exists(tmp_path / expected_file) + assert str(output).find("error") == error_code + + +# Test if RunConfig options are properly instantiated. +@pytest.mark.parametrize( + "options, expected_options", + [ + ( + None, + { + "--target": "thor", + "--name": "Basic_Job", + "--limit": 100, + "-s": HPCC_HOST, + "--port": f"{HPCC_PORT}", + "-u": HPCC_USERNAME, + "-pw": HPCC_PASSWORD, + }, + ), + ( + {"--target": "hthor", "--limit": 1000, "--name": "Basic Job 2"}, + { + "--target": "hthor", + "--name": "Basic Job 2", + "--limit": 1000, + "-s": HPCC_HOST, + "--port": f"{HPCC_PORT}", + "-u": HPCC_USERNAME, + "-pw": HPCC_PASSWORD, + }, + ), + ], +) +def test_configure_run_config(hpcc, options, expected_options): + clusters = ("thor",) + ws = WorkunitSubmit(hpcc, clusters) + ws.job_name = "Basic Job" + run_config = ws.configure_run_config(options) + assert run_config.options == expected_options diff --git a/tests/test_command_config.py b/tests/test_command_config.py new file mode 100644 index 0000000..7c32a9e --- /dev/null +++ b/tests/test_command_config.py @@ -0,0 +1,242 @@ +import pytest +from pyhpcc.command_config import CompileConfig, RunConfig +from pyhpcc.config import PASSWORD_OPTIONS, PORT_OPTION, SERVER_OPTIONS, USER_OPTIONS +from pyhpcc.errors import CompileConfigException, RunConfigException +from pyhpcc.models.auth import Auth + + +@pytest.fixture +def config_option(): + return {"--username": "testuser"} + + +@pytest.fixture +def auth(): + return Auth("university.hpccsystems.io", "8010", "testuser", "password") + + +# Test if copy is made out of provided options for creating CompileConfig +def test_compile_config_option_copy(config_option): + compile_config = CompileConfig(config_option) + config_option["-dfs"] = "website.com" + assert compile_config.options != config_option + + +# Test if compile config validation is not raising exception for correct options +@pytest.mark.parametrize( + "options", + [ + {"-user": "testuser"}, + {"-R": "r"}, + {"-Rconfig": "config"}, + {"-f": "feature"}, + {"-fconf": "fconf"}, + {"--debug": bool}, + {"--updaterepos": bool}, + {"-user": "testuser", "-R": "r", "-fconf": "feature", "--debug": bool}, + ], +) +def test_compile_config_validation_no_errors(options): + compile_config = CompileConfig(options) + try: + compile_config.validate_options() + except CompileConfigException as error: + pytest.fail(f"Faced with CompileConfig exception {str(error)}") + + +# Test if CompileConfig validate_options is raising exception for incorrect options +@pytest.mark.parametrize( + "options", + [ + {"-u": "testuser"}, + {"--manifest": "r"}, + {"--Rconfig": "config"}, + {"-userd": "testuser", "-R": "r", "-fconf": "feature", "--debug": bool}, + ], +) +def test_compile_config_validation_errors(options): + compile_config = CompileConfig(options) + with pytest.raises(CompileConfigException): + compile_config.validate_options() + + +# Test if CompileConfig create_compile_bash_command returns correct bash command +@pytest.mark.parametrize( + "file_name, options, expected_output", + [ + ( + "abc.xml", + {"--target": "thor", "-f": "dsafda"}, + "eclcc --target thor -f dsafda abc.xml", + ), + ("a.xml", {}, "eclcc a.xml"), + ( + "/usr/loc/Basic_job_submission.ecl", + { + "-platform": "thor", + "-wu": bool, + "-E": bool, + "-o": "/usr/loc/Basic_job_submission.eclxml", + }, + "eclcc -platform thor -wu -E -o /usr/loc/Basic_job_submission.eclxml /usr/loc/Basic_job_submission.ecl", + ), + ( + "/usr/loc/Basic_job_submission.ecl", + { + "-platform": "thor", + "-wu": bool, + "-E": bool, + "-o": "/usr/loc/Basic_job_submission.eclxml", + }, + "eclcc -platform thor -wu -E -o /usr/loc/Basic_job_submission.eclxml /usr/loc/Basic_job_submission.ecl", + ), + ], +) +def test_create_compile_bash_command(file_name, options, expected_output): + compile_config = CompileConfig(options) + cmd = compile_config.create_compile_bash_command(file_name) + assert set(cmd.split(" ")) == set(expected_output.split(" ")) + + +# Test if copy is made out of provided options for creating CompileConfig +def test_run_config_option_copy(config_option): + run_config = RunConfig(config_option) + config_option["--v"] = bool + assert run_config.options != config_option + + +# Test if RunConfig validate_options is raising exception for incorrect options +@pytest.mark.parametrize( + "options", + [ + {"--username": "testuser"}, + {"-X": "x"}, + {"-v": bool}, + {"--verbose": "bool"}, + {"-fconf": "fconf"}, + {"--updaterepos": bool}, + { + "--username": "testuser", + "-X": "thor", + "-fconf": "feature", + "--fetchrepos": bool, + }, + ], +) +def test_run_config_validation_no_errors(options): + run_config = RunConfig(options) + try: + run_config.validate_options() + except RunConfigException as error: + pytest.fail(f"Faced with CompileConfig exception {str(error)}") + + +# Test if RunConfig validate_options is raising errors for incorrect options +@pytest.mark.parametrize( + "options", + [ + {"--user": "testuser"}, + {"-R": "x"}, + {"--v": bool}, + {"-cacert": "bool"}, + {"-Fconf": "fconf"}, + {"abcd": bool}, + { + "--user": "testuser", + "-x": "thor", + "-Fconf": "feature", + "--Dname": "name", + }, + ], +) +def test_run_config_validation_errors(options): + run_config = RunConfig(options) + with pytest.raises(RunConfigException): + run_config.validate_options() + + +# Test if RunConfig create_run_bash_command is creating correct command +@pytest.mark.parametrize( + "file_name, options, expected_output", + [ + ( + "/usr/loc/Basic_job_submission.eclxml", + { + "--target": "thor", + "--name": "my_custom_workunit", + "--limit": 100, + "-s": "university.us-hpccsystems-dev.azure.lnrsg.io", + "--port": "8010", + "-u": "rpodugu", + "-pw": "340K5pogTWqxekqt", + }, + "ecl run /usr/loc/Basic_job_submission.eclxml --target thor --name my_custom_workunit --limit 100 -s university.us-hpccsystems-dev.azure.lnrsg.io --port 8010 -u rpodugu -pw 340K5pogTWqxekqt", + ), + ( + "/usr/loc/Basic_job_submission.eclxml", + { + "--target": "thor", + "--name": "Basic_job_submission", + "--limit": 100, + "-s": "university.us-hpccsystems-dev.azure.lnrsg.io", + "--port": "8010", + "-u": "rpodugu", + "-pw": "340K5pogTWqxekqt", + }, + "ecl run /usr/loc/Basic_job_submission.eclxml --target thor --name Basic_job_submission --limit 100 -s university.us-hpccsystems-dev.azure.lnrsg.io --port 8010 -u rpodugu -pw 340K5pogTWqxekqt", + ), + ( + "/usr/loc/Basic_job_submission.eclxml", + { + "--target": "thor", + "-v": bool, + "--name": "Basic_job_submission", + "--limit": 100, + "-s": "university.us-hpccsystems-dev.azure.lnrsg.io", + "--port": "8010", + "-u": "rpodugu", + "-pw": "340K5pogTWqxekqt", + }, + "ecl run /usr/loc/Basic_job_submission.eclxml --target thor -v --name Basic_job_submission --limit 100 -s university.us-hpccsystems-dev.azure.lnrsg.io --port 8010 -u rpodugu -pw 340K5pogTWqxekqt", + ), + ], +) +def test_create_run_bash_command(file_name, options, expected_output): + run_config = RunConfig(options) + cmd = run_config.create_run_bash_command(file_name) + assert set(cmd.split(" ")) == set(expected_output.split(" ")) + + +# Test if RunConfig create_run_bash_command raises error for incorrect options +@pytest.mark.parametrize( + "file_name, options", + [ + ( + "/usr/loc/Basic_job_submission.eclxml", + { + "--target": "thor", + "--name": "Basic_job_submission", + "--limit": 100, + "-s": "university.us-hpccsystems-dev.azure.lnrsg.io", + "--port": "8010", + "-u": "rpodugu", + "-pwd": "340K5pogTWqxekqt", + }, + ), + ], +) +def test_create_run_bash_command_errors(file_name, options): + run_config = RunConfig(options) + with pytest.raises(RunConfigException): + run_config.create_run_bash_command(file_name) + + +# Test RunConfig set_auth_params set the auth options properly +def test_set_auth_params(config_option, auth: Auth): + run_config = RunConfig(config_option) + run_config.set_auth_params(auth) + assert run_config.get_option(SERVER_OPTIONS[0]) == auth.ip + assert run_config.get_option(PASSWORD_OPTIONS[0]) == auth.password + assert run_config.get_option(PORT_OPTION) == auth.port + assert run_config.get_option(USER_OPTIONS[0]) == auth.username + assert len(run_config.options) == 4