From 7ab554c2d37eed85e623dc6b9169552def380521 Mon Sep 17 00:00:00 2001 From: jonasbn Date: Fri, 4 Oct 2024 20:38:00 +0200 Subject: [PATCH 01/16] Bumped Spellcheck action to contemporary version --- .github/workflows/ci_spelling.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_spelling.yml b/.github/workflows/ci_spelling.yml index 18df973..f148863 100644 --- a/.github/workflows/ci_spelling.yml +++ b/.github/workflows/ci_spelling.yml @@ -7,7 +7,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Check Spelling - uses: rojopolis/spellcheck-github-actions@0.23.0 + uses: rojopolis/spellcheck-github-actions@0.42.0 with: config_path: .github/workflows/.spellcheck.yml task_name: Markdown \ No newline at end of file From d9403a2da78c06aa151f026beab83b1271ae4a71 Mon Sep 17 00:00:00 2001 From: jonasbn Date: Fri, 4 Oct 2024 21:50:54 +0200 Subject: [PATCH 02/16] Moved configuration files out of workflows directory --- .github/{workflows => }/.spellcheck.yml | 2 +- .github/{workflows => }/.wordlist.txt | 0 .github/workflows/ci_spelling.yml | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename .github/{workflows => }/.spellcheck.yml (92%) rename .github/{workflows => }/.wordlist.txt (100%) diff --git a/.github/workflows/.spellcheck.yml b/.github/.spellcheck.yml similarity index 92% rename from .github/workflows/.spellcheck.yml rename to .github/.spellcheck.yml index 887fd60..1b02c7f 100644 --- a/.github/workflows/.spellcheck.yml +++ b/.github/.spellcheck.yml @@ -5,7 +5,7 @@ matrix: mode: en dictionary: wordlists: - - .github/workflows/.wordlist.txt + - .github/wordlist.txt output: wordlist.dic encoding: utf-8 pipeline: diff --git a/.github/workflows/.wordlist.txt b/.github/.wordlist.txt similarity index 100% rename from .github/workflows/.wordlist.txt rename to .github/.wordlist.txt diff --git a/.github/workflows/ci_spelling.yml b/.github/workflows/ci_spelling.yml index f148863..3da4344 100644 --- a/.github/workflows/ci_spelling.yml +++ b/.github/workflows/ci_spelling.yml @@ -9,5 +9,5 @@ jobs: - name: Check Spelling uses: rojopolis/spellcheck-github-actions@0.42.0 with: - config_path: .github/workflows/.spellcheck.yml + config_path: .github/spellcheck.yml task_name: Markdown \ No newline at end of file From 9046504d181e652ae5dd3e387df72aaec990e9b1 Mon Sep 17 00:00:00 2001 From: jonasbn Date: Fri, 4 Oct 2024 21:54:14 +0200 Subject: [PATCH 03/16] Bad a moving/renaming apparently --- .github/{.spellcheck.yml => spellcheck.yml} | 0 .github/{.wordlist.txt => wordlist.txt} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename .github/{.spellcheck.yml => spellcheck.yml} (100%) rename .github/{.wordlist.txt => wordlist.txt} (100%) diff --git a/.github/.spellcheck.yml b/.github/spellcheck.yml similarity index 100% rename from .github/.spellcheck.yml rename to .github/spellcheck.yml diff --git a/.github/.wordlist.txt b/.github/wordlist.txt similarity index 100% rename from .github/.wordlist.txt rename to .github/wordlist.txt From aa62e5f9a87089cf85da3c419c4950987e8bbdb6 Mon Sep 17 00:00:00 2001 From: Ben Schroeter Date: Sun, 6 Oct 2024 23:36:06 +1100 Subject: [PATCH 04/16] Added client variable passing to PBS command. Fixes #25 --- hpcpy/client/pbs.py | 23 +++++++++++++++++++++++ tests/test_pbs_client.py | 22 ++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/hpcpy/client/pbs.py b/hpcpy/client/pbs.py index 7f212a2..58d1b3c 100644 --- a/hpcpy/client/pbs.py +++ b/hpcpy/client/pbs.py @@ -27,6 +27,22 @@ def status(self, job_id): # Get the status out of the job ID _status = parsed.get("Jobs").get(job_id).get("job_state") return hc.PBS_STATUSES[_status] + + def _render_variables(self, variables): + """Render the variables flag for PBS. + + Parameters + ---------- + variables : dict + Dictionary of variables + + Returns + ------- + str + String formatted variables for PBS + """ + formatted = ",".join([f"{k}={v}" for k, v in variables.items()]) + return f"-v {formatted}" def submit( self, @@ -39,6 +55,7 @@ def submit( queue: str = None, walltime: timedelta = None, storage: list = None, + variables: dict = None, **context, ): """Submit a job to the scheduler. @@ -63,6 +80,8 @@ def submit( Walltime expressed as a timedelta, by default None storage: list, optional List of storage mounts to apply, by default None + variables: dict, optional + Key/value pairs added to the qsub command. **context: Additional key/value pairs to be added to command/jobscript interpolation """ @@ -110,6 +129,10 @@ def submit( directives.append(f"-l storage={storage_str}") context["storage"] = storage context["storage_str"] = storage_str + + # Add variables + if isinstance(variables, dict) and len(variables) > 0: + directives.append(self._render_variables(variables)) # Call the super return super().submit( diff --git a/tests/test_pbs_client.py b/tests/test_pbs_client.py index 64812ce..e3eb054 100644 --- a/tests/test_pbs_client.py +++ b/tests/test_pbs_client.py @@ -62,3 +62,25 @@ def test_storage(client): ) assert result == expected + +def test_variables(client): + """Test passing variables to the qsub command.""" + expected = "qsub -v var1=1234,var2=abcd test.sh" + result = client.submit( + "test.sh", dry_run=True, variables=dict(var1=1234, var2="abcd") + ) + + assert result == expected + +def test_variables_empty(client): + """Test passing empty variables dict to the qsub command works as expected.""" + expected = "qsub test.sh" + result1 = client.submit( + "test.sh", dry_run=True, variables=dict() + ) + result2 = client.submit( + "test.sh", dry_run=True + ) + + assert result1 == expected + assert result2 == expected \ No newline at end of file From 5376d99952d6a080d19b2ecb36409ac5dc96d01e Mon Sep 17 00:00:00 2001 From: Ben Schroeter Date: Sun, 6 Oct 2024 23:39:19 +1100 Subject: [PATCH 05/16] Added in shlex. --- hpcpy/utilities.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hpcpy/utilities.py b/hpcpy/utilities.py index 1c394a2..749ecd7 100644 --- a/hpcpy/utilities.py +++ b/hpcpy/utilities.py @@ -5,6 +5,7 @@ import jinja2.meta as j2m from pathlib import Path from importlib import resources +import shlex def shell( @@ -33,7 +34,7 @@ def shell( subprocess.CalledProcessError """ return sp.run( - cmd, shell=shell, check=check, capture_output=capture_output, **kwargs + shlex.split(cmd), check=check, capture_output=capture_output, **kwargs ) From 2e230bbaebd6ad10cd324805d72da73f49f0ff3a Mon Sep 17 00:00:00 2001 From: Ben Schroeter Date: Tue, 8 Oct 2024 12:00:26 +1100 Subject: [PATCH 06/16] Update hpcpy/client/pbs.py Co-authored-by: Aidan Heerdegen --- hpcpy/client/pbs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hpcpy/client/pbs.py b/hpcpy/client/pbs.py index 58d1b3c..7bf5f7a 100644 --- a/hpcpy/client/pbs.py +++ b/hpcpy/client/pbs.py @@ -81,7 +81,7 @@ def submit( storage: list, optional List of storage mounts to apply, by default None variables: dict, optional - Key/value pairs added to the qsub command. + Key/value environment variable pairs added to the qsub command. **context: Additional key/value pairs to be added to command/jobscript interpolation """ From 61ec837b98522e114d3a102c12eff4ca9ba5e639 Mon Sep 17 00:00:00 2001 From: Ben Schroeter Date: Tue, 8 Oct 2024 12:04:08 +1100 Subject: [PATCH 07/16] Update utilities.py --- hpcpy/utilities.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/hpcpy/utilities.py b/hpcpy/utilities.py index 749ecd7..90167da 100644 --- a/hpcpy/utilities.py +++ b/hpcpy/utilities.py @@ -9,7 +9,7 @@ def shell( - cmd, shell=True, check=True, capture_output=True, **kwargs + cmd, check=True, capture_output=True, **kwargs ) -> sp.CompletedProcess: """Execute a shell command. @@ -17,8 +17,6 @@ def shell( ---------- cmd : str Command - shell : bool, optional - Execute as shell, by default True check : bool, optional Check output, by default True capture_output : bool, optional From b7ecd5b4ac27cce42a1880209110c14b7b1112bf Mon Sep 17 00:00:00 2001 From: Ben Schroeter Date: Mon, 14 Oct 2024 14:14:03 +1100 Subject: [PATCH 08/16] Removed mock client, improved docs. Fixes #29 --- docs/advanced_usage.md | 35 --------- docs/usage.md | 132 +++++++++++++++++++++++++++------ hpcpy/__init__.py | 3 +- hpcpy/client/client_factory.py | 12 +-- hpcpy/client/mock.py | 31 -------- hpcpy/client/pbs.py | 4 +- hpcpy/constants.py | 8 -- hpcpy/utilities.py | 4 +- mkdocs.yml | 19 +++++ tests/test_client.py | 19 ----- tests/test_core.py | 13 ---- tests/test_pbs_client.py | 12 ++- 12 files changed, 141 insertions(+), 151 deletions(-) delete mode 100644 docs/advanced_usage.md delete mode 100644 hpcpy/client/mock.py create mode 100644 mkdocs.yml diff --git a/docs/advanced_usage.md b/docs/advanced_usage.md deleted file mode 100644 index 71cf7b0..0000000 --- a/docs/advanced_usage.md +++ /dev/null @@ -1,35 +0,0 @@ -# Advanced Usage - -The following documentation describes what we'll consider "Advanced Usage", that is, whatever might require more than a few lines of code. - -## Task dependence - -HPCpy implements a simple task-dependence strategy at the scheduler level, whereby, we can use scheduler directives to make a job dependent on another. - -Consider the following snippet: - -```python -from hpcpy import get_client -client = get_client() - -# Submit the first job -first_id = client.submit("job.sh") - -# Submit some interim jobs all requiring the first to finish -job_ids = list() -for x in range(3): - jobx_id = client.submit("job.sh", depends_on=first_id) - job_ids.append(jobx_id) - -# Submit a final job that requires everything to have finished. -job_last = client.submit("job.sh", depends_on=job_ids) -``` - -This will create 5 jobs: -- 1 x starting job -- 3 x middle jobs (which depend on the first) -- 1 x finishing job (which depends on the middle jobs to complete) - -Essentially demonstrating a "fork and join" example. - -More advanced graphs can be assembled as needed, the complexity of which is determined by your scheduler. \ No newline at end of file diff --git a/docs/usage.md b/docs/usage.md index d86c509..1dfdd7c 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -11,28 +11,59 @@ from hpcpy import get_client client = get_client() ``` -This will return the most-likely client object based on the submission commands available on the system. - -In the case of the factory being unable to return an appropriate client object (or if you need to be explicit), you may import the client explicitly for your system. - -For example: +This will return the most-likely client object based on the submission commands available on the system, raising a `NoClientException` if no host scheduler is detected. In the case of the factory being unable to return an appropriate client object (or if you need to be explicit), you may import the client explicitly for your system: ```python -from hpcpy import PBSClient -client = PBSClient() +from hpcpy import PBSClient, SLURMClient +client_pbs = PBSClient() +client_slurm = SLURMClient() ``` -You can now use the client object for the remaining examples. +!!! note -## Submitting jobs + When using this approach you are bypassing any auto-detection of the host scheduler. -The simplest way to submit a pre-written job script is via the following command: +## Submit -```python -job_id = client.submit("/path/to/script.sh") -``` +The simplest way to submit a pre-written job script is via the `submit()` command, which executes the appropriate command for the scheduler: + +=== "HPCPy (Python)" + ```python + job_id = client.submit("/path/to/script.sh") + ``` + +=== "PBS" + ```shell + JOB_ID=$(qsub /path/to/script.sh) + ``` + +=== "SLURM" + ```shell + JOB_ID=$(sbatch /path/to/script.sh) + ``` -However, oftentimes it is preferable to use a script template that is rendered with additional variables prior to submission. Depending on how this is written, a single script could be used for multiple scheduling systems. +### Environment Variables + +=== "HPCPy (Python)" + ```python + job_id = client.submit( + "/path/to/script.sh", + variables=dict(a=1, b="test") + ) + ``` + +=== "PBS" + ```shell + qsub -v a=1,b=test /path/to/script.sh + ``` + +!!! note + + All environment variables are passed to the job as strings WITHOUT treatment of commas. + +### Script templates + +Script templates can be used to generalise a single template script for use in multiple scenarios (i.e. different scheduling systems). *template.sh* ```shell @@ -52,6 +83,7 @@ job_id = client.submit( ``` This will do two things: + 1. The template will be loaded into memory, rendered, and written to a temporary file at `$HOME/.hpcpy/job_scripts` (these are periodically cleared by hpcpy). 2. The rendered jobscript will be submitted to the scheduler. @@ -68,13 +100,19 @@ job_script_filepath = client._render_job_script( ) ``` -## Checking job status +## Status Checking the status of a job that has been submitted requires the `job_id` of the job on on the scheduler. Using the `submit()` command as above will return this identifier for use with the client. -```python -status = client.status(job_id) -``` +=== "HPCPy (Python)" + ```python + status = client.status(job_id) + ``` +=== "PBS" + ```shell + STATUS=$(qstat -f -F json $JOB_ID) + # ... then grepping through to find the job_state attribute + ``` The status will be a character code as listed in `constants.py`, however, certain shortcut methods are available for the most common queries. @@ -88,12 +126,62 @@ client.is_running(job_id) More shorthand methods will be made available as required. -Note: all status related commands will poll the underlying scheduler; please be mindful of overloading the scheduling system with repeated, frequent calls. +!!! note + All status related commands will poll the underlying scheduler; please be mindful of overloading the scheduling system with repeated, frequent calls. -## Deleting jobs +## Delete Deleting a job on the system requires only the `job_id` of the job on the scheduler +=== "HPCPy (Python)" + ```python + client.delete(job_id) + ``` +=== "PBS" + ```shell + qdel $JOB_ID + ``` + +## Task dependence + +HPCpy implements a simple task-dependence strategy at the scheduler level, whereby, we can use scheduler directives to make one job dependent on another. + +=== "HPCPy (Python)" + ```python + job1 = client.submit("job1.sh") + job2 = client.submit("job2.sh", depends_on=job1) + ``` +=== "PBS" + ```shell + JOB1=$(qsub job1.sh) + JOB2=$(qsub -W depend=afterok:$JOB1 job2.sh) + ``` + +Consider the following snippet: + ```python -client.delete(job_id) -``` \ No newline at end of file +from hpcpy import get_client +client = get_client() + +# Submit the first job +first_id = client.submit("job.sh") + +# Submit some interim jobs all requiring the first to finish +job_ids = list() +for x in range(3): + jobx_id = client.submit("job.sh", depends_on=first_id) + job_ids.append(jobx_id) + +# Submit a final job that requires everything to have finished. +job_last = client.submit("job.sh", depends_on=job_ids) +``` + +This will create 5 jobs: + +- 1 x starting job +- 3 x middle jobs (which depend on the first) +- 1 x finishing job (which depends on the middle jobs to complete) + +Essentially demonstrating a "fork and join" example. + +More advanced graphs can be assembled as needed, the complexity of which is determined by your scheduler. \ No newline at end of file diff --git a/hpcpy/__init__.py b/hpcpy/__init__.py index 1c705f6..beba6d8 100644 --- a/hpcpy/__init__.py +++ b/hpcpy/__init__.py @@ -4,13 +4,12 @@ from hpcpy.client.client_factory import ClientFactory from hpcpy.client.pbs import PBSClient from hpcpy.client.slurm import SlurmClient -from hpcpy.client.mock import MockClient from typing import Union __version__ = _version.get_versions()["version"] -def get_client(*args, **kwargs) -> Union[PBSClient, SlurmClient, MockClient]: +def get_client(*args, **kwargs) -> Union[PBSClient, SlurmClient]: """Get a client object specific for the current scheduler. Returns diff --git a/hpcpy/client/client_factory.py b/hpcpy/client/client_factory.py index f62ceb0..a4f41e9 100644 --- a/hpcpy/client/client_factory.py +++ b/hpcpy/client/client_factory.py @@ -2,8 +2,6 @@ from hpcpy.client.pbs import PBSClient from hpcpy.client.slurm import SlurmClient -from hpcpy.client.mock import MockClient -import os import hpcpy.exceptions as hx from hpcpy.utilities import shell from typing import Union @@ -11,7 +9,7 @@ class ClientFactory: - def get_client(*args, **kwargs) -> Union[PBSClient, SlurmClient, MockClient]: + def get_client(*args, **kwargs) -> Union[PBSClient, SlurmClient]: """Get a client object based on what kind of scheduler we are using. Arguments: @@ -21,7 +19,7 @@ def get_client(*args, **kwargs) -> Union[PBSClient, SlurmClient, MockClient]: Returns ------- - Union[PBSClient, SlurmClient, MockClient] + Union[PBSClient, SlurmClient] Client object suitable for the detected scheduler. Raises @@ -30,11 +28,7 @@ def get_client(*args, **kwargs) -> Union[PBSClient, SlurmClient, MockClient]: When no scheduler can be detected. """ - clients = dict(ls=MockClient, qsub=PBSClient, sbatch=SlurmClient) - - # Remove the MockClient if dev mode is off - if os.getenv("HPCPY_DEV_MODE", "0") != "1": - _ = clients.pop("ls") + clients = dict(qsub=PBSClient, sbatch=SlurmClient) # Loop through the clients in order, looking for a valid scheduler for cmd, client in clients.items(): diff --git a/hpcpy/client/mock.py b/hpcpy/client/mock.py deleted file mode 100644 index d229a7e..0000000 --- a/hpcpy/client/mock.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Mock client for testing.""" - -from hpcpy.client.base import BaseClient -import hpcpy.constants as hc - - -class MockClient(BaseClient): - """Mock client object for testing.""" - - def __init__(self, *args, **kwargs): - super().__init__( - tmp_submit=hc.MOCK_SUBMIT, - tmp_status=hc.MOCK_STATUS, - tmp_delete=hc.MOCK_DELETE, - ) - - def status(self, job_id) -> str: - """Get the status of the job. - - Parameters - ---------- - job_id : str - Job ID - - Returns - ------- - str - Status. - """ - status_code = super().status(job_id=job_id) - return hc.MOCK_STATUSES[status_code] diff --git a/hpcpy/client/pbs.py b/hpcpy/client/pbs.py index 7bf5f7a..bee727e 100644 --- a/hpcpy/client/pbs.py +++ b/hpcpy/client/pbs.py @@ -27,7 +27,7 @@ def status(self, job_id): # Get the status out of the job ID _status = parsed.get("Jobs").get(job_id).get("job_state") return hc.PBS_STATUSES[_status] - + def _render_variables(self, variables): """Render the variables flag for PBS. @@ -129,7 +129,7 @@ def submit( directives.append(f"-l storage={storage_str}") context["storage"] = storage context["storage_str"] = storage_str - + # Add variables if isinstance(variables, dict) and len(variables) > 0: directives.append(self._render_variables(variables)) diff --git a/hpcpy/constants.py b/hpcpy/constants.py index 802df40..c9eab15 100644 --- a/hpcpy/constants.py +++ b/hpcpy/constants.py @@ -40,11 +40,3 @@ PBS_SUBMIT = "qsub{directives} {job_script}" PBS_STATUS = "qstat -f -F json {job_id}" PBS_DELETE = "qdel {job_id}" - -# Mock command templateds -MOCK_SUBMIT = "echo 12345" -MOCK_STATUS = "echo Q" -MOCK_DELETE = 'echo "DELETED"' - -# Mock status translation -MOCK_STATUSES = PBS_STATUSES diff --git a/hpcpy/utilities.py b/hpcpy/utilities.py index 90167da..497f7c3 100644 --- a/hpcpy/utilities.py +++ b/hpcpy/utilities.py @@ -8,9 +8,7 @@ import shlex -def shell( - cmd, check=True, capture_output=True, **kwargs -) -> sp.CompletedProcess: +def shell(cmd, check=True, capture_output=True, **kwargs) -> sp.CompletedProcess: """Execute a shell command. Parameters diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..0b0d4f8 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,19 @@ +site_name: HPCPy +nav: + - Home: index.md + - Installation: installation.md + - Usage: usage.md +theme: + name: material + +markdown_extensions: +- admonition +- pymdownx.highlight: + anchor_linenums: true + line_spans: __span + pygments_lang_class: true +- pymdownx.inlinehilite +- pymdownx.snippets +- pymdownx.superfences +- pymdownx.tabbed: + alternate_style: true \ No newline at end of file diff --git a/tests/test_client.py b/tests/test_client.py index 19ef526..7eeaa9f 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -2,7 +2,6 @@ import pytest from hpcpy import get_client -import hpcpy.constants as hc import hpcpy.utilities as hu import os @@ -28,24 +27,6 @@ def test_get_job_script_filename(client): assert len(result) == len(input_filename) + hash_length + 1 # underscore -def test_submit(client): - """Test submit.""" - result = client.submit("test.txt") - assert result == "12345" - - -def test_status(client): - """Test status.""" - result = client.status("12345") - assert result == hc.STATUS_QUEUED - - -def test_delete(client): - """Test delete.""" - result = client.delete("12345") - assert result == "DELETED" - - def test_render_job_script(client): """Test rendering the job script.""" diff --git a/tests/test_core.py b/tests/test_core.py index 2cfb985..c0b0c43 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1,6 +1,4 @@ import pytest -from hpcpy import get_client -from hpcpy.client.mock import MockClient import importlib import os @@ -15,12 +13,6 @@ def environment(): os.environ["HPCPY_DEV_MODE"] = "1" -def test_get_client(): - """Test getting a client""" - client = get_client() - assert isinstance(client, MockClient) - - def test_import_pbs_toplevel(): """Test getting a PBSClient at the top level.""" _import_from("hpcpy", "PBSClient") @@ -29,8 +21,3 @@ def test_import_pbs_toplevel(): def test_import_slurm_toplevel(): """Test getting a SlurmClient at the top level.""" _import_from("hpcpy", "SlurmClient") - - -def test_import_mock_toplevel(): - """Test getting a MockClient at the top level.""" - _import_from("hpcpy", "MockClient") diff --git a/tests/test_pbs_client.py b/tests/test_pbs_client.py index e3eb054..57a1196 100644 --- a/tests/test_pbs_client.py +++ b/tests/test_pbs_client.py @@ -63,6 +63,7 @@ def test_storage(client): assert result == expected + def test_variables(client): """Test passing variables to the qsub command.""" expected = "qsub -v var1=1234,var2=abcd test.sh" @@ -72,15 +73,12 @@ def test_variables(client): assert result == expected + def test_variables_empty(client): """Test passing empty variables dict to the qsub command works as expected.""" expected = "qsub test.sh" - result1 = client.submit( - "test.sh", dry_run=True, variables=dict() - ) - result2 = client.submit( - "test.sh", dry_run=True - ) + result1 = client.submit("test.sh", dry_run=True, variables=dict()) + result2 = client.submit("test.sh", dry_run=True) assert result1 == expected - assert result2 == expected \ No newline at end of file + assert result2 == expected From fb9737649bd22c040ade5fe95f7bd7944b3bec10 Mon Sep 17 00:00:00 2001 From: Ben Schroeter Date: Mon, 14 Oct 2024 14:17:38 +1100 Subject: [PATCH 09/16] Default to PBS client --- tests/test_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_client.py b/tests/test_client.py index 7eeaa9f..a4da8f0 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,7 +1,7 @@ """Client tests.""" import pytest -from hpcpy import get_client +from hpcpy import PBSClient import hpcpy.utilities as hu import os @@ -13,7 +13,7 @@ def environment(): @pytest.fixture() def client(): - return get_client() + return PBSClient() def test_get_job_script_filename(client): From 6dd069c82684a9d4f9a5f9a732f327039c263f88 Mon Sep 17 00:00:00 2001 From: Ben Schroeter Date: Tue, 15 Oct 2024 13:13:03 +1100 Subject: [PATCH 10/16] Added readthedocs.yml --- .readthedocs.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .readthedocs.yml diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000..495ffc0 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,20 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the version of Python and other tools you might need +build: + os: ubuntu-20.04 + tools: + python: "3.9" + +mkdocs: + configuration: mkdocs.yml + +# Optionally declare the Python requirements required to build your docs +python: + install: + - requirements: .conda/mkdocs-requirements.txt \ No newline at end of file From 764792041d83dfdf2fe0201b236a5274fa2595b6 Mon Sep 17 00:00:00 2001 From: Ben Schroeter Date: Tue, 15 Oct 2024 15:43:09 +1100 Subject: [PATCH 11/16] Create dependabot.yml --- .github/dependabot.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..5990d9c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file + +version: 2 +updates: + - package-ecosystem: "" # See documentation for possible values + directory: "/" # Location of package manifests + schedule: + interval: "weekly" From a1cb029b5ac173b431ecd492398fcc80d82b26c2 Mon Sep 17 00:00:00 2001 From: Ben Schroeter Date: Tue, 15 Oct 2024 16:10:28 +1100 Subject: [PATCH 12/16] Updated readme. Fixes #32 --- .github/wordlist.txt | 2 -- README.md | 20 ++++++++++++++++++-- docs/usage.md | 8 ++++---- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/.github/wordlist.txt b/.github/wordlist.txt index 2fb85c8..3482a12 100644 --- a/.github/wordlist.txt +++ b/.github/wordlist.txt @@ -1,6 +1,4 @@ -HPCPY HPC -hpcpy HPCpy pre py diff --git a/README.md b/README.md index 1ac2fec..2ce63fc 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,19 @@ -# HPCPY +[![CI pytest](https://github.com/ACCESS-NRI/hpcpy/actions/workflows/ci_pytest.yml/badge.svg?branch=main)](https://github.com/ACCESS-NRI/hpcpy/actions/workflows/ci_pytest.yml) +[![Documentation Status](https://readthedocs.org/projects/hpcpy/badge/?version=latest)](https://hpcpy.readthedocs.io/en/latest/?badge=latest) -HPCPY is a prototype Python client for interacting with HPC scheduling systems (i.e PBS). \ No newline at end of file +# HPCpy + +HPCpy is a Python package for interacting with HPC scheduling systems. The package provides generalised clients to communicate with HPC schedulers agnostically. + +Currently supported scheduling systems: + +- PBS +- SLURM* + +_* under development_ + +The full documentation is available at [hpcpy.readthedocs.io](https://hpcpy.readthedocs.io) + +## License + +HPCpy is distributed under the Apache Software License v2.0. Please see the [LICENSE](https://github.com/ACCESS-NRI/hpcpy/blob/main/LICENSE) file in this repository for further details. \ No newline at end of file diff --git a/docs/usage.md b/docs/usage.md index 1dfdd7c..23558a0 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -1,6 +1,6 @@ # Usage -The following describes the basic usage of hpcpy. +The following describes the basic usage of HPCpy. ## Getting a client object @@ -44,7 +44,7 @@ The simplest way to submit a pre-written job script is via the `submit()` comman ### Environment Variables -=== "HPCPy (Python)" +=== "HPCpy (Python)" ```python job_id = client.submit( "/path/to/script.sh", @@ -104,7 +104,7 @@ job_script_filepath = client._render_job_script( Checking the status of a job that has been submitted requires the `job_id` of the job on on the scheduler. Using the `submit()` command as above will return this identifier for use with the client. -=== "HPCPy (Python)" +=== "HPCpy (Python)" ```python status = client.status(job_id) ``` @@ -133,7 +133,7 @@ More shorthand methods will be made available as required. Deleting a job on the system requires only the `job_id` of the job on the scheduler -=== "HPCPy (Python)" +=== "HPCpy (Python)" ```python client.delete(job_id) ``` From 7ced15ea7a0b15652ae92e67d38c008f4cfe1cd9 Mon Sep 17 00:00:00 2001 From: Ben Schroeter Date: Thu, 17 Oct 2024 11:22:15 +1100 Subject: [PATCH 13/16] Added new exception class to reveal more information to the user. Fixes #34 --- hpcpy/client/base.py | 9 ++++++--- hpcpy/exceptions.py | 31 +++++++++++++++++++++++++++++++ hpcpy/utilities.py | 17 +++++++++++++---- tests/test_utilities.py | 9 +++++++++ 4 files changed, 59 insertions(+), 7 deletions(-) diff --git a/hpcpy/client/base.py b/hpcpy/client/base.py index ceda7cf..e585a3c 100644 --- a/hpcpy/client/base.py +++ b/hpcpy/client/base.py @@ -115,7 +115,6 @@ def status(self, job_id): job_id : str Job ID. """ - # raise NotImplementedError() cmd = self._tmp_status.format(job_id=job_id) result = self._shell(cmd) return result @@ -171,11 +170,15 @@ def _shell(self, cmd, decode=True): Command to run. decode : bool Automatically decode response with utf-8, defaults to True + Raises + ------ + hpcypy.excetions.ShellException : + When the underlying shell call fails. Returns ------- - _type_ - _description_ + str + Result from the underlying called command. """ result = shell(cmd) diff --git a/hpcpy/exceptions.py b/hpcpy/exceptions.py index 483d2f8..f2b075a 100644 --- a/hpcpy/exceptions.py +++ b/hpcpy/exceptions.py @@ -1,5 +1,36 @@ +import subprocess as sp + + class NoClientException(Exception): def __init__(self): super().__init__( "Unable to detect scheduler type, cannot determine client type." ) + + +class ShellException(Exception): + """Exception class to improve readability of the subprocess.CalledProcessError""" + + def __init__(self, called_process_error: sp.CalledProcessError): + """Constructor + + Parameters + ---------- + called_process_error : sp.CalledProcessError + Source subprocess.CalledError + """ + self.returncode = called_process_error.returncode + self.cmd = called_process_error.cmd[0] + self.stdout = called_process_error.stdout.decode() + self.stderr = called_process_error.stderr.decode() + self.output = called_process_error.output + + def __str__(self): + """Improved string representation of the error message. + + Returns + ------- + str + Improved error messaging + """ + return f"Error {self.returncode}: {self.stderr}" diff --git a/hpcpy/utilities.py b/hpcpy/utilities.py index 497f7c3..18f1dff 100644 --- a/hpcpy/utilities.py +++ b/hpcpy/utilities.py @@ -5,6 +5,7 @@ import jinja2.meta as j2m from pathlib import Path from importlib import resources +from hpcpy.exceptions import ShellException import shlex @@ -27,11 +28,19 @@ def shell(cmd, check=True, capture_output=True, **kwargs) -> sp.CompletedProcess Raises ------ - subprocess.CalledProcessError + hpcypy.excetions.ShellException : + When the shell call fails. """ - return sp.run( - shlex.split(cmd), check=check, capture_output=capture_output, **kwargs - ) + try: + return sp.run( + shlex.split(cmd), + shell=True, + check=check, + capture_output=capture_output, + **kwargs, + ) + except sp.CalledProcessError as ex: + raise ShellException(ex) def interpolate_string_template(template, **kwargs) -> str: diff --git a/tests/test_utilities.py b/tests/test_utilities.py index ac837dd..3521f32 100644 --- a/tests/test_utilities.py +++ b/tests/test_utilities.py @@ -1,8 +1,17 @@ """Tests for utilities.py""" import hpcpy.utilities as hu +import hpcpy.exceptions as hx def test_interpolate_string_template(): """Test interpolating a string template.""" assert hu.interpolate_string_template("hello {{arg}}", arg="world") == "hello world" + + +def test_shell_exception(): + """Test that error messaging is being raised to the user.""" + try: + hu.shell("blah") + except hx.ShellException as ex: + assert ex.__str__() == "Error 127: /bin/sh: blah: command not found\n" From d15ee5abe3630064ea77426626f15cf0a9228ae0 Mon Sep 17 00:00:00 2001 From: Ben Schroeter Date: Thu, 17 Oct 2024 11:31:42 +1100 Subject: [PATCH 14/16] Fixed exception test for general use. Fixes: #34 --- tests/test_utilities.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_utilities.py b/tests/test_utilities.py index 3521f32..333ba05 100644 --- a/tests/test_utilities.py +++ b/tests/test_utilities.py @@ -14,4 +14,6 @@ def test_shell_exception(): try: hu.shell("blah") except hx.ShellException as ex: - assert ex.__str__() == "Error 127: /bin/sh: blah: command not found\n" + + expected = f"Error {ex.returncode}: {ex.stderr}" + assert ex.__str__() == expected From eda8dc04688077af53ea786d98fe4ff2b37cbca0 Mon Sep 17 00:00:00 2001 From: Ben Schroeter Date: Thu, 17 Oct 2024 11:49:21 +1100 Subject: [PATCH 15/16] Update utilities.py Co-authored-by: Sean Bryan <39685865+SeanBryan51@users.noreply.github.com> --- hpcpy/utilities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hpcpy/utilities.py b/hpcpy/utilities.py index 18f1dff..6fe8e88 100644 --- a/hpcpy/utilities.py +++ b/hpcpy/utilities.py @@ -28,7 +28,7 @@ def shell(cmd, check=True, capture_output=True, **kwargs) -> sp.CompletedProcess Raises ------ - hpcypy.excetions.ShellException : + hpcpy.exceptions.ShellException : When the shell call fails. """ try: From c7281eb4a6df0bc63a2b315397506cc75fc8529c Mon Sep 17 00:00:00 2001 From: Ben Schroeter Date: Thu, 17 Oct 2024 11:49:28 +1100 Subject: [PATCH 16/16] Update base.py Co-authored-by: Sean Bryan <39685865+SeanBryan51@users.noreply.github.com> --- hpcpy/client/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hpcpy/client/base.py b/hpcpy/client/base.py index e585a3c..1605642 100644 --- a/hpcpy/client/base.py +++ b/hpcpy/client/base.py @@ -172,7 +172,7 @@ def _shell(self, cmd, decode=True): Automatically decode response with utf-8, defaults to True Raises ------ - hpcypy.excetions.ShellException : + hpcpy.exceptions.ShellException : When the underlying shell call fails. Returns