From 3e86b5d5318d757571076760097d22e3316f4b61 Mon Sep 17 00:00:00 2001 From: Benjamin Pritchard Date: Tue, 7 Jan 2025 22:02:01 -0500 Subject: [PATCH 1/4] Add function to convert duration string to int --- qcportal/qcportal/test_utils.py | 32 +++++++++++++++++++++++++++++++- qcportal/qcportal/utils.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/qcportal/qcportal/test_utils.py b/qcportal/qcportal/test_utils.py index 4f7cf03fa..337b58d09 100644 --- a/qcportal/qcportal/test_utils.py +++ b/qcportal/qcportal/test_utils.py @@ -1,4 +1,4 @@ -from qcportal.utils import chunk_iterable, seconds_to_hms, is_included +from qcportal.utils import chunk_iterable, seconds_to_hms, duration_to_seconds, is_included def test_chunk_iterable(): @@ -29,6 +29,36 @@ def test_seconds_to_hms(): assert seconds_to_hms(3670.12) == "01:01:10.12" +def test_duration_to_seconds(): + assert duration_to_seconds(0) == 0 + assert duration_to_seconds("0") == 0 + assert duration_to_seconds(17) == 17 + assert duration_to_seconds("17") == 17 + + assert duration_to_seconds("17s") == 17 + assert duration_to_seconds("70s") == 70 + assert duration_to_seconds("8m17s") == 497 + assert duration_to_seconds("80m72s") == 4872 + assert duration_to_seconds("3h8m17s") == 11297 + assert duration_to_seconds("03h08m07s") == 11287 + assert duration_to_seconds("03h08m070s") == 11350 + assert duration_to_seconds("9d03h08m070s") == 788950 + + assert duration_to_seconds("9d") == 777600 + assert duration_to_seconds("10m") == 600 + assert duration_to_seconds("90m") == 5400 + assert duration_to_seconds("04h") == 14400 + assert duration_to_seconds("4h5s") == 14405 + assert duration_to_seconds("1d9s") == 86409 + + assert duration_to_seconds("8:17") == 497 + assert duration_to_seconds("80:72") == 4872 + assert duration_to_seconds("3:8:17") == 11297 + assert duration_to_seconds("03:08:07") == 11287 + assert duration_to_seconds("03:08:070") == 11350 + assert duration_to_seconds("9:03:08:07") == 788950 + + def test_is_included(): assert is_included("test", None, None, True) is True assert is_included("test", None, None, False) is False diff --git a/qcportal/qcportal/utils.py b/qcportal/qcportal/utils.py index 89806afd1..1192212e5 100644 --- a/qcportal/qcportal/utils.py +++ b/qcportal/qcportal/utils.py @@ -8,6 +8,7 @@ import json import logging import math +import re import time from contextlib import contextmanager, redirect_stderr, redirect_stdout from hashlib import sha256 @@ -261,6 +262,37 @@ def seconds_to_hms(seconds: Union[float, int]) -> str: return f"{hours:02d}:{minutes:02d}:{seconds+fraction:02.2f}" +def duration_to_seconds(s: Union[int, str]) -> int: + """ + Parses a string in dd:hh:mm:ss or 1d2h3m4s to an integer number of seconds + """ + + # Is already an int + if isinstance(s, int): + return s + + # Plain number of seconds (as a string) + if s.isdigit(): + return int(s) + + # Handle dd:hh:mm:ss format + if ":" in s: + parts = list(map(int, s.split(":"))) + while len(parts) < 4: # Pad missing parts with zeros + parts.insert(0, 0) + days, hours, minutes, seconds = parts + return days * 86400 + hours * 3600 + minutes * 60 + seconds + + # Handle format like 3d4h7m10s + pattern = re.compile(r"(?:(\d+)d)?(?:(\d+)h)?(?:(\d+)m)?(?:(\d+)s)?") + match = pattern.fullmatch(s) + if not match: + raise ValueError(f"Invalid duration format: {s}") + + days, hours, minutes, seconds = map(lambda x: int(x) if x else 0, match.groups()) + return days * 86400 + hours * 3600 + minutes * 60 + seconds + + def recursive_normalizer(value: Any, digits: int = 10, lowercase: bool = True) -> Any: """ Prepare a structure for hashing by lowercasing all values and round all floats From 8293f0f45801b883ec0d25bd035734eb952fd940 Mon Sep 17 00:00:00 2001 From: Benjamin Pritchard Date: Wed, 8 Jan 2025 10:11:19 -0500 Subject: [PATCH 2/4] Enable more user-friendly time durations in config --- qcfractal/qcfractal/config.py | 13 +++- qcfractal/qcfractal/test_config.py | 65 +++++++++++++++++++ qcfractalcompute/qcfractalcompute/config.py | 16 ++--- .../qcfractalcompute/test_manager_config.py | 33 +++++++++- 4 files changed, 116 insertions(+), 11 deletions(-) create mode 100644 qcfractal/qcfractal/test_config.py diff --git a/qcfractal/qcfractal/config.py b/qcfractal/qcfractal/config.py index 0ae3903aa..5fc57adcf 100644 --- a/qcfractal/qcfractal/config.py +++ b/qcfractal/qcfractal/config.py @@ -21,6 +21,7 @@ from sqlalchemy.engine.url import URL, make_url from qcfractal.port_util import find_open_port +from qcportal.utils import duration_to_seconds def update_nested_dict(d, u): @@ -315,6 +316,10 @@ class WebAPIConfig(ConfigBase): None, description="Any additional options to pass directly to the waitress serve function" ) + @validator("jwt_access_token_expires", "jwt_refresh_token_expires", pre=True) + def _convert_durations(cls, v): + return duration_to_seconds(v) + class Config(ConfigCommon): env_prefix = "QCF_API_" @@ -374,7 +379,9 @@ class FractalConfig(ConfigBase): # Access logging log_access: bool = Field(False, description="Store API access in the database") - access_log_keep: int = Field(0, description="Number of days of access logs to keep. 0 means keep all") + access_log_keep: int = Field( + 0, description="How far back to keep access logs (in seconds or a string). 0 means keep all" + ) # maxmind_account_id: Optional[int] = Field(None, description="Account ID for MaxMind GeoIP2 service") maxmind_license_key: Optional[str] = Field( @@ -454,6 +461,10 @@ def _check_loglevel(cls, v): raise ValidationError(f"{v} is not a valid loglevel. Must be DEBUG, INFO, WARNING, ERROR, or CRITICAL") return v + @validator("service_frequency", "heartbeat_frequency", "access_log_keep", pre=True) + def _convert_durations(cls, v): + return duration_to_seconds(v) + class Config(ConfigCommon): env_prefix = "QCF_" diff --git a/qcfractal/qcfractal/test_config.py b/qcfractal/qcfractal/test_config.py new file mode 100644 index 000000000..f507e78ab --- /dev/null +++ b/qcfractal/qcfractal/test_config.py @@ -0,0 +1,65 @@ +import copy + +from qcfractal.config import FractalConfig + +_base_config = { + "api": { + "secret_key": "abc1234def456", + "jwt_secret_key": "abc123def456", + }, + "database": {"username": "qcfractal", "password": "abc123def456"}, +} + + +def test_config_durations_plain(tmp_path): + base_folder = str(tmp_path) + + base_config = copy.deepcopy(_base_config) + base_config["service_frequency"] = 3600 + base_config["heartbeat_frequency"] = 30 + base_config["access_log_keep"] = 100802 + base_config["api"]["jwt_access_token_expires"] = 7450 + base_config["api"]["jwt_refresh_token_expires"] = 637277 + cfg = FractalConfig(base_folder=base_folder, **base_config) + + assert cfg.service_frequency == 3600 + assert cfg.heartbeat_frequency == 30 + assert cfg.access_log_keep == 100802 + assert cfg.api.jwt_access_token_expires == 7450 + assert cfg.api.jwt_refresh_token_expires == 637277 + + +def test_config_durations_str(tmp_path): + base_folder = str(tmp_path) + + base_config = copy.deepcopy(_base_config) + base_config["service_frequency"] = "1h" + base_config["heartbeat_frequency"] = "30s" + base_config["access_log_keep"] = "1d4h2s" + base_config["api"]["jwt_access_token_expires"] = "2h4m10s" + base_config["api"]["jwt_refresh_token_expires"] = "7d9h77s" + cfg = FractalConfig(base_folder=base_folder, **base_config) + + assert cfg.service_frequency == 3600 + assert cfg.heartbeat_frequency == 30 + assert cfg.access_log_keep == 100802 + assert cfg.api.jwt_access_token_expires == 7450 + assert cfg.api.jwt_refresh_token_expires == 637277 + + +def test_config_durations_dhms(tmp_path): + base_folder = str(tmp_path) + + base_config = copy.deepcopy(_base_config) + base_config["service_frequency"] = "1:00:00" + base_config["heartbeat_frequency"] = "30" + base_config["access_log_keep"] = "1:04:00:02" + base_config["api"]["jwt_access_token_expires"] = "2:04:10" + base_config["api"]["jwt_refresh_token_expires"] = "7:09:00:77" + cfg = FractalConfig(base_folder=base_folder, **base_config) + + assert cfg.service_frequency == 3600 + assert cfg.heartbeat_frequency == 30 + assert cfg.access_log_keep == 100802 + assert cfg.api.jwt_access_token_expires == 7450 + assert cfg.api.jwt_refresh_token_expires == 637277 diff --git a/qcfractalcompute/qcfractalcompute/config.py b/qcfractalcompute/qcfractalcompute/config.py index 6419114c7..319ebc7b1 100644 --- a/qcfractalcompute/qcfractalcompute/config.py +++ b/qcfractalcompute/qcfractalcompute/config.py @@ -10,7 +10,7 @@ from pydantic import BaseModel, Field, validator from typing_extensions import Literal -from qcportal.utils import seconds_to_hms +from qcportal.utils import seconds_to_hms, duration_to_seconds def _make_abs_path(path: Optional[str], base_folder: str, default_filename: Optional[str]) -> Optional[str]: @@ -120,10 +120,7 @@ class TorqueExecutorConfig(ExecutorConfig): @validator("walltime", pre=True) def walltime_must_be_str(cls, v): - if isinstance(v, int): - return seconds_to_hms(v) - else: - return v + return seconds_to_hms(duration_to_seconds(v)) class LSFExecutorConfig(ExecutorConfig): @@ -143,10 +140,7 @@ class LSFExecutorConfig(ExecutorConfig): @validator("walltime", pre=True) def walltime_must_be_str(cls, v): - if isinstance(v, int): - return seconds_to_hms(v) - else: - return v + return seconds_to_hms(duration_to_seconds(v)) AllExecutorTypes = Union[ @@ -226,6 +220,10 @@ def _check_logfile(cls, v, values): def _check_run_dir(cls, v, values): return _make_abs_path(v, values["base_folder"], "parsl_run_dir") + @validator("update_frequency", pre=True) + def _convert_durations(cls, v): + return duration_to_seconds(v) + def read_configuration(file_paths: List[str], extra_config: Optional[Dict[str, Any]] = None) -> FractalComputeConfig: logger = logging.getLogger(__name__) diff --git a/qcfractalcompute/qcfractalcompute/test_manager_config.py b/qcfractalcompute/qcfractalcompute/test_manager_config.py index 88749c00d..8efba9838 100644 --- a/qcfractalcompute/qcfractalcompute/test_manager_config.py +++ b/qcfractalcompute/qcfractalcompute/test_manager_config.py @@ -1,9 +1,19 @@ from __future__ import annotations +import copy + import pytest import yaml -from qcfractalcompute.config import SlurmExecutorConfig +from qcfractalcompute.config import SlurmExecutorConfig, FractalComputeConfig + +_base_config = { + "cluster": "testcluster", + "server": { + "fractal_uri": "http://localhost:7777/", + }, + "executors": {}, +} @pytest.mark.parametrize("time_str", ["02:01:59", "72:00:00", "10:00:00"]) @@ -39,3 +49,24 @@ def test_manager_config_walltime(time_str): config = yaml.load(config_yaml, yaml.SafeLoader) executor_config = SlurmExecutorConfig(**config) assert executor_config.walltime == time_str + + +def test_manager_config_durations(tmp_path): + base_folder = str(tmp_path) + base_config = copy.deepcopy(_base_config) + + base_config["update_frequency"] = "900" + manager_config = FractalComputeConfig(base_folder=base_folder, **base_config) + assert manager_config.update_frequency == 900 + + base_config["update_frequency"] = 900 + manager_config = FractalComputeConfig(base_folder=base_folder, **base_config) + assert manager_config.update_frequency == 900 + + base_config["update_frequency"] = "3d4h80m09s" + manager_config = FractalComputeConfig(base_folder=base_folder, **base_config) + assert manager_config.update_frequency == 278409 + + base_config["update_frequency"] = "3:04:80:9" + manager_config = FractalComputeConfig(base_folder=base_folder, **base_config) + assert manager_config.update_frequency == 278409 From 80ce25a224c9dfaac5a150474d8c9b3ceb4cefb7 Mon Sep 17 00:00:00 2001 From: Benjamin Pritchard Date: Wed, 8 Jan 2025 11:40:52 -0500 Subject: [PATCH 3/4] Fix duration to seconds test --- qcportal/qcportal/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qcportal/qcportal/test_utils.py b/qcportal/qcportal/test_utils.py index 337b58d09..7a11de3ea 100644 --- a/qcportal/qcportal/test_utils.py +++ b/qcportal/qcportal/test_utils.py @@ -56,7 +56,7 @@ def test_duration_to_seconds(): assert duration_to_seconds("3:8:17") == 11297 assert duration_to_seconds("03:08:07") == 11287 assert duration_to_seconds("03:08:070") == 11350 - assert duration_to_seconds("9:03:08:07") == 788950 + assert duration_to_seconds("9:03:08:07") == 788887 def test_is_included(): From 9ca9085aeeecf593b9df58ff92ade6f4c8507f2b Mon Sep 17 00:00:00 2001 From: Benjamin Pritchard Date: Wed, 8 Jan 2025 17:59:53 -0500 Subject: [PATCH 4/4] Make conversion tolerant of floating point --- .../config_files/gha_fractal_compute.yaml | 2 +- qcportal/qcportal/test_utils.py | 2 ++ qcportal/qcportal/utils.py | 18 +++++++++++++++++- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/qcarchivetesting/qcarchivetesting/config_files/gha_fractal_compute.yaml b/qcarchivetesting/qcarchivetesting/config_files/gha_fractal_compute.yaml index dcdb52137..4d34959ca 100644 --- a/qcarchivetesting/qcarchivetesting/config_files/gha_fractal_compute.yaml +++ b/qcarchivetesting/qcarchivetesting/config_files/gha_fractal_compute.yaml @@ -1,6 +1,6 @@ cluster: testworker loglevel: DEBUG -update_frequency: 5.0 +update_frequency: 5 server: fractal_uri: http://localhost:7900 diff --git a/qcportal/qcportal/test_utils.py b/qcportal/qcportal/test_utils.py index 7a11de3ea..ebaadadbb 100644 --- a/qcportal/qcportal/test_utils.py +++ b/qcportal/qcportal/test_utils.py @@ -34,6 +34,8 @@ def test_duration_to_seconds(): assert duration_to_seconds("0") == 0 assert duration_to_seconds(17) == 17 assert duration_to_seconds("17") == 17 + assert duration_to_seconds(17.0) == 17 + assert duration_to_seconds("17.0") == 17 assert duration_to_seconds("17s") == 17 assert duration_to_seconds("70s") == 70 diff --git a/qcportal/qcportal/utils.py b/qcportal/qcportal/utils.py index 1192212e5..a60f0ccb0 100644 --- a/qcportal/qcportal/utils.py +++ b/qcportal/qcportal/utils.py @@ -262,7 +262,7 @@ def seconds_to_hms(seconds: Union[float, int]) -> str: return f"{hours:02d}:{minutes:02d}:{seconds+fraction:02.2f}" -def duration_to_seconds(s: Union[int, str]) -> int: +def duration_to_seconds(s: Union[int, str, float]) -> int: """ Parses a string in dd:hh:mm:ss or 1d2h3m4s to an integer number of seconds """ @@ -271,10 +271,26 @@ def duration_to_seconds(s: Union[int, str]) -> int: if isinstance(s, int): return s + # Is a float but represents an integer + if isinstance(s, float): + if s.is_integer(): + return int(s) + else: + raise ValueError(f"Invalid duration format: {s} - cannot represent fractional seconds") + # Plain number of seconds (as a string) if s.isdigit(): return int(s) + try: + f = float(s) + if f.is_integer(): + return int(f) + else: + raise ValueError(f"Invalid duration format: {s} - cannot represent fractional seconds") + except ValueError: + pass + # Handle dd:hh:mm:ss format if ":" in s: parts = list(map(int, s.split(":")))