From dffb29e5d81bbdf0e16482dde40e1baa62c1e65f Mon Sep 17 00:00:00 2001 From: SimonBoothroyd Date: Tue, 12 Oct 2021 12:01:19 +0100 Subject: [PATCH] Add initial executor CLI (#72) * Add initial executor CLI * Add simple tests * Add run tests * Clean-up launch CLI options * Fix tests * change worker default to 1, add from file Co-authored-by: Josh Horton --- devtools/conda-envs/test-env.yaml | 1 + openff/bespokefit/__init__.py | 13 ++ openff/bespokefit/cli/cli.py | 2 + openff/bespokefit/cli/executor/__init__.py | 3 + openff/bespokefit/cli/executor/executor.py | 19 ++ openff/bespokefit/cli/executor/launch.py | 114 ++++++++++++ openff/bespokefit/cli/executor/list.py | 51 ++++++ openff/bespokefit/cli/executor/run.py | 92 ++++++++++ openff/bespokefit/cli/executor/submit.py | 168 ++++++++++++++++++ openff/bespokefit/cli/executor/watch.py | 26 +++ openff/bespokefit/cli/utilities.py | 24 +++ openff/bespokefit/data/schemas/debug.json | 103 +++++++++++ openff/bespokefit/data/schemas/default.json | 103 +++++++++++ openff/bespokefit/executor/executor.py | 6 +- .../bespokefit/tests/cli/executor/__init__.py | 0 .../tests/cli/executor/test_launch.py | 52 ++++++ .../tests/cli/executor/test_list.py | 41 +++++ .../bespokefit/tests/cli/executor/test_run.py | 71 ++++++++ .../tests/cli/executor/test_submit.py | 153 ++++++++++++++++ .../tests/cli/executor/test_watch.py | 39 ++++ openff/bespokefit/tests/cli/test_utilities.py | 35 ++++ openff/bespokefit/utilities/logging.py | 6 + openff/bespokefit/workflows/bespoke.py | 11 +- 23 files changed, 1128 insertions(+), 5 deletions(-) create mode 100644 openff/bespokefit/cli/executor/__init__.py create mode 100644 openff/bespokefit/cli/executor/executor.py create mode 100644 openff/bespokefit/cli/executor/launch.py create mode 100644 openff/bespokefit/cli/executor/list.py create mode 100644 openff/bespokefit/cli/executor/run.py create mode 100644 openff/bespokefit/cli/executor/submit.py create mode 100644 openff/bespokefit/cli/executor/watch.py create mode 100644 openff/bespokefit/cli/utilities.py create mode 100644 openff/bespokefit/data/schemas/debug.json create mode 100644 openff/bespokefit/data/schemas/default.json create mode 100644 openff/bespokefit/tests/cli/executor/__init__.py create mode 100644 openff/bespokefit/tests/cli/executor/test_launch.py create mode 100644 openff/bespokefit/tests/cli/executor/test_list.py create mode 100644 openff/bespokefit/tests/cli/executor/test_run.py create mode 100644 openff/bespokefit/tests/cli/executor/test_submit.py create mode 100644 openff/bespokefit/tests/cli/executor/test_watch.py create mode 100644 openff/bespokefit/tests/cli/test_utilities.py create mode 100644 openff/bespokefit/utilities/logging.py diff --git a/devtools/conda-envs/test-env.yaml b/devtools/conda-envs/test-env.yaml index 1432afc1..37e826b7 100644 --- a/devtools/conda-envs/test-env.yaml +++ b/devtools/conda-envs/test-env.yaml @@ -19,6 +19,7 @@ dependencies: - tqdm - rich - click + - click-option-group - rdkit - openff-utilities - openff-toolkit-base >=0.10.0 diff --git a/openff/bespokefit/__init__.py b/openff/bespokefit/__init__.py index 32d13462..f8dfe184 100644 --- a/openff/bespokefit/__init__.py +++ b/openff/bespokefit/__init__.py @@ -2,6 +2,8 @@ BespokeFit Creating bespoke parameters for individual molecules. """ +import logging +import sys from ._version import get_versions @@ -9,3 +11,14 @@ __version__ = versions["version"] __git_revision__ = versions["full-revisionid"] del get_versions, versions + + +# Silence verbose messages when running the CLI otherwise you can't read the output +# without seeing tens of 'Unable to load AmberTools' or don't import simtk warnings... +if sys.argv[0].endswith("openff-bespoke"): + + from openff.bespokefit.utilities.logging import DeprecationWarningFilter + + # if "openff-bespoke" + logging.getLogger("openff.toolkit").setLevel(logging.ERROR) + logging.getLogger().addFilter(DeprecationWarningFilter()) diff --git a/openff/bespokefit/cli/cli.py b/openff/bespokefit/cli/cli.py index c700f096..fd669a20 100644 --- a/openff/bespokefit/cli/cli.py +++ b/openff/bespokefit/cli/cli.py @@ -1,5 +1,6 @@ import click +from openff.bespokefit.cli.executor import executor_cli from openff.bespokefit.cli.prepare import prepare_cli @@ -8,4 +9,5 @@ def cli(): """The root group for all CLI commands.""" +cli.add_command(executor_cli) cli.add_command(prepare_cli) diff --git a/openff/bespokefit/cli/executor/__init__.py b/openff/bespokefit/cli/executor/__init__.py new file mode 100644 index 00000000..a657093c --- /dev/null +++ b/openff/bespokefit/cli/executor/__init__.py @@ -0,0 +1,3 @@ +from openff.bespokefit.cli.executor.executor import executor_cli + +__all__ = [executor_cli] diff --git a/openff/bespokefit/cli/executor/executor.py b/openff/bespokefit/cli/executor/executor.py new file mode 100644 index 00000000..e75a1417 --- /dev/null +++ b/openff/bespokefit/cli/executor/executor.py @@ -0,0 +1,19 @@ +import click + +from openff.bespokefit.cli.executor.launch import launch_cli +from openff.bespokefit.cli.executor.list import list_cli +from openff.bespokefit.cli.executor.run import run_cli +from openff.bespokefit.cli.executor.submit import submit_cli +from openff.bespokefit.cli.executor.watch import watch_cli + + +@click.group("executor") +def executor_cli(): + """Commands for interacting with a bespoke executor.""" + + +executor_cli.add_command(launch_cli) +executor_cli.add_command(submit_cli) +executor_cli.add_command(run_cli) +executor_cli.add_command(watch_cli) +executor_cli.add_command(list_cli) diff --git a/openff/bespokefit/cli/executor/launch.py b/openff/bespokefit/cli/executor/launch.py new file mode 100644 index 00000000..a439a295 --- /dev/null +++ b/openff/bespokefit/cli/executor/launch.py @@ -0,0 +1,114 @@ +import time +from typing import Optional + +import click +import rich +from click_option_group import optgroup +from rich import pretty + +from openff.bespokefit.cli.utilities import create_command, print_header + + +# The run command inherits these options so be sure to take that into account when +# making changes here. +def launch_options( + directory: str = "bespoke-executor", + n_fragmenter_workers: Optional[int] = 1, + n_qc_compute_workers: Optional[int] = 1, + n_optimizer_workers: Optional[int] = 1, + launch_redis_if_unavailable: Optional[bool] = True, +): + + return [ + optgroup("Executor configuration"), + optgroup.option( + "--directory", + type=click.Path(exists=False, file_okay=False, dir_okay=True), + help="The directory to store any working and log files in", + required=True, + default=directory, + show_default=directory is not None, + ), + optgroup.group("Worker configuration"), + optgroup.option( + "--n-fragmenter-workers", + "n_fragmenter_workers", + type=click.INT, + help="The number of fragmentation workers to spawn", + required=n_fragmenter_workers is None, + default=n_fragmenter_workers, + show_default=n_fragmenter_workers is not None, + ), + optgroup.option( + "--n-qc-compute-workers", + "n_qc_compute_workers", + type=click.INT, + help="The number of QC compute workers to spawn", + required=n_qc_compute_workers is None, + default=n_qc_compute_workers, + show_default=n_qc_compute_workers is not None, + ), + optgroup.option( + "--n-optimizer-workers", + "n_optimizer_workers", + type=click.INT, + help="The number of optimizer workers to spawn", + required=n_optimizer_workers is None, + default=n_optimizer_workers, + show_default=n_optimizer_workers is not None, + ), + optgroup.group("Storage configuration"), + optgroup.option( + "--launch-redis/--no-launch-redis", + "launch_redis_if_unavailable", + help="Whether to launch a redis server if an already running one cannot be " + "found.", + required=launch_redis_if_unavailable is None, + default=launch_redis_if_unavailable, + show_default=launch_redis_if_unavailable is not None, + ), + ] + + +def _launch_cli( + directory: str, + n_fragmenter_workers: int, + n_qc_compute_workers: int, + n_optimizer_workers: int, + launch_redis_if_unavailable: bool, +): + """Launch a bespoke executor.""" + + pretty.install() + + console = rich.get_console() + print_header(console) + + from openff.bespokefit.executor import BespokeExecutor + + executor_status = console.status("launching the bespoke executor") + executor_status.start() + + with BespokeExecutor( + directory=directory, + n_fragmenter_workers=n_fragmenter_workers, + n_qc_compute_workers=n_qc_compute_workers, + n_optimizer_workers=n_optimizer_workers, + launch_redis_if_unavailable=launch_redis_if_unavailable, + ): + + executor_status.stop() + console.print("[[green]✓[/green]] bespoke executor launched") + + try: + while True: + time.sleep(5) + except KeyboardInterrupt: + pass + + +launch_cli = create_command( + click_command=click.command("launch"), + click_options=launch_options(), + func=_launch_cli, +) diff --git a/openff/bespokefit/cli/executor/list.py b/openff/bespokefit/cli/executor/list.py new file mode 100644 index 00000000..3e7f836e --- /dev/null +++ b/openff/bespokefit/cli/executor/list.py @@ -0,0 +1,51 @@ +import click +import requests +import rich +from rich import pretty +from rich.padding import Padding + +from openff.bespokefit.cli.utilities import print_header + + +@click.command("list") +def list_cli(): + """List the ids of any bespoke optimizations.""" + + pretty.install() + + console = rich.get_console() + print_header(console) + + from openff.bespokefit.executor.services import settings + from openff.bespokefit.executor.services.coordinator.models import ( + CoordinatorGETPageResponse, + ) + + href = ( + f"http://127.0.0.1:" + f"{settings.BEFLOW_GATEWAY_PORT}" + f"{settings.BEFLOW_API_V1_STR}/" + f"{settings.BEFLOW_COORDINATOR_PREFIX}" + ) + + try: + + request = requests.get(href) + request.raise_for_status() + + except requests.ConnectionError: + console.print( + "A connection could not be made to the bespoke executor. Please make sure " + "there is a bespoke executor running." + ) + return + + response = CoordinatorGETPageResponse.parse_raw(request.content) + response_ids = [item.id for item in response.contents] + + if len(response_ids) == 0: + console.print("No optimizations were found.") + return + + console.print(Padding("The following optimizations were found:", (0, 0, 1, 0))) + console.print("\n".join(response_ids)) diff --git a/openff/bespokefit/cli/executor/run.py b/openff/bespokefit/cli/executor/run.py new file mode 100644 index 00000000..6b8696f3 --- /dev/null +++ b/openff/bespokefit/cli/executor/run.py @@ -0,0 +1,92 @@ +from typing import Optional + +import click +import rich +from rich import pretty +from rich.padding import Padding + +from openff.bespokefit.cli.executor.launch import launch_options +from openff.bespokefit.cli.executor.submit import _submit, submit_options +from openff.bespokefit.cli.utilities import create_command, print_header + + +def _run_cli( + input_file_path: str, + output_file_path: str, + force_field_path: str, + spec_name: Optional[str], + spec_file_name: Optional[str], + directory: str, + n_fragmenter_workers: int, + n_qc_compute_workers: int, + n_optimizer_workers: int, + launch_redis_if_unavailable: bool, +): + """Run bespoke optimization using a temporary executor. + + If you are running many bespoke optimizations it is recommended that you first launch + a bespoke executor using the `launch` command and then submit the optimizations to it + using the `submit` command. + """ + + pretty.install() + + console = rich.get_console() + print_header(console) + + from openff.bespokefit.executor import BespokeExecutor, wait_until_complete + + executor_status = console.status("launching the bespoke executor") + executor_status.start() + + with BespokeExecutor( + directory=directory, + n_fragmenter_workers=n_fragmenter_workers, + n_qc_compute_workers=n_qc_compute_workers, + n_optimizer_workers=n_optimizer_workers, + launch_redis_if_unavailable=launch_redis_if_unavailable, + ): + + executor_status.stop() + console.print("[[green]✓[/green]] bespoke executor launched") + console.line() + + response = _submit( + console, + input_file_path, + force_field_path, + spec_name, + spec_file_name, + ) + + if response is None: + return + + console.print(Padding("3. running the fitting pipeline", (1, 0, 1, 0))) + + results = wait_until_complete(response.id) + + if results is None: + return + + with open(output_file_path, "w") as file: + file.write(results.json()) + + +__run_options = [*submit_options()] +__run_options.insert( + 1, + click.option( + "--output", + "output_file_path", + type=click.Path(exists=False, file_okay=True, dir_okay=False), + help="The JSON file to save the results to", + default="output.json", + show_default=True, + ), +) +__run_options.extend(launch_options()) + +run_cli = create_command( + click_command=click.command("run"), click_options=__run_options, func=_run_cli +) diff --git a/openff/bespokefit/cli/executor/submit.py b/openff/bespokefit/cli/executor/submit.py new file mode 100644 index 00000000..361c9323 --- /dev/null +++ b/openff/bespokefit/cli/executor/submit.py @@ -0,0 +1,168 @@ +import os.path +from typing import TYPE_CHECKING, Optional + +import click +import rich +from click_option_group import optgroup +from openff.utilities import get_data_file_path +from rich import pretty +from rich.padding import Padding + +from openff.bespokefit.cli.utilities import create_command, print_header + +if TYPE_CHECKING: + from openff.toolkit.topology import Molecule + + from openff.bespokefit.executor.services.coordinator.models import ( + CoordinatorPOSTResponse, + ) + from openff.bespokefit.schema.fitting import BespokeOptimizationSchema + + +# The run command inherits these options so be sure to take that into account when +# making changes here. +def submit_options(): + return [ + click.option( + "--input", + "input_file_path", + type=click.Path(exists=True, file_okay=True, dir_okay=False), + help="The file containing the molecule of interest", + ), + click.option( + "--force-field", + "force_field_path", + type=click.Path(exists=False, file_okay=True, dir_okay=False), + help="The initial force field to build upon", + default="openff-2.0.0.offxml", + show_default=True, + ), + optgroup.group("Optimization configuration"), + optgroup.option( + "--spec", + "spec_name", + type=click.Choice(choices=["default", "debug"], case_sensitive=False), + help="The name of the built-in configuration to use", + required=False, + ), + optgroup.option( + "--spec-file", + "spec_file_name", + type=click.Path(exists=False, file_okay=True, dir_okay=False), + help="The path to a serialized bespoke workflow factory", + required=False, + ), + ] + + +def _to_input_schema( + console: "rich.Console", + molecule: "Molecule", + force_field_path: str, + spec_name: Optional[str], + spec_file_name: Optional[str], +) -> Optional["BespokeOptimizationSchema"]: + + from openff.bespokefit.workflows.bespoke import BespokeWorkflowFactory + + if (spec_name is not None and spec_file_name is not None) or ( + spec_name is None and spec_file_name is None + ): + + console.print( + "[[red]ERROR[/red] The `spec` and `spec-file` arguments are mutually " + "exclusive" + ) + return None + + if spec_name is not None: + + spec_file_name = get_data_file_path( + os.path.join("schemas", f"{spec_name.lower()}.json"), + "openff.bespokefit", + ) + + workflow_factory = BespokeWorkflowFactory.parse_file(spec_file_name) + workflow_factory.initial_force_field = force_field_path + + return workflow_factory.optimization_schema_from_molecule(molecule) + + +def _submit( + console: "rich.Console", + input_file_path: str, + force_field_path: str, + spec_name: Optional[str], + spec_file_name: Optional[str], +) -> Optional["CoordinatorPOSTResponse"]: + + from openff.toolkit.topology import Molecule + + from openff.bespokefit.executor import BespokeExecutor + + console.print(Padding("1. preparing the bespoke workflow", (0, 0, 1, 0))) + + with console.status("loading the molecules"): + + molecule = Molecule.from_file(input_file_path) + + if not isinstance(molecule, Molecule) or "." in molecule.to_smiles(): + + console.print( + "[[red]ERROR[/red]] only one molecule can currently be submitted at " + "a time" + ) + return + + console.print(f"[[green]✓[/green]] [blue]{1}[/blue] molecule was found") + + with console.status("building fitting schemas"): + + input_schema = _to_input_schema( + console, molecule, force_field_path, spec_name, spec_file_name + ) + + if input_schema is None: + return + + console.print("[[green]✓[/green]] fitting schema generated") + + console.print(Padding("2. submitting the workflow", (1, 0, 1, 0))) + + executor = BespokeExecutor() # TODO: submit should be static. + executor._started = True + + response = executor.submit(input_schema) + + console.print(f"[[green]✓[/green]] workflow submitted: id={response.id}") + + return response + + +def _submit_cli( + input_file_path: str, + force_field_path: str, + spec_name: Optional[str], + spec_file_name: Optional[str], +): + """Submit a new bespoke optimization to a running executor.""" + + pretty.install() + + console = rich.get_console() + print_header(console) + + _submit( + console, + input_file_path, + force_field_path, + spec_name, + spec_file_name, + ) + + +submit_cli = create_command( + click_command=click.command("submit"), + click_options=submit_options(), + func=_submit_cli, +) diff --git a/openff/bespokefit/cli/executor/watch.py b/openff/bespokefit/cli/executor/watch.py new file mode 100644 index 00000000..e0cee435 --- /dev/null +++ b/openff/bespokefit/cli/executor/watch.py @@ -0,0 +1,26 @@ +import click +import rich +from rich import pretty + +from openff.bespokefit.cli.utilities import print_header + + +@click.command("watch") +@click.option( + "--id", + "optimization_id", + type=click.STRING, + help="The id of the optimization to watch", + required=True, +) +def watch_cli(optimization_id): + """Watch the status of a bespoke optimization.""" + + pretty.install() + + console = rich.get_console() + print_header(console) + + from openff.bespokefit.executor import wait_until_complete + + wait_until_complete(optimization_id) diff --git a/openff/bespokefit/cli/utilities.py b/openff/bespokefit/cli/utilities.py new file mode 100644 index 00000000..16661282 --- /dev/null +++ b/openff/bespokefit/cli/utilities.py @@ -0,0 +1,24 @@ +from typing import TYPE_CHECKING, Callable, List + +import click + +if TYPE_CHECKING: + import rich + + +def print_header(console: "rich.Console"): + + console.line() + console.rule("OpenFF Bespoke") + console.line() + + +def create_command( + click_command: click.command, click_options: List[click.option], func: Callable +): + """Programmatically apply click options to a function.""" + + for option in reversed(click_options): + func = option(func) + + return click_command(func) diff --git a/openff/bespokefit/data/schemas/debug.json b/openff/bespokefit/data/schemas/debug.json new file mode 100644 index 00000000..fbb644b5 --- /dev/null +++ b/openff/bespokefit/data/schemas/debug.json @@ -0,0 +1,103 @@ +{ + "initial_force_field": "openff-2.0.0.offxml", + "optimizer": { + "type": "ForceBalance", + "max_iterations": 1, + "job_type": "optimize", + "penalty_type": "L1", + "step_convergence_threshold": 0.01, + "objective_convergence_threshold": 0.01, + "gradient_convergence_threshold": 0.01, + "n_criteria": 2, + "eigenvalue_lower_bound": 0.01, + "finite_difference_h": 0.01, + "penalty_additive": 1.0, + "initial_trust_radius": 0.25, + "minimum_trust_radius": 0.05, + "error_tolerance": 1.0, + "adaptive_factor": 0.2, + "adaptive_damping": 1.0, + "normalize_weights": false, + "extras": {} + }, + "target_templates": [ + { + "weight": 1.0, + "reference_data": null, + "extras": {}, + "type": "TorsionProfile", + "attenuate_weights": true, + "energy_denominator": 1.0, + "energy_cutoff": 10.0 + } + ], + "parameter_hyperparameters": [ + { + "type": "ProperTorsions", + "priors": { + "k": 6.0 + } + } + ], + "target_torsion_smirks": [ + "[!#1]~[!$(*#*)&!D1:1]-,=;!@[!$(*#*)&!D1:2]~[!#1]" + ], + "target_smirks": [ + "ProperTorsions" + ], + "expand_torsion_terms": false, + "generate_bespoke_terms": false, + "fragmentation_engine": { + "functional_groups": { + "hydrazine": "[NX3:1][NX3:2]", + "hydrazone": "[NX3:1][NX2:2]", + "nitric_oxide": "[N:1]-[O:2]", + "amide": "[#7:1][#6:2](=[#8:3])", + "amide_n": "[#7:1][#6:2](-[O-:3])", + "amide_2": "[NX3:1][CX3:2](=[OX1:3])[NX3:4]", + "aldehyde": "[CX3H1:1](=[O:2])[#6:3]", + "sulfoxide_1": "[#16X3:1]=[OX1:2]", + "sulfoxide_2": "[#16X3+:1][OX1-:2]", + "sulfonyl": "[#16X4:1](=[OX1:2])=[OX1:3]", + "sulfinic_acid": "[#16X3:1](=[OX1:2])[OX2H,OX1H0-:3]", + "sulfinamide": "[#16X4:1](=[OX1:2])(=[OX1:3])([NX3R0:4])", + "sulfonic_acid": "[#16X4:1](=[OX1:2])(=[OX1:3])[OX2H,OX1H0-:4]", + "phosphine_oxide": "[PX4:1](=[OX1:2])([#6:3])([#6:4])([#6:5])", + "phosphonate": "[P:1](=[OX1:2])([OX2H,OX1-:3])([OX2H,OX1-:4])", + "phosphate": "[PX4:1](=[OX1:2])([#8:3])([#8:4])([#8:5])", + "carboxylic_acid": "[CX3:1](=[O:2])[OX1H0-,OX2H1:3]", + "nitro_1": "[NX3+:1](=[O:2])[O-:3]", + "nitro_2": "[NX3:1](=[O:2])=[O:3]", + "ester": "[CX3:1](=[O:2])[OX2H0:3]", + "tri_halide": "[#6:1]([F,Cl,I,Br:2])([F,Cl,I,Br:3])([F,Cl,I,Br:4])" + }, + "scheme": "WBO", + "wbo_options": { + "method": "am1-wiberg-elf10", + "max_conformers": 800, + "rms_threshold": 1.0 + }, + "threshold": 0.03, + "heuristic": "path_length", + "keep_non_rotor_ring_substituents": false + }, + "default_qc_specs": [ + { + "method": "uff", + "basis": null, + "program": "rdkit", + "spec_name": "default", + "spec_description": "Standard OpenFF optimization quantum chemistry specification.", + "store_wavefunction": "none", + "implicit_solvent": null, + "maxiter": 200, + "scf_properties": [ + "dipole", + "quadrupole", + "wiberg_lowdin_indices", + "mayer_indices" + ], + "keywords": null + } + ] +} \ No newline at end of file diff --git a/openff/bespokefit/data/schemas/default.json b/openff/bespokefit/data/schemas/default.json new file mode 100644 index 00000000..db063a80 --- /dev/null +++ b/openff/bespokefit/data/schemas/default.json @@ -0,0 +1,103 @@ +{ + "initial_force_field": "openff-2.0.0.offxml", + "optimizer": { + "type": "ForceBalance", + "max_iterations": 10, + "job_type": "optimize", + "penalty_type": "L1", + "step_convergence_threshold": 0.01, + "objective_convergence_threshold": 0.01, + "gradient_convergence_threshold": 0.01, + "n_criteria": 2, + "eigenvalue_lower_bound": 0.01, + "finite_difference_h": 0.01, + "penalty_additive": 1.0, + "initial_trust_radius": -0.25, + "minimum_trust_radius": 0.05, + "error_tolerance": 1.0, + "adaptive_factor": 0.2, + "adaptive_damping": 1.0, + "normalize_weights": false, + "extras": {} + }, + "target_templates": [ + { + "weight": 1.0, + "reference_data": null, + "extras": {}, + "type": "TorsionProfile", + "attenuate_weights": true, + "energy_denominator": 1.0, + "energy_cutoff": 10.0 + } + ], + "parameter_hyperparameters": [ + { + "type": "ProperTorsions", + "priors": { + "k": 6.0 + } + } + ], + "target_torsion_smirks": [ + "[!#1]~[!$(*#*)&!D1:1]-,=;!@[!$(*#*)&!D1:2]~[!#1]" + ], + "target_smirks": [ + "ProperTorsions" + ], + "expand_torsion_terms": true, + "generate_bespoke_terms": true, + "fragmentation_engine": { + "functional_groups": { + "hydrazine": "[NX3:1][NX3:2]", + "hydrazone": "[NX3:1][NX2:2]", + "nitric_oxide": "[N:1]-[O:2]", + "amide": "[#7:1][#6:2](=[#8:3])", + "amide_n": "[#7:1][#6:2](-[O-:3])", + "amide_2": "[NX3:1][CX3:2](=[OX1:3])[NX3:4]", + "aldehyde": "[CX3H1:1](=[O:2])[#6:3]", + "sulfoxide_1": "[#16X3:1]=[OX1:2]", + "sulfoxide_2": "[#16X3+:1][OX1-:2]", + "sulfonyl": "[#16X4:1](=[OX1:2])=[OX1:3]", + "sulfinic_acid": "[#16X3:1](=[OX1:2])[OX2H,OX1H0-:3]", + "sulfinamide": "[#16X4:1](=[OX1:2])(=[OX1:3])([NX3R0:4])", + "sulfonic_acid": "[#16X4:1](=[OX1:2])(=[OX1:3])[OX2H,OX1H0-:4]", + "phosphine_oxide": "[PX4:1](=[OX1:2])([#6:3])([#6:4])([#6:5])", + "phosphonate": "[P:1](=[OX1:2])([OX2H,OX1-:3])([OX2H,OX1-:4])", + "phosphate": "[PX4:1](=[OX1:2])([#8:3])([#8:4])([#8:5])", + "carboxylic_acid": "[CX3:1](=[O:2])[OX1H0-,OX2H1:3]", + "nitro_1": "[NX3+:1](=[O:2])[O-:3]", + "nitro_2": "[NX3:1](=[O:2])=[O:3]", + "ester": "[CX3:1](=[O:2])[OX2H0:3]", + "tri_halide": "[#6:1]([F,Cl,I,Br:2])([F,Cl,I,Br:3])([F,Cl,I,Br:4])" + }, + "scheme": "WBO", + "wbo_options": { + "method": "am1-wiberg-elf10", + "max_conformers": 800, + "rms_threshold": 1.0 + }, + "threshold": 0.03, + "heuristic": "path_length", + "keep_non_rotor_ring_substituents": false + }, + "default_qc_specs": [ + { + "method": "B3LYP-D3BJ", + "basis": "DZVP", + "program": "psi4", + "spec_name": "default", + "spec_description": "Standard OpenFF optimization quantum chemistry specification.", + "store_wavefunction": "none", + "implicit_solvent": null, + "maxiter": 200, + "scf_properties": [ + "dipole", + "quadrupole", + "wiberg_lowdin_indices", + "mayer_indices" + ], + "keywords": null + } + ] +} \ No newline at end of file diff --git a/openff/bespokefit/executor/executor.py b/openff/bespokefit/executor/executor.py index 177ec3a5..cb85af72 100644 --- a/openff/bespokefit/executor/executor.py +++ b/openff/bespokefit/executor/executor.py @@ -292,7 +292,7 @@ def wait_until_complete( else: - console.log(f"[ERROR] {str(error)}") + console.log(f"[[red]ERROR[/red]] {str(error)}") return None stage_messages = { @@ -310,7 +310,7 @@ def wait_until_complete( ) if stage_error is not None: - console.log(f"[ERROR] {str(stage_error)}") + console.log(f"[[red]ERROR[/red]] {str(stage_error)}") return None if stage is None: @@ -328,7 +328,7 @@ def wait_until_complete( final_response, error = _query_coordinator(optimization_href) if error is not None: - console.log(f"[ERROR] {str(error)}") + console.log(f"[[red]ERROR[/red]] {str(error)}") return None return final_response diff --git a/openff/bespokefit/tests/cli/executor/__init__.py b/openff/bespokefit/tests/cli/executor/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/openff/bespokefit/tests/cli/executor/test_launch.py b/openff/bespokefit/tests/cli/executor/test_launch.py new file mode 100644 index 00000000..9cce4726 --- /dev/null +++ b/openff/bespokefit/tests/cli/executor/test_launch.py @@ -0,0 +1,52 @@ +import inspect +import time + +from openff.bespokefit.cli.executor.launch import launch_cli +from openff.bespokefit.executor import BespokeExecutor + + +def test_launch(runner, monkeypatch): + + old_sleep = time.sleep + + def mock_sleep(*args, **kwargs): + + frame = inspect.stack()[1] + file_name = inspect.getmodule(frame[0]).__file__ + + if file_name.endswith("launch.py"): + raise KeyboardInterrupt() + + old_sleep(*args, **kwargs) + + def mock_start(self, *args, **kwargs): + + assert self._n_fragmenter_workers == 1 + assert self._n_qc_compute_workers == 2 + assert self._n_optimizer_workers == 3 + + assert self._directory == "mock-directory" + + assert self._launch_redis_if_unavailable is False + + self._started = True + + monkeypatch.setattr(time, "sleep", mock_sleep) + monkeypatch.setattr(BespokeExecutor, "start", mock_start) + + output = runner.invoke( + launch_cli, + args=[ + "--directory", + "mock-directory", + "--n-fragmenter-workers", + 1, + "--n-qc-compute-workers", + 2, + "--n-optimizer-workers", + 3, + "--no-launch-redis", + ], + ) + print(output.output) + assert output.exit_code == 0 diff --git a/openff/bespokefit/tests/cli/executor/test_list.py b/openff/bespokefit/tests/cli/executor/test_list.py new file mode 100644 index 00000000..3a41a1df --- /dev/null +++ b/openff/bespokefit/tests/cli/executor/test_list.py @@ -0,0 +1,41 @@ +import pytest +import requests_mock + +from openff.bespokefit.cli.executor.list import list_cli +from openff.bespokefit.executor.services import settings +from openff.bespokefit.executor.services.coordinator.models import ( + CoordinatorGETPageResponse, + CoordinatorGETResponse, +) + + +@pytest.mark.parametrize( + "n_results, expected_message", + [(0, "No optimizations were found"), (3, "The following optimizations were found")], +) +def test_list_cli(n_results, expected_message, runner): + + with requests_mock.Mocker() as m: + + mock_response = CoordinatorGETPageResponse( + self="self-page", + contents=[ + CoordinatorGETResponse(id=f"{i}", self=f"self-{i}", stages=[]) + for i in range(1, 1 + n_results) + ], + ) + + m.get( + ( + f"http://127.0.0.1:" + f"{settings.BEFLOW_GATEWAY_PORT}" + f"{settings.BEFLOW_API_V1_STR}/" + f"{settings.BEFLOW_COORDINATOR_PREFIX}" + ), + text=mock_response.json(), + ) + + output = runner.invoke(list_cli) + assert output.exit_code == 0 + + assert expected_message in output.stdout diff --git a/openff/bespokefit/tests/cli/executor/test_run.py b/openff/bespokefit/tests/cli/executor/test_run.py new file mode 100644 index 00000000..aebf0d0e --- /dev/null +++ b/openff/bespokefit/tests/cli/executor/test_run.py @@ -0,0 +1,71 @@ +import os + +import requests_mock +from openff.toolkit.topology import Molecule + +from openff.bespokefit.cli.executor.run import run_cli +from openff.bespokefit.executor.services import settings +from openff.bespokefit.executor.services.coordinator.models import ( + CoordinatorGETResponse, + CoordinatorGETStageStatus, + CoordinatorPOSTResponse, +) + + +def test_run(runner, tmpdir): + """Make sure to schema failures are cleanly handled.""" + + input_file_path = os.path.join(tmpdir, "mol.sdf") + Molecule.from_smiles("CC").to_file(input_file_path, "SDF") + + with requests_mock.Mocker() as m: + + m.post( + ( + f"http://127.0.0.1:" + f"{settings.BEFLOW_GATEWAY_PORT}" + f"{settings.BEFLOW_API_V1_STR}/" + f"{settings.BEFLOW_COORDINATOR_PREFIX}" + ), + text=CoordinatorPOSTResponse(self="", id="1").json(), + ) + m.get("http://127.0.0.1:8000/api/v1", text="") + m.get( + ( + f"http://127.0.0.1:" + f"{settings.BEFLOW_GATEWAY_PORT}" + f"{settings.BEFLOW_API_V1_STR}/" + f"{settings.BEFLOW_COORDINATOR_PREFIX}/1" + ), + text=CoordinatorGETResponse( + id="1", + self="", + stages=[ + CoordinatorGETStageStatus( + type="fragmentation", status="success", error=None, results=None + ) + ], + ).json(), + ) + + output = runner.invoke( + run_cli, + args=[ + "--input", + input_file_path, + "--spec", + "debug", + "--directory", + "mock-directory", + "--n-fragmenter-workers", + 0, + "--n-qc-compute-workers", + 0, + "--n-optimizer-workers", + 0, + "--no-launch-redis", + ], + ) + + assert output.exit_code == 0 + assert "workflow submitted: id=1" in output.output diff --git a/openff/bespokefit/tests/cli/executor/test_submit.py b/openff/bespokefit/tests/cli/executor/test_submit.py new file mode 100644 index 00000000..9c82d170 --- /dev/null +++ b/openff/bespokefit/tests/cli/executor/test_submit.py @@ -0,0 +1,153 @@ +import os.path + +import numpy +import pytest +import requests_mock +import rich +from openff.toolkit.topology import Molecule, Topology +from openmm import unit + +from openff.bespokefit.cli.executor.submit import _submit, _to_input_schema, submit_cli +from openff.bespokefit.executor.services import settings +from openff.bespokefit.executor.services.coordinator.models import ( + CoordinatorPOSTResponse, +) +from openff.bespokefit.schema.fitting import BespokeOptimizationSchema + + +@pytest.mark.parametrize( + "spec_name, spec_file_name, expected_message, output_is_none", + [ + (None, None, "The `spec` and `spec-file` arguments", True), + ("a", "b", "The `spec` and `spec-file` arguments", True), + ("debug", None, "", False), + ], +) +def test_to_input_schema_mutual_exclusive_args( + spec_name, spec_file_name, expected_message, output_is_none +): + + console = rich.get_console() + + with console.capture() as capture: + + input_schema = _to_input_schema( + console, + Molecule.from_smiles("CC"), + force_field_path="openff-2.0.0.offxml", + spec_name=spec_name, + spec_file_name=spec_file_name, + ) + + if len(expected_message) > 0: + assert expected_message in capture.get() + + assert (input_schema is None) == output_is_none + + +def test_to_input_schema(): + + input_schema = _to_input_schema( + rich.get_console(), + Molecule.from_smiles("CC"), + force_field_path="openff-1.2.1.offxml", + spec_name="debug", + spec_file_name=None, + ) + + assert isinstance(input_schema, BespokeOptimizationSchema) + assert input_schema.id == "bespoke_task_0" + + +def test_submit_multi_molecule(tmpdir): + + console = rich.get_console() + + input_file_path = os.path.join(tmpdir, "mol.pdb") + + molecules = [Molecule.from_smiles(smiles) for smiles in ("ClCl", "BrBr")] + Topology.from_molecules(molecules).to_file( + input_file_path, positions=numpy.zeros((4, 3)) * unit.angstrom + ) + + with console.capture() as capture: + + response = _submit( + console, + input_file_path=input_file_path, + force_field_path="openff-2.0.0.offxml", + spec_name="debug", + spec_file_name=None, + ) + + assert response is None + assert "only one molecule can currently" in capture.get() + + +def test_submit_invalid_schema(tmpdir): + """Make sure to schema failures are cleanly handled.""" + + input_file_path = os.path.join(tmpdir, "mol.sdf") + Molecule.from_smiles("C").to_file(input_file_path, "SDF") + + response = _submit( + rich.get_console(), + input_file_path=input_file_path, + force_field_path="openff-2.0.0.offxml", + spec_name=None, + spec_file_name=None, + ) + + assert response is None + + +def test_submit(tmpdir): + """Make sure to schema failures are cleanly handled.""" + + input_file_path = os.path.join(tmpdir, "mol.sdf") + Molecule.from_smiles("CC").to_file(input_file_path, "SDF") + + with requests_mock.Mocker() as m: + + mock_href = ( + f"http://127.0.0.1:" + f"{settings.BEFLOW_GATEWAY_PORT}" + f"{settings.BEFLOW_API_V1_STR}/" + f"{settings.BEFLOW_COORDINATOR_PREFIX}" + ) + m.post(mock_href, text=CoordinatorPOSTResponse(self="", id="1").json()) + + response = _submit( + rich.get_console(), + input_file_path=input_file_path, + force_field_path="openff-2.0.0.offxml", + spec_name="debug", + spec_file_name=None, + ) + + assert isinstance(response, CoordinatorPOSTResponse) + assert response.id == "1" + + +def test_submit_cli(runner, tmpdir): + """Make sure to schema failures are cleanly handled.""" + + input_file_path = os.path.join(tmpdir, "mol.sdf") + Molecule.from_smiles("CC").to_file(input_file_path, "SDF") + + with requests_mock.Mocker() as m: + + mock_href = ( + f"http://127.0.0.1:" + f"{settings.BEFLOW_GATEWAY_PORT}" + f"{settings.BEFLOW_API_V1_STR}/" + f"{settings.BEFLOW_COORDINATOR_PREFIX}" + ) + m.post(mock_href, text=CoordinatorPOSTResponse(self="", id="1").json()) + + output = runner.invoke( + submit_cli, args=["--input", input_file_path, "--spec", "debug"] + ) + + assert output.exit_code == 0 + assert "workflow submitted: id=1" in output.output diff --git a/openff/bespokefit/tests/cli/executor/test_watch.py b/openff/bespokefit/tests/cli/executor/test_watch.py new file mode 100644 index 00000000..cf6dbc63 --- /dev/null +++ b/openff/bespokefit/tests/cli/executor/test_watch.py @@ -0,0 +1,39 @@ +import requests_mock + +from openff.bespokefit.cli.executor.watch import watch_cli +from openff.bespokefit.executor.services import settings +from openff.bespokefit.executor.services.coordinator.models import ( + CoordinatorGETResponse, + CoordinatorGETStageStatus, +) + + +def test_watch(runner): + + with requests_mock.Mocker() as m: + + mock_href = ( + f"http://127.0.0.1:" + f"{settings.BEFLOW_GATEWAY_PORT}" + f"{settings.BEFLOW_API_V1_STR}/" + f"{settings.BEFLOW_COORDINATOR_PREFIX}/1" + ) + + mock_response = CoordinatorGETResponse( + id="1", + self=mock_href, + stages=[ + CoordinatorGETStageStatus( + type="fragmentation", status="success", error=None, results=None + ) + ], + ) + m.get( + mock_href, + text=mock_response.json(), + ) + + output = runner.invoke(watch_cli, args=["--id", "1"]) + + assert output.exit_code == 0 + assert "fragmentation successful" in output.stdout diff --git a/openff/bespokefit/tests/cli/test_utilities.py b/openff/bespokefit/tests/cli/test_utilities.py new file mode 100644 index 00000000..bd80f2c5 --- /dev/null +++ b/openff/bespokefit/tests/cli/test_utilities.py @@ -0,0 +1,35 @@ +import click +import rich + +from openff.bespokefit.cli.utilities import create_command, print_header + + +def test_print_header(): + + console = rich.get_console() + + with console.capture() as capture: + print_header(console) + + assert "OpenFF Bespoke" in capture.get() + + +def test_create_command(runner): + + mock_command = create_command( + click_command=click.command(name="test-command"), + click_options=[ + click.option("--option-2", type=click.STRING), + click.option("--option-1", type=click.STRING), + ], + func=lambda option_1, option_2: None, + ) + + help_output = runner.invoke(mock_command, ["--help"]) + assert help_output.exit_code == 0 + + assert "option-1" in help_output.output + assert "option-2" in help_output.output + + # Options should be appear in the specified order + assert help_output.output.index("option-2") < help_output.output.index("option-1") diff --git a/openff/bespokefit/utilities/logging.py b/openff/bespokefit/utilities/logging.py new file mode 100644 index 00000000..ac60428d --- /dev/null +++ b/openff/bespokefit/utilities/logging.py @@ -0,0 +1,6 @@ +import logging + + +class DeprecationWarningFilter(logging.Filter): + def filter(self, record): + return "is deprecated" not in record.getMessage() diff --git a/openff/bespokefit/workflows/bespoke.py b/openff/bespokefit/workflows/bespoke.py index 31168fec..3eb9359f 100644 --- a/openff/bespokefit/workflows/bespoke.py +++ b/openff/bespokefit/workflows/bespoke.py @@ -14,7 +14,7 @@ OptimizationResultCollection, TorsionDriveResultCollection, ) -from openff.qcsubmit.serializers import serialize +from openff.qcsubmit.serializers import deserialize, serialize from openff.qcsubmit.workflow_components import ComponentResult from openff.toolkit.topology import Molecule from openff.toolkit.typing.engines.smirnoff import ( @@ -151,7 +151,7 @@ class BespokeWorkflowFactory(ClassBase): description="The default specification (e.g. method, basis) to use when " "performing any new QC calculations. If multiple specs are provided, each spec " "will be considered in order until one is found that i) is available based on " - "the installed dependencies, and ii) is compatable with the molecule of " + "the installed dependencies, and ii) is compatible with the molecule of " "interest.", ) @@ -230,6 +230,13 @@ def export_factory(self, file_name: str) -> None: serialize(serializable=self.dict(), file_name=file_name) + @classmethod + def from_file(cls, file_name: str): + """ + Build the factory from a model serialised to file. + """ + return cls.parse_obj(deserialize(file_name=file_name)) + @classmethod def _deduplicated_list( cls, molecules: Union[Molecule, List[Molecule], str]