From d4fce465518bf0fc87a6d0671d438ee020163d14 Mon Sep 17 00:00:00 2001 From: Dilan Pathirana <59329744+dilpath@users.noreply.github.com> Date: Mon, 6 Jan 2025 16:35:58 +0100 Subject: [PATCH] Create standard/schema for `Problem` (#132) * update docs * test problem load/save round-trip * remove unused constant, update doc example * add `ModelSubspace.to_definition` * `ProblemStandard` * handle predecessor model relative path on load * add more repro info to test case 0009 * fix cli `start_iteration` * bump mkstd req * change pypesto test tolerance; add test for pypesto+cli --------- Co-authored-by: Daniel Weindl --- doc/examples/model_selection/model_space.tsv | 2 +- .../model_selection/petab_select_problem.yaml | 3 +- doc/problem_definition.rst | 26 +- doc/standard/make_schemas.py | 2 + doc/standard/problem.yaml | 35 +++ petab_select/cli.py | 18 +- petab_select/constants.py | 5 - petab_select/model.py | 48 ++-- petab_select/model_space.py | 224 ++++----------- petab_select/model_subspace.py | 52 +++- petab_select/models.py | 16 +- petab_select/problem.py | 271 +++++++++--------- pyproject.toml | 2 +- .../select/model_space_FAMoS_2019.tsv | 2 +- test/candidate_space/test_candidate_space.py | 5 +- test/model_space/model_space_file_1.tsv | 2 +- test/model_space/model_space_file_2.tsv | 2 +- test/model_space/test_model_space.py | 2 +- test/model_subspace/test_model_subspace.py | 17 +- test/problem/__init__.py | 0 test/problem/expected_output/model_space.tsv | 9 + .../expected_output/petab_select_problem.yaml | 6 + test/problem/test_problem.py | 34 +++ test/pypesto/test_pypesto.py | 126 +++++++- test_cases/0001/model_space.tsv | 2 +- test_cases/0002/model_space.tsv | 2 +- test_cases/0003/model_space.tsv | 2 +- test_cases/0004/model_space.tsv | 2 +- test_cases/0005/model_space.tsv | 2 +- test_cases/0006/model_space.tsv | 2 +- test_cases/0007/model_space.tsv | 2 +- test_cases/0008/model_space.tsv | 2 +- test_cases/0009/README.md | 2 +- test_cases/0009/model_space.tsv | 2 +- 34 files changed, 522 insertions(+), 407 deletions(-) create mode 100644 doc/standard/problem.yaml create mode 100644 test/problem/__init__.py create mode 100644 test/problem/expected_output/model_space.tsv create mode 100644 test/problem/expected_output/petab_select_problem.yaml create mode 100644 test/problem/test_problem.py diff --git a/doc/examples/model_selection/model_space.tsv b/doc/examples/model_selection/model_space.tsv index bcd8cb22..e69a4f0e 100644 --- a/doc/examples/model_selection/model_space.tsv +++ b/doc/examples/model_selection/model_space.tsv @@ -1,4 +1,4 @@ -model_subspace_id petab_yaml k1 k2 k3 +model_subspace_id model_subspace_petab_yaml k1 k2 k3 M1_0 petab_problem.yaml 0 0 0 M1_1 petab_problem.yaml 0.2 0.1 estimate M1_2 petab_problem.yaml 0.2 estimate 0 diff --git a/doc/examples/model_selection/petab_select_problem.yaml b/doc/examples/model_selection/petab_select_problem.yaml index 235b2435..360def46 100644 --- a/doc/examples/model_selection/petab_select_problem.yaml +++ b/doc/examples/model_selection/petab_select_problem.yaml @@ -1,5 +1,6 @@ -version: beta_1 +format_version: 1.0.0 criterion: AIC method: forward model_space_files: - model_space.tsv +candidate_space_arguments: {} diff --git a/doc/problem_definition.rst b/doc/problem_definition.rst index 0e2d92ca..00c22435 100644 --- a/doc/problem_definition.rst +++ b/doc/problem_definition.rst @@ -7,8 +7,8 @@ Model selection problems for PEtab Select are defined by the following files: #. a specification of the model space, and #. (optionally) a specification of the initial candidate model. -The different file formats are described below. Each file format is a YAML file -and comes with a YAML-formatted JSON schema, such that these files can be +The different file formats are described below. The YAML file formats +come with a YAML-formatted JSON schema, such that these files can be easily worked with independently of the PEtab Select library. 1. Selection problem @@ -34,14 +34,23 @@ A YAML file with a description of the model selection problem. - ``candidate_space_arguments``: Additional arguments used to generate candidate models during model selection. For example, an initial candidate model can be specified with the following code, where - ``predecessor_model.yaml`` is a valid model file. Additional arguments are - provided in the documentation of the ``CandidateSpace`` class. + ``predecessor_model.yaml`` is a valid :ref:`model file `. Additional arguments are + provided in the documentation of the ``CandidateSpace`` class, and an example is provided in + `test case 0009 `_. .. code-block:: yaml candidate_space_arguments: predecessor_model: predecessor_model.yaml +Schema +^^^^^^ + +The schema is provided as `YAML-formatted JSON schema <_static/problem.yaml>`_, which enables easy validation with various third-party tools. + +.. literalinclude:: standard/problem.yaml + :language: yaml + 2. Model space -------------- @@ -54,7 +63,7 @@ all parameters. :header-rows: 1 * - ``model_subspace_id`` - - ``petab_yaml`` + - ``model_subspace_petab_yaml`` - ``parameter_id_1`` - ... - ``parameter_id_n`` @@ -65,7 +74,7 @@ all parameters. - ... - ``model_subspace_id``: An ID for the model subspace. -- ``petab_yaml``: The PEtab YAML filename that serves as the basis of all +- ``model_subspace_petab_yaml``: The YAML filename of the PEtab problem that serves as the basis of all models in this subspace. - ``parameter_id_1`` ... ``parameter_id_n``: Specify the values that a parameter can take in the model subspace. For example, this could be: @@ -81,6 +90,9 @@ all parameters. - ``0.0;1.1;estimate`` (the parameter can take the values ``0.0`` or ``1.1``, or be estimated) +Example of concise specification +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + Using the ``;``-delimited list format, a model subspace that has two parameters (``p1, p2``) and six models: @@ -105,6 +117,8 @@ can be specified like - 0;estimate - 10;20;estimate +.. _section-model-yaml: + 3. Model(s) (Predecessor models / model interchange / report) ------------------------------------------------------------- diff --git a/doc/standard/make_schemas.py b/doc/standard/make_schemas.py index c01c62b8..d77c9730 100644 --- a/doc/standard/make_schemas.py +++ b/doc/standard/make_schemas.py @@ -1,5 +1,7 @@ from petab_select.model import ModelStandard from petab_select.models import ModelsStandard +from petab_select.problem import ProblemStandard ModelStandard.save_schema("model.yaml") ModelsStandard.save_schema("models.yaml") +ProblemStandard.save_schema("problem.yaml") diff --git a/doc/standard/problem.yaml b/doc/standard/problem.yaml new file mode 100644 index 00000000..b5529863 --- /dev/null +++ b/doc/standard/problem.yaml @@ -0,0 +1,35 @@ +description: "Handle everything related to the model selection problem.\n\nAttributes:\n\ + \ model_space:\n The model space.\n calibrated_models:\n Calibrated\ + \ models. Will be used to augment the model selection\n problem (e.g. by\ + \ excluding them from the model space).\n candidate_space_arguments:\n \ + \ Arguments are forwarded to the candidate space constructor.\n compare:\n \ + \ A method that compares models by selection criterion. See\n :func:`petab_select.model.default_compare`\ + \ for an example.\n criterion:\n The criterion used to compare models.\n\ + \ method:\n The method used to search the model space.\n version:\n\ + \ The version of the PEtab Select format.\n yaml_path:\n The location\ + \ of the selection problem YAML file. Used for relative\n paths that exist\ + \ in e.g. the model space files." +properties: + format_version: + default: 1.0.0 + title: Format Version + type: string + criterion: + type: string + method: + type: string + model_space_files: + items: + format: path + type: string + title: Model Space Files + type: array + candidate_space_arguments: + title: Candidate Space Arguments + type: object +required: +- criterion +- method +- model_space_files +title: Problem +type: object diff --git a/petab_select/cli.py b/petab_select/cli.py index d0def393..e5bbdbfc 100644 --- a/petab_select/cli.py +++ b/petab_select/cli.py @@ -12,7 +12,14 @@ from . import ui from .candidate_space import CandidateSpace -from .constants import CANDIDATE_SPACE, MODELS, PETAB_YAML, PROBLEM, TERMINATE +from .constants import ( + CANDIDATE_SPACE, + MODELS, + PETAB_YAML, + PROBLEM, + TERMINATE, + UNCALIBRATED_MODELS, +) from .model import ModelHash from .models import Models, models_to_yaml_list from .problem import Problem @@ -183,7 +190,7 @@ def start_iteration( ModelHash.from_hash(hash_str) for hash_str in excluded_model_hashes ] - ui.start_iteration( + result = ui.start_iteration( problem=problem, candidate_space=candidate_space, limit=limit, @@ -201,9 +208,8 @@ def start_iteration( ) # Save candidate models - models_to_yaml_list( - models=candidate_space.models, - output_yaml=uncalibrated_models_yaml, + result[UNCALIBRATED_MODELS].to_yaml( + filename=uncalibrated_models_yaml, relative_paths=relative_paths, ) @@ -495,7 +501,7 @@ def get_best( models=models, criterion=criterion, ) - best_model.to_yaml(output, paths_relative_to=paths_relative_to) + best_model.to_yaml(output) cli.add_command(start_iteration) diff --git a/petab_select/constants.py b/petab_select/constants.py index c25f6cfa..7ffb2f58 100644 --- a/petab_select/constants.py +++ b/petab_select/constants.py @@ -73,7 +73,6 @@ class Criterion(str, Enum): # Problem MODEL_SPACE_FILES = "model_space_files" PROBLEM = "problem" -PROBLEM_ID = "problem_id" VERSION = "version" # Candidate space @@ -174,10 +173,6 @@ class Method(str, Enum): # PEtab Select model report format. HASH = "hash" -# MODEL_SPACE_FILE_NON_PARAMETER_COLUMNS = [MODEL_ID, PETAB_YAML] -MODEL_SPACE_FILE_NON_PARAMETER_COLUMNS = [MODEL_SUBSPACE_ID, PETAB_YAML] - -# COMPARED_MODEL_ID = 'compared_'+MODEL_ID YAML_FILENAME = "yaml" # DISTANCES = { diff --git a/petab_select/model.py b/petab_select/model.py index 737e281f..149a68c6 100644 --- a/petab_select/model.py +++ b/petab_select/model.py @@ -5,12 +5,23 @@ import copy import warnings from os.path import relpath -from pathlib import Path from typing import TYPE_CHECKING, Any, ClassVar, Literal import mkstd import petab.v1 as petab +from mkstd import Path from petab.v1.C import NOMINAL_VALUE +from pydantic import ( + BaseModel, + Field, + PrivateAttr, + ValidationInfo, + ValidatorFunctionWrapHandler, + field_serializer, + field_validator, + model_serializer, + model_validator, +) from .constants import ( ESTIMATE, @@ -41,29 +52,14 @@ if TYPE_CHECKING: from .problem import Problem - -from pydantic import ( - BaseModel, - PrivateAttr, - ValidationInfo, - ValidatorFunctionWrapHandler, -) - __all__ = [ "Model", "default_compare", "ModelHash", "VIRTUAL_INITIAL_MODEL", + "ModelStandard", ] -from pydantic import ( - Field, - field_serializer, - field_validator, - model_serializer, - model_validator, -) - class ModelHash(BaseModel): """The model hash. @@ -331,9 +327,7 @@ def _fix_relative_paths( return data model = handler(data) - root_path = None - if ROOT_PATH in data: - root_path = data.pop(ROOT_PATH) + root_path = data.pop(ROOT_PATH, None) if root_path is None: return model @@ -685,17 +679,11 @@ def get_parameter_values( ] @staticmethod - def from_yaml( - filename: str | Path, - ) -> Model: - """Load a model from a YAML file. - - Args: - filename: - Location of the YAML file. - """ + def from_yaml(filename: str | Path) -> Model: + """Load a model from a YAML file.""" model = ModelStandard.load_data( - filename=filename, root_path=Path(filename).parent + filename=filename, + root_path=Path(filename).parent, ) return model diff --git a/petab_select/model_space.py b/petab_select/model_space.py index f3237ae9..83633f74 100644 --- a/petab_select/model_space.py +++ b/petab_select/model_space.py @@ -1,24 +1,19 @@ """The `ModelSpace` class and related methods.""" -import itertools +from __future__ import annotations + import logging import warnings from collections.abc import Iterable from pathlib import Path -from tempfile import NamedTemporaryFile -from typing import Any, TextIO, get_args +from typing import Any import numpy as np import pandas as pd from .candidate_space import CandidateSpace from .constants import ( - HEADER_ROW, - MODEL_ID_COLUMN, MODEL_SUBSPACE_ID, - PARAMETER_DEFINITIONS_START, - PARAMETER_VALUE_DELIMITER, - PETAB_YAML_COLUMN, TYPE_PATH, ) from .model import Model @@ -26,107 +21,9 @@ __all__ = [ "ModelSpace", - "get_model_space_df", - "read_model_space_file", - "write_model_space_df", ] -def read_model_space_file(filename: str) -> TextIO: - """Read a model space file. - - The model space specification is currently expanded and written to a - temporary file. - - Args: - filename: - The name of the file to be unpacked. - - Returns: - A temporary file object, which is the unpacked file. - """ - """ - FIXME(dilpath) - Todo: - * Consider alternatives to `_{n}` suffix for model `modelId` - * How should the selected model be reported to the user? Remove the - `_{n}` suffix and report the original `modelId` alongside the - selected parameters? Generate a set of PEtab files with the - chosen SBML file and the parameters specified in a parameter or - condition file? - * Don't "unpack" file if it is already in the unpacked format - * Sort file after unpacking - * Remove duplicates? - """ - # FIXME rewrite to just generate models from the original file, instead of - # expanding all and writing to a file. - expanded_models_file = NamedTemporaryFile(mode="r+", delete=False) - with open(filename) as fh: - with open(expanded_models_file.name, "w") as ms_f: - # could replace `else` condition with ms_f.readline() here, and - # remove `if` statement completely - for line_index, line in enumerate(fh): - # Skip empty/whitespace-only lines - if not line.strip(): - continue - if line_index != HEADER_ROW: - columns = line2row(line, unpacked=False) - parameter_definitions = [ - definition.split(PARAMETER_VALUE_DELIMITER) - for definition in columns[PARAMETER_DEFINITIONS_START:] - ] - for index, selection in enumerate( - itertools.product(*parameter_definitions) - ): - # TODO change MODEL_ID_COLUMN and YAML_ID_COLUMN - # to just MODEL_ID and YAML_FILENAME? - ms_f.write( - "\t".join( - [ - columns[MODEL_ID_COLUMN] + f"_{index}", - columns[PETAB_YAML_COLUMN], - *selection, - ] - ) - + "\n" - ) - else: - ms_f.write(line) - # FIXME replace with some 'ModelSpaceManager' object - return expanded_models_file - - -def line2row( - line: str, - delimiter: str = "\t", - unpacked: bool = True, - convert_parameters_to_float: bool = True, -) -> list: - """Parse a line from a model space file. - - Args: - line: - A line from a file with delimiter-separated columns. - delimiter: - The string that separates columns in the file. - unpacked: - Whether the line format is in the unpacked format. If ``False``, - parameter values are not converted to ``float``. - convert_parameters_to_float: - Whether parameters should be converted to ``float``. - - Returns: - A list of column values. Parameter values are converted to ``float``. - """ - columns = line.strip().split(delimiter) - metadata = columns[:PARAMETER_DEFINITIONS_START] - if unpacked and convert_parameters_to_float: - parameters = [float(p) for p in columns[PARAMETER_DEFINITIONS_START:]] - else: - parameters = columns[PARAMETER_DEFINITIONS_START:] - return metadata + parameters - - class ModelSpace: """A model space, as a collection of model subspaces. @@ -147,55 +44,71 @@ def __init__( } @staticmethod - def from_files( - filenames: list[TYPE_PATH], - ): - """Create a model space from model space files. + def load( + data: TYPE_PATH | pd.DataFrame | list[TYPE_PATH | pd.DataFrame], + root_path: TYPE_PATH = None, + ) -> ModelSpace: + """Load a model space from dataframe(s) or file(s). Args: - filenames: - The locations of the model space files. + data: + The data. TSV file(s) or pandas dataframe(s). + root_path: + Any paths in dataframe will be resolved relative to this path. + Paths in TSV files will be resolved relative to the directory + of the TSV file. Returns: - The corresponding model space. + The model space. """ - # TODO validate input? - model_space_dfs = [ - get_model_space_df(filename) for filename in filenames + if not isinstance(data, list): + data = [data] + dfs = [ + ( + root_path, + df.reset_index() if df.index.name == MODEL_SUBSPACE_ID else df, + ) + if isinstance(df, pd.DataFrame) + else (Path(df).parent, pd.read_csv(df, sep="\t")) + for df in data ] + model_subspaces = [] - for model_space_df, model_space_filename in zip( - model_space_dfs, filenames, strict=False - ): - for model_subspace_id, definition in model_space_df.iterrows(): + for root_path, df in dfs: + for _, definition in df.iterrows(): model_subspaces.append( ModelSubspace.from_definition( - model_subspace_id=model_subspace_id, definition=definition, - parent_path=Path(model_space_filename).parent, + root_path=root_path, ) ) model_space = ModelSpace(model_subspaces=model_subspaces) return model_space - @staticmethod - def from_df( - df: pd.DataFrame, - parent_path: TYPE_PATH = None, - ): - model_subspaces = [] - for model_subspace_id, definition in df.iterrows(): - model_subspaces.append( - ModelSubspace.from_definition( - model_subspace_id=model_subspace_id, - definition=definition, - parent_path=parent_path, - ) - ) - model_space = ModelSpace(model_subspaces=model_subspaces) - return model_space + def save(self, filename: TYPE_PATH | None = None) -> pd.DataFrame: + """Export the model space to a dataframe (and TSV). + + Args: + filename: + If provided, the dataframe will be saved here as a TSV. + Paths will be made relative to the parent directory of this + filename. - # TODO: `to_df` / `to_file` + Returns: + The dataframe. + """ + root_path = Path(filename).parent if filename else None + + data = [] + for model_subspace in self.model_subspaces.values(): + data.append(model_subspace.to_definition(root_path=root_path)) + df = pd.DataFrame(data) + df = df.set_index(MODEL_SUBSPACE_ID) + + if filename: + df.to_csv(filename, sep="\t") + + return df def search( self, @@ -203,7 +116,7 @@ def search( limit: int = np.inf, exclude: bool = True, ): - """...TODO + """Search all model subspaces according to a candidate space method. Args: candidate_space: @@ -249,13 +162,6 @@ def search_subspaces(only_one_subspace: bool = False): search_subspaces() - ## FIXME implement source_path.. somewhere - # if self.source_path is not None: - # for model in candidate_space.models: - # # TODO do this change elsewhere instead? - # # e.g. model subspace - # model.petab_yaml = self.source_path / model.petab_yaml - if exclude: self.exclude_models(candidate_space.models) @@ -293,27 +199,3 @@ def reset_exclusions( """Reset the exclusions in the model subspaces.""" for model_subspace in self.model_subspaces.values(): model_subspace.reset_exclusions(exclusions) - - -def get_model_space_df(df: TYPE_PATH | pd.DataFrame) -> pd.DataFrame: - # model_space_df = pd.read_csv(filename, sep='\t', index_col=MODEL_SUBSPACE_ID) # FIXME - if isinstance(df, get_args(TYPE_PATH)): - df = pd.read_csv(df, sep="\t") - if df.index.name != MODEL_SUBSPACE_ID: - df.set_index([MODEL_SUBSPACE_ID], inplace=True) - return df - - -def write_model_space_df(df: pd.DataFrame, filename: TYPE_PATH) -> None: - df.to_csv(filename, sep="\t", index=True) - - -# def get_model_space( -# filename: TYPE_PATH, -# ) -> List[ModelSubspace]: -# model_space_df = get_model_space_df(filename) -# model_subspaces = [] -# for definition in model_space_df.iterrows(): -# model_subspaces.append(ModelSubspace.from_definition(definition)) -# model_space = ModelSpace(model_subspaces=model_subspaces) -# return model_space diff --git a/petab_select/model_subspace.py b/petab_select/model_subspace.py index 1f62bd75..92a67903 100644 --- a/petab_select/model_subspace.py +++ b/petab_select/model_subspace.py @@ -2,6 +2,7 @@ import warnings from collections.abc import Iterable, Iterator from itertools import product +from os.path import relpath from pathlib import Path from typing import Any @@ -13,9 +14,9 @@ from .candidate_space import CandidateSpace from .constants import ( ESTIMATE, - MODEL_SPACE_FILE_NON_PARAMETER_COLUMNS, + MODEL_SUBSPACE_ID, + MODEL_SUBSPACE_PETAB_YAML, PARAMETER_VALUE_DELIMITER, - PETAB_YAML, STEPWISE_METHODS, TYPE_PARAMETER_DICT, TYPE_PARAMETER_OPTIONS, @@ -715,38 +716,63 @@ def reset( @staticmethod def from_definition( - model_subspace_id: str, definition: dict[str, str] | pd.Series, - parent_path: TYPE_PATH = None, + root_path: TYPE_PATH = None, ) -> "ModelSubspace": """Create a :class:`ModelSubspace` from a definition. Args: - model_subspace_id: - The model subspace ID. definition: A description of the model subspace. Keys are properties of the - model subspace, including parameters that can take different values. - parent_path: - Any paths in the definition will be set relative to this path. + model subspace, including parameters that can take different + values. + root_path: + Any paths will be resolved relative to this path. Returns: The model subspace. """ + model_subspace_id = definition.pop(MODEL_SUBSPACE_ID) + petab_yaml = definition.pop(MODEL_SUBSPACE_PETAB_YAML) parameters = { column_id: decompress_parameter_values(value) for column_id, value in definition.items() - if column_id not in MODEL_SPACE_FILE_NON_PARAMETER_COLUMNS } - petab_yaml = definition[PETAB_YAML] - if parent_path is not None: - petab_yaml = Path(parent_path) / petab_yaml + if root_path is not None: + petab_yaml = Path(root_path) / petab_yaml return ModelSubspace( model_subspace_id=model_subspace_id, petab_yaml=petab_yaml, parameters=parameters, ) + def to_definition(self, root_path: TYPE_PATH | None = None) -> pd.Series: + """Get the definition of the model subspace. + + Args: + root_path: + If provided, the ``model_subspace_petab_yaml`` will be made + relative to this path. + + Returns: + The definition. + """ + petab_yaml = self.petab_yaml + if root_path: + petab_yaml = relpath(petab_yaml, start=root_path) + return pd.Series( + { + MODEL_SUBSPACE_ID: self.model_subspace_id, + MODEL_SUBSPACE_PETAB_YAML: petab_yaml, + **{ + parameter_id: PARAMETER_VALUE_DELIMITER.join( + str(v) for v in values + ) + for parameter_id, values in self.parameters.items() + }, + } + ) + def indices_to_model(self, indices: list[int]) -> Model | None: """Get a model from the subspace, by indices of possible parameter values. diff --git a/petab_select/models.py b/petab_select/models.py index a681157c..74c38285 100644 --- a/petab_select/models.py +++ b/petab_select/models.py @@ -466,15 +466,15 @@ def to_yaml( Location of the YAML file. relative_paths: Whether to rewrite the paths in each model (e.g. the path to the - model's PEtab problem) relative to the `output_yaml` location. + model's PEtab problem) relative to the ``filename`` location. """ - models = self._models + _models = self._models if relative_paths: root_path = Path(filename).parent - models = copy.deepcopy(models) - for model in models: - model.set_relative_paths(root_path=root_path) - ModelsStandard.save_data(data=models, filename=filename) + _models = copy.deepcopy(_models) + for _model in _models: + _model.set_relative_paths(root_path=root_path) + ModelsStandard.save_data(data=Models(_models), filename=filename) def get_criterion( self, @@ -600,7 +600,7 @@ def models_from_yaml_list( stacklevel=2, ) return Models.from_yaml( - models_yaml=model_list_yaml, + filename=model_list_yaml, petab_problem=petab_problem, problem=problem, ) @@ -618,7 +618,7 @@ def models_to_yaml_list( stacklevel=2, ) Models(models=models).to_yaml( - output_yaml=output_yaml, relative_paths=relative_paths + filename=output_yaml, relative_paths=relative_paths ) diff --git a/petab_select/problem.py b/petab_select/problem.py index 5260f9e2..c8741322 100644 --- a/petab_select/problem.py +++ b/petab_select/problem.py @@ -1,23 +1,33 @@ """The model selection problem class.""" +from __future__ import annotations + +import copy import warnings from collections.abc import Callable, Iterable from functools import partial +from os.path import relpath from pathlib import Path -from typing import Any - -import yaml +from typing import Annotated, Any + +import mkstd +from pydantic import ( + BaseModel, + Field, + PlainSerializer, + PrivateAttr, + ValidationInfo, + ValidatorFunctionWrapHandler, + model_validator, +) from .analyze import get_best from .candidate_space import CandidateSpace, method_to_candidate_space_class from .constants import ( - CANDIDATE_SPACE_ARGUMENTS, CRITERION, - METHOD, - MODEL_SPACE_FILES, PREDECESSOR_MODEL, - PROBLEM_ID, - VERSION, + ROOT_PATH, + TYPE_PATH, Criterion, Method, ) @@ -27,18 +37,19 @@ __all__ = [ "Problem", + "ProblemStandard", ] -class Problem: +class Problem(BaseModel): """Handle everything related to the model selection problem. Attributes: model_space: The model space. calibrated_models: - Calibrated models. Will be used to augment the model selection problem (e.g. - by excluding them from the model space). + Calibrated models. Will be used to augment the model selection + problem (e.g. by excluding them from the model space). candidate_space_arguments: Arguments are forwarded to the candidate space constructor. compare: @@ -55,73 +66,126 @@ class Problem: paths that exist in e.g. the model space files. """ - """ - FIXME(dilpath) - Unsaved attributes: - candidate_space: - The candidate space that will be used. - Reason for not saving: - Essentially reproducible from :attr:`Problem.method` and - :attr:`Problem.calibrated_models`. - FIXME(dilpath) refactor calibrated_models out, move to e.g. candidate - space args - TODO should the relative paths be relative to the YAML or the file that contains them? - problem relative to file that contains them + format_version: str = Field(default="1.0.0") + criterion: Annotated[ + Criterion, PlainSerializer(lambda x: x.value, return_type=str) + ] + method: Annotated[ + Method, PlainSerializer(lambda x: x.value, return_type=str) + ] + model_space_files: list[Path] + candidate_space_arguments: dict[str, Any] = Field(default_factory=dict) + + _compare: Callable[[Model, Model], bool] = PrivateAttr(default=None) + + @model_validator(mode="wrap") + def _check_input( + data: dict[str, Any] | Problem, + handler: ValidatorFunctionWrapHandler, + info: ValidationInfo, + ) -> Problem: + if isinstance(data, Problem): + return data + + compare = data.pop("compare", None) or data.pop("_compare", None) + root_path = Path(data.pop(ROOT_PATH, "")) + + problem = handler(data) + + if compare is None: + compare = partial(default_compare, criterion=problem.criterion) + problem._compare = compare + + problem._model_space = ModelSpace.load( + [ + root_path / model_space_file + for model_space_file in problem.model_space_files + ] + ) - """ + if PREDECESSOR_MODEL in problem.candidate_space_arguments: + problem.candidate_space_arguments[PREDECESSOR_MODEL] = ( + root_path + / problem.candidate_space_arguments[PREDECESSOR_MODEL] + ) - def __init__( - self, - model_space: ModelSpace, - candidate_space_arguments: dict[str, Any] = None, - compare: Callable[[Model, Model], bool] = None, - criterion: Criterion = None, - problem_id: str = None, - method: str = None, - version: str = None, - yaml_path: Path | str = None, - ): - self.model_space = model_space - self.criterion = criterion - self.problem_id = problem_id - self.method = method - self.version = version - self.yaml_path = Path(yaml_path) - - self.candidate_space_arguments = candidate_space_arguments - if self.candidate_space_arguments is None: - self.candidate_space_arguments = {} - - self.compare = compare - if self.compare is None: - self.compare = partial(default_compare, criterion=self.criterion) + return problem - def __str__(self): - return ( - f"YAML: {self.yaml_path}\n" - f"Method: {self.method}\n" - f"Criterion: {self.criterion}\n" - f"Version: {self.version}\n" + @staticmethod + def from_yaml(filename: TYPE_PATH) -> Problem: + """Load a problem from a YAML file.""" + problem = ProblemStandard.load_data( + filename=filename, + root_path=Path(filename).parent, ) + return problem - def get_path(self, relative_path: str | Path) -> Path: - """Get the path to a resource, from a relative path. + def to_yaml( + self, + filename: str | Path, + ) -> None: + """Save a problem to a YAML file. - Args: - relative_path: - The path to the resource, that is relative to the PEtab Select - problem YAML file location. + All paths will be made relative to the ``filename`` directory. - Returns: - The path to the resource. - """ + Args: + filename: + Location of the YAML file. """ - TODO: - Unused? + root_path = Path(filename).parent + + problem = copy.deepcopy(self) + problem.model_space_files = [ + relpath( + model_space_file.resolve(), + start=root_path, + ) + for model_space_file in problem.model_space_files + ] + ProblemStandard.save_data(data=problem, filename=filename) + + def save( + self, + directory: str | Path, + ) -> None: + """Save all data (problem and model space) to a ``directory``. + + Inside the directory, two files will be created: + (1) ``petab_select_problem.yaml``, and + (2) ``model_space.tsv``. + + All paths will be made relative to the ``directory``. """ - if self.yaml_path is None: - return Path(relative_path) - return self.yaml_path.parent / relative_path + directory = Path(directory) + directory.mkdir(exist_ok=True, parents=True) + + problem = copy.deepcopy(self) + problem.model_space_files = ["model_space.tsv"] + if PREDECESSOR_MODEL in problem.candidate_space_arguments: + problem.candidate_space_arguments[PREDECESSOR_MODEL] = relpath( + problem.candidate_space_arguments[PREDECESSOR_MODEL], + start=directory, + ) + ProblemStandard.save_data( + data=problem, filename=directory / "petab_select_problem.yaml" + ) + + problem.model_space.save(filename=directory / "model_space.tsv") + + @property + def compare(self): + return self._compare + + @property + def model_space(self): + return self._model_space + + def __str__(self): + return ( + f"Method: {self.method}\n" + f"Criterion: {self.criterion}\n" + f"Format version: {self.format_version}\n" + ) def exclude_models( self, @@ -153,72 +217,6 @@ def exclude_model_hashes( ) self.exclude_models(models=Models(models=model_hashes, problem=self)) - @staticmethod - def from_yaml( - yaml_path: str | Path, - ) -> "Problem": - """Generate a problem from a PEtab Select problem YAML file. - - Args: - yaml_path: - The location of the PEtab Select problem YAML file. - - Returns: - A `Problem` instance. - """ - yaml_path = Path(yaml_path) - with open(yaml_path) as f: - problem_specification = yaml.safe_load(f) - - if not problem_specification.get(MODEL_SPACE_FILES, []): - raise KeyError( - "The model selection problem specification file is missing " - "model space files." - ) - - model_space = ModelSpace.from_files( - # problem_specification[MODEL_SPACE_FILES], - [ - # `pathlib.Path` appears to handle absolute `model_space_file` paths - # correctly, even if used as a relative path. - # TODO test - # This is similar to the `Problem.get_path` method. - yaml_path.parent / model_space_file - for model_space_file in problem_specification[ - MODEL_SPACE_FILES - ] - ], - # source_path=yaml_path.parent, - ) - - criterion = problem_specification.get(CRITERION, None) - if criterion is not None: - criterion = Criterion(criterion) - - problem_id = problem_specification.get(PROBLEM_ID, None) - - candidate_space_arguments = problem_specification.get( - CANDIDATE_SPACE_ARGUMENTS, - None, - ) - if candidate_space_arguments is not None: - if PREDECESSOR_MODEL in candidate_space_arguments: - candidate_space_arguments[PREDECESSOR_MODEL] = ( - yaml_path.parent - / candidate_space_arguments[PREDECESSOR_MODEL] - ) - - return Problem( - model_space=model_space, - candidate_space_arguments=candidate_space_arguments, - criterion=criterion, - # TODO refactor method to use enum - method=problem_specification.get(METHOD, None), - problem_id=problem_id, - version=problem_specification.get(VERSION, None), - yaml_path=yaml_path, - ) - def get_best( self, models: Models, @@ -311,3 +309,6 @@ def new_candidate_space( **candidate_space_kwargs, ) return candidate_space + + +ProblemStandard = mkstd.YamlStandard(model=Problem) diff --git a/pyproject.toml b/pyproject.toml index 7043546f..65e8a770 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ dependencies = [ "pyyaml>=6.0.2", "click>=8.1.7", "dill>=0.3.9", - "mkstd>=0.0.7", + "mkstd>=0.0.8", ] [project.optional-dependencies] plot = [ diff --git a/test/candidate_space/input/famos_synthetic/select/model_space_FAMoS_2019.tsv b/test/candidate_space/input/famos_synthetic/select/model_space_FAMoS_2019.tsv index 9d602c3f..1411532a 100644 --- a/test/candidate_space/input/famos_synthetic/select/model_space_FAMoS_2019.tsv +++ b/test/candidate_space/input/famos_synthetic/select/model_space_FAMoS_2019.tsv @@ -1,2 +1,2 @@ -model_subspace_id petab_yaml ro_A ro_B ro_C ro_D mu_AB mu_BA mu_AC mu_CA mu_AD mu_DA mu_BC mu_CB mu_BD mu_DB mu_CD mu_DC +model_subspace_id model_subspace_petab_yaml ro_A ro_B ro_C ro_D mu_AB mu_BA mu_AC mu_CA mu_AD mu_DA mu_BC mu_CB mu_BD mu_DB mu_CD mu_DC model_subspace_1 ../petab/FAMoS_2019_problem.yaml 0;estimate 0;estimate 0;estimate 0;estimate 0;estimate 0;estimate 0;estimate 0;estimate 0;estimate 0;estimate 0;estimate 0;estimate 0;estimate 0;estimate 0;estimate 0;estimate diff --git a/test/candidate_space/test_candidate_space.py b/test/candidate_space/test_candidate_space.py index 970a3e00..0cca4104 100644 --- a/test/candidate_space/test_candidate_space.py +++ b/test/candidate_space/test_candidate_space.py @@ -12,7 +12,7 @@ from petab_select.constants import ( ESTIMATE, ) -from petab_select.model_space import ModelSpace, get_model_space_df +from petab_select.model_space import ModelSpace @pytest.fixture @@ -98,6 +98,5 @@ def model_space(calibrated_model_space) -> pd.DataFrame: data["k4"].append(k4) data["k5"].append(k5) df = pd.DataFrame(data=data) - df = get_model_space_df(df) - model_space = ModelSpace.from_df(df) + model_space = ModelSpace.load(df) return model_space diff --git a/test/model_space/model_space_file_1.tsv b/test/model_space/model_space_file_1.tsv index 8b6c9f1a..6e04853e 100644 --- a/test/model_space/model_space_file_1.tsv +++ b/test/model_space/model_space_file_1.tsv @@ -1,3 +1,3 @@ -model_subspace_id petab_yaml k1 k2 k3 k4 +model_subspace_id model_subspace_petab_yaml k1 k2 k3 k4 model_subspace_1 ../../doc/examples/model_selection/petab_problem.yaml 0.2;estimate 0.1;estimate estimate 0;0.1;estimate model_subspace_2 ../../doc/examples/model_selection/petab_problem.yaml 0 0 0 estimate diff --git a/test/model_space/model_space_file_2.tsv b/test/model_space/model_space_file_2.tsv index 315fb50f..b02c3a8d 100644 --- a/test/model_space/model_space_file_2.tsv +++ b/test/model_space/model_space_file_2.tsv @@ -1,2 +1,2 @@ -model_subspace_id petab_yaml k1 k2 k3 k4 +model_subspace_id model_subspace_petab_yaml k1 k2 k3 k4 model_subspace_3 ../../doc/examples/model_selection/petab_problem.yaml estimate estimate 0.3;estimate estimate diff --git a/test/model_space/test_model_space.py b/test/model_space/test_model_space.py index ace5560e..3250be9f 100644 --- a/test/model_space/test_model_space.py +++ b/test/model_space/test_model_space.py @@ -26,7 +26,7 @@ def model_space_files() -> list[Path]: @pytest.fixture def model_space(model_space_files) -> ModelSpace: - return ModelSpace.from_files(model_space_files) + return ModelSpace.load(model_space_files) def test_model_space_forward_virtual(model_space): diff --git a/test/model_subspace/test_model_subspace.py b/test/model_subspace/test_model_subspace.py index 5d3d6de9..cdbe94c7 100644 --- a/test/model_subspace/test_model_subspace.py +++ b/test/model_subspace/test_model_subspace.py @@ -13,8 +13,9 @@ ) from petab_select.constants import ( ESTIMATE, + MODEL_SUBSPACE_ID, + MODEL_SUBSPACE_PETAB_YAML, PARAMETER_VALUE_DELIMITER, - PETAB_YAML, Criterion, ) from petab_select.model import Model @@ -22,10 +23,10 @@ @pytest.fixture -def model_subspace_id_and_definition() -> pd.Series: - model_subspace_id = "model_subspace_1" +def model_subspace_definition() -> pd.Series: data = { - PETAB_YAML: Path(__file__).parent.parent.parent + MODEL_SUBSPACE_ID: "model_subspace_1", + MODEL_SUBSPACE_PETAB_YAML: Path(__file__).parent.parent.parent / "doc" / "examples" / "model_selection" @@ -35,15 +36,13 @@ def model_subspace_id_and_definition() -> pd.Series: "k3": ESTIMATE, "k4": PARAMETER_VALUE_DELIMITER.join(["0", "0.1", ESTIMATE]), } - return model_subspace_id, pd.Series(data=data, dtype=str) + return pd.Series(data=data, dtype=str) @pytest.fixture -def model_subspace(model_subspace_id_and_definition) -> ModelSubspace: - model_subspace_id, definition = model_subspace_id_and_definition +def model_subspace(model_subspace_definition) -> ModelSubspace: return petab_select.model_subspace.ModelSubspace.from_definition( - model_subspace_id=model_subspace_id, - definition=definition, + definition=model_subspace_definition, ) diff --git a/test/problem/__init__.py b/test/problem/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/problem/expected_output/model_space.tsv b/test/problem/expected_output/model_space.tsv new file mode 100644 index 00000000..32422e01 --- /dev/null +++ b/test/problem/expected_output/model_space.tsv @@ -0,0 +1,9 @@ +model_subspace_id model_subspace_petab_yaml k1 k2 k3 +M1_0 ../../../doc/examples/model_selection/petab_problem.yaml 0 0 0 +M1_1 ../../../doc/examples/model_selection/petab_problem.yaml 0.2 0.1 estimate +M1_2 ../../../doc/examples/model_selection/petab_problem.yaml 0.2 estimate 0 +M1_3 ../../../doc/examples/model_selection/petab_problem.yaml estimate 0.1 0 +M1_4 ../../../doc/examples/model_selection/petab_problem.yaml 0.2 estimate estimate +M1_5 ../../../doc/examples/model_selection/petab_problem.yaml estimate 0.1 estimate +M1_6 ../../../doc/examples/model_selection/petab_problem.yaml estimate estimate 0 +M1_7 ../../../doc/examples/model_selection/petab_problem.yaml estimate estimate estimate diff --git a/test/problem/expected_output/petab_select_problem.yaml b/test/problem/expected_output/petab_select_problem.yaml new file mode 100644 index 00000000..360def46 --- /dev/null +++ b/test/problem/expected_output/petab_select_problem.yaml @@ -0,0 +1,6 @@ +format_version: 1.0.0 +criterion: AIC +method: forward +model_space_files: +- model_space.tsv +candidate_space_arguments: {} diff --git a/test/problem/test_problem.py b/test/problem/test_problem.py new file mode 100644 index 00000000..f9f68811 --- /dev/null +++ b/test/problem/test_problem.py @@ -0,0 +1,34 @@ +from pathlib import Path + +import petab_select + +test_path = Path(__file__).parent + +problem_yaml = ( + test_path.parent.parent + / "doc" + / "examples" + / "model_selection" + / "petab_select_problem.yaml" +) + + +def test_round_trip(): + """Test storing/loading of a full problem.""" + problem0 = petab_select.Problem.from_yaml(problem_yaml) + problem0.save(test_path / "output") + + with open(test_path / "expected_output/petab_select_problem.yaml") as f: + problem_yaml0 = f.read() + with open(test_path / "expected_output/model_space.tsv") as f: + model_space_tsv0 = f.read() + + with open(test_path / "output/petab_select_problem.yaml") as f: + problem_yaml1 = f.read() + with open(test_path / "output/model_space.tsv") as f: + model_space_tsv1 = f.read() + + # The exported problem YAML is as expected, with updated relative paths. + assert problem_yaml1 == problem_yaml0 + # The exported model space TSV is as expected, with updated relative paths. + assert model_space_tsv1 == model_space_tsv0 diff --git a/test/pypesto/test_pypesto.py b/test/pypesto/test_pypesto.py index e670d8cc..efaf10a4 100644 --- a/test/pypesto/test_pypesto.py +++ b/test/pypesto/test_pypesto.py @@ -1,4 +1,7 @@ import os +import shlex +import shutil +import subprocess from pathlib import Path import numpy as np @@ -7,12 +10,14 @@ import pypesto.optimize import pypesto.select import pytest +import yaml import petab_select from petab_select import Model from petab_select.constants import ( CRITERIA, ESTIMATED_PARAMETERS, + TERMINATE, ) os.environ["AMICI_EXPERIMENTAL_SBML_NONCONST_CLS"] = "1" @@ -47,6 +52,12 @@ def objective_customizer(obj): obj.amici_solver.setRelativeTolerance(1e-12) +model_problem_options = { + "minimize_options": minimize_options, + "objective_customizer": objective_customizer, +} + + @pytest.mark.parametrize( "test_case_path_stem", sorted( @@ -54,6 +65,7 @@ def objective_customizer(obj): ), ) def test_pypesto(test_case_path_stem): + """Run all test cases with pyPESTO.""" if test_cases and test_case_path_stem not in test_cases: pytest.skip("Test excluded from subset selected for debugging.") @@ -72,14 +84,118 @@ def test_pypesto(test_case_path_stem): # Run the selection process until "exhausted". pypesto_select_problem.select_to_completion( - minimize_options=minimize_options, - objective_customizer=objective_customizer, + model_problem_options=model_problem_options, ) # Get the best model - best_model = petab_select_problem.get_best( + best_model = petab_select.analyze.get_best( models=pypesto_select_problem.calibrated_models, + criterion=petab_select_problem.criterion, + compare=petab_select_problem.compare, + ) + + # Load the expected model. + expected_model = Model.from_yaml(expected_model_yaml) + + def get_series(model, dict_attribute) -> pd.Series: + return pd.Series( + getattr(model, dict_attribute), + dtype=np.float64, + ).sort_index() + + # The estimated parameters and criteria values are as expected. + for dict_attribute in [CRITERIA, ESTIMATED_PARAMETERS]: + pd.testing.assert_series_equal( + get_series(expected_model, dict_attribute), + get_series(best_model, dict_attribute), + rtol=1e-2, + ) + # FIXME ensure `current model criterion` trajectory also matches, in summary.tsv file, + # for test case 0009, after summary format is revised + + +@pytest.mark.skipif( + os.getenv("GITHUB_ACTIONS") == "true", + reason="Too CPU heavy for CI.", +) +def test_famos_cli(): + """Run test case 0009 with pyPESTO and the CLI interface.""" + test_case_path = test_cases_path / "0009" + expected_model_yaml = test_case_path / "expected.yaml" + problem_yaml = test_case_path / "petab_select_problem.yaml" + + problem = petab_select.Problem.from_yaml(problem_yaml) + + # Setup working directory for intermediate files + work_dir = Path(__file__).parent / "output_famos_cli" + work_dir_str = str(work_dir) + if work_dir.exists(): + shutil.rmtree(work_dir_str) + work_dir.mkdir(exist_ok=True, parents=True) + + models_yamls = [] + metadata_yaml = work_dir / "metadata.yaml" + state_dill = work_dir / "state.dill" + iteration = 0 + while True: + iteration += 1 + uncalibrated_models_yaml = ( + work_dir / f"uncalibrated_models_{iteration}.yaml" + ) + calibrated_models_yaml = ( + work_dir / f"calibrated_models_{iteration}.yaml" + ) + models_yaml = work_dir / f"models_{iteration}.yaml" + models_yamls.append(models_yaml) + # Start iteration + subprocess.run( # noqa: S603 + shlex.split( + f"""petab_select start_iteration + --problem {problem_yaml} + --state {state_dill} + --output-uncalibrated-models {uncalibrated_models_yaml} + """ + ) + ) + # Calibrate models + models = petab_select.Models.from_yaml(uncalibrated_models_yaml) + for model in models: + pypesto.select.ModelProblem( + model=model, + criterion=problem.criterion, + **model_problem_options, + ) + models.to_yaml(filename=calibrated_models_yaml) + # End iteration + subprocess.run( # noqa: S603 + shlex.split( + f"""petab_select end_iteration + --output-models {models_yaml} + --output-metadata {metadata_yaml} + --state {state_dill} + --calibrated-models {calibrated_models_yaml} + """ + ) + ) + with open(metadata_yaml) as f: + metadata = yaml.safe_load(f) + if metadata[TERMINATE]: + break + + # Get the best model + models_yamls_arg = " ".join( + f"--models {models_yaml}" for models_yaml in models_yamls + ) + subprocess.run( # noqa: S603 + shlex.split( + f"""petab_select get_best + --problem {problem_yaml} + {models_yamls_arg} + --output {work_dir / "best_model.yaml"} + """ + ) ) + best_model = petab_select.Model.from_yaml(work_dir / "best_model.yaml") # Load the expected model. expected_model = Model.from_yaml(expected_model_yaml) @@ -95,5 +211,7 @@ def get_series(model, dict_attribute) -> pd.Series: pd.testing.assert_series_equal( get_series(expected_model, dict_attribute), get_series(best_model, dict_attribute), - atol=1e-2, + rtol=1e-2, ) + # FIXME ensure `current model criterion` trajectory also matches, in summary.tsv file, + # after summary format is revised diff --git a/test_cases/0001/model_space.tsv b/test_cases/0001/model_space.tsv index dff3e821..d7a994ec 100644 --- a/test_cases/0001/model_space.tsv +++ b/test_cases/0001/model_space.tsv @@ -1,2 +1,2 @@ -model_subspace_id petab_yaml k1 k2 k3 +model_subspace_id model_subspace_petab_yaml k1 k2 k3 M1_1 petab/petab_problem.yaml 0.2 0.1 0 diff --git a/test_cases/0002/model_space.tsv b/test_cases/0002/model_space.tsv index 31f0474d..fbcdbd83 100644 --- a/test_cases/0002/model_space.tsv +++ b/test_cases/0002/model_space.tsv @@ -1,4 +1,4 @@ -model_subspace_id petab_yaml k1 k2 k3 +model_subspace_id model_subspace_petab_yaml k1 k2 k3 M1_0 ../0001/petab/petab_problem.yaml 0 0 0 M1_1 ../0001/petab/petab_problem.yaml 0.2 0.1 estimate M1_2 ../0001/petab/petab_problem.yaml 0.2 estimate 0 diff --git a/test_cases/0003/model_space.tsv b/test_cases/0003/model_space.tsv index 39f8ae9a..bde9182f 100644 --- a/test_cases/0003/model_space.tsv +++ b/test_cases/0003/model_space.tsv @@ -1,2 +1,2 @@ -model_subspace_id petab_yaml k1 k2 k3 +model_subspace_id model_subspace_petab_yaml k1 k2 k3 M1 ../0001/petab/petab_problem.yaml 0;0.2;estimate 0;0.1;estimate 0;estimate diff --git a/test_cases/0004/model_space.tsv b/test_cases/0004/model_space.tsv index 31f0474d..fbcdbd83 100644 --- a/test_cases/0004/model_space.tsv +++ b/test_cases/0004/model_space.tsv @@ -1,4 +1,4 @@ -model_subspace_id petab_yaml k1 k2 k3 +model_subspace_id model_subspace_petab_yaml k1 k2 k3 M1_0 ../0001/petab/petab_problem.yaml 0 0 0 M1_1 ../0001/petab/petab_problem.yaml 0.2 0.1 estimate M1_2 ../0001/petab/petab_problem.yaml 0.2 estimate 0 diff --git a/test_cases/0005/model_space.tsv b/test_cases/0005/model_space.tsv index 31f0474d..fbcdbd83 100644 --- a/test_cases/0005/model_space.tsv +++ b/test_cases/0005/model_space.tsv @@ -1,4 +1,4 @@ -model_subspace_id petab_yaml k1 k2 k3 +model_subspace_id model_subspace_petab_yaml k1 k2 k3 M1_0 ../0001/petab/petab_problem.yaml 0 0 0 M1_1 ../0001/petab/petab_problem.yaml 0.2 0.1 estimate M1_2 ../0001/petab/petab_problem.yaml 0.2 estimate 0 diff --git a/test_cases/0006/model_space.tsv b/test_cases/0006/model_space.tsv index 6a2deec2..5c2190f1 100644 --- a/test_cases/0006/model_space.tsv +++ b/test_cases/0006/model_space.tsv @@ -1,4 +1,4 @@ -model_subspace_id petab_yaml k1 k2 k3 +model_subspace_id model_subspace_petab_yaml k1 k2 k3 M1_0 ../0001/petab/petab_problem.yaml 0.2 0.1 0 M1_1 ../0001/petab/petab_problem.yaml 0.2 0.1 estimate M1_2 ../0001/petab/petab_problem.yaml 0.2 estimate 0 diff --git a/test_cases/0007/model_space.tsv b/test_cases/0007/model_space.tsv index 1c09d4b8..a066a8e4 100644 --- a/test_cases/0007/model_space.tsv +++ b/test_cases/0007/model_space.tsv @@ -1,4 +1,4 @@ -model_subspace_id petab_yaml k1 k2 k3 +model_subspace_id model_subspace_petab_yaml k1 k2 k3 M1_0 petab/petab_problem.yaml 0.2 0.1 0 M1_1 petab/petab_problem.yaml 0.2 0.1 estimate M1_2 petab/petab_problem.yaml 0.2 estimate 0 diff --git a/test_cases/0008/model_space.tsv b/test_cases/0008/model_space.tsv index 01e8de35..55888545 100644 --- a/test_cases/0008/model_space.tsv +++ b/test_cases/0008/model_space.tsv @@ -1,4 +1,4 @@ -model_subspace_id petab_yaml k1 k2 k3 +model_subspace_id model_subspace_petab_yaml k1 k2 k3 M1_0 ../0007/petab/petab_problem.yaml 0.2 0.1 0 M1_1 ../0007/petab/petab_problem.yaml 0.2 0.1 estimate M1_2 ../0007/petab/petab_problem.yaml 0.2 estimate 0 diff --git a/test_cases/0009/README.md b/test_cases/0009/README.md index 37243b6e..207ece74 100644 --- a/test_cases/0009/README.md +++ b/test_cases/0009/README.md @@ -2,4 +2,4 @@ N.B. This original Blasi et al. problem is difficult to solve with a stepwise me 1. performing 100 FAMoS starts, initialized at random models. Usually <5% of the starts ended at the best model. 2. assessing reproducibility. Most of the starts that end at the best model are not reproducible. Instead, the path through model space can differ a lot despite "good" calibration, because many pairs of models differ in AICc by less than numerical noise. -1 start was found that reproducibly ends at the best model. The initial model of that start is the predecessor model in this test case. However, the path through model space is not reproducible -- there are at least two possibilities, perhaps more, depending on simulation tolerances. Hence, you should expect to produce a similar `expected_summary.tsv`, but perhaps with a few rows swapped. If you see a different summary.tsv, please report (or retry a few times). +1 start was found that reproducibly ends at the best model. The initial model of that start is the predecessor model in this test case. However, the path through model space is not reproducible -- there are at least two possibilities, perhaps more, depending on simulation tolerances. Hence, you should expect to produce a similar `expected_summary.tsv`, but perhaps with a few rows swapped. If you see a different `summary.tsv`, please report (or retry a few times). In particular, a different `summary.tsv` file will have a different sequence of values in the `current model criterion` column (accounting for numerical noise). diff --git a/test_cases/0009/model_space.tsv b/test_cases/0009/model_space.tsv index a2cb8ac2..1dd28263 100644 --- a/test_cases/0009/model_space.tsv +++ b/test_cases/0009/model_space.tsv @@ -1,2 +1,2 @@ -model_subspace_id petab_yaml a_0ac_k05 a_0ac_k08 a_0ac_k12 a_0ac_k16 a_k05_k05k08 a_k05_k05k12 a_k05_k05k16 a_k08_k05k08 a_k08_k08k12 a_k08_k08k16 a_k12_k05k12 a_k12_k08k12 a_k12_k12k16 a_k16_k05k16 a_k16_k08k16 a_k16_k12k16 a_k05k08_k05k08k12 a_k05k08_k05k08k16 a_k05k12_k05k08k12 a_k05k12_k05k12k16 a_k05k16_k05k08k16 a_k05k16_k05k12k16 a_k08k12_k05k08k12 a_k08k12_k08k12k16 a_k08k16_k05k08k16 a_k08k16_k08k12k16 a_k12k16_k05k12k16 a_k12k16_k08k12k16 a_k05k08k12_4ac a_k05k08k16_4ac a_k05k12k16_4ac a_k08k12k16_4ac +model_subspace_id model_subspace_petab_yaml a_0ac_k05 a_0ac_k08 a_0ac_k12 a_0ac_k16 a_k05_k05k08 a_k05_k05k12 a_k05_k05k16 a_k08_k05k08 a_k08_k08k12 a_k08_k08k16 a_k12_k05k12 a_k12_k08k12 a_k12_k12k16 a_k16_k05k16 a_k16_k08k16 a_k16_k12k16 a_k05k08_k05k08k12 a_k05k08_k05k08k16 a_k05k12_k05k08k12 a_k05k12_k05k12k16 a_k05k16_k05k08k16 a_k05k16_k05k12k16 a_k08k12_k05k08k12 a_k08k12_k08k12k16 a_k08k16_k05k08k16 a_k08k16_k08k12k16 a_k12k16_k05k12k16 a_k12k16_k08k12k16 a_k05k08k12_4ac a_k05k08k16_4ac a_k05k12k16_4ac a_k08k12k16_4ac M petab/petab_problem.yaml 1.0;estimate 1.0;estimate 1.0;estimate 1.0;estimate 1.0;estimate 1.0;estimate 1.0;estimate 1.0;estimate 1.0;estimate 1.0;estimate 1.0;estimate 1.0;estimate 1.0;estimate 1.0;estimate 1.0;estimate 1.0;estimate 1.0;estimate 1.0;estimate 1.0;estimate 1.0;estimate 1.0;estimate 1.0;estimate 1.0;estimate 1.0;estimate 1.0;estimate 1.0;estimate 1.0;estimate 1.0;estimate 1.0;estimate 1.0;estimate 1.0;estimate 1.0;estimate