From 0228bcfb571ecb95a8663932a2ea47b06ec27b89 Mon Sep 17 00:00:00 2001 From: Dhanshree Arora Date: Sat, 26 Oct 2024 15:44:25 +0530 Subject: [PATCH 1/7] Update config.yml - fix Circle CI pipeline --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2d109bd06..d7b530402 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,7 +3,7 @@ version: 2 jobs: installation: docker: - - image: ersiliaos/conda:3.7 + - image: continuumio/miniconda3:24.1.2-0 steps: - checkout - run: @@ -18,7 +18,7 @@ jobs: ersilia --help test: docker: - - image: ersiliaos/conda:3.7 + - image: continuumio/miniconda3:24.1.2-0 steps: - checkout - run: @@ -66,7 +66,7 @@ jobs: ersilia -v delete molecular-weight docs-build: docker: - - image: ersiliaos/conda:3.7 + - image: continuumio/miniconda3:24.1.2-0 steps: - checkout - run: From f43fc072ae4d08b1d43d7794aa3a836ba0d7eeeb Mon Sep 17 00:00:00 2001 From: Abellegese Date: Wed, 20 Nov 2024 23:13:25 +0300 Subject: [PATCH 2/7] Model Tester: An model testing CLI --- ersilia/cli/commands/test.py | 83 ++- ersilia/publish/test.py | 960 ++----------------------------- ersilia/publish/test_services.py | 933 ++++++++++++++++++++++++++++++ pyproject.toml | 3 +- 4 files changed, 1037 insertions(+), 942 deletions(-) create mode 100644 ersilia/publish/test_services.py diff --git a/ersilia/cli/commands/test.py b/ersilia/cli/commands/test.py index 8dbb580b7..3b56791b4 100644 --- a/ersilia/cli/commands/test.py +++ b/ersilia/cli/commands/test.py @@ -1,23 +1,9 @@ -import os import click -import json -import tempfile - from ...cli import echo from . import ersilia_cli -from ersilia.cli.commands.run import run_cmd -from ersilia.core.base import ErsiliaBase from ...publish.test import ModelTester -from ersilia.utils.exceptions_utils import throw_ersilia_exception - -from ersilia.utils.exceptions_utils.test_exceptions import WrongCardIdentifierError - -from ersilia.default import INFORMATION_FILE - - def test_cmd(): - """Test a model""" # Example usage: ersilia test {MODEL} @ersilia_cli.command( @@ -26,19 +12,58 @@ def test_cmd(): ) @click.argument("model", type=click.STRING) @click.option( - "-o", "--output", "output", required=False, default=None, type=click.STRING + "-e", + "--env", + "env", + required=False, + default=None, + type=click.STRING ) - def test(model, output): - mdl = ModelTester(model) - model_id = mdl.model_id - - if model_id is None: - echo( - "No model seems to be served. Please run 'ersilia serve ...' before.", - fg="red", - ) - return - - mt = ModelTester(model_id=model_id) - # click.echo("Checking model information") - mt.run(output) # pass in the output here + @click.option( + "-t", + "--type", + "type", + required=False, + default=None, + type=click.STRING + ) + @click.option( + "-l", + "--level", + "level", + required=False, + default=None, + type=click.STRING + ) + @click.option( + "-d", + "--dir", + "dir", + required=False, + default=None, + type=click.STRING + ) + @click.option( + "-o", + "--output", + "output", + required=False, + default=None, + type=click.STRING + ) + def test( + model, + env, + type, + level, + dir, + output + ): + mt = ModelTester( + model_id=model, + env=env, + type=type, + level=level, + dir=dir + ) + mt.run(output) diff --git a/ersilia/publish/test.py b/ersilia/publish/test.py index 3a619e006..18c601fbe 100644 --- a/ersilia/publish/test.py +++ b/ersilia/publish/test.py @@ -1,920 +1,56 @@ -# TODO adapt to input-type agnostic. For now, it works only with Compound input types. -import json -import os -import subprocess -import tempfile -import time -import click -import types -from collections import defaultdict -from datetime import datetime - -from ersilia.utils.conda import SimpleConda -from .. import ErsiliaBase, ErsiliaModel, throw_ersilia_exception -from ..cli import echo -from ..core.session import Session -from ..default import EOS, INFORMATION_FILE -from ..io.input import ExampleGenerator -from ..utils.exceptions_utils import test_exceptions as texc -from ..utils.logging import make_temp_dir -from ..utils.terminal import run_command_check_output - - - -# Check if we have the required imports in the environment -MISSING_PACKAGES = False -try: - from scipy.stats import spearmanr - from fuzzywuzzy import fuzz -except ImportError: - MISSING_PACKAGES = True - -RUN_FILE = "run.sh" -DATA_FILE = "data.csv" -NUM_SAMPLES = 5 -BOLD = "\033[1m" -RESET = "\033[0m" +from .test_services import IOService, CheckService, RunnerService +from .. import ErsiliaBase +from ..utils.logging import logger class ModelTester(ErsiliaBase): - def __init__(self, model_id, config_json=None): - ErsiliaBase.__init__(self, config_json=config_json, credentials_json=None) + def __init__( + self, + model_id, + env, + type, + level, + dir + ): + ErsiliaBase.__init__( + self, + config_json=None, + credentials_json=None + ) self.model_id = model_id - self.model_size = 0 - self.tmp_folder = make_temp_dir(prefix="ersilia-") - self._info = self._read_information() - self._input = self._info["card"]["Input"] - self._output_type = self._info["card"]["Output Type"] - self.RUN_FILE = "run.sh" - self.information_check = False - self.single_input = False - self.example_input = False - self.consistent_output = False - self.run_using_bash = False - - def _read_information(self): - json_file = os.path.join(self._dest_dir, self.model_id, INFORMATION_FILE) - self.logger.debug("Reading model information from {0}".format(json_file)) - if not os.path.exists(json_file): - raise texc.InformationFileNotExist(self.model_id) - with open(json_file, "r") as f: - data = json.load(f) - return data - - """ - This function uses the fuzzy wuzzy package to compare the differences between outputs when - they're strings and not floats. The fuzz.ratio gives the percent of similarity between the two outputs. - Example: two strings that are the exact same will return 100 - """ - - def _compare_output_strings(self, output1, output2): - if output1 is None and output2 is None: - return 100 - else: - return fuzz.ratio(output1, output2) - - - - """ - When the user specifies an output file, the file will show the user how big the model is. This function - calculates the size of the model to allow this. - """ - - def _set_model_size(self, directory): - for dirpath, dirnames, filenames in os.walk(directory): - for filename in filenames: - file_path = os.path.join(dirpath, filename) - self.model_size += os.path.getsize(file_path) - - """ - This helper method was taken from the run.py file, and just prints the output for the user - """ - - def _print_output(self, result, output): - echo("Printing output...") - - if isinstance(result, types.GeneratorType): - for r in result: - if r is not None: - if output is not None: - with open(output.name, "w") as file: - json.dump(r, output.name) - else: - echo(json.dumps(r, indent=4)) - else: - if output is not None: - message = echo("Something went wrong", fg="red") - with open(output.name, "w") as file: - json.dump(message, output.name) - else: - echo("Something went wrong", fg="red") - - else: - echo(result) - - """ - This helper method checks that the model ID is correct. - """ - - def _check_model_id(self, data): - self.logger.debug("Checking model ID...") - if data["card"]["Identifier"] != self.model_id: - raise texc.WrongCardIdentifierError(self.model_id) - - """ - This helper method checks that the slug field is non-empty. - """ - - def _check_model_slug(self, data): - self.logger.debug("Checking model slug...") - if not data["card"]["Slug"]: - raise texc.EmptyField("slug") - - """ - This helper method checks that the description field is non-empty. - """ - - def _check_model_description(self, data): - self.logger.debug("Checking model description...") - if not data["card"]["Description"]: - raise texc.EmptyField("Description") - - """ - This helper method checks that the model task is one of the following valid entries: - - Classification - - Regression - - Generative - - Representation - - Similarity - - Clustering - - Dimensionality reduction - """ - - def _check_model_task(self, data): - self.logger.debug("Checking model task...") - valid_tasks = [ - "Classification", - "Regression", - "Generative", - "Representation", - "Similarity", - "Clustering", - "Dimensionality reduction", - ] - sep = ", " - tasks = [] - if sep in data["card"]["Task"]: - tasks = data["card"]["Task"].split(sep) - else: - tasks = data["card"]["Task"] - for task in tasks: - if task not in valid_tasks: - raise texc.InvalidEntry("Task") - - """ - This helper method checks that the input field is one of the following valid entries: - - Compound - - Protein - - Text - """ - - def _check_model_input(self, data): - self.logger.debug("Checking model input...") - valid_inputs = [["Compound"], ["Protein"], ["Text"]] - if data["card"]["Input"] not in valid_inputs: - raise texc.InvalidEntry("Input") - - """ - This helper method checks that the input shape field is one of the following valid entries: - - Single - - Pair - - List - - Pair of Lists - - List of Lists - """ - - def _check_model_input_shape(self, data): - self.logger.debug("Checking model input shape...") - valid_input_shapes = [ - "Single", - "Pair", - "List", - "Pair of Lists", - "List of Lists", - ] - if data["card"]["Input Shape"] not in valid_input_shapes: - raise texc.InvalidEntry("Input Shape") - - """ - This helper method checks the the output is one of the following valid entries: - - Boolean - - Compound - - Descriptor - - Distance - - Experimental value - - Image - - Other value - - Probability - - Protein - - Score - - Text - """ - - def _check_model_output(self, data): - self.logger.debug("Checking model output...") - valid_outputs = [ - "Boolean", - "Compound", - "Descriptor", - "Distance", - "Experimental value", - "Image", - "Other value", - "Probability", - "Protein", - "Score", - "Text", - ] - sep = ", " - outputs = [] - if sep in data["card"]["Output"]: - outputs = data["card"]["Output"].split(sep) - else: - outputs = data["card"]["Output"] - for output in outputs: - if output not in valid_outputs: - raise texc.InvalidEntry("Output") - - """ - This helper method checks that the output type is one of the following valid entries: - - String - - Float - - Integer - """ - - def _check_model_output_type(self, data): - self.logger.debug("Checking model output type...") - valid_output_types = [["String"], ["Float"], ["Integer"]] - if data["card"]["Output Type"] not in valid_output_types: - raise texc.InvalidEntry("Output Type") - - """ - This helper method checks that the output shape is one of the following valid entries: - - Single - - List - - Flexible List - - Matrix - - Serializable Object - """ - - def _check_model_output_shape(self, data): - self.logger.debug("Checking model output shape...") - valid_output_shapes = [ - "Single", - "List", - "Flexible List", - "Matrix", - "Serializable Object", - ] - if data["card"]["Output Shape"] not in valid_output_shapes: - raise texc.InvalidEntry("Output Shape") - - """ - Check the model information to make sure it's correct. Performs the following checks: - - Checks that model ID is correct - - Checks that model slug is non-empty - - Checks that model description is non-empty - - Checks that the model task is valid - - Checks that the model input, input shape is valid - - Checks that the model output, output type, output shape is valid - """ - - @throw_ersilia_exception() - def check_information(self, output): - - - self.logger.debug("Checking that model information is correct") - self.logger.debug( - BOLD - + "Beginning checks for {0} model information:".format(self.model_id) - + RESET + self.env = env + self.type = type + self.level = level + self.dir = dir + self.ios = IOService( + self.logger, + self._dest_dir, + self._model_path, + self._get_bundle_location, + self._get_bentoml_location, + self.model_id, + self.env, + self.type ) - json_file = os.path.join(self._dest_dir, self.model_id, INFORMATION_FILE) - with open(json_file, "r") as f: - data = json.load(f) - - self._check_model_id(data) - self._check_model_slug(data) - self._check_model_description(data) - self._check_model_task(data) - self._check_model_input(data) - self._check_model_input_shape(data) - self._check_model_output(data) - self._check_model_output_type(data) - self._check_model_output_shape(data) - click.echo(BOLD + "Test: Model information, SUCCESS! ✅\n" + RESET) - - if output is not None: - self.information_check = True - - - """ - Runs the model on a single smiles string and prints to the user if no output is specified. - """ - - @throw_ersilia_exception() - def check_single_input(self, output): - session = Session(config_json=None) - service_class = session.current_service_class() - input = "COc1ccc2c(NC(=O)Nc3cccc(C(F)(F)F)n3)ccnc2c1" - - self.logger.debug(BOLD + "Testing model on single smiles input...\n" + RESET) - mdl = ErsiliaModel(self.model_id, service_class=service_class, config_json=None) - result = mdl.run(input=input, output=output, batch_size=100) - - if output is not None: - self.single_input = True - click.echo(BOLD + "Test: Single SMILES input, SUCCESS! ✅\n" + RESET) - else: - self._print_output(result, output) - - """ - Generates an example input of 5 smiles using the 'example' command, and then tests the model on that input and prints it - to the consol if no output file is specified by the user. - """ - - @throw_ersilia_exception() - def check_example_input(self, output): - session = Session(config_json=None) - service_class = session.current_service_class() - eg = ExampleGenerator(model_id=self.model_id) - input = eg.example(n_samples=NUM_SAMPLES, file_name=None, simple=True, try_predefined=False) - self.logger.debug( - BOLD - + "\nTesting model on input of 5 smiles given by 'example' command...\n" - + RESET + self.checks = CheckService( + self.logger, + self.model_id, + self._dest_dir, + self.dir, + self.ios, + self.type ) - self.logger.debug("This is the input: {0}".format(input)) - mdl = ErsiliaModel(self.model_id, service_class=service_class, config_json=None) - result = mdl.run(input=input, output=output, batch_size=100) - if output is not None: - self.example_input = True - click.echo(BOLD + "Test: 5 SMILES input, SUCCESS! ✅\n") - else: - self._print_output(result, output) - - """ - Gets an example input of 5 smiles using the 'example' command, and then runs this same input on the - model twice. Then, it checks if the outputs are consistent or not and specifies that to the user. If - it is not consistent, an InconsistentOutput error is raised. Lastly, it makes sure that the number of - outputs equals the number of inputs. - """ - - - - - - - @throw_ersilia_exception() - def check_consistent_output(self): - def compute_rmse(y_true, y_pred): - squared_errors = [(yt - yp) ** 2 for yt, yp in zip(y_true, y_pred)] - mse = sum(squared_errors) / len(squared_errors) - return mse - - - self.logger.debug(BOLD + "\nConfirming model produces consistent output..." + RESET) - - session = Session(config_json=None) - service_class = session.current_service_class() - - eg = ExampleGenerator(model_id=self.model_id) - input = eg.example(n_samples=NUM_SAMPLES, file_name=None, simple=True, try_predefined=False) - - mdl1 = ErsiliaModel(self.model_id, service_class=service_class, config_json=None) - mdl2 = ErsiliaModel(self.model_id, service_class=service_class, config_json=None) - result = mdl1.run(input=input, output=None, batch_size=100) - result2 = mdl2.run(input=input, output=None, batch_size=100) - - zipped = list(zip(result, result2)) - - for item1, item2 in zipped: - output1 = item1["output"] - output2 = item2["output"] - - keys1 = list(output1.keys()) - keys2 = list(output2.keys()) - - for key1, key2 in zip(keys1, keys2): - if not isinstance(output1[key1], type(output2[key2])): - for item1, item2 in zipped: - self.logger.debug(item1) - self.logger.debug(item2) - self.logger.debug("\n") - raise texc.InconsistentOutputTypes(self.model_id) - - if output1[key1] is None: - continue - - elif isinstance(output1[key1], (float, int)): - # Calculate RMSE - rmse = compute_rmse([output1[key1]], [output2[key2]]) - self.logger.debug(f"RMSE for {key1}: {rmse}") - if rmse > 0.1: # Adjust the threshold as needed - self.logger.debug( - BOLD - + "\nBash run and Ersilia run produce inconsistent results (Root Mean Square Error difference exceeds 10%)." - + RESET - ) - raise texc.InconsistentOutputs(self.model_id) - - # Calculate Spearman's correlation - rho, p_value = spearmanr([output1[key1]], [output2[key2]]) - self.logger.debug(f"Spearman's correlation for {key1}: {rho}") - if rho < 0.5: # Adjust the threshold as needed - self.logger.debug( - BOLD - + "\nBash run and Ersilia run produce inconsistent results (Spearman's correlation below threshold)." - + RESET - ) - raise texc.InconsistentOutputs(self.model_id) - - elif isinstance(output1[key1], list): - ls1 = output1[key1] - ls2 = output2[key2] - - # Calculate rmse for lists - rmse = compute_rmse(ls1, ls2) - self.logger.debug(f"RMSE for {key1}: {rmse}") - if rmse > 0.1: # Adjust the threshold as needed - self.logger.debug( - BOLD - + "\nBash run and Ersilia run produce inconsistent results (Root Mean Square Error exceeded threshold of 10% for list)." - + RESET - ) - raise texc.InconsistentOutputs(self.model_id) - - # Calculate Spearman's correlation for lists - rho, p_value = spearmanr(ls1, ls2) - self.logger.debug(f"Spearman's correlation for {key1}: {rho}") - if rho < 0.5: # Adjust the threshold as needed - self.logger.debug( - BOLD - + "\nBash run and Ersilia run produce inconsistent results (Spearman's correlation below threshold for list)." - + RESET - ) - raise texc.InconsistentOutputs(self.model_id) - # Check String Outputs - else: - if self._compare_output_strings(output1[key1], output2[key2]) <= 95: - self.logger.debug(f"output1 value: {output1[key1]}") - self.logger.debug(f"output2 value: {output2[key2]}") - raise texc.InconsistentOutputs(self.model_id) - self.consistent_output = True - click.echo( - BOLD + "Test: Output Consistency, SUCCESS! ✅\n" + RESET - ) - - self.logger.debug( - BOLD + "\nConfirming there are same number of outputs as inputs..." + RESET + self.runner = RunnerService( + self.model_id, + self.logger, + self.ios, + self.checks, + self._model_path, + self.env, + self.type, + self.level, + self.dir ) - self.logger.debug(f"Number of inputs: {NUM_SAMPLES}") - self.logger.debug(f"Number of outputs: {len(zipped)}") - - if NUM_SAMPLES != len(zipped): - self.logger.debug("Number of inputs: {NUM_SAMPLES}") - self.logger.debug("Number of outputs: {len(zipped)}") - raise texc.MissingOutputs() - else: - click.echo(BOLD + "\nTest: Equal Inputs and Outputs, SUCCESS! ✅\n" + RESET) - - @staticmethod - def default_env(): - if "CONDA_DEFAULT_ENV" in os.environ: - return os.environ["CONDA_DEFAULT_ENV"] - else: - return BASE - - @staticmethod - def conda_prefix(is_base): - o = run_command_check_output("which conda").rstrip() - if o: - o = os.path.abspath(os.path.join(o, "..", "..")) - return o - if is_base: - o = run_command_check_output("echo $CONDA_PREFIX").rstrip() - return o - else: - o = run_command_check_output("echo $CONDA_PREFIX_1").rstrip() - return o - - def is_base(self): - default_env = self.default_env() - if default_env == "base": - return True - else: - return False - - - def _compare_string_similarity(self, str1, str2, similarity_threshold): - similarity = fuzz.ratio(str1, str2) - return similarity >= similarity_threshold - - - @staticmethod - def get_directory_size_without_symlinks(directory): - if directory is None: - return 0, {}, {} - if not os.path.exists(directory): - return 0, {}, {} - - total_size = 0 - file_types = defaultdict(int) - file_sizes = defaultdict(int) - - for dirpath, dirnames, filenames in os.walk(directory): - for filename in filenames: - filepath = os.path.join(dirpath, filename) - if not os.path.islink(filepath): - size = os.path.getsize(filepath) - total_size += size - file_extension = os.path.splitext(filename)[1] - file_types[file_extension] += 1 - file_sizes[file_extension] += size - - return total_size, file_types, file_sizes - - - @staticmethod - def get_directory_size_with_symlinks(directory): - if directory is None: - return 0, {}, {} - if not os.path.exists(directory): - return 0, {}, {} - - total_size = 0 - file_types = defaultdict(int) - file_sizes = defaultdict(int) - - for dirpath, dirnames, filenames in os.walk(directory): - for filename in filenames: - filepath = os.path.join(dirpath, filename) - size = os.path.getsize(filepath) - total_size += size - file_extension = os.path.splitext(filename)[1] - file_types[file_extension] += 1 - file_sizes[file_extension] += size - return total_size, file_types, file_sizes - - # Get location of conda environment - def _get_environment_location(self): - conda = SimpleConda() - python_path = conda.get_python_path_env(environment=self.model_id) - env_dir = os.path.dirname(python_path).split("/") - env_dir = "/".join(env_dir[:-1]) - return env_dir - - - @throw_ersilia_exception() - def get_directories_sizes(self): - self.logger.debug(BOLD + "Calculating model size... " + RESET) - def log_file_analysis(size, file_types, file_sizes, label): - self.logger.debug(f"Analyzing files in {label}:") - self.logger.debug(f"File types & counts: {dict(file_types)}") - self.logger.debug(f"Total size: {size} bytes") - dest_dir = self._model_path(model_id=self.model_id) - bundle_dir = self._get_bundle_location(model_id=self.model_id) - bentoml_dir = self._get_bentoml_location(model_id=self.model_id) - env_dir = self._get_environment_location() - dest_size, dest_file_types, dest_file_sizes = self.get_directory_size_without_symlinks(dest_dir) - bundle_size, bundle_file_types, bundle_file_sizes = self.get_directory_size_without_symlinks(bundle_dir) - bentoml_size, bentoml_file_types, bentoml_file_sizes = self.get_directory_size_without_symlinks(bentoml_dir) - env_size, env_file_types, env_file_sizes = self.get_directory_size_with_symlinks(env_dir) - - log_file_analysis(dest_size, dest_file_types, dest_file_sizes, "dest_dir") - log_file_analysis(bundle_size, bundle_file_types, bundle_file_sizes, "bundle_dir") - log_file_analysis(bentoml_size, bentoml_file_types, bentoml_file_sizes, "bentoml_dir") - log_file_analysis(env_size, env_file_types, env_file_sizes, "env_dir") - - model_size = dest_size + bundle_size + bentoml_size + env_size - self.model_size = model_size - size_kb = model_size / 1024 - size_mb = size_kb / 1024 - size_gb = size_mb / 1024 - - self.logger.debug(BOLD + "Model Size Breakdown:" + RESET) - self.logger.debug(f"KB: {size_kb}") - self.logger.debug(f"MB: {size_mb}") - self.logger.debug(f"GB: {size_gb}") - - self.logger.debug("Sizes of directories:") - self.logger.debug(f"dest_size: {dest_size} bytes") - self.logger.debug(f"bundle_size: {bundle_size} bytes") - self.logger.debug(f"bentoml_size: {bentoml_size} bytes") - self.logger.debug(f"env_size: {env_size} bytes") - return { - "dest_size": dest_size, - "bundle_size": bundle_size, - "bentoml_size": bentoml_size, - "env_size": env_size - } - - - @throw_ersilia_exception() - def run_bash(self): - def updated_read_csv(self, file_path, ersilia_flag = False): - data = [] - with open(file_path, "r") as file: - lines = file.readlines() - headers = lines[0].strip().split(",") - if ersilia_flag: - headers = headers[2:] - - print("\n", "\n") - - for line in lines[1:]: - self.logger.debug(f"Processing line: {line}") - values = line.strip().split(",") - if ersilia_flag: - selected_values = values[2:] - else: - selected_values = values - self.logger.debug(f"Selected Values: {selected_values} and their type {self._output_type}") - - if self._output_type == ["Float"]: - selected_values = [float(x) for x in selected_values] - self.logger.debug(f"Converted to floats: {selected_values}") - elif self._output_type == ["Integer"]: - selected_values = [int(x) for x in selected_values] - self.logger.debug(f"Converted to integers: {selected_values}") - else: - self.logger.debug(f"Unknown type, keeping as strings: {selected_values}") - - row_data = dict(zip(headers, selected_values)) - self.logger.debug(f"Appending row data: {row_data}") - data.append(row_data) - - return data - - with tempfile.TemporaryDirectory() as temp_dir: - self.logger.debug(BOLD + "\nRunning the model bash script..." + RESET) - model_path = os.path.join(EOS, "dest", self.model_id) - - # Create an example input - eg = ExampleGenerator(model_id=self.model_id) - input = eg.example(n_samples=NUM_SAMPLES, file_name=None, simple=True, try_predefined=False) - # Read it into a temp file - ex_file = os.path.abspath(os.path.join(temp_dir, "example_file.csv")) - - - with open(ex_file, "w") as f: - f.write("smiles") - for item in input: - f.write(str(item) + "\n") - - run_sh_path = os.path.join(model_path, "model", "framework", "run.sh") - run_sh_path = os.path.join(model_path, "model", "framework", "run.sh") - # Halt this check if the run.sh file does not exist (e.g. eos3b5e) - if not os.path.exists (run_sh_path): - self.logger.debug( - BOLD + "\n ⛔️ Check halted. Either run.sh file does not exist, or model was not fetched via --from_github or --from_s3. ⛔️" + RESET - ) - return - - # Navigate into the temporary directory - - subdirectory_path = os.path.join(model_path, "model", "framework") - self.logger.debug(f"Changing directory to: {subdirectory_path}") - os.chdir(subdirectory_path) - try: - run_path = os.path.abspath(subdirectory_path) - tmp_script = os.path.abspath(os.path.join(temp_dir, "script.sh")) - bash_output_path = os.path.abspath(os.path.join(temp_dir, "bash_output.csv")) - output_log = os.path.abspath(os.path.join(temp_dir, "output.txt")) - error_log = os.path.abspath(os.path.join(temp_dir, "error.txt")) - bash_script = """ - source {0}/etc/profile.d/conda.sh - conda activate {1} - cd {2} - bash run.sh . {3} {4} > {5} 2> {6} - conda deactivate - """.format( - self.conda_prefix(self.is_base()), - self.model_id, - run_path, - ex_file, - bash_output_path, - output_log, - error_log, - ) - self.logger.debug(f"Script path: {tmp_script}") - self.logger.debug(f"bash output path: {bash_output_path}") - self.logger.debug(f"Output log path: {output_log}") - self.logger.debug(f"Error log path: {error_log}") - with open(tmp_script, "w") as f: - f.write(bash_script) - - self.logger.debug(BOLD + "\nExecuting 'bash run.sh'..." + RESET) - try: - bash_result = subprocess.run( - ["bash", tmp_script], capture_output=True, text=True, check=True - ) - except subprocess.CalledProcessError as e: - self.logger.debug(f"STDOUT: {e.stdout}") - raise RuntimeError("Error encountered while running the bash script.") from e - - if os.path.exists(bash_output_path): - with open(bash_output_path, "r") as bash_output_file: - output_content = bash_output_file.read() - self.logger.debug(BOLD + "\nBash execution completed!" + RESET) - self.logger.debug("Captured Raw Bash Output:") - self.logger.debug(output_content) - else: - click.echo(BOLD + "\nWARNING: Bash output file not found when reading the path:"+RESET) - self.logger.debug(bash_output_path) - click.echo(BOLD + "\n Ersilia and Bash comparison will raise an error" + RESET) - self.logger.debug(f"Generating bash script content for debugging: {tmp_script}\n") - - with open(error_log, "r") as error_file: - error_content = error_file.read() - self.logger.debug(BOLD + "\nCaptured Error:" + RESET) - if error_content == "": - click.echo("No errors on bash run found 😄 ✅\n") - self.run_using_bash = True # bash run was successful - else: - self.logger.debug(error_content) - - except Exception as e: - raise RuntimeError(f"Error while activating the conda environment: {e}") - - self.logger.debug(BOLD + "\nExecuting ersilia run..."+RESET) - ersilia_output_path = os.path.abspath(os.path.join(temp_dir, "ersilia_output.csv")) - self.logger.debug(f"Ersilia output will be written to: {ersilia_output_path}") - - session = Session(config_json=None) - service_class = session.current_service_class() - mdl = ErsiliaModel( - self.model_id, service_class=service_class, config_json=None - ) - result = mdl.run(input=ex_file, output=ersilia_output_path, batch_size=100) - - - - if os.path.exists(ersilia_output_path): - with open(ersilia_output_path, "r") as ersilia_output_file: - output_content = ersilia_output_file.read() - click.echo("No errors on Ersilia run found 😄 ✅") - self.logger.debug("Captured Raw Ersilia Output:") - self.logger.debug(output_content) - else: - self.logger.debug(BOLD+f"Ersilia output file not found from the path: {ersilia_output_path}"+RESET) - self.logger.debug("Processing ersilia csv output...") - ersilia_run = updated_read_csv(self, ersilia_output_path, True) - - - with open(bash_output_path, "r") as bash_output_file: - output_content = bash_output_file.read() - self.logger.debug("Captured Raw Bash Output:") - self.logger.debug(output_content) - self.logger.debug("Processing raw bash output...: ") - bash_run = updated_read_csv(self, bash_output_path, False) - self.logger.debug(f"\nBash output:\n {bash_run}") - self.logger.debug(f"\nErsilia output:\n {ersilia_run}") - - # Select common columns for comparison - ersilia_columns = set() - for row in ersilia_run: - ersilia_columns.update(row.keys()) - self.logger.debug(f"\n Ersilia columns: {ersilia_columns}") - - bash_columns = set() - for row in bash_run: - bash_columns.update(row.keys()) - self.logger.debug(f"\n Bash columns: {bash_columns}") - - common_columns = ersilia_columns & bash_columns - def compute_rmse(y_true, y_pred): - squared_errors = [(yt - yp) ** 2 for yt, yp in zip(y_true, y_pred)] - mse = sum(squared_errors) / len(squared_errors) - return mse - - idx = 1 - self.logger.debug(BOLD + "\nComparing outputs from Ersilia and Bash runs..." + RESET) - for column in common_columns: - for i in range(len(ersilia_run)): - if isinstance(ersilia_run[i][column], (float, int)) and isinstance( - bash_run[i][column], (float, int) - ): - values1 = [row[column] for row in ersilia_run] - values2 = [row[column] for row in bash_run] - rmse = compute_rmse(values1, values2) - self.logger.debug(f"Root Mean Square Error for {column}: {rmse}") - if rmse > 0.10: - self.logger.debug( - BOLD - + "\nBash run and Ersilia run produce inconsistent results (Root Mean Square Error difference exceeds 10%)." - + RESET - ) - self.logger.debug(f"Values that raised error: {values1}, {values2}") - raise texc.InconsistentOutputs(self.model_id) - rho, p_value = spearmanr(values1, values2) - self.logger.debug(f"Spearman's correlation for {column}: {rho}\n") - if rho < 0.5: # Adjust the threshold as needed - self.logger.debug( - BOLD - + "\nBash run and Ersilia run produce inconsistent results (Spearman's correlation below threshold)." - + RESET - ) - raise texc.InconsistentOutputs(self.model_id) - # Both instances are strings - elif isinstance(ersilia_run[i][column], str) and isinstance( - bash_run[i][column], str - ): - if not all( - self._compare_string_similarity(a, b, 95) - for a, b in zip(ersilia_run[i][column], bash_run[i][column]) - ): - self.logger.debug( - BOLD - + "\nBash run and Ersilia run produce inconsistent results." - + RESET - ) - self.logger.debug(f"Error in the following column: {column}") - self.logger.debug(ersilia_run[i][column]) - self.logger.debug(bash_run[i][column]) - raise texc.InconsistentOutputs(self.model_id) - elif isinstance(ersilia_run[i][column], bool) and isinstance( - ersilia_run[i][column], bool - ): - if not ersilia_run[i][column].equals(bash_run[i][column]): - self.logger.debug( - BOLD - + "\nBash run and Ersilia run produce inconsistent results." - + RESET - ) - self.logger.debug("Error in the following column: ", column) - self.logger.debug(ersilia_run[i][column]) - self.logger.debug(bash_run[i][column]) - raise texc.InconsistentOutputs(self.model_id) - - click.echo( - BOLD - + "Test: Bash and Ersilia run comparison check, SUCCESS! ✅ Test Complete 🎉!" - + RESET - ) - - """ - writes to the .json file all the basic information received from the test module: - - size of the model - - did the basic checks pass? True or False - - time to run the model - - did the single input run without error? True or False - - did the run bash run without error? True or False - - did the example input run without error? True or False - - are the outputs consistent? True or False - """ - - def make_output(self, output, time): - size_kb = self.model_size / 1024 - size_mb = size_kb / 1024 - size_gb = size_mb / 1024 - - timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - - - data = { - "date and time run": timestamp, # Add date and time field - "model size": {"KB": size_kb, "MB": size_mb, "GB": size_gb}, - "time to run tests (seconds)": time, - "basic checks passed": self.information_check, - "single input run without error": self.single_input, - "example input run without error": self.example_input, - "outputs consistent": self.consistent_output, - "bash run without error": self.run_using_bash, - } - with open(output, "w") as json_file: - json.dump(data, json_file, indent=4) - def run(self, output_file): - if MISSING_PACKAGES: - raise ImportError( - "Missing packages required for testing, please install test extras as 'pip install ersilia[test]'" - ) - output_file = os.path.join(self._model_path(self.model_id), "TEST_MODULE_OUTPUT.csv") - start = time.time() - self.check_information(output_file) - self.check_single_input(output_file) - self.check_example_input(output_file) - self.check_consistent_output() - self.get_directories_sizes() - self.get_directories_sizes() - self.run_bash() - end = time.time() - seconds_taken = end - start - - - if output_file: - self.make_output(output_file, seconds_taken) - click.echo(f"📁 The output file is located at: {output_file}") - else: - click.echo("No output file specified! Skipping output file generation.") + def run(self, output_file=None): + self.runner.run(output_file) \ No newline at end of file diff --git a/ersilia/publish/test_services.py b/ersilia/publish/test_services.py new file mode 100644 index 000000000..fa566fa9c --- /dev/null +++ b/ersilia/publish/test_services.py @@ -0,0 +1,933 @@ +# TODO adapt to input-type agnostic. For now, it works only with Compound input types. +import json +import os +import csv +import subprocess +import tempfile +import asyncio +import time +import click +import types +from collections import defaultdict +from datetime import datetime + +from ersilia.utils.conda import SimpleConda +from .. import ErsiliaModel, throw_ersilia_exception +from ..hub.fetch.fetch import ModelFetcher +from ..cli import echo +from ..core.session import Session +from ..default import EOS, INFORMATION_FILE +from ..io.input import ExampleGenerator +from ..utils.exceptions_utils import test_exceptions as texc +from ..utils.terminal import run_command_check_output + +# Check if we have the required imports in the environment +MISSING_PACKAGES = False +try: + from scipy.stats import spearmanr + from fuzzywuzzy import fuzz + from rich.console import Console + from rich.table import Table + from rich.box import SIMPLE +except ImportError: + MISSING_PACKAGES = True + +RUN_FILE = "run.sh" +DATA_FILE = "data.csv" +NUM_SAMPLES = 5 +BOLD = "\033[1m" +RESET = "\033[0m" +BASE = "base" + +TEST_MESSAGES = { + "pkg_err": "Missing packages required for testing. \ + Please install test extras with 'pip install ersilia[test]'." +} + +class IOService: + def __init__( + self, logger, + dest_dir, + model_path, + bundle_path, + bentoml_path, + model_id, + env, + type + ): + self.logger = logger + self.model_id = model_id + self.env = env if env is not None else BASE + self.model_size = 0 + self.console = Console() + self.check_results = [] + self.type = type + self._model_path = model_path + self._bundle_path = bundle_path + self._bentoml_path = bentoml_path + self._dest_dir = dest_dir + + + def _run_check( + self, + check_function, + data, + check_name, + additional_info=None + ): + + try: + if additional_info is not None: + check_function(additional_info) + else: + check_function(data) + self.check_results.append((check_name, "[green]✔[/green]")) + return True + except Exception as e: + self.logger.error(f"Check '{check_name,additional_info}' failed: {e}") + self.check_results.append((check_name, "[red]✖[/red]")) + return False + + def _generate_table(self, title, headers, rows): + table = Table( + title=title, + border_style="#FFC0CB" + ) + for header in headers: + table.add_column( + header, + justify="center", + no_wrap=True, + width=20 + ) + + for row in rows: + table.add_row(*[str(cell) for cell in row]) + # render + self.console.print(table) + + def _get_file_requirements(self): + + if self.type.lower() == "bentoml": + return [ + "src/service.py", + "pack.py", + "Dockerfile", + "metadata.json", + "README.md", + "model/framework/run.sh", + "LICENSE", + ] + elif self.type.lower() == "ersilia": + return [ + "install.yml", + "metadata.json", + "metadata.yml", + "model/framework/example/input.csv", + "model/framework/example/output.csv", + "README.md", + "model/framework/run.sh", + "LICENSE", + ] + else: + raise ValueError(f"Unsupported model type: {self.type}") + + + def _get_environment_location(self): + conda = SimpleConda() + python_path = conda.get_python_path_env(environment=self.env) + return os.path.dirname(os.path.dirname(python_path)) + + def read_information(self): + file = os.path.join( + self._dest_dir, + self.model_id, + INFORMATION_FILE + ) + self.logger.info(f"Dest: {self._dest_dir}") + if not os.path.exists(file): + raise FileNotFoundError(f"Information file does not exist for model {self.model_id}") + with open(file, "r") as f: + return json.load(f) + + def set_model_size(self, directory): + return sum( + os.path.getsize(os.path.join(dirpath, filename)) + for dirpath, _, filenames in os.walk(directory) + for filename in filenames + ) + + def print_output(self, result, output): + def write_output(data): + if output is not None: + with open(output.name, "w") as file: + json.dump(data, file) + else: + print(json.dumps(data, indent=4)) + + if isinstance(result, types.GeneratorType): + for r in result: + write_output(r if r is not None else "Something went wrong") + else: + print(result) + + def _get_environment_location(self): + conda = SimpleConda() + python_path = conda.get_python_path_env(environment=BASE) + return os.path.dirname(os.path.dirname(python_path)) + + + @throw_ersilia_exception() + def get_directories_sizes(self): + self.logger.debug(BOLD + "Calculating model size... " + RESET) + + def format_size(size_in_bytes): + if size_in_bytes < 1024: + return f"{size_in_bytes}B" # Bytes + size_kb = size_in_bytes / 1024 + if size_kb < 1024: + return f"{size_kb:.2f}KB" # Kilobytes + size_mb = size_kb / 1024 + if size_mb < 1024: + return f"{size_mb:.2f}MB" # Megabytes + size_gb = size_mb / 1024 + return f"{size_gb:.2f}GB" # Gigabytes + + def get_directory_size(directory, include_symlinks=True): + if not directory or not os.path.exists(directory): + return 0, defaultdict(int), defaultdict(int) + + _types, _sizes, total_sz = defaultdict(int), defaultdict(int), 0 + + for dirpath, _, filenames in os.walk(directory): + for filename in filenames: + filepath = os.path.join(dirpath, filename) + + if include_symlinks or not os.path.islink(filepath): + try: + size = os.path.getsize(filepath) + total_sz += size + file_extension = os.path.splitext(filename)[1] + _types[file_extension] += 1 + _sizes[file_extension] += size + except OSError as e: + self.logger.warning(f"Failed to access file {filepath}: {e}") + + return total_sz, _types, _sizes + + directories = { + "dest_dir": (self._model_path(model_id=self.model_id), False), + "bundle_dir": (self._bundle_path(model_id=self.model_id), False), + "bentoml_dir": (self._bentoml_path(model_id=self.model_id), False), + "env_dir": (self._get_environment_location(), True), + } + + total_size = 0 + directory_sizes = {} + + for label, (directory, include_symlinks) in directories.items(): + size, file_types, _ = get_directory_size( + directory=directory, + include_symlinks=include_symlinks + ) + formatted_size = format_size(size) + directory_sizes[label] = formatted_size + total_size += size + + total_size_formatted = format_size(total_size) + + self.logger.debug(BOLD + "Model Size Breakdown:" + RESET) + self.logger.debug(f"Total: {total_size_formatted}") + + self.logger.debug("Sizes of directories:") + for label, size in directory_sizes.items(): + self.logger.debug(f"{label}: {size}") + + self.model_size = total_size_formatted + return directory_sizes + + +class CheckService: + def __init__( + self, + logger, + model_id, + dest_dir, + dir, + ios, + type + ): + self.logger = logger + self.model_id = model_id + self._dest_dir = dest_dir + self.dir = dir + self.type = type + self._run_check = ios._run_check + self._generate_table = ios._generate_table + self._get_file_requirements = ios._get_file_requirements + self.console = ios.console + self.check_results = ios.check_results + self.information_check = False + self.single_input = False + self.example_input = False + self.consistent_output = False + + def _check_file_existence(self, file_path): + + if file_path == "metadata.json" and "ersilia" in self.type.lower(): + json_exists = os.path.exists(os.path.join(self.dir, "metadata.json")) + yaml_exists = os.path.exists(os.path.join(self.dir, "metadata.yml")) + if not (json_exists or yaml_exists): + raise FileNotFoundError(f"Neither 'metadata.json' nor 'metadata.yml' found.") + else: + if not os.path.exists(os.path.join(self.dir, file_path)): + raise FileNotFoundError(f"File '{file_path}' does not exist.") + + def check_files(self): + + self.logger.debug(f"Checking file requirements for {self.type} model.") + fr = self._get_file_requirements() + + for fp in fr: + self.logger.debug(f"Checking file: {fp}") + self._run_check( + self._check_file_existence, + None, + f"File: {fp}", + fp + ) + + def _check_model_id(self, data): + self.logger.debug("Checking model ID...") + if data["card"]["Identifier"] != self.model_id: + raise texc.WrongCardIdentifierError(self.model_id) + + + def _check_model_slug(self, data): + self.logger.debug("Checking model slug...") + if not data["card"]["Slug"]: + raise texc.EmptyField("slug") + + + def _check_model_description(self, data): + self.logger.debug("Checking model description...") + if not data["card"]["Description"]: + raise texc.EmptyField("Description") + + def _check_model_task(self, data): + self.logger.debug("Checking model task...") + valid_tasks = { + "Classification", + "Regression", + "Generative", + "Representation", + "Similarity", + "Clustering", + "Dimensionality reduction", + } + + raw_tasks = data.get("card", {}).get("Task", "") + if isinstance(raw_tasks, str): + tasks = [ + task.strip() + for task + in raw_tasks.split(",") + if task.strip() + ] + elif isinstance(raw_tasks, list): + tasks = [ + task.strip() + for task + in raw_tasks + if isinstance(task, str) and task.strip() + ] + else: + raise texc.InvalidEntry( + "Task", + message="Task field must be a string or list." + ) + + if not tasks: + raise texc.InvalidEntry( + "Task", + message="Task field is missing or empty." + ) + + invalid_tasks = [task for task in tasks if task not in valid_tasks] + if invalid_tasks: + raise texc.InvalidEntry( + "Task", message=f"Invalid tasks: {', '.join(invalid_tasks)}" + ) + + self.logger.debug("All tasks are valid.") + + + + def _check_model_input(self, data): + self.logger.debug("Checking model input...") + valid_inputs = [{"Compound"}, {"Protein"}, {"Text"}] + if set(data["card"]["Input"]) not in valid_inputs: + raise texc.InvalidEntry("Input") + + def _check_model_input_shape(self, data): + self.logger.debug("Checking model input shape...") + valid_input_shapes = { + "Single", + "Pair", + "List", + "Pair of Lists", + "List of Lists" + } + if data["card"]["Input Shape"] not in valid_input_shapes: + raise texc.InvalidEntry("Input Shape") + + + def _check_model_output(self, data): + self.logger.debug("Checking model output...") + valid_outputs = { + "Boolean", + "Compound", + "Descriptor", + "Distance", + "Experimental value", + "Image", + "Other value", + "Probability", + "Protein", + "Score", + "Text", + } + + raw_outputs = data.get("card", {}).get("Output", "") + if isinstance(raw_outputs, str): + outputs = [ + output.strip() + for output + in raw_outputs.split(",") + if output.strip() + ] + elif isinstance(raw_outputs, list): + outputs = [ + output.strip() + for output + in raw_outputs + if isinstance(output, str) and output.strip() + ] + else: + raise texc.InvalidEntry( + "Output", + message="Output field must be a string or list." + ) + + if not outputs: + raise texc.InvalidEntry( + "Output", + message="Output field is missing or empty." + ) + + invalid_outputs = [ + output + for output + in outputs + if output not in valid_outputs + ] + if invalid_outputs: + raise texc.InvalidEntry( + "Output", + message=f"Invalid outputs: {', '.join(invalid_outputs)}" + ) + + self.logger.debug("All outputs are valid.") + + + def _check_model_output_type(self, data): + self.logger.debug("Checking model output type...") + valid_output_types = [{"String"}, {"Float"}, {"Integer"}] + if set(data["card"]["Output Type"]) not in valid_output_types: + raise texc.InvalidEntry("Output Type") + + def _check_model_output_shape(self, data): + self.logger.debug("Checking model output shape...") + valid_output_shapes = { + "Single", + "List", + "Flexible List", + "Matrix", + "Serializable Object" + } + if data["card"]["Output Shape"] not in valid_output_shapes: + raise texc.InvalidEntry("Output Shape") + + + + @throw_ersilia_exception() + def check_information(self, output): + self.logger.debug("Checking that model information is correct") + self.logger.debug( + BOLD + f"Beginning checks for {self.model_id} model information:" + RESET + ) + file = os.path.join( + self._dest_dir, + self.model_id, + INFORMATION_FILE + ) + with open(file, "r") as f: + data = json.load(f) + + self._run_check(self._check_model_id, data, "Model ID") + self._run_check(self._check_model_slug, data, "Model Slug") + self._run_check(self._check_model_description, data, "Model Description") + self._run_check(self._check_model_task, data, "Model Task") + self._run_check(self._check_model_input, data, "Model Input") + self._run_check(self._check_model_input_shape, data, "Model Input Shape") + self._run_check(self._check_model_output, data, "Model Output") + self._run_check(self._check_model_output_type, data, "Model Output Type") + self._run_check(self._check_model_output_shape, data, "Model Output Shape") + + if output is not None: + self.information_check = True + + @throw_ersilia_exception() + def check_single_input(self, output, fetch_and_serve, run_model): + + input_smiles = "COc1ccc2c(NC(=O)Nc3cccc(C(F)(F)F)n3)ccnc2c1" + + self.logger.debug(BOLD + "Testing model on single smiles input...\n" + RESET) + fetch_and_serve() + result = run_model( + input=input_smiles, + output=output, + batch=100 + ) + + if output is not None: + self.single_input = True + else: + self._print_output(result, output) + + @throw_ersilia_exception() + def check_example_input(self, output, run_model, run_example): + input_samples = run_example( + n_samples=NUM_SAMPLES, + file_name=None, + simple=True, + try_predefined=False + ) + + self.logger.debug( + BOLD + "\nTesting model on input of 5 smiles given by 'example' command...\n" + RESET + ) + self.logger.debug(f"This is the input: {input_samples}") + + result = run_model( + input=input_samples, + output=output, + batch=100 + ) + + if output is not None: + self.example_input = True + else: + self._print_output(result, output) + + @throw_ersilia_exception() + def check_consistent_output(self, run_example, run_model): + def compute_rmse(y_true, y_pred): + return sum((yt - yp) ** 2 for yt, yp in zip(y_true, y_pred)) ** 0.5 / len(y_true) + def _compare_output_strings(output1, output2): + if output1 is None and output2 is None: + return 100 + else: + return fuzz.ratio(output1, output2) + def validate_output(output1, output2): + if not isinstance(output1, type(output2)): + raise texc.InconsistentOutputTypes(self.model_id) + + if output1 is None: + return + + if isinstance(output1, (float, int)): + rmse = compute_rmse([output1], [output2]) + if rmse > 0.1: + raise texc.InconsistentOutputs(self.model_id) + + rho, _ = spearmanr([output1], [output2]) + if rho < 0.5: + raise texc.InconsistentOutputs(self.model_id) + + elif isinstance(output1, list): + rmse = compute_rmse(output1, output2) + if rmse > 0.1: + raise texc.InconsistentOutputs(self.model_id) + + rho, _ = spearmanr(output1, output2) + if rho < 0.5: + raise texc.InconsistentOutputs(self.model_id) + + elif isinstance(output1, str): + if _compare_output_strings(output1, output2) <= 95: + raise texc.InconsistentOutputs(self.model_id) + + def read_csv(file_path): + if not os.path.exists(file_path): + raise FileNotFoundError(f"File not found: {file_path}") + with open(file_path, mode='r') as csv_file: + reader = csv.DictReader(csv_file) + return [row for row in reader] + + def is_json(data): + try: + if isinstance(data, str): + json.loads(data) + elif isinstance(data, list): + json.dumps(data) + return True + except (ValueError, TypeError): + return False + + + + self.logger.debug(BOLD + "\nConfirming model produces consistent output..." + RESET) + + input_samples = run_example( + n_samples=NUM_SAMPLES, + file_name=None, + simple=True, + try_predefined=False + ) + result1 = run_model( + input=input_samples, + output="output1.csv", + batch=100 + ) + result2 = run_model( + input=input_samples, + output="output2.csv", + batch=100 + ) + + if is_json(result1) and is_json(result2): + data1, data2 = result1, result2 + else: + data1 = read_csv("output1.csv") + data2 = read_csv("output2.csv") + + for res1, res2 in zip(data1, data2): + for key in res1: + if key in res2: + validate_output(res1[key], res2[key]) + else: + raise KeyError(f"Key '{key}' not found in second result.") + + self.consistent_output = True + +class RunnerService: + def __init__( + self, + model_id, + logger, + ios_service, + checkup_service, + model_path, + env, + type, + level, + dir + ): + self.model_id = model_id + self.logger = logger + self.ios_service = ios_service + self.console = ios_service.console + self.checkup_service = checkup_service + self._model_path = model_path + self.env = env + self.type = type + self.level = level + self.dir = dir + self._output_type = self.ios_service.read_information()["card"]["Output Type"] + session = Session(config_json=None) + service_class = session.current_service_class() + self.model = ErsiliaModel( + self.model_id, + service_class=service_class + ) + self.example = ExampleGenerator(model_id=self.model_id) + self.fetcher = ModelFetcher(repo_path=self.dir) + self.run_using_bash = False + + def run_model(self, input, output, batch): + return self.model.run( + input=input, + output=output, + batch_size=batch + ) + def fetch_and_serve(self): + asyncio.run(self.fetcher.fetch(self.model_id + )) + self.model.serve() + def serve_model(self, input, output, batch): + return self.model.run( + input=input, + output=output, + batch_size=batch + ) + def run_exampe( + self, + n_samples, + file_name=None, + simple=True, + try_predefined=False + ): + return self.example.example( + n_samples=n_samples, + file_name=file_name, + simple=simple, + try_predefined=try_predefined + ) + @throw_ersilia_exception() + def run_bash(self): + def compute_rmse(y_true, y_pred): + return sum((yt - yp) ** 2 for yt, yp in zip(y_true, y_pred)) ** 0.5 / len(y_true) + + def compare_outputs(bsh_data, ers_data): + columns = set(bsh_data[0].keys()) & set(data[0].keys()) + self.logger.debug(f"Common columns: {columns}") + + for column in columns: + bv = [row[column] for row in bsh_data] + ev = [row[column] for row in ers_data] + + if all(isinstance(val, (int, float)) for val in bv + ev): + rmse = compute_rmse(bv, ev) + self.logger.debug(f"RMSE for {column}: {rmse}") + + if rmse > 0.1: + raise texc.InconsistentOutputs(self.model_id) + elif all(isinstance(val, str) for val in bv + ev): + if not all(self._compare_string_similarity(a, b, 95) for a, b in zip(bv, ev)): + raise texc.InconsistentOutputs(self.model_id) + + def read_csv(path, flag=False): + try: + with open(path, "r") as file: + lines = file.readlines() + + if not lines: + self.logger.error(f"File at {path} is empty.") + return [] + + headers = lines[0].strip().split(",") + if flag: + headers = headers[2:] + + data = [] + for line in lines[1:]: + self.logger.debug(f"Processing line: {line.strip()}") + values = line.strip().split(",") + values = values[2:] if flag else values + + try: + if self._output_type == ["Float"]: + _values = [float(x) for x in values] + elif self._output_type == ["Integer"]: + _values = [int(x) for x in values] + except ValueError as e: + self.logger.warning(f"Value conversion error: {e}") + data.append(dict(zip(headers, _values))) + + return data + except Exception as e: + raise RuntimeError(f"Failed to read CSV from {path}.") from e + + def run_subprocess(command, env_vars=None): + try: + result = subprocess.run( + command, + capture_output=True, + text=True, + check=True, + env=env_vars + ) + self.logger.debug(f"Subprocess output: {result.stdout}") + return result.stdout + except subprocess.CalledProcessError as e: + raise RuntimeError("Subprocess execution failed.") from e + + with tempfile.TemporaryDirectory() as temp_dir: + + model_path = os.path.join(self.dir) + temp_script_path = os.path.join(temp_dir, "script.sh") + bash_output_path = os.path.join(temp_dir, "bash_output.csv") + output_path = os.path.join(temp_dir, "ersilia_output.csv") + output_log_path = os.path.join(temp_dir, "output.txt") + error_log_path = os.path.join(temp_dir, "error.txt") + + input = self.run_exampe( + n_samples=NUM_SAMPLES, + file_name=None, + simple=True, + try_predefined=False + ) + + ex_file = os.path.join(temp_dir, "example_file.csv") + + with open(ex_file, "w") as example_file: + example_file.write("smiles\n" + "\n".join(map(str, input))) + + run_sh_path = os.path.join( + model_path, + "model", + "framework", + "run.sh" + ) + if not os.path.exists(run_sh_path): + self.logger.warning(f"run.sh not found at {run_sh_path}. Skipping bash run.") + return + + bash_script = f""" + source {self.conda_prefix(self.is_base())}/etc/profile.d/conda.sh + conda activate {BASE} + cd {os.path.dirname(run_sh_path)} + bash run.sh . {ex_file} {bash_output_path} > {output_log_path} 2> {error_log_path} + conda deactivate + """ + + with open(temp_script_path, "w") as script_file: + script_file.write(bash_script) + + self.logger.debug(f"Running bash script: {temp_script_path}") + run_subprocess(["bash", temp_script_path]) + + bsh_data = read_csv(bash_output_path) + + self.run_model( + ex_file, + output_path, + 100 + ) + data = read_csv(output_path, flag=True) + + compare_outputs(bsh_data, data) + + self.run_using_bash = True + + @staticmethod + def default_env(): + if "CONDA_DEFAULT_ENV" in os.environ: + return os.environ["CONDA_DEFAULT_ENV"] + else: + return BASE + + @staticmethod + def conda_prefix(is_base): + o = run_command_check_output("which conda").rstrip() + if o: + o = os.path.abspath(os.path.join(o, "..", "..")) + return o + if is_base: + o = run_command_check_output("echo $CONDA_PREFIX").rstrip() + return o + else: + o = run_command_check_output("echo $CONDA_PREFIX_1").rstrip() + return o + + def is_base(self): + default_env = self.default_env() + self.logger.debug(f"Default environment: {default_env}") + if default_env == "base": + return True + else: + return False + + + def _compare_string_similarity( + self, + str1, + str2, + threshold + ): + similarity = fuzz.ratio(str1, str2) + return similarity >= threshold + + + def make_output(self, time, model_size): + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + self.information_check = self.checkup_service.information_check + self.single_input = self.checkup_service.single_input + self.example_input = self.checkup_service.example_input + self.consistent_output = self.checkup_service.consistent_output + data = [ + ("Date and Time Run", timestamp), + ("Model Size (MB)", model_size), + ("Time to Run Tests (seconds)", time), + ("Basic Checks Passed", self.information_check), + ("Single Input Run Without Error", self.single_input), + ("Example Input Run Without Error", self.example_input), + ("Outputs Consistent", self.consistent_output), + ("Bash Run Without Error", self.run_using_bash), + ] + + headers = ["Check Type", "Status"] + + self.ios_service._generate_table("Test Run Summary", headers, data) + + def run(self, output_file=None): + if MISSING_PACKAGES: + raise ImportError(TEST_MESSAGES["pkg_err"]) + + if not output_file: + output_file = os.path.join( + self._model_path(self.model_id), "result.csv" + ) + + st = time.time() + try: + self.checkup_service.check_information(output_file) + self.ios_service._generate_table( + title="Model Information Checks", + headers=["Check", "Status"], + rows=self.ios_service.check_results + ) + self.ios_service.check_results.clear() + self.checkup_service.check_files() + self.ios_service._generate_table( + title="Model Information Checks", + headers=["Check", "Status"], + rows=self.ios_service.check_results + ) + ds = self.ios_service.get_directories_sizes() + self.ios_service._generate_table( + title="Model Directory Sizes", + headers=["Dest dir", "Env Dir"], + rows=[(ds["dest_dir"], ds["env_dir"])] + ) + if self.level == "deep": + self.checkup_service.check_single_input( + output_file, + self.fetch_and_serve, + self.run_model + ) + self.ios_service._generate_table( + title="Runner Checkup Status", + headers=["Runner", "Status"], + rows=[["Fetch", "[green]✔[/green]"], ["Serve", "[green]✔[/green]"], ["Run", "[green]✔[/green]"]] + ) + self.checkup_service.check_example_input( + output_file, + self.run_model, + self.run_exampe + ) + self.checkup_service.check_consistent_output( + self.run_exampe, + self.run_model + ) + model_size = self.ios_service.model_size + self.run_bash() + et = time.time() + elapsed = et - st + self.make_output(elapsed, model_size) + + except Exception as e: + click.echo(f"❌ An error occurred: {e}") + finally: + click.echo("🏁 Run process finished.") \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 0e82b9eed..9e0cf4ce8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,12 +62,13 @@ fuzzywuzzy = { version = "^0.18.0", optional = true } sphinx = { version = ">=6.0.0", optional = true } # for minimum version and support for Python 3.10 jinja2 = { version = "^3.1.2", optional = true } scipy = { version = "<=1.10.0", optional = true } +rich = { version = "*", optional = true } [tool.poetry.extras] # Instead of using poetry dependency groups, we use extras to make it pip installable lake = ["isaura"] docs = ["sphinx", "jinja2"] -test = ["pytest", "pytest-asyncio", "pytest-benchmark", "fuzzywuzzy", "scipy"] +test = ["pytest", "pytest-asyncio", "pytest-benchmark", "fuzzywuzzy", "scipy", "nox"] #all = [lake, docs, test] [tool.poetry.scripts] From 2802e2671164e2fcc5827290137a36056d96b495 Mon Sep 17 00:00:00 2001 From: Abellegese Date: Wed, 20 Nov 2024 23:31:36 +0300 Subject: [PATCH 3/7] Model Tester: An model testing CLI --- ersilia/publish/test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ersilia/publish/test.py b/ersilia/publish/test.py index 18c601fbe..0076a2f41 100644 --- a/ersilia/publish/test.py +++ b/ersilia/publish/test.py @@ -1,6 +1,5 @@ -from .test_services import IOService, CheckService, RunnerService +from test_services import IOService, CheckService, RunnerService from .. import ErsiliaBase -from ..utils.logging import logger class ModelTester(ErsiliaBase): From 7ff9ae6df8817fb9269286c5400f9ed530766fe2 Mon Sep 17 00:00:00 2001 From: Abellegese Date: Wed, 20 Nov 2024 23:38:55 +0300 Subject: [PATCH 4/7] Model Tester: An model testing CLI --- ersilia/publish/test.py | 11 ++++++++++- ersilia/publish/test_services.py | 5 ++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/ersilia/publish/test.py b/ersilia/publish/test.py index 0076a2f41..3344e3bbf 100644 --- a/ersilia/publish/test.py +++ b/ersilia/publish/test.py @@ -1,4 +1,13 @@ -from test_services import IOService, CheckService, RunnerService +import importlib + +def get_service_class(class_name): + module = importlib.import_module('ersilia.publish.test_services') + return getattr(module, class_name) + +IOService = get_service_class('IOService') +CheckService = get_service_class('CheckService') +RunnerService = get_service_class('RunnerService') + from .. import ErsiliaBase diff --git a/ersilia/publish/test_services.py b/ersilia/publish/test_services.py index fa566fa9c..9b0d9d214 100644 --- a/ersilia/publish/test_services.py +++ b/ersilia/publish/test_services.py @@ -850,7 +850,7 @@ def _compare_string_similarity( return similarity >= threshold - def make_output(self, time, model_size): + def make_output(self, time): timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") self.information_check = self.checkup_service.information_check self.single_input = self.checkup_service.single_input @@ -858,7 +858,6 @@ def make_output(self, time, model_size): self.consistent_output = self.checkup_service.consistent_output data = [ ("Date and Time Run", timestamp), - ("Model Size (MB)", model_size), ("Time to Run Tests (seconds)", time), ("Basic Checks Passed", self.information_check), ("Single Input Run Without Error", self.single_input), @@ -925,7 +924,7 @@ def run(self, output_file=None): self.run_bash() et = time.time() elapsed = et - st - self.make_output(elapsed, model_size) + self.make_output(elapsed) except Exception as e: click.echo(f"❌ An error occurred: {e}") From f2e8adcf821f264e639993ab839fd85c4b36c212 Mon Sep 17 00:00:00 2001 From: Abellegese Date: Wed, 20 Nov 2024 23:45:22 +0300 Subject: [PATCH 5/7] Model Tester: An model testing CLI --- ersilia/publish/test.py | 937 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 929 insertions(+), 8 deletions(-) diff --git a/ersilia/publish/test.py b/ersilia/publish/test.py index 3344e3bbf..d668192e2 100644 --- a/ersilia/publish/test.py +++ b/ersilia/publish/test.py @@ -1,14 +1,935 @@ -import importlib +# TODO adapt to input-type agnostic. For now, it works only with Compound input types. +import json +import os +import csv +import subprocess +import tempfile +import asyncio +import time +import click +import types +from collections import defaultdict +from datetime import datetime -def get_service_class(class_name): - module = importlib.import_module('ersilia.publish.test_services') - return getattr(module, class_name) +from ersilia.utils.conda import SimpleConda +from .. import ErsiliaModel, throw_ersilia_exception, ErsiliaBase +from ..hub.fetch.fetch import ModelFetcher +from ..cli import echo +from ..core.session import Session +from ..default import EOS, INFORMATION_FILE +from ..io.input import ExampleGenerator +from ..utils.exceptions_utils import test_exceptions as texc +from ..utils.terminal import run_command_check_output -IOService = get_service_class('IOService') -CheckService = get_service_class('CheckService') -RunnerService = get_service_class('RunnerService') +# Check if we have the required imports in the environment +MISSING_PACKAGES = False +try: + from scipy.stats import spearmanr + from fuzzywuzzy import fuzz + from rich.console import Console + from rich.table import Table + from rich.box import SIMPLE +except ImportError: + MISSING_PACKAGES = True -from .. import ErsiliaBase +RUN_FILE = "run.sh" +DATA_FILE = "data.csv" +NUM_SAMPLES = 5 +BOLD = "\033[1m" +RESET = "\033[0m" +BASE = "base" + +TEST_MESSAGES = { + "pkg_err": "Missing packages required for testing. \ + Please install test extras with 'pip install ersilia[test]'." +} + +class IOService: + def __init__( + self, logger, + dest_dir, + model_path, + bundle_path, + bentoml_path, + model_id, + env, + type + ): + self.logger = logger + self.model_id = model_id + self.env = env if env is not None else BASE + self.model_size = 0 + self.console = Console() + self.check_results = [] + self.type = type + self._model_path = model_path + self._bundle_path = bundle_path + self._bentoml_path = bentoml_path + self._dest_dir = dest_dir + + + def _run_check( + self, + check_function, + data, + check_name, + additional_info=None + ): + + try: + if additional_info is not None: + check_function(additional_info) + else: + check_function(data) + self.check_results.append((check_name, "[green]✔[/green]")) + return True + except Exception as e: + self.logger.error(f"Check '{check_name,additional_info}' failed: {e}") + self.check_results.append((check_name, "[red]✖[/red]")) + return False + + def _generate_table(self, title, headers, rows): + table = Table( + title=title, + border_style="#FFC0CB" + ) + for header in headers: + table.add_column( + header, + justify="center", + no_wrap=True, + width=20 + ) + + for row in rows: + table.add_row(*[str(cell) for cell in row]) + # render + self.console.print(table) + + def _get_file_requirements(self): + + if self.type.lower() == "bentoml": + return [ + "src/service.py", + "pack.py", + "Dockerfile", + "metadata.json", + "README.md", + "model/framework/run.sh", + "LICENSE", + ] + elif self.type.lower() == "ersilia": + return [ + "install.yml", + "metadata.json", + "metadata.yml", + "model/framework/example/input.csv", + "model/framework/example/output.csv", + "README.md", + "model/framework/run.sh", + "LICENSE", + ] + else: + raise ValueError(f"Unsupported model type: {self.type}") + + + def _get_environment_location(self): + conda = SimpleConda() + python_path = conda.get_python_path_env(environment=self.env) + return os.path.dirname(os.path.dirname(python_path)) + + def read_information(self): + file = os.path.join( + self._dest_dir, + self.model_id, + INFORMATION_FILE + ) + self.logger.info(f"Dest: {self._dest_dir}") + if not os.path.exists(file): + raise FileNotFoundError(f"Information file does not exist for model {self.model_id}") + with open(file, "r") as f: + return json.load(f) + + def set_model_size(self, directory): + return sum( + os.path.getsize(os.path.join(dirpath, filename)) + for dirpath, _, filenames in os.walk(directory) + for filename in filenames + ) + + def print_output(self, result, output): + def write_output(data): + if output is not None: + with open(output.name, "w") as file: + json.dump(data, file) + else: + print(json.dumps(data, indent=4)) + + if isinstance(result, types.GeneratorType): + for r in result: + write_output(r if r is not None else "Something went wrong") + else: + print(result) + + def _get_environment_location(self): + conda = SimpleConda() + python_path = conda.get_python_path_env(environment=BASE) + return os.path.dirname(os.path.dirname(python_path)) + + + @throw_ersilia_exception() + def get_directories_sizes(self): + self.logger.debug(BOLD + "Calculating model size... " + RESET) + + def format_size(size_in_bytes): + if size_in_bytes < 1024: + return f"{size_in_bytes}B" # Bytes + size_kb = size_in_bytes / 1024 + if size_kb < 1024: + return f"{size_kb:.2f}KB" # Kilobytes + size_mb = size_kb / 1024 + if size_mb < 1024: + return f"{size_mb:.2f}MB" # Megabytes + size_gb = size_mb / 1024 + return f"{size_gb:.2f}GB" # Gigabytes + + def get_directory_size(directory, include_symlinks=True): + if not directory or not os.path.exists(directory): + return 0, defaultdict(int), defaultdict(int) + + _types, _sizes, total_sz = defaultdict(int), defaultdict(int), 0 + + for dirpath, _, filenames in os.walk(directory): + for filename in filenames: + filepath = os.path.join(dirpath, filename) + + if include_symlinks or not os.path.islink(filepath): + try: + size = os.path.getsize(filepath) + total_sz += size + file_extension = os.path.splitext(filename)[1] + _types[file_extension] += 1 + _sizes[file_extension] += size + except OSError as e: + self.logger.warning(f"Failed to access file {filepath}: {e}") + + return total_sz, _types, _sizes + + directories = { + "dest_dir": (self._model_path(model_id=self.model_id), False), + "bundle_dir": (self._bundle_path(model_id=self.model_id), False), + "bentoml_dir": (self._bentoml_path(model_id=self.model_id), False), + "env_dir": (self._get_environment_location(), True), + } + + total_size = 0 + directory_sizes = {} + + for label, (directory, include_symlinks) in directories.items(): + size, file_types, _ = get_directory_size( + directory=directory, + include_symlinks=include_symlinks + ) + formatted_size = format_size(size) + directory_sizes[label] = formatted_size + total_size += size + + total_size_formatted = format_size(total_size) + + self.logger.debug(BOLD + "Model Size Breakdown:" + RESET) + self.logger.debug(f"Total: {total_size_formatted}") + + self.logger.debug("Sizes of directories:") + for label, size in directory_sizes.items(): + self.logger.debug(f"{label}: {size}") + + self.model_size = total_size_formatted + return directory_sizes + + +class CheckService: + def __init__( + self, + logger, + model_id, + dest_dir, + dir, + ios, + type + ): + self.logger = logger + self.model_id = model_id + self._dest_dir = dest_dir + self.dir = dir + self.type = type + self._run_check = ios._run_check + self._generate_table = ios._generate_table + self._get_file_requirements = ios._get_file_requirements + self.console = ios.console + self.check_results = ios.check_results + self.information_check = False + self.single_input = False + self.example_input = False + self.consistent_output = False + + def _check_file_existence(self, file_path): + + if file_path == "metadata.json" and "ersilia" in self.type.lower(): + json_exists = os.path.exists(os.path.join(self.dir, "metadata.json")) + yaml_exists = os.path.exists(os.path.join(self.dir, "metadata.yml")) + if not (json_exists or yaml_exists): + raise FileNotFoundError(f"Neither 'metadata.json' nor 'metadata.yml' found.") + else: + if not os.path.exists(os.path.join(self.dir, file_path)): + raise FileNotFoundError(f"File '{file_path}' does not exist.") + + def check_files(self): + + self.logger.debug(f"Checking file requirements for {self.type} model.") + fr = self._get_file_requirements() + + for fp in fr: + self.logger.debug(f"Checking file: {fp}") + self._run_check( + self._check_file_existence, + None, + f"File: {fp}", + fp + ) + + def _check_model_id(self, data): + self.logger.debug("Checking model ID...") + if data["card"]["Identifier"] != self.model_id: + raise texc.WrongCardIdentifierError(self.model_id) + + + def _check_model_slug(self, data): + self.logger.debug("Checking model slug...") + if not data["card"]["Slug"]: + raise texc.EmptyField("slug") + + + def _check_model_description(self, data): + self.logger.debug("Checking model description...") + if not data["card"]["Description"]: + raise texc.EmptyField("Description") + + def _check_model_task(self, data): + self.logger.debug("Checking model task...") + valid_tasks = { + "Classification", + "Regression", + "Generative", + "Representation", + "Similarity", + "Clustering", + "Dimensionality reduction", + } + + raw_tasks = data.get("card", {}).get("Task", "") + if isinstance(raw_tasks, str): + tasks = [ + task.strip() + for task + in raw_tasks.split(",") + if task.strip() + ] + elif isinstance(raw_tasks, list): + tasks = [ + task.strip() + for task + in raw_tasks + if isinstance(task, str) and task.strip() + ] + else: + raise texc.InvalidEntry( + "Task", + message="Task field must be a string or list." + ) + + if not tasks: + raise texc.InvalidEntry( + "Task", + message="Task field is missing or empty." + ) + + invalid_tasks = [task for task in tasks if task not in valid_tasks] + if invalid_tasks: + raise texc.InvalidEntry( + "Task", message=f"Invalid tasks: {', '.join(invalid_tasks)}" + ) + + self.logger.debug("All tasks are valid.") + + + + def _check_model_input(self, data): + self.logger.debug("Checking model input...") + valid_inputs = [{"Compound"}, {"Protein"}, {"Text"}] + if set(data["card"]["Input"]) not in valid_inputs: + raise texc.InvalidEntry("Input") + + def _check_model_input_shape(self, data): + self.logger.debug("Checking model input shape...") + valid_input_shapes = { + "Single", + "Pair", + "List", + "Pair of Lists", + "List of Lists" + } + if data["card"]["Input Shape"] not in valid_input_shapes: + raise texc.InvalidEntry("Input Shape") + + + def _check_model_output(self, data): + self.logger.debug("Checking model output...") + valid_outputs = { + "Boolean", + "Compound", + "Descriptor", + "Distance", + "Experimental value", + "Image", + "Other value", + "Probability", + "Protein", + "Score", + "Text", + } + + raw_outputs = data.get("card", {}).get("Output", "") + if isinstance(raw_outputs, str): + outputs = [ + output.strip() + for output + in raw_outputs.split(",") + if output.strip() + ] + elif isinstance(raw_outputs, list): + outputs = [ + output.strip() + for output + in raw_outputs + if isinstance(output, str) and output.strip() + ] + else: + raise texc.InvalidEntry( + "Output", + message="Output field must be a string or list." + ) + + if not outputs: + raise texc.InvalidEntry( + "Output", + message="Output field is missing or empty." + ) + + invalid_outputs = [ + output + for output + in outputs + if output not in valid_outputs + ] + if invalid_outputs: + raise texc.InvalidEntry( + "Output", + message=f"Invalid outputs: {', '.join(invalid_outputs)}" + ) + + self.logger.debug("All outputs are valid.") + + + def _check_model_output_type(self, data): + self.logger.debug("Checking model output type...") + valid_output_types = [{"String"}, {"Float"}, {"Integer"}] + if set(data["card"]["Output Type"]) not in valid_output_types: + raise texc.InvalidEntry("Output Type") + + def _check_model_output_shape(self, data): + self.logger.debug("Checking model output shape...") + valid_output_shapes = { + "Single", + "List", + "Flexible List", + "Matrix", + "Serializable Object" + } + if data["card"]["Output Shape"] not in valid_output_shapes: + raise texc.InvalidEntry("Output Shape") + + + + @throw_ersilia_exception() + def check_information(self, output): + self.logger.debug("Checking that model information is correct") + self.logger.debug( + BOLD + f"Beginning checks for {self.model_id} model information:" + RESET + ) + file = os.path.join( + self._dest_dir, + self.model_id, + INFORMATION_FILE + ) + with open(file, "r") as f: + data = json.load(f) + + self._run_check(self._check_model_id, data, "Model ID") + self._run_check(self._check_model_slug, data, "Model Slug") + self._run_check(self._check_model_description, data, "Model Description") + self._run_check(self._check_model_task, data, "Model Task") + self._run_check(self._check_model_input, data, "Model Input") + self._run_check(self._check_model_input_shape, data, "Model Input Shape") + self._run_check(self._check_model_output, data, "Model Output") + self._run_check(self._check_model_output_type, data, "Model Output Type") + self._run_check(self._check_model_output_shape, data, "Model Output Shape") + + if output is not None: + self.information_check = True + + @throw_ersilia_exception() + def check_single_input(self, output, fetch_and_serve, run_model): + + input_smiles = "COc1ccc2c(NC(=O)Nc3cccc(C(F)(F)F)n3)ccnc2c1" + + self.logger.debug(BOLD + "Testing model on single smiles input...\n" + RESET) + fetch_and_serve() + result = run_model( + input=input_smiles, + output=output, + batch=100 + ) + + if output is not None: + self.single_input = True + else: + self._print_output(result, output) + + @throw_ersilia_exception() + def check_example_input(self, output, run_model, run_example): + input_samples = run_example( + n_samples=NUM_SAMPLES, + file_name=None, + simple=True, + try_predefined=False + ) + + self.logger.debug( + BOLD + "\nTesting model on input of 5 smiles given by 'example' command...\n" + RESET + ) + self.logger.debug(f"This is the input: {input_samples}") + + result = run_model( + input=input_samples, + output=output, + batch=100 + ) + + if output is not None: + self.example_input = True + else: + self._print_output(result, output) + + @throw_ersilia_exception() + def check_consistent_output(self, run_example, run_model): + def compute_rmse(y_true, y_pred): + return sum((yt - yp) ** 2 for yt, yp in zip(y_true, y_pred)) ** 0.5 / len(y_true) + def _compare_output_strings(output1, output2): + if output1 is None and output2 is None: + return 100 + else: + return fuzz.ratio(output1, output2) + def validate_output(output1, output2): + if not isinstance(output1, type(output2)): + raise texc.InconsistentOutputTypes(self.model_id) + + if output1 is None: + return + + if isinstance(output1, (float, int)): + rmse = compute_rmse([output1], [output2]) + if rmse > 0.1: + raise texc.InconsistentOutputs(self.model_id) + + rho, _ = spearmanr([output1], [output2]) + if rho < 0.5: + raise texc.InconsistentOutputs(self.model_id) + + elif isinstance(output1, list): + rmse = compute_rmse(output1, output2) + if rmse > 0.1: + raise texc.InconsistentOutputs(self.model_id) + + rho, _ = spearmanr(output1, output2) + if rho < 0.5: + raise texc.InconsistentOutputs(self.model_id) + + elif isinstance(output1, str): + if _compare_output_strings(output1, output2) <= 95: + raise texc.InconsistentOutputs(self.model_id) + + def read_csv(file_path): + if not os.path.exists(file_path): + raise FileNotFoundError(f"File not found: {file_path}") + with open(file_path, mode='r') as csv_file: + reader = csv.DictReader(csv_file) + return [row for row in reader] + + def is_json(data): + try: + if isinstance(data, str): + json.loads(data) + elif isinstance(data, list): + json.dumps(data) + return True + except (ValueError, TypeError): + return False + + + + self.logger.debug(BOLD + "\nConfirming model produces consistent output..." + RESET) + + input_samples = run_example( + n_samples=NUM_SAMPLES, + file_name=None, + simple=True, + try_predefined=False + ) + result1 = run_model( + input=input_samples, + output="output1.csv", + batch=100 + ) + result2 = run_model( + input=input_samples, + output="output2.csv", + batch=100 + ) + + if is_json(result1) and is_json(result2): + data1, data2 = result1, result2 + else: + data1 = read_csv("output1.csv") + data2 = read_csv("output2.csv") + + for res1, res2 in zip(data1, data2): + for key in res1: + if key in res2: + validate_output(res1[key], res2[key]) + else: + raise KeyError(f"Key '{key}' not found in second result.") + + self.consistent_output = True + +class RunnerService: + def __init__( + self, + model_id, + logger, + ios_service, + checkup_service, + model_path, + env, + type, + level, + dir + ): + self.model_id = model_id + self.logger = logger + self.ios_service = ios_service + self.console = ios_service.console + self.checkup_service = checkup_service + self._model_path = model_path + self.env = env + self.type = type + self.level = level + self.dir = dir + self._output_type = self.ios_service.read_information()["card"]["Output Type"] + session = Session(config_json=None) + service_class = session.current_service_class() + self.model = ErsiliaModel( + self.model_id, + service_class=service_class + ) + self.example = ExampleGenerator(model_id=self.model_id) + self.fetcher = ModelFetcher(repo_path=self.dir) + self.run_using_bash = False + + def run_model(self, input, output, batch): + return self.model.run( + input=input, + output=output, + batch_size=batch + ) + def fetch_and_serve(self): + asyncio.run(self.fetcher.fetch(self.model_id + )) + self.model.serve() + def serve_model(self, input, output, batch): + return self.model.run( + input=input, + output=output, + batch_size=batch + ) + def run_exampe( + self, + n_samples, + file_name=None, + simple=True, + try_predefined=False + ): + return self.example.example( + n_samples=n_samples, + file_name=file_name, + simple=simple, + try_predefined=try_predefined + ) + @throw_ersilia_exception() + def run_bash(self): + def compute_rmse(y_true, y_pred): + return sum((yt - yp) ** 2 for yt, yp in zip(y_true, y_pred)) ** 0.5 / len(y_true) + + def compare_outputs(bsh_data, ers_data): + columns = set(bsh_data[0].keys()) & set(data[0].keys()) + self.logger.debug(f"Common columns: {columns}") + + for column in columns: + bv = [row[column] for row in bsh_data] + ev = [row[column] for row in ers_data] + + if all(isinstance(val, (int, float)) for val in bv + ev): + rmse = compute_rmse(bv, ev) + self.logger.debug(f"RMSE for {column}: {rmse}") + + if rmse > 0.1: + raise texc.InconsistentOutputs(self.model_id) + elif all(isinstance(val, str) for val in bv + ev): + if not all(self._compare_string_similarity(a, b, 95) for a, b in zip(bv, ev)): + raise texc.InconsistentOutputs(self.model_id) + + def read_csv(path, flag=False): + try: + with open(path, "r") as file: + lines = file.readlines() + + if not lines: + self.logger.error(f"File at {path} is empty.") + return [] + + headers = lines[0].strip().split(",") + if flag: + headers = headers[2:] + + data = [] + for line in lines[1:]: + self.logger.debug(f"Processing line: {line.strip()}") + values = line.strip().split(",") + values = values[2:] if flag else values + + try: + if self._output_type == ["Float"]: + _values = [float(x) for x in values] + elif self._output_type == ["Integer"]: + _values = [int(x) for x in values] + except ValueError as e: + self.logger.warning(f"Value conversion error: {e}") + data.append(dict(zip(headers, _values))) + + return data + except Exception as e: + raise RuntimeError(f"Failed to read CSV from {path}.") from e + + def run_subprocess(command, env_vars=None): + try: + result = subprocess.run( + command, + capture_output=True, + text=True, + check=True, + env=env_vars + ) + self.logger.debug(f"Subprocess output: {result.stdout}") + return result.stdout + except subprocess.CalledProcessError as e: + raise RuntimeError("Subprocess execution failed.") from e + + with tempfile.TemporaryDirectory() as temp_dir: + + model_path = os.path.join(self.dir) + temp_script_path = os.path.join(temp_dir, "script.sh") + bash_output_path = os.path.join(temp_dir, "bash_output.csv") + output_path = os.path.join(temp_dir, "ersilia_output.csv") + output_log_path = os.path.join(temp_dir, "output.txt") + error_log_path = os.path.join(temp_dir, "error.txt") + + input = self.run_exampe( + n_samples=NUM_SAMPLES, + file_name=None, + simple=True, + try_predefined=False + ) + + ex_file = os.path.join(temp_dir, "example_file.csv") + + with open(ex_file, "w") as example_file: + example_file.write("smiles\n" + "\n".join(map(str, input))) + + run_sh_path = os.path.join( + model_path, + "model", + "framework", + "run.sh" + ) + if not os.path.exists(run_sh_path): + self.logger.warning(f"run.sh not found at {run_sh_path}. Skipping bash run.") + return + + bash_script = f""" + source {self.conda_prefix(self.is_base())}/etc/profile.d/conda.sh + conda activate {BASE} + cd {os.path.dirname(run_sh_path)} + bash run.sh . {ex_file} {bash_output_path} > {output_log_path} 2> {error_log_path} + conda deactivate + """ + + with open(temp_script_path, "w") as script_file: + script_file.write(bash_script) + + self.logger.debug(f"Running bash script: {temp_script_path}") + run_subprocess(["bash", temp_script_path]) + + bsh_data = read_csv(bash_output_path) + + self.run_model( + ex_file, + output_path, + 100 + ) + data = read_csv(output_path, flag=True) + + compare_outputs(bsh_data, data) + + self.run_using_bash = True + + @staticmethod + def default_env(): + if "CONDA_DEFAULT_ENV" in os.environ: + return os.environ["CONDA_DEFAULT_ENV"] + else: + return BASE + + @staticmethod + def conda_prefix(is_base): + o = run_command_check_output("which conda").rstrip() + if o: + o = os.path.abspath(os.path.join(o, "..", "..")) + return o + if is_base: + o = run_command_check_output("echo $CONDA_PREFIX").rstrip() + return o + else: + o = run_command_check_output("echo $CONDA_PREFIX_1").rstrip() + return o + + def is_base(self): + default_env = self.default_env() + self.logger.debug(f"Default environment: {default_env}") + if default_env == "base": + return True + else: + return False + + + def _compare_string_similarity( + self, + str1, + str2, + threshold + ): + similarity = fuzz.ratio(str1, str2) + return similarity >= threshold + + + def make_output(self, time): + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + self.information_check = self.checkup_service.information_check + self.single_input = self.checkup_service.single_input + self.example_input = self.checkup_service.example_input + self.consistent_output = self.checkup_service.consistent_output + data = [ + ("Date and Time Run", timestamp), + ("Time to Run Tests (seconds)", time), + ("Basic Checks Passed", self.information_check), + ("Single Input Run Without Error", self.single_input), + ("Example Input Run Without Error", self.example_input), + ("Outputs Consistent", self.consistent_output), + ("Bash Run Without Error", self.run_using_bash), + ] + + headers = ["Check Type", "Status"] + + self.ios_service._generate_table("Test Run Summary", headers, data) + + def run(self, output_file=None): + if MISSING_PACKAGES: + raise ImportError(TEST_MESSAGES["pkg_err"]) + + if not output_file: + output_file = os.path.join( + self._model_path(self.model_id), "result.csv" + ) + + st = time.time() + try: + self.checkup_service.check_information(output_file) + self.ios_service._generate_table( + title="Model Information Checks", + headers=["Check", "Status"], + rows=self.ios_service.check_results + ) + self.ios_service.check_results.clear() + self.checkup_service.check_files() + self.ios_service._generate_table( + title="Model Information Checks", + headers=["Check", "Status"], + rows=self.ios_service.check_results + ) + ds = self.ios_service.get_directories_sizes() + self.ios_service._generate_table( + title="Model Directory Sizes", + headers=["Dest dir", "Env Dir"], + rows=[(ds["dest_dir"], ds["env_dir"])] + ) + if self.level == "deep": + self.checkup_service.check_single_input( + output_file, + self.fetch_and_serve, + self.run_model + ) + self.ios_service._generate_table( + title="Runner Checkup Status", + headers=["Runner", "Status"], + rows=[["Fetch", "[green]✔[/green]"], ["Serve", "[green]✔[/green]"], ["Run", "[green]✔[/green]"]] + ) + self.checkup_service.check_example_input( + output_file, + self.run_model, + self.run_exampe + ) + self.checkup_service.check_consistent_output( + self.run_exampe, + self.run_model + ) + model_size = self.ios_service.model_size + self.run_bash() + et = time.time() + elapsed = et - st + self.make_output(elapsed) + + except Exception as e: + click.echo(f"❌ An error occurred: {e}") + finally: + click.echo("🏁 Run process finished.") class ModelTester(ErsiliaBase): From 84377a02442568f2289701ff34ddb472db85dc95 Mon Sep 17 00:00:00 2001 From: Abellegese Date: Thu, 21 Nov 2024 18:01:50 +0300 Subject: [PATCH 6/7] Model Tester: An model testing CLI --- ersilia/publish/test.py | 93 +++++++++++++++++++---------------------- 1 file changed, 43 insertions(+), 50 deletions(-) diff --git a/ersilia/publish/test.py b/ersilia/publish/test.py index d668192e2..65d313ae3 100644 --- a/ersilia/publish/test.py +++ b/ersilia/publish/test.py @@ -12,7 +12,7 @@ from datetime import datetime from ersilia.utils.conda import SimpleConda -from .. import ErsiliaModel, throw_ersilia_exception, ErsiliaBase +from .. import ErsiliaModel, ErsiliaBase, throw_ersilia_exception from ..hub.fetch.fetch import ModelFetcher from ..cli import echo from ..core.session import Session @@ -21,7 +21,6 @@ from ..utils.exceptions_utils import test_exceptions as texc from ..utils.terminal import run_command_check_output -# Check if we have the required imports in the environment MISSING_PACKAGES = False try: from scipy.stats import spearmanr @@ -32,12 +31,11 @@ except ImportError: MISSING_PACKAGES = True -RUN_FILE = "run.sh" -DATA_FILE = "data.csv" +RUN_FILE = "run.sh" +DATA_FILE = "data.csv" NUM_SAMPLES = 5 -BOLD = "\033[1m" -RESET = "\033[0m" -BASE = "base" +BOLD = "\033[1m" +BASE = "base" TEST_MESSAGES = { "pkg_err": "Missing packages required for testing. \ @@ -179,19 +177,19 @@ def _get_environment_location(self): @throw_ersilia_exception() def get_directories_sizes(self): - self.logger.debug(BOLD + "Calculating model size... " + RESET) - - def format_size(size_in_bytes): - if size_in_bytes < 1024: - return f"{size_in_bytes}B" # Bytes - size_kb = size_in_bytes / 1024 - if size_kb < 1024: - return f"{size_kb:.2f}KB" # Kilobytes - size_mb = size_kb / 1024 - if size_mb < 1024: - return f"{size_mb:.2f}MB" # Megabytes - size_gb = size_mb / 1024 - return f"{size_gb:.2f}GB" # Gigabytes + self.logger.debug("Calculating model size") + + def format_size(szb): + if szb < 1024: + return f"{szb}B" # Bytes + szkb = szb / 1024 + if szkb < 1024: + return f"{szkb:.2f}KB" # Kilobytes + szmb = szkb / 1024 + if szmb < 1024: + return f"{szmb:.2f}MB" # Megabytes + szgb = szmb / 1024 + return f"{szgb:.2f}GB" # Gigabytes def get_directory_size(directory, include_symlinks=True): if not directory or not os.path.exists(directory): @@ -222,28 +220,23 @@ def get_directory_size(directory, include_symlinks=True): "env_dir": (self._get_environment_location(), True), } - total_size = 0 - directory_sizes = {} + total, directory_sizes = 0, {} for label, (directory, include_symlinks) in directories.items(): - size, file_types, _ = get_directory_size( + size, _ , _ = get_directory_size( directory=directory, include_symlinks=include_symlinks ) - formatted_size = format_size(size) - directory_sizes[label] = formatted_size - total_size += size + size = format_size(size) + directory_sizes[label] = size + total += size - total_size_formatted = format_size(total_size) + formatted = format_size(total) - self.logger.debug(BOLD + "Model Size Breakdown:" + RESET) - self.logger.debug(f"Total: {total_size_formatted}") - - self.logger.debug("Sizes of directories:") for label, size in directory_sizes.items(): self.logger.debug(f"{label}: {size}") - self.model_size = total_size_formatted + self.model_size = formatted return directory_sizes @@ -364,13 +357,13 @@ def _check_model_task(self, data): def _check_model_input(self, data): - self.logger.debug("Checking model input...") + self.logger.debug("Checking model input") valid_inputs = [{"Compound"}, {"Protein"}, {"Text"}] if set(data["card"]["Input"]) not in valid_inputs: raise texc.InvalidEntry("Input") def _check_model_input_shape(self, data): - self.logger.debug("Checking model input shape...") + self.logger.debug("Checking model input shape") valid_input_shapes = { "Single", "Pair", @@ -462,10 +455,7 @@ def _check_model_output_shape(self, data): @throw_ersilia_exception() def check_information(self, output): - self.logger.debug("Checking that model information is correct") - self.logger.debug( - BOLD + f"Beginning checks for {self.model_id} model information:" + RESET - ) + self.logger.debug(f"Beginning checks for {self.model_id} model information") file = os.path.join( self._dest_dir, self.model_id, @@ -492,7 +482,7 @@ def check_single_input(self, output, fetch_and_serve, run_model): input_smiles = "COc1ccc2c(NC(=O)Nc3cccc(C(F)(F)F)n3)ccnc2c1" - self.logger.debug(BOLD + "Testing model on single smiles input...\n" + RESET) + self.logger.debug("Testing model on single smiles input") fetch_and_serve() result = run_model( input=input_smiles, @@ -514,10 +504,7 @@ def check_example_input(self, output, run_model, run_example): try_predefined=False ) - self.logger.debug( - BOLD + "\nTesting model on input of 5 smiles given by 'example' command...\n" + RESET - ) - self.logger.debug(f"This is the input: {input_samples}") + self.logger.debug("Testing model on input of 5 smiles given by 'example' command") result = run_model( input=input_samples, @@ -585,9 +572,7 @@ def is_json(data): except (ValueError, TypeError): return False - - - self.logger.debug(BOLD + "\nConfirming model produces consistent output..." + RESET) + self.logger.debug("Confirming model produces consistent output...") input_samples = run_example( n_samples=NUM_SAMPLES, @@ -868,7 +853,11 @@ def make_output(self, time): headers = ["Check Type", "Status"] - self.ios_service._generate_table("Test Run Summary", headers, data) + self.ios_service._generate_table( + "Test Run Summary", + headers, + data + ) def run(self, output_file=None): if MISSING_PACKAGES: @@ -876,7 +865,8 @@ def run(self, output_file=None): if not output_file: output_file = os.path.join( - self._model_path(self.model_id), "result.csv" + self._model_path(self.model_id), + "result.csv" ) st = time.time() @@ -909,7 +899,11 @@ def run(self, output_file=None): self.ios_service._generate_table( title="Runner Checkup Status", headers=["Runner", "Status"], - rows=[["Fetch", "[green]✔[/green]"], ["Serve", "[green]✔[/green]"], ["Run", "[green]✔[/green]"]] + rows=[ + ["Fetch", "[green]✔[/green]"], + ["Serve", "[green]✔[/green]"], + ["Run", "[green]✔[/green]"] + ] ) self.checkup_service.check_example_input( output_file, @@ -920,7 +914,6 @@ def run(self, output_file=None): self.run_exampe, self.run_model ) - model_size = self.ios_service.model_size self.run_bash() et = time.time() elapsed = et - st From 449603417fdd02c446ac32357b3a9f25e4926180 Mon Sep 17 00:00:00 2001 From: Dhanshree Arora Date: Fri, 29 Nov 2024 11:11:18 +0530 Subject: [PATCH 7/7] Make the serializer use the fields from the header (#1406) --- ersilia/serve/standard_api.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/ersilia/serve/standard_api.py b/ersilia/serve/standard_api.py index 9cddc33ef..3a98a27ed 100644 --- a/ersilia/serve/standard_api.py +++ b/ersilia/serve/standard_api.py @@ -263,21 +263,17 @@ def is_amenable(self, output_data): return True def serialize_to_csv(self, input_data, result, output_data): - k = list(result[0].keys())[0] - v = result[0][k] - if type(v) is list: - is_list = True - else: - is_list = False with open(output_data, "w") as f: writer = csv.writer(f) writer.writerow(self.header) for i_d, r_d in zip(input_data, result): - v = r_d[k] - if not is_list: - r = [i_d["key"], i_d["input"]] + [v] - else: - r = [i_d["key"], i_d["input"]] + v + r = [i_d["key"], i_d["input"]] + for k in self.header[2:]: + v = r_d[k] + if isinstance(v, list): + r+=v + else: + r+=[v] writer.writerow(r) return output_data