diff --git a/.github/workflows/code_quality.yaml b/.github/workflows/code_quality.yaml index f3084be..f3a4cb9 100644 --- a/.github/workflows/code_quality.yaml +++ b/.github/workflows/code_quality.yaml @@ -58,8 +58,8 @@ jobs: with: os: ${{ job.os }} python-version: '3.11' - poetry-install-options: "--only=types --no-root" - poetry-export-options: "--only=types" + poetry-install-options: "--only=main --only=types --no-root" + poetry-export-options: "--only=main --only=types" - name: Check types run: poetry run mypy tesk/ diff --git a/.github/workflows/endpoint_test.yaml b/.github/workflows/endpoint_test.yaml index 472bae5..90ccf55 100644 --- a/.github/workflows/endpoint_test.yaml +++ b/.github/workflows/endpoint_test.yaml @@ -43,4 +43,4 @@ jobs: done echo "API failed to start in time" exit 1 -... \ No newline at end of file +... diff --git a/deployment/config.yaml b/deployment/config.yaml index e102bf3..466aea6 100644 --- a/deployment/config.yaml +++ b/deployment/config.yaml @@ -1,3 +1,4 @@ +--- # FOCA configuration # Available in app context as attributes of `current_app.config.foca` # Automatically validated via FOCA @@ -71,9 +72,10 @@ log: # Exception configuration # Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html#foca.models.config.ExceptionConfig exceptions: - required_members: [['detail'], ['status'], ['title']] - status_member: ['status'] - exceptions: tesk.exceptions.exceptions + required_members: [['detail'], ['status'], ['title']] + status_member: ['status'] + exceptions: tesk.exceptions.exceptions storeLogs: execution_trace: True +... diff --git a/docs/source/index.rst b/docs/source/index.rst index 2845e52..9df0581 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -4,11 +4,11 @@ TESK .. toctree:: :maxdepth: 1 :caption: TESK - + .. mdinclude:: ../../README.md .. mdinclude:: ../tesintro.md -.. Not adding a heading to separate deployment docs because +.. Not adding a heading to separate deployment docs because .. these md files already have heading. .. mdinclude:: ../../deployment/documentation/deployment.md .. mdinclude:: ../../deployment/documentation/integrated_wes_tes.md @@ -19,7 +19,7 @@ Package contents .. toctree:: :maxdepth: 2 - :caption: package + :caption: API reference pages/tesk/modules diff --git a/docs/source/pages/tesk/modules.rst b/docs/source/pages/tesk/modules.rst index cd5ce57..b4db9ce 100644 --- a/docs/source/pages/tesk/modules.rst +++ b/docs/source/pages/tesk/modules.rst @@ -1,7 +1,14 @@ -tesk +TESK ==== .. toctree:: :maxdepth: 4 + :caption: Services (filer and taskmaster) - tesk + tesk.services + +.. toctree:: + :maxdepth: 4 + :caption: API + + tesk.api diff --git a/docs/source/pages/tesk/tesk.api.ga4gh.rst b/docs/source/pages/tesk/tesk.api.ga4gh.rst new file mode 100644 index 0000000..8c96efd --- /dev/null +++ b/docs/source/pages/tesk/tesk.api.ga4gh.rst @@ -0,0 +1,18 @@ +tesk.api.ga4gh package +====================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + tesk.api.ga4gh.tes + +Module contents +--------------- + +.. automodule:: tesk.api.ga4gh + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pages/tesk/tesk.api.ga4gh.tes.base.rst b/docs/source/pages/tesk/tesk.api.ga4gh.tes.base.rst new file mode 100644 index 0000000..9fd8ef0 --- /dev/null +++ b/docs/source/pages/tesk/tesk.api.ga4gh.tes.base.rst @@ -0,0 +1,21 @@ +tesk.api.ga4gh.tes.base package +=============================== + +Submodules +---------- + +tesk.api.ga4gh.tes.base.base\_tesk\_request module +-------------------------------------------------- + +.. automodule:: tesk.api.ga4gh.tes.base.base_tesk_request + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: tesk.api.ga4gh.tes.base + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pages/tesk/tesk.api.ga4gh.tes.rst b/docs/source/pages/tesk/tesk.api.ga4gh.tes.rst new file mode 100644 index 0000000..1da9b71 --- /dev/null +++ b/docs/source/pages/tesk/tesk.api.ga4gh.tes.rst @@ -0,0 +1,29 @@ +tesk.api.ga4gh.tes package +========================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + tesk.api.ga4gh.tes.base + +Submodules +---------- + +tesk.api.ga4gh.tes.controllers module +------------------------------------- + +.. automodule:: tesk.api.ga4gh.tes.controllers + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: tesk.api.ga4gh.tes + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pages/tesk/tesk.api.rst b/docs/source/pages/tesk/tesk.api.rst new file mode 100644 index 0000000..e9d2f31 --- /dev/null +++ b/docs/source/pages/tesk/tesk.api.rst @@ -0,0 +1,18 @@ +tesk.api package +================ + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + tesk.api.ga4gh + +Module contents +--------------- + +.. automodule:: tesk.api + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pages/tesk/tesk.rst b/docs/source/pages/tesk/tesk.rst index b023c8f..30517ad 100644 --- a/docs/source/pages/tesk/tesk.rst +++ b/docs/source/pages/tesk/tesk.rst @@ -7,8 +7,36 @@ Subpackages .. toctree:: :maxdepth: 4 + tesk.api tesk.services +Submodules +---------- + +tesk.app module +--------------- + +.. automodule:: tesk.app + :members: + :undoc-members: + :show-inheritance: + +tesk.exceptions module +---------------------- + +.. automodule:: tesk.exceptions + :members: + :undoc-members: + :show-inheritance: + +tesk.tesk\_app module +--------------------- + +.. automodule:: tesk.tesk_app + :members: + :undoc-members: + :show-inheritance: + Module contents --------------- diff --git a/mypy.ini b/mypy.ini index 00fe171..ca440d8 100644 --- a/mypy.ini +++ b/mypy.ini @@ -10,5 +10,5 @@ ignore_missing_imports = True [mypy-connexion.*] ignore_missing_imports = True -[mypy-foca] +[mypy-foca.*] ignore_missing_imports = True diff --git a/poetry.lock b/poetry.lock index 3143c30..2b6dd05 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2980,6 +2980,17 @@ files = [ [package.dependencies] botocore-stubs = "*" +[[package]] +name = "types-pyyaml" +version = "6.0.12.20240311" +description = "Typing stubs for PyYAML" +optional = false +python-versions = ">=3.8" +files = [ + {file = "types-PyYAML-6.0.12.20240311.tar.gz", hash = "sha256:a9e0f0f88dc835739b0c1ca51ee90d04ca2a897a71af79de9aec5f38cb0a5342"}, + {file = "types_PyYAML-6.0.12.20240311-py3-none-any.whl", hash = "sha256:b845b06a1c7e54b8e5b4c683043de0d9caf205e7434b3edc678ff2411979b8f6"}, +] + [[package]] name = "types-requests" version = "2.32.0.20240523" @@ -3165,4 +3176,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "f84324e2321c3fc0bd7e89e46294a494af5805b4bc1845adeb7d08d5ffe7fdfb" +content-hash = "f6edc31e4b0fa5e68bb0d33605194c85b653d4d9f3f9bca1c977136950ec6247" diff --git a/pyproject.toml b/pyproject.toml index f92be0a..2019a5f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,7 @@ boto3-stubs = "^1.34.108" kubernetes-stubs = "^22.6.0.post1" mypy = "^1.10.0" types-botocore = "^1.0.2" +types-pyyaml = "^6.0.12.20240311" types-requests = "^2.31.0.20240406" types-urllib3 = "^1.26.25.14" types-werkzeug = "^1.0.9" diff --git a/tesk/api/ga4gh/tes/base/__init__.py b/tesk/api/ga4gh/tes/base/__init__.py new file mode 100644 index 0000000..cafd7b3 --- /dev/null +++ b/tesk/api/ga4gh/tes/base/__init__.py @@ -0,0 +1 @@ +"""Package for base class for TESK API request.""" diff --git a/tesk/api/ga4gh/tes/base/base_tesk_request.py b/tesk/api/ga4gh/tes/base/base_tesk_request.py new file mode 100644 index 0000000..d0b691d --- /dev/null +++ b/tesk/api/ga4gh/tes/base/base_tesk_request.py @@ -0,0 +1,53 @@ +"""Base class for the TES API request.""" + +from abc import ABC, abstractmethod +from typing import Any, final + +from pydantic import BaseModel + +from tesk.tesk_app import TeskApp + + +class BaseTeskRequest(ABC, TeskApp): + """Base class for the TES API. + + This class is an abstract class that defines the common properties and + methods needed by all of the TES API endpoint business logic. + + Notes: This initializes the parent class `TeskApp` and will give + all the subclasses access to the configuration object `conf` and + config path. + """ + + def __init__(self) -> None: + """Initializes the BaseTeskRequest class.""" + super().__init__() + + @abstractmethod + def api_response(self) -> BaseModel: + """Returns the response as Pydantic model. + + Should be implemented by the child class as final + business logic for the specific endpoint. + + Returns: + BaseModel: API response for the specific endpoint. + + Raises: + Exception: If API response break due to something unhandled. + """ + return BaseModel() + + @final + def response(self) -> dict[str, Any]: + """Returns serialized response. + + Should be invoked by controller. + + Returns: + dict: Serialized response for the specific endpoint. + """ + _response = self.api_response() + if not isinstance(_response, BaseModel): + raise TypeError('API response must be a Pydantic model.') + return _response.dict() diff --git a/tesk/api/ga4gh/tes/models/validators/base/__init__.py b/tesk/api/ga4gh/tes/models/validators/base/__init__.py new file mode 100644 index 0000000..52748c3 --- /dev/null +++ b/tesk/api/ga4gh/tes/models/validators/base/__init__.py @@ -0,0 +1 @@ +"""Package for base class for custom pydantic validators.""" diff --git a/tesk/api/ga4gh/tes/models/validators/base/base_validator.py b/tesk/api/ga4gh/tes/models/validators/base/base_validator.py new file mode 100644 index 0000000..5ad0f15 --- /dev/null +++ b/tesk/api/ga4gh/tes/models/validators/base/base_validator.py @@ -0,0 +1,79 @@ +"""Base validator class, all custom validators must implement it.""" + +import logging +from abc import ABC, abstractmethod +from typing import Any, Generic, TypeVar + +from pydantic import ValidationError + +T = TypeVar('T') +logger = logging.getLogger(__name__) + + +class BaseValidator(ABC, Generic[T]): + """Base custom validator class. + + Base validator class for fields in Pydantic models, + if the field is empty, ie None, then it is returned as is without any validation. + otherwise the validation logic is applied to the field. + """ + + @property + @abstractmethod + def error_message(self) -> str: + """Returns the error message. + + Returns: + str: The error message to be used when validation fails. + """ + pass + + @abstractmethod + def validation_logic(self, value: T) -> bool: + """Validation logic for the field. + + Args: + value: The value of the field being validated. + + Returns: + bool: True if the validation is successful, False otherwise. + """ + pass + + def _raise_error(self, cls: Any, value: T) -> None: + """Raise a validation error. + + Args: + cls: The class being validated. + value: The value of the field being validated. + + Raises: + ValidationError: Raised when the validation fails. + """ + logger.error(f'Validation failed for {value} in {cls.__name__}.') + raise ValidationError( + self.error_message, + model=cls, + ) + + def validate(self, cls: Any, value: T) -> T: + """Validate the value. + + If the value is None, that is the fields is optional + in the model, then it is returned as is without any validation. + + Args: + cls: The class being validated. + value: The value of the field being validated. + + Returns: + value: The validated value (if valid). + + Raises: + ValidationError: If the value is not valid. + """ + if not value: + return value + elif not self.validation_logic(value): + self._raise_error(cls, value) + return value diff --git a/tesk/api/specs/task_execution_service.117cd92.openapi.yaml b/tesk/api/specs/task_execution_service.117cd92.openapi.yaml index fb2a875..a6ac84a 100644 --- a/tesk/api/specs/task_execution_service.117cd92.openapi.yaml +++ b/tesk/api/specs/task_execution_service.117cd92.openapi.yaml @@ -120,7 +120,7 @@ paths: ?tag_key=foo1&tag_value=bar1&tag_key=foo2&tag_value=bar2 ``` Should be constructed into the structure { "foo1" : "bar1", "foo2" : "bar2"} - + ``` ?tag_key=foo1 ``` @@ -884,4 +884,4 @@ components: System logs are only included in the FULL task view. items: type: string - description: TaskLog describes logging information related to a Task. \ No newline at end of file + description: TaskLog describes logging information related to a Task. diff --git a/tesk/app.py b/tesk/app.py index 52e3d02..2df417f 100644 --- a/tesk/app.py +++ b/tesk/app.py @@ -1,52 +1,23 @@ -"""API server entry point.""" +"""App entry point.""" import logging -import os -from pathlib import Path -from connexion import FlaskApp -from foca import Foca +from werkzeug.exceptions import InternalServerError -logger = logging.getLogger(__name__) - - -def init_app() -> FlaskApp: - """Initialize and return the FOCA app. - - This function initializes the FOCA app by loading the configuration - from the environment variable `TESK_FOCA_CONFIG_PATH` if set, or from - the default path if not. It raises a `FileNotFoundError` if the - configuration file is not found. +from tesk.tesk_app import TeskApp - Returns: - FlaskApp: A Connexion application instance. - - Raises: - FileNotFoundError: If the configuration file is not found. - """ - # Determine the configuration path - config_path_env = os.getenv('TESK_FOCA_CONFIG_PATH') - if config_path_env: - config_path = Path(config_path_env).resolve() - else: - config_path = ( - Path(__file__).parents[1] / 'deployment' / 'config.yaml' - ).resolve() - - # Check if the configuration file exists - if not config_path.exists(): - raise FileNotFoundError(f'Config file not found at: {config_path}') - - foca = Foca( - config_file=config_path, - ) - return foca.create_app() +logger = logging.getLogger(__name__) def main() -> None: """Run FOCA application.""" - app = init_app() - app.run(port=app.port) + try: + TeskApp().run() + except Exception as error: + logger.exception('An error occurred while running the application.') + raise InternalServerError( + 'An error occurred while running the application.' + ) from error if __name__ == '__main__': diff --git a/tesk/exceptions.py b/tesk/exceptions.py index 793c033..42192d6 100644 --- a/tesk/exceptions.py +++ b/tesk/exceptions.py @@ -7,12 +7,18 @@ OAuthProblem, Unauthorized, ) +from pydantic import ValidationError from werkzeug.exceptions import ( BadRequest, InternalServerError, NotFound, ) + +class ConfigNotFoundError(FileNotFoundError): + """Configuration file not found error.""" + + # exceptions raised in app context exceptions = { Exception: { @@ -60,9 +66,14 @@ 'detail': 'An unexpected error occurred.', 'status': 500, }, + ValidationError: { + 'title': 'Validation error', + 'detail': 'Value or object is not compatible with required type or schema.', + 'status': 400, + }, + ConfigNotFoundError: { + 'title': 'Configuration file not found', + 'detail': 'Configuration file not found.', + 'status': 500, + }, } - - -# exceptions raised outside of app context -class ValidationError(Exception): - """Value or object is not compatible with required type or schema.""" diff --git a/tesk/tesk_app.py b/tesk/tesk_app.py new file mode 100644 index 0000000..706c94b --- /dev/null +++ b/tesk/tesk_app.py @@ -0,0 +1,123 @@ +"""Base class for the APP used in initialization of API.""" + +import logging +import os +from pathlib import Path +from typing import Optional, final + +from foca import Foca +from foca.config.config_parser import ConfigParser + +from tesk.exceptions import ConfigNotFoundError + +logger = logging.getLogger(__name__) + + +class TeskApp(Foca): + """TESK API class extending the Foca framework. + + This class is used to initialize the TESK API application, contains + business logic for parsing configuration files and starting the + application server. + + Attributes: + config_file (Path): Path to the configuration file. + custom_config_model (Path): Path to the custom configuration model file. + conf (Any): Configuration object. + + Args: + config_file (Optional[Path]): Path to the configuration file. + Defaults to None. + custom_config_model (Optional[Path]): Path to the custom + configuration model file. Defaults to None. + + Notes: TeskApp class uses environment variables to load the configuration + file and custom configuration model file with `TESK_FOCA_CONFIG_FILE` + and `TESK_FOCA_CUSTOM_CONFIG_MODEL` respectively. + + Raises: + ConfigNotFoundError: If the configuration file is not found. + + Method: + run: Run the application. + + Example: + >>> from tesk.tesk_app import TeskApp + >>> app = TeskApp() + >>> app.run() + """ + + def __init__( + self, + config_file: Optional[Path] = None, + custom_config_model: Optional[Path] = None, + ) -> None: + """Initialize the TeskApp class. + + Args: + config_file (Optional[Path]): Path to the configuration file. + Defaults to None. + custom_config_model (Optional[Path]): Path to the custom + configuration model file. Defaults to None. + """ + if not config_file: + self._load_config_file() + else: + self.config_file = config_file + if not custom_config_model: + self.custom_config_model = self._load_custom_config_model() + else: + self.custom_config_model = custom_config_model + self.conf = ConfigParser( + config_file=self.config_file, + custom_config_model=self.custom_config_model, + format_logs=True, + ).config + + @final + def run(self) -> None: + """Run the application.""" + _environment = self.conf.server.environment or 'production' + logger.info(f'Running application in {_environment} environment...') + _debug = self.conf.server.debug or False + _app = self.create_app() + _app.run(host=self.conf.server.host, port=self.conf.server.port, debug=_debug) + + @final + def _load_config_file(self) -> None: + """Load the configuration file path from env variable or default location. + + Raises: + ConfigNotFoundError: If the configuration file is not found. + """ + logger.info('Loading configuration path...') + if config_path_env := os.getenv('TESK_FOCA_CONFIG_FILE'): + self.config_file = Path(config_path_env).resolve() + else: + self.config_file = ( + Path(__file__).parents[1] / 'deployment' / 'config.yaml' + ).resolve() + + if not self.config_file.exists(): + raise ConfigNotFoundError( + f'Config file not found at: {self.config_file}', + ) + + @final + def _load_custom_config_model(self) -> None: + """Load the custom configuration model path from environment variable or None. + + Raises: + ConfigNotFoundError: If the custom configuration model is specified + and found. + """ + logger.info('Loading custom configuration model path...') + if custom_config_model_env := os.getenv('TESK_FOCA_CUSTOM_CONFIG_MODEL'): + self.custom_config_model = Path(custom_config_model_env).resolve() + else: + self.custom_config_model = None + + if self.custom_config_model and not self.custom_config_model.exists(): + raise ConfigNotFoundError( + f'Custom configuration model not found at: {self.custom_config_model}', + )