From a9b83aebaba1340e4834b808a2b59ddc53d12abd Mon Sep 17 00:00:00 2001 From: Alex Kanitz Date: Thu, 10 Nov 2022 15:06:48 +0100 Subject: [PATCH] feat: add entry point for returning Celery app (#160) --- README.md | 14 ++- foca/foca.py | 157 ++++++++++++++++------------- foca/version.py | 2 +- tests/test_files/conf_no_jobs.yaml | 62 ++++++++++++ tests/test_foca.py | 137 +++++++++++++------------ 5 files changed, 234 insertions(+), 138 deletions(-) create mode 100644 tests/test_files/conf_no_jobs.yaml diff --git a/README.md b/README.md index c28d4a30..dd4e5118 100644 --- a/README.md +++ b/README.md @@ -255,10 +255,16 @@ jobs: > FOCA and registers the tasks found in modules `my_task_1` and `my_task_2`. > > Cf. the [API model][docs-models-jobs] for further details. -> -> **WARNING:** Note that FOCA's support for asynchronous tasks is currently -> still experimental and thus hasn't been tested extensively. Please use with -> caution. + +The `foca.Foca` class provides a method `.create_celery_app()` that you can +use in your Celery worker entry point to crate a Celery app, like so: + +```py +from foca import Foca + +foca = Foca(config="my_app/config.yaml") +my_celery_app = foca.create_celery_app() +``` ### Configuring logging diff --git a/foca/foca.py b/foca/foca.py index dce1b3a4..d4284308 100644 --- a/foca/foca.py +++ b/foca/foca.py @@ -4,10 +4,11 @@ from pathlib import Path from typing import Optional +from celery import Celery from connexion import App from foca.security.access_control.register_access_control import ( - register_access_control + register_access_control, ) from foca.security.access_control.constants import ( DEFAULT_SPEC_CONTROLLER, @@ -27,7 +28,6 @@ class Foca: - def __init__( self, config_file: Optional[Path] = None, @@ -35,52 +35,45 @@ def __init__( ) -> None: """Instantiate FOCA class. - Args: - config_file: Path to application configuration file in YAML - format. Cf. :py:class:`foca.models.config.Config` for - required file structure. - custom_config_model: Path to model to be used for custom config - parameter validation, supplied in "dot notation", e.g., - ``myapp.config.models.CustomConfig`, where ``CustomConfig`` - is the actual importable name of a `pydantic` model for - your custom configuration, deriving from ``BaseModel``. - FOCA will attempt to instantiate the model with the values - passed to the ``custom`` section in the application's - configuration, if present. Wherever possible, make sure - that default values are supplied for each config - parameters, so as to make it easier for others to - write/modify their app configuration. - - Attributes: - config_file: Path to application configuration file in YAML - format. Cf. :py:class:`foca.models.config.Config` for - required file structure. - custom_config_model: Path to model to be used for custom config - parameter validation, supplied in "dot notation", e.g., - ``myapp.config.models.CustomConfig`, where ``CustomConfig`` - is the actual importable name of a `pydantic` model for - your custom configuration, deriving from ``BaseModel``. - FOCA will attempt to instantiate the model with the values - passed to the ``custom`` section in the application's - configuration, if present. Wherever possible, make sure - that default values are supplied for each config - parameters, so as to make it easier for others to - write/modify their app configuration. + Args: + config_file: Path to application configuration file in YAML + format. Cf. :py:class:`foca.models.config.Config` for + required file structure. + custom_config_model: Path to model to be used for custom config + parameter validation, supplied in "dot notation", e.g., + ``myapp.config.models.CustomConfig`, where ``CustomConfig`` + is the actual importable name of a `pydantic` model for + your custom configuration, deriving from ``BaseModel``. + FOCA will attempt to instantiate the model with the values + passed to the ``custom`` section in the application's + configuration, if present. Wherever possible, make sure + that default values are supplied for each config + parameters, so as to make it easier for others to + write/modify their app configuration. + + Attributes: + config_file: Path to application configuration file in YAML + format. Cf. :py:class:`foca.models.config.Config` for + required file structure. + custom_config_model: Path to model to be used for custom config + parameter validation, supplied in "dot notation", e.g., + ``myapp.config.models.CustomConfig`, where ``CustomConfig`` + is the actual importable name of a `pydantic` model for + your custom configuration, deriving from ``BaseModel``. + FOCA will attempt to instantiate the model with the values + passed to the ``custom`` section in the application's + configuration, if present. Wherever possible, make sure + that default values are supplied for each config + parameters, so as to make it easier for others to + write/modify their app configuration. + conf: App configuration. Instance of + :py:class:`foca.models.config.Config`. """ - self.config_file: Optional[Path] = Path( - config_file - ) if config_file is not None else None + self.config_file: Optional[Path] = ( + Path(config_file) if config_file is not None else None + ) self.custom_config_model: Optional[str] = custom_config_model - - def create_app(self) -> App: - """Set up and initialize FOCA-based microservice. - - Returns: - Connexion application instance. - """ - - # Parse config parameters and format logging - conf = ConfigParser( + self.conf = ConfigParser( config_file=self.config_file, custom_config_model=self.custom_config_model, format_logs=True, @@ -91,8 +84,14 @@ def create_app(self) -> App: else: logger.info("Default app configuration used.") + def create_app(self) -> App: + """Set up and initialize FOCA-based Connexion app. + + Returns: + Connexion application instance. + """ # Create Connexion app - cnx_app = create_connexion_app(conf) + cnx_app = create_connexion_app(self.conf) logger.info("Connexion app created.") # Register error handlers @@ -100,71 +99,89 @@ def create_app(self) -> App: logger.info("Error handler registered.") # Enable cross-origin resource sharing - if(conf.security.cors.enabled is True): + if self.conf.security.cors.enabled is True: enable_cors(cnx_app.app) logger.info("CORS enabled.") else: logger.info("CORS not enabled.") # Register OpenAPI specs - if conf.api.specs: + if self.conf.api.specs: cnx_app = register_openapi( app=cnx_app, - specs=conf.api.specs, + specs=self.conf.api.specs, ) else: logger.info("No OpenAPI specifications provided.") # Register MongoDB - if conf.db: + if self.conf.db: cnx_app.app.config.foca.db = register_mongodb( app=cnx_app.app, - conf=conf.db, + conf=self.conf.db, ) logger.info("Database registered.") else: logger.info("No database support configured.") # Register permission management and casbin enforcer - if conf.security.auth.required: + if self.conf.security.auth.required: if ( - conf.security.access_control.api_specs is None or - conf.security.access_control.api_controllers is None + self.conf.security.access_control.api_specs is None + or self.conf.security.access_control.api_controllers is None ): - conf.security.access_control.api_controllers = ( + self.conf.security.access_control.api_controllers = ( DEFAULT_SPEC_CONTROLLER ) - if conf.security.access_control.db_name is None: - conf.security.access_control.db_name = ( + if self.conf.security.access_control.db_name is None: + self.conf.security.access_control.db_name = ( DEFAULT_ACCESS_CONTROL_DB_NAME ) - if conf.security.access_control.collection_name is None: - conf.security.access_control.collection_name = ( + if self.conf.security.access_control.collection_name is None: + self.conf.security.access_control.collection_name = ( DEFAULT_ACESS_CONTROL_COLLECTION_NAME ) cnx_app = register_access_control( cnx_app=cnx_app, - mongo_config=conf.db, - access_control_config=conf.security.access_control + mongo_config=self.conf.db, + access_control_config=self.conf.security.access_control, ) else: if ( - conf.security.access_control.api_specs or - conf.security.access_control.api_controllers + self.conf.security.access_control.api_specs + or self.conf.security.access_control.api_controllers ): logger.error( - "Please enable security config to register " - "access control." + "Please enable security config to register access control." ) + return cnx_app + + def create_celery_app(self) -> Celery: + """Set up and initialize FOCA-based Celery app. + + Returns: + Celery application instance. + """ + # Create Connexion app + cnx_app = create_connexion_app(self.conf) + logger.info("Connexion app created.") + + # Register error handlers + cnx_app = register_exception_handler(cnx_app) + logger.info("Error handler registered.") + # Create Celery app - if conf.jobs: - create_celery_app(cnx_app.app) + if self.conf.jobs: + celery_app = create_celery_app(cnx_app.app) logger.info("Support for background tasks set up.") else: - logger.info("No support for background tasks configured.") + raise ValueError( + "No support for background tasks configured. Please use the " + "'jobs' keyword section in your configuration file." + ) - return cnx_app + return celery_app diff --git a/foca/version.py b/foca/version.py index 4d743afd..1b183907 100644 --- a/foca/version.py +++ b/foca/version.py @@ -1,3 +1,3 @@ """Single source of truth for package version.""" -__version__ = '0.10.0' +__version__ = '0.11.0' diff --git a/tests/test_files/conf_no_jobs.yaml b/tests/test_files/conf_no_jobs.yaml new file mode 100644 index 00000000..6528d3e2 --- /dev/null +++ b/tests/test_files/conf_no_jobs.yaml @@ -0,0 +1,62 @@ +server: + host: '0.0.0.0' + port: 8080 + debug: True + environment: development + testing: False + use_reloader: True + +api: + specs: + - path: my_specs.yaml + path_out: my_specs.modified.yaml + append: null + add_operation_fields: null + disable_auth: False + connexion: null + +security: + auth: + add_key_to_claims: True + algorithms: + - RS256 + allow_expired: False + audience: null + claim_identity: sub + claim_issuer: iss + validation_methods: + - userinfo + - public_key + validation_checks: all + +db: + host: mongo + port: 27017 + dbs: + my_db: + collections: + my-col-1: + indexes: null + +custom: + my_custom_field_1: my_custom_value_1 + my_custom_field_2: my_custom_value_2 + my_custom_field_3: my_custom_value_3 + +log: + version: 1 + disable_existing_loggers: False + formatters: + standard: + class: logging.Formatter + style: "{" + format: "[{asctime}: {levelname:<8}] {message} [{name}]" + handlers: + console: + class: logging.StreamHandler + level: 20 + formatter: standard + stream: ext://sys.stderr + root: + level: 10 + handlers: [console] diff --git a/tests/test_foca.py b/tests/test_foca.py index 6461b269..2ba7d186 100644 --- a/tests/test_foca.py +++ b/tests/test_foca.py @@ -4,6 +4,7 @@ import pytest import shutil +from celery import Celery from connexion import App from pydantic import ValidationError from pymongo.collection import Collection @@ -18,27 +19,26 @@ DIR = Path(__file__).parent / "test_files" PATH_SPECS_2_YAML_ORIGINAL = str(DIR / "openapi_2_petstore.original.yaml") PATH_SPECS_2_YAML_MODIFIED = str(DIR / "openapi_2_petstore.modified.yaml") -PATH_SPECS_INVALID_OPENAPI = str(DIR / "invalid_openapi_2.yaml") -EMPTY_CONF = str(DIR / "empty_conf.yaml") -INVALID_CONF = str(DIR / "invalid_conf.yaml") -VALID_CONF = str(DIR / "conf_valid.yaml") -VALID_DB_CONF = str(DIR / "conf_db.yaml") -INVALID_DB_CONF = str(DIR / "invalid_conf_db.yaml") -INVALID_LOG_CONF = str(DIR / "conf_invalid_log_level.yaml") -JOBS_CONF = str(DIR / "conf_jobs.yaml") -INVALID_JOBS_CONF = str(DIR / "conf_invalid_jobs.yaml") -API_CONF = str(DIR / "conf_api.yaml") -VALID_CORS_CONF_DISABLED = str(DIR / "conf_valid_cors_disabled.yaml") -VALID_CORS_CONF_ENABLED = str(DIR / "conf_valid_cors_enabled.yaml") -INVALID_ACCESS_CONTROL_CONF = str(DIR / "conf_invalid_access_control.yaml") -VALID_ACCESS_CONTROL_CONF = str(DIR / "conf_valid_access_control.yaml") +EMPTY_CONF = DIR / "empty_conf.yaml" +INVALID_CONF = DIR / "invalid_conf.yaml" +VALID_DB_CONF = DIR / "conf_db.yaml" +INVALID_DB_CONF = DIR / "invalid_conf_db.yaml" +INVALID_LOG_CONF = DIR / "conf_invalid_log_level.yaml" +JOBS_CONF = DIR / "conf_jobs.yaml" +NO_JOBS_CONF = DIR / "conf_no_jobs.yaml" +INVALID_JOBS_CONF = DIR / "conf_invalid_jobs.yaml" +API_CONF = DIR / "conf_api.yaml" +VALID_CORS_CONF_DISABLED = DIR / "conf_valid_cors_disabled.yaml" +VALID_CORS_CONF_ENABLED = DIR / "conf_valid_cors_enabled.yaml" +INVALID_ACCESS_CONTROL_CONF = DIR / "conf_invalid_access_control.yaml" +VALID_ACCESS_CONTROL_CONF = DIR / "conf_valid_access_control.yaml" def create_modified_api_conf(path, temp_dir, api_specs_in, api_specs_out): - """Create a copy of a configuration YAML file""" - temp_path = Path(temp_dir) / 'temp_test.yaml' + """Create a copy of a configuration YAML file.""" + temp_path = Path(temp_dir) / "temp_test.yaml" shutil.copy2(path, temp_path) - with open(temp_path, 'r+') as conf_file: + with open(temp_path, "r+") as conf_file: conf = safe_load(conf_file) conf["api"]["specs"][0]["path"] = api_specs_in conf["api"]["specs"][0]["path_out"] = api_specs_out @@ -48,51 +48,52 @@ def create_modified_api_conf(path, temp_dir, api_specs_in, api_specs_out): return temp_path -def test_foca_output_defaults(): - """Ensure foca() returns a Connexion app instance; defaults only.""" - foca = Foca() - app = foca.create_app() - assert isinstance(app, App) +def test_foca_constructor_empty_conf(): + """Empty config.""" + with pytest.raises(ValidationError): + Foca(config_file=EMPTY_CONF) + + +def test_foca_constructor_invalid_conf(): + """Invalid config file format.""" + with pytest.raises(ValueError): + Foca(config_file=INVALID_CONF) -def test_foca_empty_conf(): - """Ensure foca() require non-empty configuration parameters (if a field is - present)""" - foca = Foca(config_file=EMPTY_CONF) +def test_foca_constructor_invalid_conf_log(): + """Invalid 'log' field.""" with pytest.raises(ValidationError): - foca.create_app() + Foca(config_file=INVALID_LOG_CONF) -def test_foca_invalid_structure(): - """Test foca(); invalid configuration file structure.""" - foca = Foca(config_file=INVALID_CONF) - with pytest.raises(ValueError): - foca.create_app() +def test_foca_constructor_invalid_conf_jobs(): + """Invalid 'jobs' field.""" + with pytest.raises(ValidationError): + Foca(config_file=INVALID_JOBS_CONF) -def test_foca_invalid_log(): - """Test foca(); invalid log field""" - foca = Foca(config_file=INVALID_LOG_CONF) +def test_foca_constructor_invalid_conf_db(): + """Invalid 'db' field.""" with pytest.raises(ValidationError): - foca.create_app() + Foca(config_file=INVALID_DB_CONF) -def test_foca_jobs(): - """Ensure foca() returns a Connexion app instance; valid jobs field""" - foca = Foca(config_file=JOBS_CONF) +def test_foca_create_app_output_defaults(): + """Ensure a Connexion app instance is returned; defaults only.""" + foca = Foca() app = foca.create_app() assert isinstance(app, App) -def test_foca_invalid_jobs(): - """Test foca(); invalid jobs field""" - foca = Foca(config_file=INVALID_JOBS_CONF) - with pytest.raises(ValidationError): - foca.create_app() +def test_foca_create_app_jobs(): + """Ensure a Connexion app instance is returned; valid 'jobs' field.""" + foca = Foca(config_file=JOBS_CONF) + app = foca.create_app() + assert isinstance(app, App) -def test_foca_api(tmpdir): - """Ensure foca() returns a Connexion app instance; valid api field""" +def test_foca_create_app_api(tmpdir): + """Ensure a Connexion app instance is returned; valid 'api' field.""" temp_file = create_modified_api_conf( API_CONF, tmpdir, @@ -105,50 +106,60 @@ def test_foca_api(tmpdir): def test_foca_db(): - """Ensure foca() returns a Connexion app instance; valid db field""" + """Ensure a Connexion app instance is returned; valid 'db' field.""" foca = Foca(config_file=VALID_DB_CONF) app = foca.create_app() - my_db = app.app.config.foca.db.dbs['my-db'] - my_coll = my_db.collections['my-col-1'] + my_db = app.app.config.foca.db.dbs["my-db"] + my_coll = my_db.collections["my-col-1"] assert isinstance(my_db.client, Database) assert isinstance(my_coll.client, Collection) assert isinstance(app, App) -def test_foca_invalid_db(): - """Test foca(); invalid db field""" - foca = Foca(config_file=INVALID_DB_CONF) - with pytest.raises(ValidationError): - foca.create_app() - - -def test_foca_CORS_enabled(): - """Ensures CORS is enabled for Microservice""" +def test_foca_cors_config_flag_enabled(): + """Ensures CORS config flag is set correctly (enabled).""" foca = Foca(config_file=VALID_CORS_CONF_ENABLED) app = foca.create_app() assert app.app.config.foca.security.cors.enabled is True def test_foca_CORS_disabled(): - """Ensures CORS is disabled if user explicitly mentions""" + """Ensures CORS config flag is set correctly (disabled).""" foca = Foca(config_file=VALID_CORS_CONF_DISABLED) app = foca.create_app() assert app.app.config.foca.security.cors.enabled is False def test_foca_invalid_access_control(): - """Ensures access control is not enabled if auth is disabled""" + """Ensures access control is not enabled if auth flag is disabled.""" foca = Foca(config_file=INVALID_ACCESS_CONTROL_CONF) app = foca.create_app() assert app.app.config.foca.db is None def test_foca_valid_access_control(): - """Ensures access control enabled.""" + """Ensures access control settings are set correctly.""" foca = Foca(config_file=VALID_ACCESS_CONTROL_CONF) app = foca.create_app() - my_db = app.app.config.foca.db.dbs['test_db'] - my_coll = my_db.collections['test_collection'] + my_db = app.app.config.foca.db.dbs["test_db"] + my_coll = my_db.collections["test_collection"] assert isinstance(my_db.client, Database) assert isinstance(my_coll.client, Collection) assert isinstance(app, App) + + +def test_foca_create_celery_app(): + """Ensure a Celery app instance is returned.""" + foca = Foca(config_file=JOBS_CONF) + app = foca.create_celery_app() + assert isinstance(app, Celery) + + +def test_foca_create_celery_app_without_jobs_field(): + """Ensure that Celery app creation fails if 'jobs' field missing.""" + foca = Foca(config_file=NO_JOBS_CONF) + with pytest.raises(ValueError): + foca.create_celery_app() + + +# INVALID_JOBS_CONF = DIR / "conf_invalid_jobs.yaml"