From f3974dd215563b8633e9d1869665fe4004ab48c5 Mon Sep 17 00:00:00 2001 From: Mirek Simek Date: Thu, 16 Jan 2025 14:28:09 +0100 Subject: [PATCH 1/2] Removed proxy, added gitlab urls --- pyproject.toml | 2 +- src/nrp_devtools/cli/__init__.py | 4 +- src/nrp_devtools/cli/alembic.py | 5 +- src/nrp_devtools/cli/build.py | 38 ++++--- src/nrp_devtools/cli/check.py | 46 ++++++-- src/nrp_devtools/cli/develop.py | 53 +++++---- src/nrp_devtools/cli/forks.py | 16 ++- src/nrp_devtools/cli/image.py | 14 ++- src/nrp_devtools/cli/initialize.py | 9 +- src/nrp_devtools/cli/model.py | 40 ++++--- src/nrp_devtools/cli/proxy.py | 14 --- src/nrp_devtools/cli/upgrade.py | 12 +- src/nrp_devtools/commands/alembic.py | 56 ++++++---- src/nrp_devtools/commands/check_old.py | 5 +- src/nrp_devtools/commands/develop/runner.py | 115 +++++++++++++------- src/nrp_devtools/commands/docker.py | 9 +- src/nrp_devtools/commands/invenio.py | 5 +- src/nrp_devtools/commands/types.py | 10 ++ src/nrp_devtools/commands/ui/assets.py | 25 +++-- src/nrp_devtools/commands/ui/build.py | 4 +- src/nrp_devtools/commands/ui/link_assets.py | 3 +- src/nrp_devtools/commands/utils.py | 90 +++++++++++---- src/nrp_devtools/config/config.py | 44 +++++--- src/nrp_devtools/config/enums.py | 8 +- src/nrp_devtools/config/model_config.py | 7 +- src/nrp_devtools/pypi_proxy/__init__.py | 0 src/nrp_devtools/pypi_proxy/proxy.py | 84 -------------- 27 files changed, 410 insertions(+), 308 deletions(-) delete mode 100644 src/nrp_devtools/cli/proxy.py create mode 100644 src/nrp_devtools/commands/types.py delete mode 100644 src/nrp_devtools/pypi_proxy/__init__.py delete mode 100644 src/nrp_devtools/pypi_proxy/proxy.py diff --git a/pyproject.toml b/pyproject.toml index acdca77..c46c696 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "nrp-devtools" -version = "1.0.18" +version = "1.0.19" description = "NRP repository development tools" readme = "README.md" authors = [{ name = "Miroslav Simek", email = "miroslav.simek@cesnet.cz" }] diff --git a/src/nrp_devtools/cli/__init__.py b/src/nrp_devtools/cli/__init__.py index fe2741a..2eced50 100644 --- a/src/nrp_devtools/cli/__init__.py +++ b/src/nrp_devtools/cli/__init__.py @@ -12,7 +12,6 @@ from .ui import ui_group from .upgrade import upgrade_command from .watch import watch_command -from .proxy import start_proxy __all__ = [ "nrp_command", @@ -28,6 +27,5 @@ "alembic_command", "image_command", "forks_group", - "watch_command" - "start_proxy" + "watch_command", ] diff --git a/src/nrp_devtools/cli/alembic.py b/src/nrp_devtools/cli/alembic.py index 9fe8e47..377e8bb 100644 --- a/src/nrp_devtools/cli/alembic.py +++ b/src/nrp_devtools/cli/alembic.py @@ -1,10 +1,13 @@ +from typing import Any + from ..commands.alembic import build_alembic +from ..commands.types import StepFunctions from ..config import OARepoConfig from .base import command_sequence, nrp_command @nrp_command.command(name="alembic") @command_sequence() -def alembic_command(*, config: OARepoConfig, **kwargs): +def alembic_command(*, config: OARepoConfig, **kwargs: Any) -> StepFunctions: """Builds the repository""" return (build_alembic,) diff --git a/src/nrp_devtools/cli/build.py b/src/nrp_devtools/cli/build.py index 22d3c16..53f89ac 100644 --- a/src/nrp_devtools/cli/build.py +++ b/src/nrp_devtools/cli/build.py @@ -1,41 +1,49 @@ from functools import partial +from typing import Any import click from ..commands.invenio import install_invenio_cfg from ..commands.resolver import get_resolver +from ..commands.types import StepFunctions from ..commands.ui import build_production_ui, collect_assets, install_npm_packages from ..commands.utils import make_step, no_args, run_fixup from ..config import OARepoConfig from .base import command_sequence, nrp_command -from ..pypi_proxy.proxy import start_pypi_proxy @nrp_command.command(name="build") @command_sequence() -def build_command(*, config: OARepoConfig, **kwargs): +def build_command(*, config: OARepoConfig, **kwargs: Any) -> StepFunctions: """Builds the repository""" - start_pypi_proxy(config.pypi_proxy_target) return build_command_internal(config=config, **kwargs) -def build_command_internal(*, config: OARepoConfig, **kwargs): +def build_command_internal(*, config: OARepoConfig, **kwargs: Any) -> StepFunctions: resolver = get_resolver(config) return ( no_args( partial(click.secho, "Building repository for production", fg="yellow"), - name="display_message" + name="display_message", + ), + make_step( + lambda config, **kwargs: resolver.clean_previous_installation(), + name="clean_previous_installation", + ), + make_step( + lambda config, **kwargs: resolver.create_empty_venv(), + name="create_empty_venv", + ), + run_fixup( + lambda config, **kwargs: resolver.check_requirements(), + lambda config, **kwargs: resolver.build_requirements(), + fix=True, + name="check_and_build_requirements", + ), + make_step( + lambda config, **kwargs: resolver.install_python_repository(), + name="install_python_repository", ), - make_step(lambda config, **kwargs: resolver.clean_previous_installation(), - name="clean_previous_installation"), - make_step(lambda config, **kwargs: resolver.create_empty_venv(), - name="create_empty_venv"), - run_fixup(lambda config, **kwargs: resolver.check_requirements(), - lambda config, **kwargs: resolver.build_requirements(), - fix=True, - name="check_and_build_requirements"), - make_step(lambda config, **kwargs: resolver.install_python_repository(), - name="install_python_repository"), install_invenio_cfg, collect_assets, install_npm_packages, diff --git a/src/nrp_devtools/cli/check.py b/src/nrp_devtools/cli/check.py index a09b2d9..0afcb55 100644 --- a/src/nrp_devtools/cli/check.py +++ b/src/nrp_devtools/cli/check.py @@ -1,4 +1,5 @@ from functools import partial +from typing import Any import click @@ -23,27 +24,36 @@ fix_s3_bucket_exists, fix_s3_location_in_database, ) +from ..commands.types import StepFunctions from ..commands.ui import check_ui, fix_ui from ..commands.utils import make_step, no_args, run_fixup from ..config import OARepoConfig from .base import command_sequence, nrp_command -from ..pypi_proxy.proxy import start_pypi_proxy @nrp_command.command(name="check") @click.option("--fix", is_flag=True, default=False) @click.option("--local-packages", "-e", multiple=True) @command_sequence() -def check_command(*, config: OARepoConfig, local_packages=None, fix=False, **kwargs): +def check_command( + *, + config: OARepoConfig, + local_packages: list[str] | None = None, + fix: bool = False, + **kwargs: Any, +) -> StepFunctions: "Checks prerequisites for running the repository, initializes a build environment and rebuilds the repository." context = {} return check_commands(context, config, local_packages, fix) -def check_commands(context, config, local_packages, fix): +def check_commands( + context: dict[str, Any], + config: OARepoConfig, + local_packages: list[str] | None, + fix: bool, +) -> StepFunctions: resolver = get_resolver(config) - if fix: - start_pypi_proxy(config.pypi_proxy_target) return ( # # infrastructure checks @@ -58,13 +68,19 @@ def check_commands(context, config, local_packages, fix): # # virtualenv exists # - run_fixup(lambda config, **kwargs: resolver.check_virtualenv(**kwargs), - lambda config, **kwargs: resolver.fix_virtualenv(**kwargs), fix=fix), + run_fixup( + lambda config, **kwargs: resolver.check_virtualenv(**kwargs), + lambda config, **kwargs: resolver.fix_virtualenv(**kwargs), + fix=fix, + ), # # requirements have been built # - run_fixup(lambda config, **kwargs: resolver.check_requirements(**kwargs), - lambda config, **kwargs: resolver.build_requirements(**kwargs), fix=fix), + run_fixup( + lambda config, **kwargs: resolver.check_requirements(**kwargs), + lambda config, **kwargs: resolver.build_requirements(**kwargs), + fix=fix, + ), # # invenio.cfg and variables are inside virtual environment # @@ -72,12 +88,18 @@ def check_commands(context, config, local_packages, fix): # # can run invenio command # - run_fixup(lambda config, **kwargs: resolver.check_invenio_callable(**kwargs), - lambda config, **kwargs: resolver.install_python_repository(**kwargs), fix=fix), + run_fixup( + lambda config, **kwargs: resolver.check_invenio_callable(**kwargs), + lambda config, **kwargs: resolver.install_python_repository(**kwargs), + fix=fix, + ), # # any local packages are installed inside the virtual environment # - make_step(lambda config, **kwargs: resolver.install_local_packages(**kwargs), local_packages=local_packages), + make_step( + lambda config, **kwargs: resolver.install_local_packages(**kwargs), + local_packages=local_packages, + ), # # check that docker containers are running # diff --git a/src/nrp_devtools/cli/develop.py b/src/nrp_devtools/cli/develop.py index 0665e12..aab2dba 100644 --- a/src/nrp_devtools/cli/develop.py +++ b/src/nrp_devtools/cli/develop.py @@ -1,13 +1,15 @@ +from typing import Any + import click from ..commands.develop import Runner from ..commands.develop.controller import run_develop_controller +from ..commands.types import StepFunction, StepFunctions from ..commands.ui.link_assets import copy_assets_to_webpack_build_dir from ..commands.utils import make_step from ..config import OARepoConfig from .base import command_sequence, nrp_command from .check import check_commands -from ..pypi_proxy.proxy import start_pypi_proxy @nrp_command.command(name="develop") @@ -27,35 +29,42 @@ "--shell/--no-shell", default=False, help="Start the new development environment that is based on " - "shell script and does not suffer with problems with stdout/stderr", + "shell script and does not suffer with problems with stdout/stderr", ) @command_sequence() def develop_command( - *, config: OARepoConfig, local_packages=None, checks=True, shell=False, **kwargs -): + *, + config: OARepoConfig, + local_packages: list[str] | None = None, + checks: bool = True, + shell: bool = False, + **kwargs: Any, +) -> StepFunctions: """Starts the development environment for the repository.""" - context = {} - start_pypi_proxy(config.pypi_proxy_target) - commands = [ + context: dict[str, Any] = {} + commands: list[StepFunction] = [ *(check_commands(context, config, local_packages, fix=True) if checks else ()), copy_assets_to_webpack_build_dir, ] if not shell: runner = Runner(config) - commands.extend([ - make_step( - lambda config=None, runner=None: runner.start_python_server( - development_mode=True + commands.extend( + [ + make_step( + lambda config=None, _runner=None: runner.start_python_server( + development_mode=True + ), + runner=runner, + ), + make_step( + lambda config=None, _runner=None: runner.start_webpack_server(), + runner=runner, + ), + make_step( + lambda config=None, _runner=None: runner.start_file_watcher(), + runner=runner, ), - runner=runner, - ), - make_step( - lambda config=None, runner=None: runner.start_webpack_server(), - runner=runner, - ), - make_step( - lambda config=None, runner=None: runner.start_file_watcher(), runner=runner - ), - make_step(run_develop_controller, runner=runner, development_mode=True), - ]) + make_step(run_develop_controller, runner=runner, development_mode=True), + ] + ) return commands diff --git a/src/nrp_devtools/cli/forks.py b/src/nrp_devtools/cli/forks.py index a83cca4..61e4443 100644 --- a/src/nrp_devtools/cli/forks.py +++ b/src/nrp_devtools/cli/forks.py @@ -1,13 +1,15 @@ +from typing import Any + import click from ..commands.forks import apply_forks +from ..commands.resolver import get_resolver from ..config import OARepoConfig from .base import command_sequence, nrp_command -from ..commands.resolver import get_resolver @nrp_command.group(name="forks") -def forks_group(**kwargs): +def forks_group(**kwargs: Any): pass @@ -15,26 +17,28 @@ def forks_group(**kwargs): @click.argument("python_package") @click.argument("git_fork_url") @command_sequence(save=True) -def add_fork_command(*, config: OARepoConfig, python_package, git_fork_url, **kwargs): +def add_fork_command( + *, config: OARepoConfig, python_package: str, git_fork_url: str, **kwargs: Any +): config.add_fork(python_package, git_fork_url) @forks_group.command(name="remove", help="Remove a fork") @click.argument("python_package") @command_sequence(save=True) -def remove_fork_command(*, config: OARepoConfig, python_package, **kwargs): +def remove_fork_command(*, config: OARepoConfig, python_package: str, **kwargs: Any): config.remove_fork(python_package) @forks_group.command(name="list", help="List all forks") @command_sequence() -def list_forks_command(*, config: OARepoConfig, **kwargs): +def list_forks_command(*, config: OARepoConfig, **kwargs: Any): for package, url in config.forks.items(): click.echo(f"{package}: {url}") @forks_group.command(name="apply", help="Apply all forks") @command_sequence() -def apply_forks_command(*, config: OARepoConfig, **kwargs): +def apply_forks_command(*, config: OARepoConfig, **kwargs: Any): resolver = get_resolver(config) apply_forks(config, resolver) diff --git a/src/nrp_devtools/cli/image.py b/src/nrp_devtools/cli/image.py index 9a25576..fafbfcd 100644 --- a/src/nrp_devtools/cli/image.py +++ b/src/nrp_devtools/cli/image.py @@ -1,3 +1,5 @@ +from typing import Any + import click from ..commands.utils import make_step, run_cmdline @@ -7,10 +9,16 @@ @nrp_command.command(name="image") @command_sequence() -def image_command(*, config: OARepoConfig, local_packages=None, checks=True, **kwargs): +def image_command( + *, + config: OARepoConfig, + local_packages: list[str] | None = None, + checks: bool = True, + **kwargs: Any, +): """Starts the repository. Make sure to run `nrp check` first.""" - def build_image(*args, **kwargs): + def build_image(*args: Any, **kwargs: Any): info_string = run_cmdline("docker", "info", grab_stdout=True) architecture_line = [ x for x in info_string.split("\n") if "Architecture:" in x @@ -36,7 +44,7 @@ def build_image(*args, **kwargs): "BUILDKIT_PROGRESS": "plain", "BUILDPLATFORM": build_platform, "DOCKER_BUILDKIT": "1", - **config.global_environment() + **config.global_environment(), }, ) diff --git a/src/nrp_devtools/cli/initialize.py b/src/nrp_devtools/cli/initialize.py index a42a41c..e79490c 100644 --- a/src/nrp_devtools/cli/initialize.py +++ b/src/nrp_devtools/cli/initialize.py @@ -27,7 +27,12 @@ repository_dir_as_argument=True, repository_dir_must_exist=False, save=True ) def initialize_command( - *, repository_dir, config: OARepoConfig, verbose, initial_config, no_input + *, + repository_dir: Path, + config: OARepoConfig, + verbose: bool, + initial_config: Path, + no_input: bool, ): """ Initialize a new nrp project. Note: the project directory must be empty. @@ -53,7 +58,7 @@ def initialize_step(config: OARepoConfig): initialize_repository(config) click.secho( - f""" + """ Your repository is now initialized. You can start the repository in development mode diff --git a/src/nrp_devtools/cli/model.py b/src/nrp_devtools/cli/model.py index d55a81d..11c9ef0 100644 --- a/src/nrp_devtools/cli/model.py +++ b/src/nrp_devtools/cli/model.py @@ -1,8 +1,9 @@ +import shutil import tempfile from pathlib import Path +from typing import Any import click -import shutil from ..commands.model.compile import ( add_model_to_i18n, @@ -13,11 +14,11 @@ ) from ..commands.model.create import create_model from ..commands.resolver import get_resolver +from ..commands.types import StepFunctions from ..commands.utils import make_step from ..config import OARepoConfig, ask_for_configuration -from ..config.model_config import ModelConfig, ModelFeature, BaseModel +from ..config.model_config import BaseModel, ModelConfig, ModelFeature from .base import command_sequence, nrp_command -from ..pypi_proxy.proxy import start_pypi_proxy @nrp_command.group(name="model") @@ -31,16 +32,22 @@ def model_group(): @click.argument("model_name") @click.option("--copy-model-config", help="Use a configuration file", type=click.Path()) @command_sequence(save=True) -def create_model_command(*, config: OARepoConfig, model_name, copy_model_config=None, **kwargs): +def create_model_command( + *, + config: OARepoConfig, + model_name: str, + copy_model_config: Path | None = None, + **kwargs: Any, +) -> StepFunctions: for model in config.models: if model.model_name == model_name: click.secho(f"Model {model_name} already exists", fg="red", err=True) - return + return () if copy_model_config: # if the config file is ready, just copy and add note to oarepo.yaml # TODO: if possible, parse the values below from the copied config file - values = { + values: dict[str, Any] = { "base_model": BaseModel.empty, "model_name": model_name, "model_description": "", @@ -50,8 +57,8 @@ def create_model_command(*, config: OARepoConfig, model_name, copy_model_config= ModelFeature.requests, ModelFeature.files, ModelFeature.drafts, - ModelFeature.relations - ] + ModelFeature.relations, + ], } values["model_package"] = ModelConfig.default_model_package(config, values) values["api_prefix"] = ModelConfig.default_api_prefix(config, values) @@ -75,19 +82,22 @@ def set_model_configuration(config: OARepoConfig, *args, **kwargs): @model_group.command(name="compile", help="Compile a model") @click.argument("model_name") -@click.option("--reinstall-builder/--keep-builder", - is_flag=True, help="Reinstall the model builder", default=True) +@click.option( + "--reinstall-builder/--keep-builder", + is_flag=True, + help="Reinstall the model builder", + default=True, +) @command_sequence() -def compile_model_command(*, config: OARepoConfig, model_name, reinstall_builder, **kwargs): - start_pypi_proxy(config.pypi_proxy_target) +def compile_model_command( + *, config: OARepoConfig, model_name, reinstall_builder, **kwargs +): model = config.get_model(model_name) # create a temporary directory using tempfile tempdir = str(Path(tempfile.mkdtemp()).resolve()) if reinstall_builder: - steps = [ - make_step(install_model_compiler, model=model) - ] + steps = [make_step(install_model_compiler, model=model)] else: steps = [] diff --git a/src/nrp_devtools/cli/proxy.py b/src/nrp_devtools/cli/proxy.py deleted file mode 100644 index fec28b4..0000000 --- a/src/nrp_devtools/cli/proxy.py +++ /dev/null @@ -1,14 +0,0 @@ -import click - -from ..config import OARepoConfig -from .base import command_sequence, nrp_command -from .build import build_command_internal -from time import sleep -from ..pypi_proxy.proxy import start_pypi_proxy - -@nrp_command.command(name="proxy") -@click.argument("time") -@command_sequence() -def start_proxy(*, config: OARepoConfig, time, **kwargs): - start_pypi_proxy(config.pypi_proxy_target) - sleep(int(time)) \ No newline at end of file diff --git a/src/nrp_devtools/cli/upgrade.py b/src/nrp_devtools/cli/upgrade.py index 8f5c208..736780d 100644 --- a/src/nrp_devtools/cli/upgrade.py +++ b/src/nrp_devtools/cli/upgrade.py @@ -2,9 +2,6 @@ from ..config import OARepoConfig from .base import command_sequence, nrp_command from .build import build_command_internal -from ..pypi_proxy.proxy import start_pypi_proxy - - @nrp_command.command(name="upgrade") @@ -14,9 +11,6 @@ def upgrade_command(*, config: OARepoConfig, **kwargs): Resolves the newest applicable packages, downloads them and rebuilds the repository. """ - start_pypi_proxy(config.pypi_proxy_target) - return (lambda config: get_resolver(config).build_requirements(),) + build_command_internal(config=config) - - - - + return ( + lambda config: get_resolver(config).build_requirements(), + ) + build_command_internal(config=config) diff --git a/src/nrp_devtools/commands/alembic.py b/src/nrp_devtools/commands/alembic.py index 94b65d6..0f9632e 100644 --- a/src/nrp_devtools/commands/alembic.py +++ b/src/nrp_devtools/commands/alembic.py @@ -1,6 +1,7 @@ import os import re from pathlib import Path +from typing import Any, cast import click @@ -11,17 +12,18 @@ from nrp_devtools.config import OARepoConfig -def build_alembic(config): - click.secho(f"Generating alembic", fg="yellow") +def build_alembic(config: OARepoConfig, **kwargs: Any) -> None: + click.secho("Generating alembic", fg="yellow") alembic_path = config.shared_dir / "alembic" + assert config.repository branch = config.repository.repository_package setup_alembic(config, branch, alembic_path) - click.secho(f"Alembic successfully generated", fg="green") + click.secho("Alembic successfully generated", fg="green") -def setup_alembic(config: OARepoConfig, branch: str, alembic_path: Path): +def setup_alembic(config: OARepoConfig, branch: str, alembic_path: Path) -> None: filecount = len( [x for x in alembic_path.iterdir() if x.is_file() and x.name.endswith(".py")] ) @@ -32,13 +34,13 @@ def setup_alembic(config: OARepoConfig, branch: str, alembic_path: Path): update_alembic(config, branch, alembic_path) -def update_alembic(config: OARepoConfig, branch, alembic_path): +def update_alembic(config: OARepoConfig, branch: str, alembic_path: Path) -> None: # alembic has been initialized, update heads and generate invenio_alembic_path = convert_to_invenio_alembic_path(config, alembic_path) files = [file_path.name for file_path in alembic_path.iterdir()] - file_numbers = [] + file_numbers: list[int] = [] for file in files: file_number_regex = re.findall(f"(?<={branch}_)\d+", file) if file_number_regex: @@ -48,32 +50,44 @@ def update_alembic(config: OARepoConfig, branch, alembic_path): "Nrp install revision." ) run_cmdline( - config.invenio_command, "alembic", "upgrade", "heads", cwd=config.repository_dir + config.invenio_command, + "alembic", + "upgrade", + "heads", + cwd=str(config.repository_dir), ) new_revision = get_revision_number( - run_cmdline( - config.invenio_command, - "alembic", - "revision", - revision_message, - "-b", - branch, - "--path", - invenio_alembic_path, - grab_stdout=True, - cwd=config.repository_dir, + cast( + str, + run_cmdline( + config.invenio_command, + "alembic", + "revision", + revision_message, + "-b", + branch, + "--path", + invenio_alembic_path, + grab_stdout=True, + cwd=str(config.repository_dir), + ), ), file_revision_name_suffix, ) rewrite_revision_file(alembic_path, new_file_number, branch, new_revision) fix_sqlalchemy_utils(alembic_path) run_cmdline( - config.invenio_command, "alembic", "upgrade", "heads", cwd=config.repository_dir + config.invenio_command, + "alembic", + "upgrade", + "heads", + cwd=str(config.repository_dir), ) -def convert_to_invenio_alembic_path(config, alembic_path): +def convert_to_invenio_alembic_path(config: OARepoConfig, alembic_path: Path) -> str: + alembic_locations: list[str] (alembic_locations,) = get_invenio_configuration(config, {}, "ALEMBIC_LOCATIONS") resolved_alembic_path = alembic_path.resolve() for loc in alembic_locations: @@ -84,7 +98,7 @@ def convert_to_invenio_alembic_path(config, alembic_path): ) -def intialize_alembic(config, branch, alembic_path): +def intialize_alembic(config: OARepoConfig, branch: str, alembic_path: Path): initialize_db_if_not_initialized(config) # alembic has not been initialized yet ... diff --git a/src/nrp_devtools/commands/check_old.py b/src/nrp_devtools/commands/check_old.py index 340b832..8de46b2 100644 --- a/src/nrp_devtools/commands/check_old.py +++ b/src/nrp_devtools/commands/check_old.py @@ -1,8 +1,11 @@ +from typing import Any + from nrp_devtools.commands.check import check_failed from nrp_devtools.commands.utils import run_cmdline +from nrp_devtools.config.config import OARepoConfig -def check_imagemagick_callable(config): +def check_imagemagick_callable(config: OARepoConfig, **kwargs: Any): try: run_cmdline("convert", "--version", grab_stdout=True, raise_exception=True) except: diff --git a/src/nrp_devtools/commands/develop/runner.py b/src/nrp_devtools/commands/develop/runner.py index f27fe7d..55a34ff 100644 --- a/src/nrp_devtools/commands/develop/runner.py +++ b/src/nrp_devtools/commands/develop/runner.py @@ -10,8 +10,20 @@ import click import psutil -from watchdog.events import FileSystemEventHandler +from watchdog.events import ( + DirCreatedEvent, + DirDeletedEvent, + DirModifiedEvent, + DirMovedEvent, + FileClosedEvent, + FileCreatedEvent, + FileDeletedEvent, + FileModifiedEvent, + FileMovedEvent, + FileSystemEventHandler, +) from watchdog.observers import Observer +from watchdog.observers.api import BaseObserver from nrp_devtools.commands.ui.assets import load_watched_paths from nrp_devtools.commands.ui.link_assets import copy_assets_to_webpack_build_dir @@ -19,16 +31,16 @@ class Runner: - python_server_process: Optional[subprocess.Popen] = None - webpack_server_process: Optional[subprocess.Popen] = None + python_server_process: Optional[subprocess.Popen[bytes]] = None + webpack_server_process: Optional[subprocess.Popen[bytes]] = None file_copier: Optional["FileCopier"] = None def __init__(self, config: OARepoConfig): self.config = config - def start_python_server(self, development_mode=False): + def start_python_server(self, development_mode: bool = False) -> None: click.secho("Starting python server", fg="yellow") - environment = {} + environment: dict[str, str] = {} if development_mode: environment["FLASK_DEBUG"] = "1" environment["INVENIO_TEMPLATES_AUTO_RELOAD"] = "1" @@ -50,7 +62,7 @@ def start_python_server(self, development_mode=False): target=self._read_python_output, daemon=True ) self.python_reader_thread.start() - for i in range(5): + for _i in range(5): time.sleep(2) if self.python_server_process.poll() is not None: click.secho( @@ -63,7 +75,7 @@ def start_python_server(self, development_mode=False): break click.secho("Python server started", fg="green") - def start_webpack_server(self): + def start_webpack_server(self) -> None: click.secho("Starting webpack server", fg="yellow") manifest_path = ( self.config.invenio_instance_path / "static" / "dist" / "manifest.json" @@ -81,7 +93,7 @@ def start_webpack_server(self): pass_fds=(sys.stdin.fileno(), sys.stdout.fileno(), sys.stderr.fileno()), ) # wait at most a minute for webpack to start - for i in range(60): + for _i in range(60): time.sleep(2) if self.webpack_server_process.poll() is not None: click.secho( @@ -100,24 +112,24 @@ def start_webpack_server(self): break click.secho("Webpack server started", fg="green") - def start_file_watcher(self): + def start_file_watcher(self) -> None: click.secho("Starting file watcher", fg="yellow") self.file_copier = FileCopier(self.config) click.secho("File watcher started", fg="green") - def stop(self): + def stop(self) -> None: self.stop_python_server() self.stop_webpack_server() self.stop_file_watcher() - def restart_python_server(self, development_mode=False): + def restart_python_server(self, development_mode: bool = False) -> None: try: self.stop_python_server() self.start_python_server(development_mode=development_mode) except: traceback.print_exc() - def restart_webpack_server(self): + def restart_webpack_server(self) -> None: try: self.stop_webpack_server() self.stop_file_watcher() @@ -129,7 +141,7 @@ def restart_webpack_server(self): except: traceback.print_exc() - def stop_python_server(self): + def stop_python_server(self) -> None: click.secho("Stopping python server", fg="yellow") if self.python_server_process: self.python_server_process.terminate() @@ -142,7 +154,7 @@ def stop_python_server(self): self._kill_process_tree(self.python_server_process) self.python_server_process = None - def stop_webpack_server(self): + def stop_webpack_server(self) -> None: click.secho("Stopping webpack server", fg="yellow") if self.webpack_server_process: self.webpack_server_process.terminate() @@ -155,20 +167,20 @@ def stop_webpack_server(self): self._kill_process_tree(self.webpack_server_process) self.webpack_server_process = None - def stop_file_watcher(self): + def stop_file_watcher(self) -> None: click.secho("Stopping file watcher", fg="yellow") if self.file_copier: self.file_copier.join() self.file_copier = None - def _kill_process_tree(self, process_tree: subprocess.Popen): + def _kill_process_tree(self, process_tree: subprocess.Popen[bytes]) -> None: parent_pid = process_tree.pid parent = psutil.Process(parent_pid) for child in parent.children(recursive=True): child.kill() parent.kill() - def _read_python_output(self): + def _read_python_output(self) -> None: while True: try: if ( @@ -178,7 +190,7 @@ def _read_python_output(self): break line: bytes = self.python_server_process.stdout.readline() if line: - for r in range(5): + for _r in range(5): try: sys.stdout.buffer.write(line) except: @@ -195,13 +207,18 @@ def _read_python_output(self): class FileCopier: class Handler(FileSystemEventHandler): - def __init__(self, source_path: Path, target_path: Path, watcher): - self.source_root_path = source_path - self.target_root_path = target_path + def __init__( + self, + source_path: str | bytes | Path, + target_path: str | bytes | Path, + watcher: BaseObserver, + ): + self.source_root_path = self._convert_to_path(source_path) + self.target_root_path = self._convert_to_path(target_path) self.watcher = watcher - print(f"Watching {source_path} -> {target_path}") + print(f"Watching {self.source_root_path} -> {self.target_root_path}") - def on_closed(self, event): + def on_closed(self, event: FileClosedEvent): if event.is_directory: return @@ -211,7 +228,7 @@ def on_closed(self, event): except: traceback.print_exc() - def on_modified(self, event): + def on_modified(self, event: DirModifiedEvent | FileModifiedEvent): if event.is_directory: return @@ -221,7 +238,7 @@ def on_modified(self, event): except: traceback.print_exc() - def on_moved(self, event): + def on_moved(self, event: DirMovedEvent | FileMovedEvent): try: time.sleep(0.01) self.remove_file(event.src_path, self.make_target_path(event.src_path)) @@ -229,7 +246,7 @@ def on_moved(self, event): except: traceback.print_exc() - def on_created(self, event): + def on_created(self, event: DirCreatedEvent | FileCreatedEvent): """When a new directory is created, add a watch for it""" if event.is_directory: self.watcher.schedule( @@ -238,35 +255,49 @@ def on_created(self, event): self.make_target_path(event.src_path), self.watcher, ), - event.src_path, + str(self._convert_to_path(event.src_path)), recursive=True, ) - def on_deleted(self, event): + def on_deleted(self, event: DirDeletedEvent | FileDeletedEvent): try: time.sleep(0.01) self.remove_file(event.src_path, self.make_target_path(event.src_path)) except: traceback.print_exc() - def make_target_path(self, source_path): - return self.target_root_path / Path(source_path).relative_to( - self.source_root_path - ) - - def copy_file(self, source_path, target_path): + def _convert_to_path(self, path: bytes | str | Path) -> Path: + if isinstance(path, bytes): + path = path.decode("utf-8") + assert isinstance(path, (str, Path)) + return Path(path) + + def make_target_path(self, source_path: bytes | str | Path) -> Path: + return self.target_root_path / self._convert_to_path( + source_path + ).relative_to(self.source_root_path) + + def copy_file( + self, source_path: bytes | str | Path, target_path: bytes | str | Path + ): + source_path = self._convert_to_path(source_path) + target_path = self._convert_to_path(target_path) if str(source_path).endswith("~"): return print(f"Copying {source_path} to {target_path}") target_path.parent.mkdir(parents=True, exist_ok=True) shutil.copy(source_path, target_path) - def remove_file(self, source_path, target_path): + def remove_file( + self, source_path: bytes | str | Path, target_path: bytes | str | Path + ): + source_path = self._convert_to_path(source_path) + target_path = self._convert_to_path(target_path) print(f"Removing {target_path}") if target_path.exists(): target_path.unlink() - def __init__(self, config): + def __init__(self, config: OARepoConfig): self.config = config static = (config.ui_dir / "static").resolve() @@ -280,16 +311,16 @@ def __init__(self, config): static_target_path = self.config.invenio_instance_path / "static" assets_target_path = self.config.invenio_instance_path / "assets" - for path, kind in self.watched_paths.items(): - path = Path(path).resolve() + for watched_path, kind in self.watched_paths.items(): + path = Path(watched_path).resolve() if not path.exists(): - click.secho(f">>>> Watcher error:", fg="red") - click.secho(f">>>>", fg="red") + click.secho(">>>> Watcher error:", fg="red") + click.secho(">>>>", fg="red") click.secho( f">>>> Path {path} does not exist, will not watch it!", fg="red" ) - click.secho(f">>>>", fg="red") - click.secho(f">>>>", fg="red") + click.secho(">>>>", fg="red") + click.secho(">>>>", fg="red") continue if kind == "static": diff --git a/src/nrp_devtools/commands/docker.py b/src/nrp_devtools/commands/docker.py index e63de13..ec30333 100644 --- a/src/nrp_devtools/commands/docker.py +++ b/src/nrp_devtools/commands/docker.py @@ -1,6 +1,7 @@ import re import time import traceback +from typing import Any from urllib.parse import urlparse import click @@ -31,7 +32,7 @@ def fix_docker_env(config: OARepoConfig, **kwargs): ) -def check_docker_callable(config: OARepoConfig, will_fix=False, **kwargs): +def check_docker_callable(config: OARepoConfig, will_fix: bool = False, **kwargs: Any): try: run_cmdline("docker", "ps", grab_stdout=True, raise_exception=True) except: # noqa @@ -89,16 +90,16 @@ def retry(fn, config, context, tries=10, timeout=1, verbose=False): for i in range(tries): try: fn(config, context) - click.secho(f" ... alive", fg="green") + click.secho(" ... alive", fg="green") return except KeyboardInterrupt: raise - except BaseException as e: + except BaseException: # catch SystemExit as well if verbose: click.secho(traceback.format_exc(), fg="red") if i == tries - 1: - click.secho(f" ... failed", fg="red") + click.secho(" ... failed", fg="red") raise click.secho( f" ... not yet ready, sleeping for {int(timeout)} seconds", diff --git a/src/nrp_devtools/commands/invenio.py b/src/nrp_devtools/commands/invenio.py index 0d1def9..4b26cd2 100644 --- a/src/nrp_devtools/commands/invenio.py +++ b/src/nrp_devtools/commands/invenio.py @@ -1,5 +1,6 @@ import json import tempfile +from typing import Any from nrp_devtools.commands.utils import run_cmdline from nrp_devtools.config import OARepoConfig @@ -27,7 +28,7 @@ def get_repository_info(config, context=None): return repository_info -def get_invenio_configuration(config, context, *args): +def get_invenio_configuration(config, context, *args) -> Any: def _get_config(config, context): if context is not None and "repository_configuration" in context: return context["repository_configuration"] @@ -67,7 +68,7 @@ def check_invenio_cfg(config: OARepoConfig, will_fix=False, **kwargs): ) -def install_invenio_cfg(config: OARepoConfig, **kwargs): +def install_invenio_cfg(config: OARepoConfig, **kwargs: Any): instance_dir = config.invenio_instance_path instance_dir.mkdir(exist_ok=True, parents=True) diff --git a/src/nrp_devtools/commands/types.py b/src/nrp_devtools/commands/types.py new file mode 100644 index 0000000..39ce621 --- /dev/null +++ b/src/nrp_devtools/commands/types.py @@ -0,0 +1,10 @@ +from typing import Any, Protocol + +from ..config import OARepoConfig + + +class StepFunction(Protocol): + def __call__(self, config: OARepoConfig, **kwargs: Any) -> None: ... + + +StepFunctions = tuple[StepFunction, ...] | list[StepFunction] diff --git a/src/nrp_devtools/commands/ui/assets.py b/src/nrp_devtools/commands/ui/assets.py index 388d7c0..c871200 100644 --- a/src/nrp_devtools/commands/ui/assets.py +++ b/src/nrp_devtools/commands/ui/assets.py @@ -1,26 +1,31 @@ import json import shutil from pathlib import Path +from typing import Any + +from nrp_devtools.config.config import OARepoConfig from ..utils import run_cmdline from .less import register_less_components -def load_watched_paths(paths_json, extra_paths): - watched_paths = {} +def load_watched_paths( + paths_json: str | Path, extra_paths: list[str] | None = None +) -> dict[str, str]: + watched_paths: dict[str, str] = {} with open(paths_json) as f: for target, paths in json.load(f).items(): if target.startswith("@"): continue for pth in paths: watched_paths[pth] = target - for e in extra_paths: + for e in extra_paths or []: source, target = e.split("=", maxsplit=1) watched_paths[source] = target return watched_paths -def collect_assets(config): +def collect_assets(config: OARepoConfig, **kwargs: Any): invenio_instance_path = config.invenio_instance_path shutil.rmtree(invenio_instance_path / "assets", ignore_errors=True) shutil.rmtree(invenio_instance_path / "static", ignore_errors=True) @@ -33,7 +38,10 @@ def collect_assets(config): "assets", "collect", f"{invenio_instance_path}/watch.list.json", - environ={"INVENIO_INSTANCE_PATH": str(config.invenio_instance_path), "FLASK_DEBUG": None}, + environ={ + "INVENIO_INSTANCE_PATH": str(config.invenio_instance_path), + "FLASK_DEBUG": None, + }, ) run_cmdline( @@ -43,7 +51,10 @@ def collect_assets(config): "create", # during create, if FLASK_DEBUG is set, invenio links directories, instead of copying # disabling this as it brings problems if directories overlap (the case with templates) - environ={"INVENIO_INSTANCE_PATH": str(config.invenio_instance_path), "FLASK_DEBUG": None}, + environ={ + "INVENIO_INSTANCE_PATH": str(config.invenio_instance_path), + "FLASK_DEBUG": None, + }, ) # "invenio collect" does collect assets, but links them instead of copying, @@ -54,7 +65,7 @@ def collect_assets(config): copy_assets_to_webpack_build_dir(config) -def install_npm_packages(config): +def install_npm_packages(config: OARepoConfig, **kwargs: Any): run_cmdline( config.invenio_command, "webpack", diff --git a/src/nrp_devtools/commands/ui/build.py b/src/nrp_devtools/commands/ui/build.py index 559f85b..79c7792 100644 --- a/src/nrp_devtools/commands/ui/build.py +++ b/src/nrp_devtools/commands/ui/build.py @@ -1,8 +1,10 @@ +from typing import Any + from nrp_devtools.commands.utils import run_cmdline from nrp_devtools.config import OARepoConfig -def build_production_ui(config: OARepoConfig): +def build_production_ui(config: OARepoConfig, **kwargs: Any): run_cmdline( config.invenio_command, "webpack", diff --git a/src/nrp_devtools/commands/ui/link_assets.py b/src/nrp_devtools/commands/ui/link_assets.py index 7349e16..369e771 100644 --- a/src/nrp_devtools/commands/ui/link_assets.py +++ b/src/nrp_devtools/commands/ui/link_assets.py @@ -1,5 +1,6 @@ import shutil from pathlib import Path +from typing import Any import click from tqdm import tqdm @@ -9,7 +10,7 @@ from .assets import load_watched_paths -def copy_assets_to_webpack_build_dir(config: OARepoConfig): +def copy_assets_to_webpack_build_dir(config: OARepoConfig, **kwargs: Any): # assets = (config.site_dir / "assets").resolve() static = (config.ui_dir / "static").resolve() diff --git a/src/nrp_devtools/commands/utils.py b/src/nrp_devtools/commands/utils.py index 7a85e3f..4599467 100644 --- a/src/nrp_devtools/commands/utils.py +++ b/src/nrp_devtools/commands/utils.py @@ -4,7 +4,7 @@ import traceback from functools import wraps from pathlib import Path -from typing import Callable, Union +from typing import Any, Callable, Literal, Union, overload import caseconverter import click @@ -12,6 +12,8 @@ from nrp_devtools.config import OARepoConfig +from .types import StepFunction + def run_cookiecutter( output_dir, @@ -28,21 +30,66 @@ def run_cookiecutter( ) +@overload def run_cmdline( - *cmdline, - cwd=".", - environ=None, - check_only=False, - grab_stdout=False, - grab_stderr=False, - discard_output=False, - raise_exception=False, - no_input=False, - no_environment=False, -): + *_cmdline: str, + cwd: str = ".", + environ: dict[str, str | None] | None = None, + check_only: bool = False, + grab_stdout: Literal[True], + grab_stderr: bool = False, + discard_output: bool = False, + raise_exception: bool = False, + no_input: bool = False, + no_environment: bool = False, +) -> str: ... + + +@overload +def run_cmdline( + *_cmdline: str, + cwd: str = ".", + environ: dict[str, str | None] | None = None, + check_only: bool = False, + grab_stdout: Literal[False], + grab_stderr: bool = False, + discard_output: bool = False, + raise_exception: bool = False, + no_input: bool = False, + no_environment: bool = False, +) -> bool: ... + + +@overload +def run_cmdline( + *_cmdline: str, + cwd: str = ".", + environ: dict[str, str | None] | None = None, + check_only: bool = False, + grab_stdout: bool = False, + grab_stderr: bool = False, + discard_output: bool = False, + raise_exception: bool = False, + no_input: bool = False, + no_environment: bool = False, +) -> bool: ... + + +def run_cmdline( + *_cmdline: str, + cwd: str = ".", + environ: dict[str, str | None] | None = None, + check_only: bool = False, + grab_stdout: bool = False, + grab_stderr: bool = False, + discard_output: bool = False, + raise_exception: bool = False, + no_input: bool = False, + no_environment: bool = False, +) -> Union[str, bool]: + cmdline = list(_cmdline) if no_environment: - env = { - } + env = {} else: env = os.environ.copy() @@ -56,7 +103,7 @@ def run_cmdline( else: env[k] = v - cwd = Path(cwd).absolute() + cwd = str(Path(cwd).absolute()) cmdline = [str(x) for x in cmdline] click.secho(f"Running {' '.join(cmdline)}", fg="blue", err=True) @@ -72,7 +119,7 @@ def run_cmdline( if grab_stderr or discard_output: kwargs["stderr"] = subprocess.PIPE - ret = subprocess.run( + ret: Any = subprocess.run( cmdline, check=True, cwd=cwd, @@ -166,7 +213,7 @@ def make_step( _swallow_errors=False, name=None, **global_kwargs, -): +) -> StepFunction: @wraps(fun) def step(config, **kwargs): should_call = _if if not callable(_if) else _if(config) @@ -182,21 +229,25 @@ def step(config, **kwargs): ) else: raise + if name: step.__name__ = name return step -def no_args(fun, name=None): +def no_args(fun, name=None) -> StepFunction: @wraps(fun) def _no_args(*args, **kwargs): fun() + _no_args.__name__ = name or getattr(fun, "__name__", "no_args") return _no_args -def run_fixup(check_function, fix_function, fix=True, name=None, **global_kwargs): +def run_fixup( + check_function, fix_function, fix=True, name=None, **global_kwargs +) -> StepFunction: @wraps(check_function) def _run_fixup(config, **kwargs): try: @@ -212,6 +263,7 @@ def _run_fixup(config, **kwargs): check_function( config, fast_fail=False, will_fix=False, **kwargs, **global_kwargs ) + if name: _run_fixup.__name__ = name return _run_fixup diff --git a/src/nrp_devtools/config/config.py b/src/nrp_devtools/config/config.py index cf1dc55..b02696e 100644 --- a/src/nrp_devtools/config/config.py +++ b/src/nrp_devtools/config/config.py @@ -2,7 +2,7 @@ from enum import Enum from io import StringIO from pathlib import Path -from typing import Dict, List, Optional, Set +from typing import Any, Dict, List, Optional, Set, cast import dacite import yaml @@ -14,19 +14,19 @@ from .ui_config import UIConfig serialization_config = dacite.Config() -serialization_config.type_hooks = { +serialization_config.type_hooks = { # type: ignore Path: lambda x: Path(x), - ModelFeature: lambda x: ModelFeature[x] if isinstance(x, str) else x, - BaseModel: lambda x: BaseModel[x] if isinstance(x, str) else x, + ModelFeature: lambda x: ModelFeature[x] if isinstance(x, str) else x, # type: ignore + BaseModel: lambda x: BaseModel[x] if isinstance(x, str) else x, # type: ignore Set[ModelFeature]: lambda x: set(x), } -def Enum_representer(dumper, data): +def Enum_representer(dumper: Any, data: Any): return dumper.represent_scalar("tag:yaml.org,2002:str", data.value) -def Set_representer(dumper, data): +def Set_representer(dumper: Any, data: Any): return dumper.represent_sequence( "tag:yaml.org,2002:seq", list(data), flow_style=True ) @@ -35,8 +35,13 @@ def Set_representer(dumper, data): SafeRepresenter.add_multi_representer(Enum, Enum_representer) SafeRepresenter.add_representer(set, Set_representer) + +class UnknownSentinel: + pass + + UNKNOWN = ( - object() + UnknownSentinel() ) # marker for unknown default value in get_model which will emit KeyError @@ -52,7 +57,7 @@ class OARepoConfig: python = "python3" python_version = ">=3.9,<3.11" - overrides = {} + overrides: dict[str, str] = {} @property def venv_dir(self): @@ -66,14 +71,17 @@ def pdm_dir(self): @property def ui_dir(self): + assert self.repository return self.repository_dir / self.repository.ui_package @property def shared_dir(self): + assert self.repository return self.repository_dir / self.repository.shared_package @property def models_dir(self): + assert self.repository return self.repository_dir / self.repository.model_package @property @@ -93,12 +101,14 @@ def theme_dir_name(self): def add_model(self, model: ModelConfig): self.models.append(model) - def get_model(self, model_name: str, default=UNKNOWN) -> ModelConfig: + def get_model( + self, model_name: str, default: ModelConfig | UnknownSentinel = UNKNOWN + ) -> ModelConfig: for model in self.models: if model.model_name == model_name: return model if default is not UNKNOWN: - return default + return cast(ModelConfig, default) known_models = ", ".join(sorted([model.model_name for model in self.models])) raise KeyError( f"Model {model_name} not found. Known models are: {known_models}" @@ -107,12 +117,14 @@ def get_model(self, model_name: str, default=UNKNOWN) -> ModelConfig: def add_ui(self, ui: UIConfig): self.uis.append(ui) - def get_ui(self, ui_name: str, default=UNKNOWN) -> UIConfig: + def get_ui( + self, ui_name: str, default: UIConfig | UnknownSentinel = UNKNOWN + ) -> UIConfig: for ui in self.uis: if ui.name == ui_name: return ui if default is not UNKNOWN: - return default + return cast(UIConfig, default) known_uis = ", ".join(sorted([ui.name for ui in self.uis])) raise KeyError(f"UI {ui_name} not found. Known UIs are: {known_uis}") @@ -126,7 +138,7 @@ def remove_fork(self, python_package: str): def config_file(self): return self.repository_dir / "oarepo.yaml" - def load(self, extra_config=None): + def load(self, extra_config: Path | None = None): if extra_config: config_file = extra_config else: @@ -167,8 +179,6 @@ def save(self): @classmethod def global_environment(cls): return { - "PIP_EXTRA_INDEX_URL": "http://127.0.0.1:4549/simple", - "UV_EXTRA_INDEX_URL": "http://127.0.0.1:4549/simple", + "PIP_EXTRA_INDEX_URL": "https://gitlab.cesnet.cz/api/v4/projects/1408/packages/pypi", + "UV_EXTRA_INDEX_URL": "https://gitlab.cesnet.cz/api/v4/projects/1408/packages/pypi", } - - pypi_proxy_target = "https://oarepo.github.io/pypi/packages/simple" \ No newline at end of file diff --git a/src/nrp_devtools/config/enums.py b/src/nrp_devtools/config/enums.py index 5599079..99b26bc 100644 --- a/src/nrp_devtools/config/enums.py +++ b/src/nrp_devtools/config/enums.py @@ -1,11 +1,12 @@ from enum import Enum +from typing import cast -def enum(value, *, description): +def enum(value: str, *, description: str) -> tuple[str, str]: return (value, description) -def enum_with_descriptions(clz) -> Enum: +def enum_with_descriptions(clz: type) -> type: clz_name = clz.__name__ descriptions = {} new_class_members = {} @@ -14,6 +15,7 @@ def enum_with_descriptions(clz) -> Enum: new_class_members[name] = value[0] descriptions[name] = value[1] ret = Enum(clz_name, new_class_members) - for name, desc in descriptions.items(): + + for name, desc in cast(dict[str, str], descriptions).items(): getattr(ret, name).description = desc return ret diff --git a/src/nrp_devtools/config/model_config.py b/src/nrp_devtools/config/model_config.py index 0546aa6..9593e6e 100644 --- a/src/nrp_devtools/config/model_config.py +++ b/src/nrp_devtools/config/model_config.py @@ -1,6 +1,6 @@ import dataclasses import re -from typing import Set +from typing import Any, Set from caseconverter import kebabcase, snakecase @@ -68,8 +68,7 @@ class BaseModel: @dataclasses.dataclass class ModelConfig: - prompts = {} - options = {} + prompts: dict[str, str] = {} base_model: BaseModel prompts["base_model"] = "Base model to use for the repository" @@ -92,6 +91,8 @@ class ModelConfig: features: Set[ModelFeature] prompts["features"] = "Model features" + options: dict[str, Any] = {} + @classmethod def default_model_package(cls, config, values): return snakecase(values["model_name"]) diff --git a/src/nrp_devtools/pypi_proxy/__init__.py b/src/nrp_devtools/pypi_proxy/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/nrp_devtools/pypi_proxy/proxy.py b/src/nrp_devtools/pypi_proxy/proxy.py deleted file mode 100644 index 788c238..0000000 --- a/src/nrp_devtools/pypi_proxy/proxy.py +++ /dev/null @@ -1,84 +0,0 @@ -from functools import partial - -from flask import Flask, current_app -import requests -from lxml import etree -import flask.cli -import logging - -# do not show werkzeug logs -log = logging.getLogger('werkzeug') -log.disabled = True -flask.cli.show_server_banner = lambda *args: None - -app = Flask(__name__) - -@app.route('/') -def welcome(): - return "PyPI Proxy to filter requests to pypi site. Try /simple", 200, {"Content-Type": "text/html"} - -@app.route('/simple') -def simple(): - url = current_app.config["PYPI_SERVER_URL"] - atags, html_root = read_url_links(url) - pypi_packages = set() - for x in atags: - package_name = x.attrib["href"].strip('/').split('/')[-1] - pypi_packages.add(package_name) - x.attrib["href"] = f"/simple/{package_name}" - current_app.pypi_packages = pypi_packages - return etree.tostring(html_root, encoding=str), 200, {"Content-Type": "text/html"} - - -def read_url_links(url): - content_html = requests.get(url).text - # replace malformed
tags, without this they would get stripped out - content_html = content_html.replace('
', '
') - parser = etree.HTMLParser() - html_root = etree.fromstring(content_html, parser) - atags = html_root.findall(".//a") - return atags, html_root - - -@app.route('/simple//') -def package(package): - if not hasattr(current_app, "pypi_packages"): - simple() - if package not in current_app.pypi_packages: - return "Package not found", 404, {"Content-Type": "text/plain"} - - url = f"{current_app.config['PYPI_SERVER_URL']}/{package}" - atags, html_root = read_url_links(url) - for x in atags: - x.attrib["href"] = f"/simple/{package}/{x.attrib['href'].split('/')[-1]}" - return etree.tostring(html_root, encoding=str), 200, {"Content-Type": "text/html"} - -@app.route('/simple//') -def package_version(package, subpath): - if not hasattr(current_app, "pypi_packages"): - simple() - if package not in current_app.pypi_packages: - return "Package not found", 404, {"Content-Type": "text/plain"} - - # on github pages, packages are on the same level as 'simple' - # and PYPI_SERVER_URL ends with 'simple', that's why /../ is needed - url = f"{current_app.config['PYPI_SERVER_URL']}/../{subpath}" - resp = requests.get(url) - return resp.content, resp.status_code, {"Content-Type": resp.headers.get("Content-Type")} - -def run(pypi_server_url): - app.config["PYPI_SERVER_URL"] = pypi_server_url - app.run(host="127.0.0.1", port=4549, debug=False) - -_pypi_proxy_thread = None - -def start_pypi_proxy(pypi_server_url): - global _pypi_proxy_thread - if not _pypi_proxy_thread: - import threading - _pypi_proxy_thread = threading.Thread(target=partial(run, pypi_server_url), daemon=True) - _pypi_proxy_thread.start() - - -if __name__ == '__main__': - run("https://oarepo.github.io/pypi/packages/simple") \ No newline at end of file From 260377298b9ec738044a7536367f4633bbb43424 Mon Sep 17 00:00:00 2001 From: Mirek Simek Date: Thu, 16 Jan 2025 15:15:59 +0100 Subject: [PATCH 2/2] dataclasses fixes --- src/nrp_devtools/config/config.py | 6 +++--- src/nrp_devtools/config/model_config.py | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/nrp_devtools/config/config.py b/src/nrp_devtools/config/config.py index b02696e..0cde429 100644 --- a/src/nrp_devtools/config/config.py +++ b/src/nrp_devtools/config/config.py @@ -57,7 +57,7 @@ class OARepoConfig: python = "python3" python_version = ">=3.9,<3.11" - overrides: dict[str, str] = {} + overrides = {} # untyped so that it is not generated as a member of the dataclass @property def venv_dir(self): @@ -179,6 +179,6 @@ def save(self): @classmethod def global_environment(cls): return { - "PIP_EXTRA_INDEX_URL": "https://gitlab.cesnet.cz/api/v4/projects/1408/packages/pypi", - "UV_EXTRA_INDEX_URL": "https://gitlab.cesnet.cz/api/v4/projects/1408/packages/pypi", + "PIP_EXTRA_INDEX_URL": "https://gitlab.cesnet.cz/api/v4/projects/1408/packages/pypi/simple", + "UV_EXTRA_INDEX_URL": "https://gitlab.cesnet.cz/api/v4/projects/1408/packages/pypi/simple", } diff --git a/src/nrp_devtools/config/model_config.py b/src/nrp_devtools/config/model_config.py index 9593e6e..81c1d99 100644 --- a/src/nrp_devtools/config/model_config.py +++ b/src/nrp_devtools/config/model_config.py @@ -1,6 +1,6 @@ import dataclasses import re -from typing import Any, Set +from typing import Set from caseconverter import kebabcase, snakecase @@ -68,7 +68,8 @@ class BaseModel: @dataclasses.dataclass class ModelConfig: - prompts: dict[str, str] = {} + prompts = {} # untyped so that it is not generated as a member of a dataclass + options = {} # untyped so that it is not generated as a member of a dataclass base_model: BaseModel prompts["base_model"] = "Base model to use for the repository" @@ -91,8 +92,6 @@ class ModelConfig: features: Set[ModelFeature] prompts["features"] = "Model features" - options: dict[str, Any] = {} - @classmethod def default_model_package(cls, config, values): return snakecase(values["model_name"])