diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ec03517 --- /dev/null +++ b/.gitignore @@ -0,0 +1,73 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] + +# Idea software family +.idea/ +# VSCode +.vscode/ + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover + +# Django stuff: +*.log + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Vim swapfiles +.*.sw? +venv/ +.envrc +.venv/ +.venv-builder/ +node_modules/ + +.DS_Store + +docker/.env + +temporary_test_repository/ +temporary_test_directory/ \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..f2719d1 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +graft oarepo_tools +global-exclude oarepo_tools/i18next/node_modules/** +global-exclude **/__pycache__/** \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..140a4e1 --- /dev/null +++ b/README.md @@ -0,0 +1,34 @@ +# NRP repository development tools + +This tool is used to setup and run an Invenio based NRP instance. + +## Prerequisites + +- Python 3.10 +- node version 16 or greater, npm version 7, 8 or 10 +- imagemagick and development packages for imagemagick +- standard build tools (gcc, make, ...), on ubuntu build-essential +- Docker 20.10.10+ and docker-compose 1.17.0+ (or OrbStack on Mac) + +## Creating a new repository + +1. Download the installer + +```bash +curl -sSL https://raw.githubusercontent.com/oarepo/nrp-tools/main/nrp-installer.sh +``` + +2. Run the installer with a directory where the repository will be created + +```bash +bash nrp-installer.sh my-repo +``` + +After asking a couple of questions, the installer will create the +repository in the `my-repo` directory. + +It will also initialize git version control system and commit the initial +sources. + +3. Go to the my-repo directory and see the README.md file there for further instructions + (or have a peek at [](src/nrp_devtools/...) ) \ No newline at end of file diff --git a/nrp-installer.sh b/nrp-installer.sh new file mode 100755 index 0000000..397c8f3 --- /dev/null +++ b/nrp-installer.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# This script installs the NRP repository tools + +# Environment variables +# PYTHON: +# python executable to use for installation and running the NRP tools +# NRP_GIT_URL: +# URL of the NRP git repository +# NRP_GIT_BRANCH: +# branch of the NRP git repository to use +# LOCAL_NRP_DEVTOOLS_LOCATION: +# location of the local NRP repository. +# If set, do not clone the NRP repository but use the local one. + +set -e + +NRP_GIT_URL=${NRP_GIT_URL:-https://github.com/oarepo/nrp-devtools.git} +NRP_GIT_BRANCH=${NRP_GIT_BRANCH:-main} + +SUPPORTED_PYTHON_VERSIONS=(3.12 3.11 3.10 3.9) + +if [ -z "$PYTHON" ] ; then + + # find a supported python + for version in "${SUPPORTED_PYTHON_VERSIONS[@]}"; do + if command -v python$version >/dev/null 2>&1; then + PYTHON=python$version + break + fi + done + + if [ -z "$PYTHON" ] ; then + echo "No supported python version found. Please install python 3.9 or higher + or set the PYTHON environment variable to the python executable." + exit 1 + fi +fi + +# clone nrp tool to a temporary directory +ACTUAL_DIR="$(pwd)" +TMP_DIR=$(mktemp -d) + +trap 'rm -rf "$TMP_DIR"' EXIT + +echo "Installing temporary NRP CLI to $TMP_DIR, will clean it up on exit." + + +$PYTHON -m venv "$TMP_DIR/.venv" +source "$TMP_DIR/.venv/bin/activate" +pip install -U setuptools pip wheel + +if [ -z "$LOCAL_NRP_DEVTOOLS_LOCATION" ] ; then + LOCAL_NRP_DEVTOOLS_LOCATION="$TMP_DIR/nrp-devtools" + git clone "$NRP_GIT_URL" --branch "$NRP_GIT_BRANCH" --depth 1 "$LOCAL_NRP_DEVTOOLS_LOCATION" +fi +pip install -e "$LOCAL_NRP_DEVTOOLS_LOCATION" + +"$TMP_DIR"/.venv/bin/nrp-devtools initialize "$@" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e1c3951 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,60 @@ +[project] +name = "nrp-devtools" +version = "0.1.0" +description = "NRP repository development tools" +readme = "README.md" +authors = [ + {name = "Miroslav Simek", email = "miroslav.simek@cesnet.cz" } +] +dependencies = [ + "setuptools", + "pip", + "wheel", + "click", + "aenum", + + # config loading/writing + "PyYAML", + "dacite", + "ruamel.yaml", + + # for code scaffolding + "case-converter", + "cookiecutter", + "cryptography", + + # for managing requirements / installation + "pdm", + "tomli", + "tomli-w", + "requirements-parser", + + # progress bar everywhere + "tqdm", + + # for develop (webpack) + "watchdog", + "pytimedinput", + "psutil", + + # nrp makemessages + "oarepo-tools", + + # for checks + "minio", + "redis", + "psycopg[binary]", + "pika", + "opensearch-py", + + # testing + "pytest", +] + +[project.scripts] +nrp-devtools = "nrp_devtools.main:nrp_command" + + +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..c1057cf --- /dev/null +++ b/setup.py @@ -0,0 +1,3 @@ +import setuptools + +setuptools.setup() \ No newline at end of file diff --git a/src/nrp_devtools/__init__.py b/src/nrp_devtools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/nrp_devtools/cli/__init__.py b/src/nrp_devtools/cli/__init__.py new file mode 100644 index 0000000..e0f8b26 --- /dev/null +++ b/src/nrp_devtools/cli/__init__.py @@ -0,0 +1,21 @@ +from .base import nrp_command +from .build import build_command +from .check import check_command +from .develop import develop_command +from .initialize import initialize_command +from .model import model_group +from .run import run_command +from .ui import ui_group +from .upgrade import upgrade_command + +__all__ = [ + "nrp_command", + "initialize_command", + "upgrade_command", + "ui_group", + "develop_command", + "check_command", + "build_command", + "run_command", + "model_group", +] diff --git a/src/nrp_devtools/cli/base.py b/src/nrp_devtools/cli/base.py new file mode 100644 index 0000000..4b03535 --- /dev/null +++ b/src/nrp_devtools/cli/base.py @@ -0,0 +1,97 @@ +""" +This is the main entry point for the nrp devtools command line interface. +""" +import functools +import sys +from pathlib import Path +from typing import Any, Callable, Dict, Union + +import click + +from nrp_devtools.commands.utils import run_steps +from nrp_devtools.config import OARepoConfig + + +@click.group() +def nrp_command(**kwargs): + """NRP devtools command line interface.""" + + +def command_sequence( + repository_dir_must_exist=True, + repository_dir_as_argument=False, + continue_on_errors: Union[ + bool, Callable[[OARepoConfig, Dict[str, Any]], bool] + ] = False, +): + def wrapper(command): + command = click.option( + "--verbose", "-v", is_flag=True, help="Enables verbose mode." + )(command) + command = click.option("--step", help="Run only this step", multiple=True)( + command + ) + command = click.option( + "--dry-run", + is_flag=True, + help="Show steps that would be run and exit.", + )(command) + command = click.option( + "--steps", + "show_steps", + is_flag=True, + help="Show steps that would be run and exit.", + )(command) + if repository_dir_as_argument: + command = click.argument( + "repository_dir", + type=click.Path(exists=repository_dir_must_exist), + )(command) + else: + command = click.option( + "--repository-dir", + "-d", + default=".", + help="Repository directory (default is the current directory).", + type=click.Path(exists=False), + )(command) + + @functools.wraps(command) + def proxied(*args, **kwargs): + repository_dir = kwargs["repository_dir"] + steps = kwargs.pop("step", None) + dry_run = kwargs.pop("dry_run", False) or kwargs.pop("show_steps", False) + if repository_dir: + kwargs["repository_dir"] = repository_dir = Path( + repository_dir + ).resolve() + + if repository_dir_must_exist and not repository_dir.exists(): + click.secho("Project directory must exist", fg="red") + sys.exit(1) + + config = OARepoConfig(repository_dir) + config.load() + + # run the command + step_commands = command(*args, config=config, **kwargs) + if dry_run: + click.secho("Steps that would be run:\n", fg="green") + for idx, step_cmd in enumerate(step_commands): + click.secho(f"{idx+1:3d} {step_cmd.__name__}", fg="green") + return + elif step_commands: + _continue_on_errors = ( + continue_on_errors(config, kwargs) + if callable(continue_on_errors) + else continue_on_errors + ) + + run_steps( + config, steps, step_commands, continue_on_errors=_continue_on_errors + ) + config.save() + + return proxied + + return wrapper diff --git a/src/nrp_devtools/cli/build.py b/src/nrp_devtools/cli/build.py new file mode 100644 index 0000000..4a1153e --- /dev/null +++ b/src/nrp_devtools/cli/build.py @@ -0,0 +1,36 @@ +from functools import partial + +import click + +from ..commands.invenio import install_invenio_cfg +from ..commands.pdm import ( + build_requirements, + check_requirements, + clean_previous_installation, + create_empty_venv, + install_python_repository, +) +from ..commands.ui import build_production_ui, collect_assets, install_npm_packages +from ..commands.utils import make_step, no_args, run_fixup +from ..config import OARepoConfig +from .base import command_sequence, nrp_command + + +@nrp_command.command(name="build") +@command_sequence() +def build_command(*, config: OARepoConfig, **kwargs): + """Builds the repository""" + return ( + no_args( + partial(click.secho, "Building repository for production", fg="yellow") + ), + make_step(clean_previous_installation), + make_step(create_empty_venv), + run_fixup(check_requirements, build_requirements, fix=True), + install_python_repository, + install_invenio_cfg, + collect_assets, + install_npm_packages, + build_production_ui, + no_args(partial(click.secho, "Successfully built the repository", fg="green")), + ) diff --git a/src/nrp_devtools/cli/check.py b/src/nrp_devtools/cli/check.py new file mode 100644 index 0000000..fc3ab5c --- /dev/null +++ b/src/nrp_devtools/cli/check.py @@ -0,0 +1,123 @@ +from functools import partial + +import click + +from ..commands.check_old import check_imagemagick_callable +from ..commands.db import check_db, fix_db +from ..commands.docker import ( + check_containers, + check_docker_callable, + check_docker_compose_version, + check_node_version, + check_npm_version, + fix_containers, +) +from ..commands.invenio import check_invenio_cfg, install_invenio_cfg +from ..commands.opensearch import check_search, fix_custom_fields, fix_search +from ..commands.pdm import ( + build_requirements, + check_invenio_callable, + check_requirements, + check_virtualenv, + fix_virtualenv, + install_local_packages, + install_python_repository, +) +from ..commands.s3 import ( + check_s3_bucket_exists, + check_s3_location_in_database, + fix_s3_bucket_exists, + fix_s3_location_in_database, +) +from ..commands.ui import check_ui, fix_ui +from ..commands.utils import make_step, no_args, run_fixup +from ..config import OARepoConfig +from .base import command_sequence, nrp_command + + +@nrp_command.command(name="check") +@click.option("--fix", is_flag=True, default=False) +@click.option("--local-packages", "-e", multiple=True) +@command_sequence() +def check_command(*, config: OARepoConfig, local_packages=None, fix=False, **kwargs): + "Checks prerequisites for running the repository, initializes a build environment and rebuilds the repository." + context = {} + return check_commands(context, local_packages, fix) + + +def check_commands(context, local_packages, fix): + return ( + # + # infrastructure checks + # + no_args(partial(click.secho, "Checking repository requirements", fg="yellow")), + check_docker_callable, + make_step(check_docker_compose_version, expected_major=1, expected_minor=17), + make_step(check_node_version, supported_versions=(14, 16, 20, 21)), + make_step(check_npm_version, supported_versions=(6, 7, 8, 10)), + check_imagemagick_callable, + # + # virtualenv exists + # + run_fixup(check_virtualenv, fix_virtualenv, fix=fix), + # + # requirements have been built + # + run_fixup(check_requirements, build_requirements, fix=fix), + # + # any local packages are installed inside the virtual environment + # + make_step(install_local_packages, local_packages=local_packages), + # + # invenio.cfg is inside virtual environment + # + run_fixup(check_invenio_cfg, install_invenio_cfg, fix=fix), + # + # can run invenio command + # + run_fixup(check_invenio_callable, install_python_repository, fix=fix), + # + # check that docker containers are running + # + run_fixup(check_containers, fix_containers, context=context, fix=fix), + # + # check that ui is compiled + # + run_fixup(check_ui, fix_ui, fix=fix), + # + # check that database is initialized + # + run_fixup(check_db, fix_db, context=context, fix=fix), + # + # check that opensearch is initialized and contains all indices and custom fields + # + run_fixup(check_search, fix_search, context=context, fix=fix), + # + # check that s3 location inside invenio is initialized + # + run_fixup( + check_s3_location_in_database, + fix_s3_location_in_database, + fix=fix, + context=context, + ), + # + # check that s3 bucket exists + # + run_fixup( + check_s3_bucket_exists, fix_s3_bucket_exists, fix=fix, context=context + ), + # + # check that custom fields have been put to opensearch. + # currently there is no way of checking that, so will always fix it + # (does noop if already fixed) + # + make_step(fix_custom_fields, context=context), + # + # check that fixtures are loaded into the database + # TODO: can not do this now as the fixtures do not have a way of getting + # their identifier. This might be fixed in the future. + # + # make_step(check_fixtures, context=context, fix=fix), + no_args(partial(click.secho, "Repository ready to be run", fg="yellow")), + ) diff --git a/src/nrp_devtools/cli/develop.py b/src/nrp_devtools/cli/develop.py new file mode 100644 index 0000000..2a5416d --- /dev/null +++ b/src/nrp_devtools/cli/develop.py @@ -0,0 +1,49 @@ +import click + +from ..commands.develop import Runner +from ..commands.develop.controller import run_develop_controller +from ..commands.ui.link_assets import link_assets +from ..commands.utils import make_step +from ..config import OARepoConfig +from .base import command_sequence, nrp_command +from .check import check_commands + + +@nrp_command.command(name="develop") +@click.option( + "--extra-library", + "-e", + "local_packages", + multiple=True, + help="Path to a local package to install", +) +@click.option( + "--checks/--skip-checks", + default=True, + help="Check the environment before starting (default is to check, disable to get a faster startup)", +) +@command_sequence() +def develop_command( + *, config: OARepoConfig, local_packages=None, checks=True, **kwargs +): + """Starts the development environment for the repository.""" + runner = Runner(config) + context = {} + return ( + *(check_commands(context, local_packages, fix=True) if checks else ()), + link_assets, + make_step( + lambda config=None, runner=None: runner.start_python_server( + development_mode=True + ), + runner=runner, + ), + make_step( + lambda config=None, runner=None: runner.start_webpack_server(), + runner=runner, + ), + make_step( + lambda config=None, runner=None: runner.start_file_watcher(), runner=runner + ), + make_step(run_develop_controller, runner=runner, development_mode=True), + ) diff --git a/src/nrp_devtools/cli/initialize.py b/src/nrp_devtools/cli/initialize.py new file mode 100644 index 0000000..46ca0f2 --- /dev/null +++ b/src/nrp_devtools/cli/initialize.py @@ -0,0 +1,70 @@ +import sys +from pathlib import Path + +import click + +from ..commands.initialize import initialize_repository +from ..config import OARepoConfig +from ..config.repository_config import RepositoryConfig +from ..config.wizard import ask_for_configuration +from .base import command_sequence, nrp_command + + +@nrp_command.command(name="initialize") +@click.option( + "--initial-config", + default=None, + help="Initial configuration file", + type=click.Path(exists=True), +) +@click.option( + "--no-input", + default=None, + help="Do not ask for input, use the initial config only", + is_flag=True, +) +@command_sequence(repository_dir_as_argument=True, repository_dir_must_exist=False) +def initialize_command( + *, repository_dir, config: OARepoConfig, verbose, initial_config, no_input +): + """ + Initialize a new nrp project. Note: the project directory must be empty. + """ + if repository_dir.exists() and len(list(repository_dir.iterdir())) > 0: + click.secho( + f"Project directory {repository_dir} must be empty", fg="red", err=True + ) + sys.exit(1) + + def initialize_step(config: OARepoConfig): + if initial_config: + config.load(Path(initial_config)) + + if not no_input: + click.secho( + """Please answer a few questions to configure your repository.""" + ) + config.repository = ask_for_configuration( + config, + RepositoryConfig, + ) + + initialize_repository(config) + click.secho( + f""" +Your repository is now initialized. + +You can start the repository in development mode +via ./nrp develop and head to https://127.0.0.1:5000/ +to check that everything has been installed correctly. + +Then add metadata models via ./nrp model add , +edit the model and compile it via ./nrp model compile . + +Add jinjax template pages via ./nrp ui page create +and model ui pages via ./nrp ui model create . +""", + fg="green", + ) + + return (initialize_step,) diff --git a/src/nrp_devtools/cli/model.py b/src/nrp_devtools/cli/model.py new file mode 100644 index 0000000..d0d6130 --- /dev/null +++ b/src/nrp_devtools/cli/model.py @@ -0,0 +1,65 @@ +import tempfile +from pathlib import Path + +import click + +from ..commands.model.compile import ( + add_requirements_and_entrypoints, + compile_model_to_tempdir, + copy_compiled_model, + generate_alembic, + install_model_compiler, +) +from ..commands.model.create import create_model +from ..commands.pdm import install_python_repository +from ..commands.utils import make_step +from ..config import OARepoConfig, ask_for_configuration +from ..config.model_config import ModelConfig +from .base import command_sequence, nrp_command + + +@nrp_command.group(name="model") +def model_group(): + """ + Model management commands + """ + + +@model_group.command(name="create", help="Create a new model") +@click.argument("model_name") +@command_sequence() +def create_model_command(*, config: OARepoConfig, model_name, **kwargs): + for model in config.models: + if model.model_name == model_name: + click.secho(f"Model {model_name} already exists", fg="red", err=True) + return + + def set_model_configuration(config: OARepoConfig, *args, **kwargs): + config.add_model( + ask_for_configuration( + config, ModelConfig, initial_values={"model_name": model_name} + ) + ) + + return ( + set_model_configuration, + make_step(create_model, model_name=model_name), + ) + + +@model_group.command(name="compile", help="Compile a model") +@click.argument("model_name") +@command_sequence() +def compile_model_command(*, config: OARepoConfig, model_name, **kwargs): + model = config.get_model(model_name) + # create a temporary directory using tempfile + tempdir = str(Path(tempfile.mkdtemp()).resolve()) + + return ( + make_step(install_model_compiler, model=model), + make_step(compile_model_to_tempdir, model=model, tempdir=tempdir), + make_step(copy_compiled_model, model=model, tempdir=tempdir), + make_step(add_requirements_and_entrypoints, model=model, tempdir=tempdir), + make_step(install_python_repository), + make_step(generate_alembic, model=model), + ) diff --git a/src/nrp_devtools/cli/run.py b/src/nrp_devtools/cli/run.py new file mode 100644 index 0000000..02b58a1 --- /dev/null +++ b/src/nrp_devtools/cli/run.py @@ -0,0 +1,19 @@ + +from ..commands.develop import Runner +from ..commands.develop.controller import run_develop_controller +from ..commands.utils import make_step +from ..config import OARepoConfig +from .base import command_sequence, nrp_command + + +@nrp_command.command(name="run") +@command_sequence() +def run_command(*, config: OARepoConfig, local_packages=None, checks=True, **kwargs): + """Starts the repository. Make sure to run `nrp check` first.""" + runner = Runner(config) + return ( + make_step( + lambda config=None, runner=None: runner.start_python_server(), runner=runner + ), + make_step(run_develop_controller, runner=runner, development_mode=False), + ) diff --git a/src/nrp_devtools/cli/ui.py b/src/nrp_devtools/cli/ui.py new file mode 100644 index 0000000..ca97e4c --- /dev/null +++ b/src/nrp_devtools/cli/ui.py @@ -0,0 +1,146 @@ +import json + +import click + +from ..commands.ui.create import ( + create_model_ui, + create_page_ui, + register_model_ui, + register_page_ui, +) +from ..commands.utils import make_step +from ..config import OARepoConfig +from ..config.ui_config import UIConfig +from .base import command_sequence, nrp_command + + +@nrp_command.group(name="ui") +def ui_group(): + """ + UI management commands. + """ + + +""" +nrp ui pages create +``` + +The `ui-endpoint` is the endpoint of the root for pages, for example +`/docs` or `/search`. The `ui-name` is the name of the collection of pages, +such as `docs` or `search`. + +If `ui-endpoint` is not specified, it will be the same as +`ui-name` with '/' prepended. +""" + + +@ui_group.group(name="pages") +def pages_group(): + """ + UI pages management commands. + """ + + +@pages_group.command( + name="create", + help="""Create a new UI pages collection. + The ui-name is the name of the collection of pages, such as docs or search, + ui-endpoint is the url path of the pages' root, for example /docs or /search. + If not specified, it will be the same as ui-name with '/' prepended. + """, +) +@click.argument("ui-name") +@click.argument( + "ui-endpoint", + required=False, +) +@command_sequence() +def create_pages(config: OARepoConfig, ui_name, ui_endpoint, **kwargs): + """ + Create a new UI pages collection. + """ + ui_name = ui_name.replace("-", "_") + ui_endpoint = ui_endpoint or ("/" + ui_name.replace("_", "-")) + if not ui_endpoint.startswith("/"): + ui_endpoint = "/" + ui_endpoint + + if config.get_ui(ui_name, default=None): + click.secho(f"UI {ui_name} already exists", fg="red", err=True) + return + + def set_ui_configuration(config: OARepoConfig, *args, **kwargs): + config.add_ui(UIConfig(name=ui_name, endpoint=ui_endpoint)) + + return ( + set_ui_configuration, + make_step(create_page_ui, ui_name=ui_name), + make_step(register_page_ui, ui_name=ui_name), + ) + + +@ui_group.group(name="model") +def model_group(): + """ + UI model management commands + """ + + +@model_group.command( + name="create", + help="""Create a new UI for metadata model. + The model-name is the name of the model, such as documents or records, + ui-name is the name of the ui (default is the same as model-name). + ui-endpoint, if not passed, is taken from the model's resource-config, + field base-html-url. + """, +) +@click.argument("model-name") +@click.argument("ui-name", required=False) +@click.argument( + "ui-endpoint", + required=False, +) +@command_sequence() +def create_model(config: OARepoConfig, model_name, ui_name, ui_endpoint, **kwargs): + """ + Create a new UI model. + """ + if not ui_name: + ui_name = model_name + + ui_name = ui_name.replace("-", "_") + + if config.get_ui(ui_name, default=None): + click.secho(f"UI {ui_name} already exists", fg="red", err=True) + return + + def set_ui_configuration(config: OARepoConfig, *args, **kwargs): + nonlocal ui_endpoint + + model = config.get_model(model_name) + + model_data = json.loads( + ( + config.repository_dir / model.model_package / "models" / "records.json" + ).read_text() + ) + api_service = model_data["model"]["service-config"]["service-id"] + ui_serializer_class = model_data["model"]["json-serializer"]["class"] + if not ui_endpoint: + ui_endpoint = model_data["model"]["resource-config"]["base-html-url"] + + config.add_ui( + UIConfig( + name=ui_name, + endpoint=ui_endpoint, + model=model_name, + api_service=api_service, + ui_serializer_class=ui_serializer_class, + ) + ) + + return ( + set_ui_configuration, + make_step(create_model_ui, ui_name=ui_name), + make_step(register_model_ui, ui_name=ui_name), + ) diff --git a/src/nrp_devtools/cli/upgrade.py b/src/nrp_devtools/cli/upgrade.py new file mode 100644 index 0000000..be97d59 --- /dev/null +++ b/src/nrp_devtools/cli/upgrade.py @@ -0,0 +1,20 @@ +from . import build_command +from ..commands.invenio import install_invenio_cfg +from ..commands.pdm import build_requirements, install_python_repository, clean_previous_installation, create_empty_venv +from ..commands.ui.assets import collect_assets, install_npm_packages +from ..commands.ui.build import build_production_ui +from ..commands.utils import make_step +from ..config import OARepoConfig +from .base import command_sequence, nrp_command + + +@nrp_command.command(name="upgrade") +@command_sequence() +def upgrade_command(*, config: OARepoConfig, **kwargs): + """Upgrades the repository. + + Resolves the newest applicable packages, downloads them and rebuilds the repository. + """ + return ( + build_requirements, + ) + build_command(config=config) diff --git a/src/nrp_devtools/commands/__init__.py b/src/nrp_devtools/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/nrp_devtools/commands/check.py b/src/nrp_devtools/commands/check.py new file mode 100644 index 0000000..b0129dd --- /dev/null +++ b/src/nrp_devtools/commands/check.py @@ -0,0 +1,8 @@ +import sys + +import click + + +def check_failed(message): + click.secho(message, fg="red", err=True) + sys.exit(1) diff --git a/src/nrp_devtools/commands/check_old.py b/src/nrp_devtools/commands/check_old.py new file mode 100644 index 0000000..340b832 --- /dev/null +++ b/src/nrp_devtools/commands/check_old.py @@ -0,0 +1,11 @@ +from nrp_devtools.commands.check import check_failed +from nrp_devtools.commands.utils import run_cmdline + + +def check_imagemagick_callable(config): + try: + run_cmdline("convert", "--version", grab_stdout=True, raise_exception=True) + except: + check_failed( + "ImageMagick is not callable. Please install ImageMagick on your system." + ) diff --git a/src/nrp_devtools/commands/db.py b/src/nrp_devtools/commands/db.py new file mode 100644 index 0000000..acf0f8d --- /dev/null +++ b/src/nrp_devtools/commands/db.py @@ -0,0 +1,29 @@ +from nrp_devtools.commands.check import check_failed +from nrp_devtools.commands.invenio import get_repository_info +from nrp_devtools.commands.utils import run_cmdline +from nrp_devtools.config import OARepoConfig + + +def check_db(config: OARepoConfig, context=None, **kwargs): + repository_info = get_repository_info(config, context) + db_status = repository_info["db"] + + if db_status != "ok": + check_failed( + f"Database is not ready, it reports status {db_status}.", + ) + + +def fix_db(config: OARepoConfig, context=None, **kwargs): + repository_info = get_repository_info(config, context) + db_status = repository_info["db"] + + if db_status == "not_initialized": + run_cmdline(config.invenio_command, "db", "create") + elif db_status == "migration_pending": + run_cmdline(config.invenio_command, "alembic", "upgrade", "heads") + else: + raise Exception(f'db error not handled: "{db_status}"') + + # make the repository info reinitialize during the next check + context.pop("repository_info") diff --git a/src/nrp_devtools/commands/develop/__init__.py b/src/nrp_devtools/commands/develop/__init__.py new file mode 100644 index 0000000..a338d40 --- /dev/null +++ b/src/nrp_devtools/commands/develop/__init__.py @@ -0,0 +1,7 @@ +from .controller import run_develop_controller +from .runner import Runner + +__all__ = [ + "Runner", + "run_develop_controller", +] diff --git a/src/nrp_devtools/commands/develop/controller.py b/src/nrp_devtools/commands/develop/controller.py new file mode 100644 index 0000000..9106698 --- /dev/null +++ b/src/nrp_devtools/commands/develop/controller.py @@ -0,0 +1,44 @@ +import click +from pytimedinput import timedInput + +from nrp_devtools.commands.develop import Runner +from nrp_devtools.config import OARepoConfig + + +def show_menu(server: bool, ui: bool, development_mode: bool): + click.secho("") + click.secho("") + if development_mode: + click.secho("Development server is running", fg="green") + else: + click.secho("Production server is running", fg="yellow") + click.secho("") + click.secho("=======================================") + click.secho("") + if server: + click.secho("1 or server - restart python server", fg="green") + if ui and not development_mode: + click.secho("2 or ui - restart webpack server", fg="green") + + click.secho("0 or stop - stop the server (Ctrl-C also works)", fg="red") + click.secho("") + click.secho("") + + +def run_develop_controller( + config: OARepoConfig, runner: Runner, server=True, ui=True, development_mode=False +): + while True: + show_menu(server, ui, development_mode) + (choice, timed_out) = timedInput(prompt="Your choice: ", timeout=60) + if timed_out: + continue + choice = choice.strip() + print("Got choice", choice) + if choice in ("0", "stop"): + runner.stop() + break + elif choice in ("1", "server"): + runner.restart_python_server(development_mode=development_mode) + elif choice in ("2", "ui"): + runner.restart_webpack_server() diff --git a/src/nrp_devtools/commands/develop/runner.py b/src/nrp_devtools/commands/develop/runner.py new file mode 100644 index 0000000..5d5d62a --- /dev/null +++ b/src/nrp_devtools/commands/develop/runner.py @@ -0,0 +1,162 @@ +import os +import subprocess +import threading +import time +from threading import RLock +from typing import Optional + +import click +import psutil + +from nrp_devtools.commands.ui.link_assets import link_assets +from nrp_devtools.config import OARepoConfig + + +class Runner: + python_server_process: Optional[subprocess.Popen] = None + webpack_server_process: Optional[subprocess.Popen] = None + file_watcher_thread: Optional[threading.Thread] = None + file_watcher_stopping = None + + def __init__(self, config: OARepoConfig): + self.config = config + + def start_python_server(self, development_mode=False): + click.secho("Starting python server", fg="yellow") + environment = {} + if development_mode: + environment["FLASK_DEBUG"] = "1" + environment["INVENIO_TEMPLATES_AUTO_RELOAD"] = "1" + self.python_server_process = subprocess.Popen( + [ + self.config.invenio_command, + "run", + "--cert", + self.config.repository_dir / "docker" / "development.crt", + "--key", + self.config.repository_dir / "docker" / "development.key", + ], + env={**os.environ, **environment}, + ) + for i in range(5): + time.sleep(2) + if self.python_server_process.poll() is not None: + click.secho( + "Python server failed to start. Fix the problem and type 'server' to reload", + fg="red", + ) + self.python_server_process.wait() + self.python_server_process = None + time.sleep(10) + break + click.secho("Python server started", fg="green") + + def start_webpack_server(self): + click.secho("Starting webpack server", fg="yellow") + manifest_path = ( + self.config.invenio_instance_path / "static" / "dist" / "manifest.json" + ) + if manifest_path.exists(): + manifest_path.unlink() + + self.webpack_server_process = subprocess.Popen( + [ + "npm", + "run", + "start", + ], + cwd=self.config.invenio_instance_path / "assets", + ) + # wait at most a minute for webpack to start + for i in range(60): + time.sleep(2) + if self.webpack_server_process.poll() is not None: + click.secho( + "Webpack server failed to start. Fix the problem and type 'ui' to reload", + fg="red", + ) + self.webpack_server_process.wait() + self.webpack_server_process = None + time.sleep(10) + break + + if manifest_path.exists(): + manifest_data = manifest_path.read_text() + if '"status": "done"' in manifest_data: + click.secho("Webpack server is running", fg="green") + break + click.secho("Webpack server started", fg="green") + + def start_file_watcher(self): + click.secho("Starting file watcher", fg="yellow") + + def watch_files(): + while True: + if self.file_watcher_stopping.acquire(timeout=1): + break + + self.file_watcher_stopping = RLock() + self.file_watcher_stopping.acquire() + + self.file_watcher_thread = threading.Thread(target=watch_files, daemon=True) + self.file_watcher_thread.start() + click.secho("File watcher started", fg="green") + + def stop(self): + self.stop_python_server() + self.stop_webpack_server() + self.stop_file_watcher() + + def restart_python_server(self, development_mode=False): + self.stop_python_server() + self.start_python_server(development_mode=development_mode) + + def restart_webpack_server(self): + self.stop_webpack_server() + self.stop_file_watcher() + # just for being sure, link assets + # (they might have changed and were not registered before) + link_assets(self.config) + self.start_file_watcher() + self.start_webpack_server() + + def stop_python_server(self): + click.secho("Stopping python server", fg="yellow") + if self.python_server_process: + self.python_server_process.terminate() + try: + self.python_server_process.wait(timeout=5) + except subprocess.TimeoutExpired: + click.secho( + "Python server did not stop in time, killing it", fg="yellow" + ) + self._kill_process_tree(self.python_server_process) + self.python_server_process = None + + def stop_webpack_server(self): + click.secho("Stopping webpack server", fg="yellow") + if self.webpack_server_process: + self.webpack_server_process.terminate() + try: + self.webpack_server_process.wait(timeout=5) + except subprocess.TimeoutExpired: + click.secho( + "Webpack server did not stop in time, killing it", fg="yellow" + ) + self._kill_process_tree(self.webpack_server_process) + self.webpack_server_process = None + + def stop_file_watcher(self): + click.secho("Stopping file watcher", fg="yellow") + if self.file_watcher_thread: + self.file_watcher_stopping.release() + self.file_watcher_thread.join() + self.file_watcher_thread = None + self.file_watcher_stopping = None + + def _kill_process_tree(self, process_tree: subprocess.Popen): + parent_pid = process_tree.pid + parent = psutil.Process(parent_pid) + for child in parent.children(recursive=True): + child.kill() + parent.kill() diff --git a/src/nrp_devtools/commands/docker.py b/src/nrp_devtools/commands/docker.py new file mode 100644 index 0000000..59d2610 --- /dev/null +++ b/src/nrp_devtools/commands/docker.py @@ -0,0 +1,248 @@ +import re +import time +import traceback +from urllib.parse import urlparse + +import click +import pika +import psycopg +import redis +from minio import Minio +from opensearchpy import OpenSearch + +from nrp_devtools.commands.utils import run_cmdline +from nrp_devtools.config import OARepoConfig + +from .check import check_failed +from .invenio import get_invenio_configuration, get_repository_info + + +def check_docker_callable(config: OARepoConfig): + try: + run_cmdline("docker", "ps", grab_stdout=True, raise_exception=True) + except: # noqa + check_failed( + "Docker is not callable. Please install docker and make sure it is running." + ) + + +def check_version(*args, expected_major, expected_minor=None, strict=False): + try: + result = run_cmdline(*args, grab_stdout=True, raise_exception=True) + except: + check_failed(f"Command {' '.join(args)} is not callable.") + version_result = re.search(r".*?([0-9]+)\.([0-9]+)\.([0-9]+)", result) + major = int(version_result.groups()[0]) + minor = int(version_result.groups()[1]) + if strict: + if isinstance(expected_major, (list, tuple)): + assert ( + major in expected_major + ), f"Expected major version to be one of {expected_major}, found {major}" + else: + assert ( + major == expected_major + ), f"Expected major version to be one {expected_major}, found {major}" + if expected_minor: + assert minor == expected_minor + elif not ( + major > expected_major or (major == expected_major and minor >= expected_minor) + ): + raise Exception("Expected docker compose version ") + + +def check_docker_compose_version(config, expected_major, expected_minor): + check_version( + "docker", + "compose", + "version", + expected_major=expected_major, + expected_minor=expected_minor, + ) + + +def check_node_version(config, supported_versions): + check_version("node", "--version", expected_major=supported_versions, strict=True) + + +def check_npm_version(config, supported_versions): + check_version("npm", "--version", expected_major=supported_versions, strict=True) + + +def retry(fn, config, context, tries=10, timeout=1, verbose=False): + click.secho(f"Calling {fn.__name__}", fg="yellow") + for i in range(tries): + try: + fn(config, context) + click.secho(f" ... alive", fg="green") + return + except KeyboardInterrupt: + raise + except BaseException as e: + # catch SystemExit as well + if verbose: + click.secho(traceback.format_exc(), fg="red") + if i == tries - 1: + click.secho(f" ... failed", fg="red") + raise + click.secho( + f" ... not yet ready, sleeping for {int(timeout)} seconds", + fg="yellow", + ) + time.sleep(int(timeout)) + nt = timeout * 1.5 + if int(nt) == int(timeout): + timeout = timeout + 1 + else: + timeout = nt + + +def check_containers( + config: OARepoConfig, *, context, fast_fail=False, verbose=False, **kwargs +): + def test_docker_containers_accessible(*_, **__): + # pass empty context to prevent caching of the repository info + repository_info = get_repository_info(config, context={}) + if repository_info["db"] == "connection_error": + check_failed("Database container is not running or is not accessible.") + else: + click.secho(" Database is alive", fg="green") + if repository_info["opensearch"] == "connection_error": + check_failed("OpenSearch container is not running or is not accessible.") + else: + click.secho(" OpenSearch is alive", fg="green") + if repository_info["files"] == "connection_error": + check_failed("S3 container (minio) is not running or is not accessible.") + else: + click.secho(" S3 is alive", fg="green") + if repository_info["mq"] == "connection_error": + check_failed( + "Message queue container (rabbitmq) is not running or is not accessible." + ) + else: + click.secho(" Message queue is alive", fg="green") + if repository_info["cache"] == "connection_error": + check_failed("Cache container (redis) is not running or is not accessible.") + else: + click.secho(" Cache is alive", fg="green") + + if fast_fail: + test_docker_containers_accessible() + else: + retry( + test_docker_containers_accessible, + config, + context, + verbose=verbose, + ) + + +def fix_containers(config: OARepoConfig, *, context, **kwargs): + run_cmdline( + "docker", + "compose", + "up", + "-d", + "cache", + "db", + "mq", + "search", + "s3", + cwd=config.repository_dir / "docker", + ) + + +def split_url(url): + parsed_url = urlparse(url) + return ( + parsed_url.hostname, + parsed_url.port, + parsed_url.username, + parsed_url.password, + parsed_url.path[1:], + ) + + +def check_docker_redis(config, context): + (redis_url,) = get_invenio_configuration(config, context, "CACHE_REDIS_URL") + host, port, username, password, db = split_url(redis_url) + pool = redis.ConnectionPool( + host=host, port=port, db=db, username=username, password=password + ) + r = redis.Redis(connection_pool=pool) + r.keys("blahblahblah") # fails if there is no connection + pool.disconnect() + + +def check_docker_db(config, context): + host, port, user, password, dbname = split_url( + get_invenio_configuration( + config, + context, + "SQLALCHEMY_DATABASE_URI", + )[0] + ) + + with psycopg.connect( + dbname=dbname, host=host, port=port, user=user, password=password + ) as conn: + assert conn.execute("SELECT 1").fetchone()[0] == 1 + + +def check_docker_mq(config, context): + host, port, user, password, _ = split_url( + get_invenio_configuration( + config, + context, + "CELERY_BROKER_URL", + )[0] + ) + connection = pika.BlockingConnection( + pika.ConnectionParameters( + host=host, + port=port, + credentials=pika.credentials.PlainCredentials(user, password), + ) + ) + channel = connection.channel() + connection.process_data_events(2) + assert connection.is_open + connection.close() + + +def check_docker_s3(config, context): + endpoint_url, access_key, secret_key = get_invenio_configuration( + config, context, "S3_ENDPOINT_URL", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY" + ) + (host, port, *_) = split_url(endpoint_url) + + client = Minio( + f"{host}:{port}", + access_key=access_key, + secret_key=secret_key, + secure=False, + ) + client.list_buckets() + + +def check_docker_search(config, context): + ( + search_hosts, + search_config, + prefix, + ) = get_invenio_configuration( + config, + context, + "SEARCH_HOSTS", + "SEARCH_CLIENT_CONFIG", + "SEARCH_INDEX_PREFIX", + ) + client = OpenSearch( + hosts=[{"host": search_hosts[0]["host"], "port": search_hosts[0]["port"]}], + use_ssl=search_config.get("use_ssl", False), + verify_certs=search_config.get("verify_certs", False), + ssl_assert_hostname=search_config.get("ssl_assert_hostname", False), + ssl_show_warn=search_config.get("ssl_show_warn", False), + ca_certs=search_config.get("ca_certs", None), + ) + info = client.info(pretty=True) diff --git a/src/nrp_devtools/commands/initialize.py b/src/nrp_devtools/commands/initialize.py new file mode 100644 index 0000000..e77be43 --- /dev/null +++ b/src/nrp_devtools/commands/initialize.py @@ -0,0 +1,33 @@ +from pathlib import Path + +from nrp_devtools.commands.utils import run_cookiecutter +from nrp_devtools.config import OARepoConfig +from nrp_devtools.x509 import generate_selfsigned_cert + + +def initialize_repository(config: OARepoConfig): + run_cookiecutter( + config.repository_dir.parent, + template=Path(__file__).parent.parent / "templates" / "repository", + extra_context={ + "repository_name": config.repository_dir.name, + "shared_package": config.repository.shared_package, + "repository_package": config.repository.repository_package, + "repository_human_name": config.repository.repository_human_name, + "ui_package": config.repository.ui_package, + "oarepo_version": config.repository.oarepo_version, + }, + ) + + # generate the certificate + cert, key = generate_selfsigned_cert("localhost", ["127.0.0.1"]) + (config.repository_dir / "docker" / "development.crt").write_bytes(cert) + (config.repository_dir / "docker" / "development.key").write_bytes(key) + + # link the variables + (config.repository_dir / "docker" / ".env").symlink_to( + config.repository_dir / "variables" + ) + + # mark the nrp command executable + (config.repository_dir / "nrp").chmod(0o755) diff --git a/src/nrp_devtools/commands/invenio.py b/src/nrp_devtools/commands/invenio.py new file mode 100644 index 0000000..6aa7b19 --- /dev/null +++ b/src/nrp_devtools/commands/invenio.py @@ -0,0 +1,78 @@ +import json +import tempfile + +from nrp_devtools.commands.utils import run_cmdline +from nrp_devtools.config import OARepoConfig + +from .check import check_failed + + +def get_repository_info(config, context=None): + if context is not None and "repository_info" in context: + return context["repository_info"] + + with tempfile.NamedTemporaryFile(suffix=".json") as f: + run_cmdline( + config.invenio_command, + "oarepo", + "check", + f.name, + raise_exception=True, + grab_stdout=True, + ) + f.seek(0) + repository_info = json.load(f) + if context is not None: + context["repository_info"] = repository_info + return repository_info + + +def get_invenio_configuration(config, context, *args): + def _get_config(config, context): + if context is not None and "repository_configuration" in context: + return context["repository_configuration"] + + with tempfile.NamedTemporaryFile(suffix=".json") as f: + run_cmdline( + config.invenio_command, + "oarepo", + "configuration", + f.name, + raise_exception=True, + grab_stdout=True, + ) + f.seek(0) + configuration = json.load(f) + if context is not None: + context["repository_configuration"] = configuration + return configuration + + configuration = _get_config(config, context) + return [configuration[x] for x in args] + + +def check_invenio_cfg(config: OARepoConfig, **kwargs): + instance_dir = config.invenio_instance_path + target_invenio_cfg = instance_dir / "invenio.cfg" + target_variables = instance_dir / "variables" + if not target_invenio_cfg.exists(): + check_failed( + f"Site directory {instance_dir} does not contain invenio.cfg", + ) + if not target_variables.exists(): + check_failed( + f"Site directory {instance_dir} does not contain variables file", + ) + + +def install_invenio_cfg(config: OARepoConfig, **kwargs): + instance_dir = config.invenio_instance_path + instance_dir.mkdir(exist_ok=True, parents=True) + + target_invenio_cfg = instance_dir / "invenio.cfg" + target_variables = instance_dir / "variables" + + if not target_invenio_cfg.exists(): + target_invenio_cfg.symlink_to(config.repository_dir / "invenio.cfg") + if not target_variables.exists(): + target_variables.symlink_to(config.repository_dir / "variables") diff --git a/src/nrp_devtools/commands/model/__init__.py b/src/nrp_devtools/commands/model/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/nrp_devtools/commands/model/compile.py b/src/nrp_devtools/commands/model/compile.py new file mode 100644 index 0000000..4efe85a --- /dev/null +++ b/src/nrp_devtools/commands/model/compile.py @@ -0,0 +1,391 @@ +import configparser +import json +import os +import re +import shutil +import venv +from pathlib import Path + +import click +import yaml + +from nrp_devtools.commands.pyproject import PyProject +from nrp_devtools.commands.utils import run_cmdline +from nrp_devtools.config import OARepoConfig +from nrp_devtools.config.model_config import ModelConfig + + +def model_compiler_venv_dir(config: OARepoConfig, model): + venv_dir = ( + config.repository_dir / ".nrp" / f"oarepo-model-builder-{model.model_name}" + ) + return venv_dir.resolve() + + +def install_model_compiler(config: OARepoConfig, *, model: ModelConfig): + venv_dir = model_compiler_venv_dir(config, model) + print("model builder venv dir", venv_dir) + click.secho(f"Installing model compiler to {venv_dir}", fg="yellow") + + if venv_dir.exists(): + shutil.rmtree(venv_dir) + + venv_args = [str(venv_dir)] + venv.main(venv_args) + + run_cmdline( + venv_dir / "bin" / "pip", + "install", + "-U", + "setuptools", + "pip", + "wheel", + ) + + run_cmdline( + venv_dir / "bin" / "pip", + "install", + f"oarepo-model-builder", + ) + + with open(config.models_dir / model.model_config_file) as f: + model_data = yaml.safe_load(f) + + # install plugins from model.yaml + _install_plugins_from_model(model_data, venv_dir) + + # install plugins from included files + uses = model_data.get("use") or [] + if not isinstance(uses, list): + uses = [uses] + + for use in uses: + if not use.startswith("."): + # can not currently find plugins in uses + # that are registered as entrypoints + continue + with open(config.models_dir / use) as f: + used_data = yaml.safe_load(f) + _install_plugins_from_model(used_data, venv_dir) + + click.secho(f"Model compiler installed to {venv_dir}", fg="green") + + +def _install_plugins_from_model(model_data, venv_dir): + plugins = model_data.get("plugins", {}).get("packages", []) + for package in plugins: + run_cmdline( + venv_dir / "bin" / "pip", + "install", + package, + ) + + +def compile_model_to_tempdir(config: OARepoConfig, *, model: ModelConfig, tempdir): + click.secho(f"Compiling model {model.model_name} to {tempdir}", fg="yellow") + venv_dir = model_compiler_venv_dir(config, model) + run_cmdline( + venv_dir / "bin" / "oarepo-compile-model", + "-vvv", + str(config.models_dir / model.model_config_file), + "--output-directory", + str(tempdir), + ) + click.secho( + f"Model {model.model_name} successfully compiled to {tempdir}", fg="green" + ) + + +def copy_compiled_model(config: OARepoConfig, *, model: ModelConfig, tempdir): + click.secho( + f"Copying compiled model {model.model_name} from {tempdir} to {model.model_package}", + fg="yellow", + ) + alembic_path = Path(_get_alembic_path(tempdir, model.model_package)).resolve() + + remove_all_files_in_directory( + config.repository_dir / model.model_package, except_of=alembic_path + ) + + copy_all_files_but_keep_existing( + Path(tempdir) / model.model_package, config.repository_dir / model.model_package + ) + + click.secho( + f"Compiled model {model.model_name} successfully copied to {model.model_package}", + fg="green", + ) + + +def _get_alembic_path(rootdir, package_name): + model_file = Path(rootdir) / package_name / "models" / "records.json" + + with open(model_file) as f: + model_data = json.load(f) + + return model_data["model"]["record-metadata"]["alembic"].replace(".", "/") + + +def remove_all_files_in_directory(directory: Path, except_of=None): + if not directory.exists(): + return True + + remove_this_directory = True + for path in directory.iterdir(): + if path.resolve() == except_of: + remove_this_directory = False + continue + if path.is_file(): + path.unlink() + else: + remove_this_directory = ( + remove_all_files_in_directory(path) and remove_this_directory + ) + if remove_this_directory: + directory.rmdir() + return remove_this_directory + + +def copy_all_files_but_keep_existing(src: Path, dst: Path): + def non_overwriting_copy(src, dst, *, follow_symlinks=True): + if Path(dst).exists(): + return + shutil.copy2(src, dst, follow_symlinks=follow_symlinks) + + return shutil.copytree( + src, dst, copy_function=non_overwriting_copy, dirs_exist_ok=True + ) + + +def add_requirements_and_entrypoints( + config: OARepoConfig, *, model: ModelConfig, tempdir +): + click.secho( + f"Adding requirements and entrypoints from {model.model_name}", fg="yellow" + ) + + setup_cfg = Path(tempdir) / "setup.cfg" + # load setup.cfg via configparser + config_parser = configparser.ConfigParser() + config_parser.read(setup_cfg) + dependencies = config_parser["options"].get("install_requires", "").split("\n") + test_depedencies = ( + config_parser["options.extras_require"].get("tests", "").split("\n") + ) + entrypoints = {} + for ep_name, ep_values in config_parser["options.entry_points"].items(): + entrypoints[ep_name] = ep_values.split("\n") + + pyproject = PyProject(config.repository_dir / "pyproject.toml") + + pyproject.add_dependencies(*dependencies) + pyproject.add_optional_dependencies("tests", *test_depedencies) + + for ep_name, ep_values in entrypoints.items(): + for val in ep_values: + if not val: + continue + val = [x.strip() for x in val.split("=")] + pyproject.add_entry_point(ep_name, val[0], val[1]) + + pyproject.save() + + click.secho( + f"Requirements and entrypoint successfully copied from {model.model_name}", + fg="green", + ) + + +def generate_alembic(config: OARepoConfig, *, model: ModelConfig): + click.secho(f"Generating alembic for {model.model_name}", fg="yellow") + alembic_path = config.repository_dir / _get_alembic_path( + config.repository_dir, model.model_package + ) + branch = model.model_name + setup_alembic(config, branch, alembic_path, model) + click.secho(f"Alembic for {model.model_name} successfully generated", fg="green") + + +def setup_alembic( + config: OARepoConfig, branch: str, alembic_path: Path, model: ModelConfig +): + filecount = len( + [x for x in alembic_path.iterdir() if x.is_file() and x.name.endswith(".py")] + ) + + if filecount < 2: + intialize_alembic(config, branch, alembic_path, model) + else: + update_alembic(config, branch, alembic_path) + + +def update_alembic(config: OARepoConfig, branch, alembic_path): + # alembic has been initialized, update heads and generate + files = [file_path.name for file_path in alembic_path.iterdir()] + file_numbers = [] + for file in files: + file_number_regex = re.findall(f"(?<={branch}_)\d+", file) + if file_number_regex: + file_numbers.append(int(file_number_regex[0])) + new_file_number = max(file_numbers) + 1 + revision_message, file_revision_name_suffix = get_revision_names( + "Nrp install revision." + ) + run_cmdline( + config.invenio_command, "alembic", "upgrade", "heads", cwd=config.repository_dir + ) + + new_revision = get_revision_number( + run_cmdline( + config.invenio_command, + "alembic", + "revision", + revision_message, + "-b", + branch, + grab_stdout=True, + cwd=config.repository_dir, + ), + file_revision_name_suffix, + ) + rewrite_revision_file(alembic_path, new_file_number, branch, new_revision) + fix_sqlalchemy_utils(alembic_path) + run_cmdline( + config.invenio_command, "alembic", "upgrade", "heads", cwd=config.repository_dir + ) + + +def intialize_alembic(config, branch, alembic_path, model): + # alembic has not been initialized yet ... + run_cmdline( + config.invenio_command, + "alembic", + "upgrade", + "heads", + cwd=config.repository_dir, + ) + # create model branch + revision_message, file_revision_name_suffix = get_revision_names( + f"Create {branch} branch for {model.model_package}." + ) + new_revision = get_revision_number( + run_cmdline( + config.invenio_command, + "alembic", + "revision", + revision_message, + "-b", + branch, + "-p", + "dbdbc1b19cf2", + "--empty", + cwd=config.repository_dir, + grab_stdout=True, + ), + file_revision_name_suffix, + ) + rewrite_revision_file(alembic_path, "1", branch, new_revision) + fix_sqlalchemy_utils(alembic_path) + run_cmdline( + config.invenio_command, "alembic", "upgrade", "heads", cwd=config.repository_dir + ) + + revision_message, file_revision_name_suffix = get_revision_names( + "Initial revision." + ) + new_revision = get_revision_number( + run_cmdline( + config.invenio_command, + "alembic", + "revision", + revision_message, + "-b", + branch, + cwd=config.repository_dir, + grab_stdout=True, + ), + file_revision_name_suffix, + ) + rewrite_revision_file(alembic_path, "2", branch, new_revision) + # the link to down-revision is created correctly after alembic upgrade heads + # on the corrected file, explicit rewrite of down-revision is not needed + fix_sqlalchemy_utils(alembic_path) + run_cmdline( + config.invenio_command, "alembic", "upgrade", "heads", cwd=config.repository_dir + ) + + +def fix_sqlalchemy_utils(alembic_path): + for fn in alembic_path.iterdir(): + if not fn.name.endswith(".py"): + continue + data = fn.read_text() + + empty_migration = ''' +def upgrade(): +"""Upgrade database.""" +# ### commands auto generated by Alembic - please adjust! ### +pass +# ### end Alembic commands ###''' + + if re.sub(r"\s", "", empty_migration) in re.sub(r"\s", "", data): + click.secho(f"Found empty migration in file {fn}, deleting it", fg="yellow") + fn.unlink() + continue + + modified = False + if "import sqlalchemy_utils" not in data: + data = "import sqlalchemy_utils\n" + data + modified = True + if "import sqlalchemy_utils.types" not in data: + data = "import sqlalchemy_utils.types\n" + data + modified = True + if modified: + fn.write_text(data) + + +def get_revision_number(stdout_str, file_suffix): + mtch = re.search(f"(\w{{12}}){file_suffix}", stdout_str) + if not mtch: + raise ValueError("Revision number was not found in revision create stdout") + return mtch.group(1) + + +def get_revision_names(revision_message): + file_name = revision_message[0].lower() + revision_message[1:] + file_name = "_" + file_name.replace(" ", "_") + if file_name[-1] == ".": + file_name = file_name[:-1] + + file_name = file_name[:30] # there seems to be maximum length for the file name + idx = file_name.rfind("_") + file_name = file_name[:idx] # and all words after it are cut + return revision_message, file_name + + +def rewrite_revision_file( + alembic_path, new_id_number, revision_id_prefix, current_revision_id +): + files = list(alembic_path.iterdir()) + files_with_this_revision_id = [ + file_name for file_name in files if current_revision_id in str(file_name) + ] + + if not files_with_this_revision_id: + raise ValueError( + "Alembic file rewrite couldn't find the generated revision file" + ) + + if len(files_with_this_revision_id) > 1: + raise ValueError("More alembic files with the same revision number found") + + target_file = str(files_with_this_revision_id[0]) + new_id = f"{revision_id_prefix}_{new_id_number}" + with open(target_file, "r") as f: + file_text = f.read() + file_text = file_text.replace( + f"revision = '{current_revision_id}'", f"revision = '{new_id}'" + ) + with open(target_file.replace(current_revision_id, new_id), "w") as f: + f.write(file_text) + os.remove(target_file) diff --git a/src/nrp_devtools/commands/model/create.py b/src/nrp_devtools/commands/model/create.py new file mode 100644 index 0000000..18f433e --- /dev/null +++ b/src/nrp_devtools/commands/model/create.py @@ -0,0 +1,171 @@ +import yaml + +from nrp_devtools.config import OARepoConfig +from nrp_devtools.config.model_config import BaseModel, ModelConfig, ModelFeature + + +def create_model(config: OARepoConfig, *, model_name): + model: ModelConfig = config.get_model(model_name) + + profiles = ["record"] + profile_specific = {} + + plugins = { + "oarepo-model-builder-ui", + } + permission_presets = ["authenticated"] + extra_includes = [] + runtime_dependencies = {} + settings = {} + + if ModelFeature.drafts in model.features: + profiles.append("draft") + profile_specific["draft"] = {} + plugins.add("oarepo-model-builder-drafts") + + if ModelFeature.files in model.features: + profiles.append("files") + plugins.add("oarepo-model-builder-files") + if ModelFeature.drafts in model.features: + profiles.append("draft_files") + profile_specific["draft-files"] = {} + plugins.add("oarepo-model-builder-drafts-files") + + if ModelFeature.communities in model.features: + permission_presets = ["communities"] + + if ModelFeature.requests in model.features: + extra_includes.append(f"./{model.model_name}-requests.yaml") + plugins.add("oarepo-model-builder-requests") + + if ModelFeature.custom_fields in model.features: + extra_includes.append(f"./{model.model_name}-custom_fields.yaml") + plugins.add("oarepo-model-builder-cf") + + if ModelFeature.files in model.features: + profile_specific["files"] = { + "use": ["invenio_files"], + "properties": {"use": [f"./{model.model_name}-files.yaml"]}, + } + + if ModelFeature.vocabularies in model.features: + plugins.add("oarepo-model-builder-vocabularies") + runtime_dependencies["nr-vocabularies"] = "2.0.0" + + if ModelFeature.relations in model.features: + plugins.add("oarepo-model-builder-relations") + + if ModelFeature.nr_vocabularies in model.features: + runtime_dependencies["nr-vocabularies"] = "2.0.8" + + settings["i18n-languages"] = ["en"] + + if ModelFeature.multilingual in model.features: + plugins.add("oarepo-model-builder-multilingual") + settings.update( + { + "supported-langs": { + "en": {"text": {}}, + }, + } + ) + + if model.base_model == BaseModel.data: + extend = { + "extend": '"nr-data#DataModel"', + } + plugins.add("oarepo-model-builder-nr") + runtime_dependencies["nr-metadata"] = "2.0.0" + elif model.base_model == BaseModel.documents: + extend = { + "extend": '"nr-documents#DocumentModel"', + } + plugins.add("oarepo-model-builder-nr") + runtime_dependencies["nr-metadata"] = "2.0.0" + elif model.base_model == BaseModel.empty: + extend = {} + else: + raise ValueError(f"Unknown base model: {model.base_model}") + + model_schema = { + "profiles": profiles, + "record": { + "module": {"qualified": model.model_package}, + **extend, + "permissions": {"presets": permission_presets}, + "pid": {"type": model.pid_type}, + "properties": {"metadata": {"properties": {}}}, + "resource-config": {"base-html-url": f"/{model.api_prefix}/"}, + **profile_specific, + "use": ["invenio", *extra_includes], + }, + "plugins": { + "builder": {"disable": ["script_sample_data"]}, + "packages": list(plugins), + }, + "runtime-dependencies": runtime_dependencies, + "settings": settings, + } + + save_schema(config.models_dir / model.model_config_file, model_schema) + + if ModelFeature.files in model.features: + save_schema( + config.models_dir / f"{model.model_name}-files.yaml", + {"caption": {"type": "keyword"}}, + ) + + if ModelFeature.requests in model.features: + save_schema( + config.models_dir / f"{model.model_name}-requests.yaml", + { + "service-config": { + "components": [ + '{{oarepo_requests.services.components.PublishDraftComponent}}("publish_draft", "delete_record")' + ] + }, + "draft": { + "requests": { + "publish-draft": { + "type": { + "base-classes": [ + "oarepo_requests.types.publish_draft.PublishDraftRequestType" + ] + }, + "actions": { + "submit": { + "class": "oarepo_requests.actions.publish_draft.PublishDraftSubmitAction", + "generate": False, + } + }, + } + } + }, + "requests": { + "delete-record": { + "type": { + "base-classes": [ + "oarepo_requests.types.delete_record.DeleteRecordRequestType" + ] + }, + "actions": { + "submit": { + "class": "oarepo_requests.actions.delete_topic.DeleteTopicSubmitAction", + "generate": False, + } + }, + } + }, + }, + ) + + if ModelFeature.custom_fields in model.features: + save_schema( + config.models_dir / f"{model.model_name}-custom_fields.yaml", + {"custom-fields": []}, + ) + + +def save_schema(file, schema): + with open(file, "w") as f: + yaml.dump(schema, f) diff --git a/src/nrp_devtools/commands/opensearch.py b/src/nrp_devtools/commands/opensearch.py new file mode 100644 index 0000000..b715b7e --- /dev/null +++ b/src/nrp_devtools/commands/opensearch.py @@ -0,0 +1,29 @@ +from nrp_devtools.commands.check import check_failed +from nrp_devtools.commands.invenio import get_repository_info +from nrp_devtools.commands.utils import run_cmdline +from nrp_devtools.config import OARepoConfig + + +def check_search(config: OARepoConfig, context=None, **kwargs): + opensearch_status = get_repository_info(config, context)["opensearch"] + if opensearch_status != "ok": + check_failed( + f"Search is not ready, it reports status {opensearch_status}.", + ) + + +def fix_search(config: OARepoConfig, context=None, **kwargs): + opensearch_status = get_repository_info(config, context)["opensearch"] + if opensearch_status != "ok": + run_cmdline(config.invenio_command, "oarepo", "index", "init") + run_cmdline(config.invenio_command, "oarepo", "cf", "init") + + # make the repository info reinitialize during the next check + context.pop("repository_info") + + +def fix_custom_fields(config: OARepoConfig, context=None, **kwargs): + run_cmdline(config.invenio_command, "oarepo", "cf", "init") + + # make the repository info reinitialize during the next check + context.pop("repository_info") diff --git a/src/nrp_devtools/commands/pdm.py b/src/nrp_devtools/commands/pdm.py new file mode 100644 index 0000000..851aded --- /dev/null +++ b/src/nrp_devtools/commands/pdm.py @@ -0,0 +1,236 @@ +import os +import re +import shutil +import sys + +import click +import requirements +import tomli +import tomli_w + +from nrp_devtools.commands.check import check_failed +from nrp_devtools.commands.utils import install_python_modules, run_cmdline +from nrp_devtools.config import OARepoConfig + + +def clean_previous_installation(config: OARepoConfig, **kwargs): + destroy_venv(config) + + +def create_empty_venv(config): + install_python_modules(config, config.pdm_dir, "setuptools", "pip", "wheel", "pdm") + install_python_modules( + config, + config.venv_dir, + "setuptools", + "pip", + "wheel", + ) + + +def destroy_venv(config): + if config.pdm_dir.exists(): + shutil.rmtree(config.pdm_dir) + if config.venv_dir.exists(): + shutil.rmtree(config.venv_dir) + + +def build_requirements(config, **kwargs): + destroy_venv(config) + create_empty_venv(config) + create_pdm_file(config, ".nrp/oarepo-pdm") + lock_python_repository(config, ".nrp/oarepo-pdm") + oarepo_requirements = export_pdm_requirements(config, ".nrp/oarepo-pdm") + + lock_python_repository(config) + all_requirements = export_pdm_requirements(config) + + oarepo_requirements = list(requirements.parse(oarepo_requirements)) + all_requirements = list(requirements.parse(all_requirements)) + + # get the current version of oarepo + oarepo_requirement = [x for x in oarepo_requirements if x.name == "oarepo"][0] + + # now make the difference of those two (we do not want to have oarepo dependencies in the result) + # as oarepo will be installed to virtualenv separately (to handle system packages) + oarepo_requirements_names = {x.name for x in oarepo_requirements} + non_oarepo_requirements = [ + x for x in all_requirements if x.name not in oarepo_requirements_names + ] + + # remove local packages + non_oarepo_requirements = [ + x for x in non_oarepo_requirements if "file://" not in x.line + ] + + # and generate final requirements + resolved_requirements = "\n".join( + [oarepo_requirement.line, *[x.line for x in non_oarepo_requirements]] + ) + (config.repository_dir / "requirements.txt").write_text(resolved_requirements) + + +def lock_python_repository(config, subdir=None): + run_pdm(config, "lock", subdir=subdir) + + +def check_invenio_callable(config, **kwargs): + try: + run_cmdline( + config.venv_dir / "bin" / "invenio", + "oarepo", + "version", + raise_exception=True, + grab_stdout=True, + ) + except: + check_failed( + f"Virtualenv directory {config.venv_dir} does not contain a callable invenio installation", + ) + + +def install_python_repository(config, **kwargs): + run_pdm(config, "install", "--dev", "--no-lock") + + +def create_pdm_file(config: OARepoConfig, output_directory: str): + original_pdm_file = tomli.loads( + (config.repository_dir / "pyproject.toml").read_text() + ) + dependencies = original_pdm_file["project"]["dependencies"] + oarepo_dependency = [ + x + for x in dependencies + if re.match(r"^\s*oarepo\s*==.*", x) # take only oarepo package, discard others + ][0] + + original_pdm_file["project"]["dependencies"] = [oarepo_dependency] + + output_path = config.repository_dir / output_directory + output_path.mkdir(parents=True, exist_ok=True) + + (output_path / "pyproject.toml").write_text(tomli_w.dumps(original_pdm_file)) + + +def remove_virtualenv_from_env(): + current_env = dict(os.environ) + virtual_env_dir = current_env.pop("VIRTUAL_ENV", None) + if not virtual_env_dir: + return current_env + current_env.pop("PYTHONHOME", None) + current_env.pop("PYTHON", None) + path = current_env.pop("PATH", None) + split_path = path.split(os.pathsep) + split_path = [x for x in split_path if not x.startswith(virtual_env_dir)] + current_env["PATH"] = os.pathsep.join(split_path) + return current_env + + +def run_pdm(config, *args, subdir=None, **kwargs): + cwd = config.repository_dir + if subdir: + cwd = cwd / subdir + + if (cwd / "__pypackages__").exists(): + shutil.rmtree(cwd / "__pypackages__") + + environ = { + "PDM_IGNORE_ACTIVE_VENV": "1", + "PDM_IGNORE_SAVED_PYTHON": "1", + **remove_virtualenv_from_env(), + } + + if (cwd / ".venv").exists(): + environ.pop("PDM_IGNORE_ACTIVE_VENV", None) + environ["VIRTUAL_ENV"] = str(cwd / ".venv") + print(f"Using venv for pdm: {environ['VIRTUAL_ENV']}") + + return run_cmdline( + config.pdm_dir / "bin" / "pdm", + *args, + cwd=cwd, + environ=environ, + no_environment=True, + raise_exception=True, + **kwargs, + ) + + +def export_pdm_requirements(config, subdir=None): + return run_pdm( + config, + "export", + "-f", + "requirements", + "--without-hashes", + grab_stdout=True, + subdir=subdir, + ) + + +def install_local_packages(config, local_packages=None): + if not local_packages: + return + for lp in local_packages: + run_cmdline( + config.venv_dir / "bin" / "pip", + "install", + "--config-settings", + "editable_mode=compat", + "-e", + lp, + cwd=config.repository_dir, + ) + + +def check_virtualenv(config: OARepoConfig, **kwargs): + if not config.venv_dir.exists(): + click.secho( + f"Virtualenv directory {config.venv_dir} does not exist", fg="red", err=True + ) + sys.exit(1) + + try: + run_cmdline( + config.venv_dir / "bin" / "python", + "--version", + raise_exception=True, + ) + except: # noqa + check_failed( + f"Virtualenv directory {config.venv_dir} does not contain a python installation", + ) + + try: + run_cmdline( + config.venv_dir / "bin" / "pip", + "list", + raise_exception=True, + grab_stdout=True, + ) + except: # noqa + check_failed( + f"Virtualenv directory {config.venv_dir} does not contain a pip installation", + ) + + +def fix_virtualenv(config: OARepoConfig, **kwargs): + destroy_venv(config) + create_empty_venv(config) + + +def check_requirements(config: OARepoConfig, **kwargs): + reqs_file = config.repository_dir / "requirements.txt" + if not reqs_file.exists(): + check_failed(f"Requirements file {reqs_file} does not exist") + + # if any pyproject.toml is newer than requirements.txt, we need to rebuild + pyproject = config.repository_dir / "pyproject.toml" + + if pyproject.exists() and pyproject.stat().st_mtime > reqs_file.stat().st_mtime: + check_failed( + f"Requirements file {reqs_file} is out of date, {pyproject} has been modified", + ) + + +# aa diff --git a/src/nrp_devtools/commands/pyproject.py b/src/nrp_devtools/commands/pyproject.py new file mode 100644 index 0000000..b927dc0 --- /dev/null +++ b/src/nrp_devtools/commands/pyproject.py @@ -0,0 +1,50 @@ +import re + +import tomli +import tomli_w + + +class PyProject: + def __init__(self, path): + self.pyproject_file = path + with open(self.pyproject_file, "rb") as f: + self.pyproject_data = tomli.load(f) + + def save(self): + with open(self.pyproject_file, "wb") as f: + tomli_w.dump(self.pyproject_data, f) + + def add_dependencies(self, *dependencies): + self.update_dependencies_array( + self.pyproject_data["project"].setdefault("dependencies", []), dependencies + ) + return self + + def add_optional_dependencies(self, group, *dependencies): + self.update_dependencies_array( + self.pyproject_data["project"] + .setdefault("optional-dependencies", {}) + .setdefault(group, []), + dependencies, + ) + return self + + def add_entry_point(self, group, name, value): + toml_entry_points = self.pyproject_data["project"].setdefault( + "entry-points", {} + ) + + ep_group = toml_entry_points.setdefault(group, {}) + ep_group[name] = value + return self + + def update_dependencies_array(self, target_array, new_values): + target_packages = [re.split("[<>=]", ta)[0] for ta in target_array] + for value in new_values: + if not value: + continue + value_package = re.split("[<>=]", value)[0] + if value_package not in target_packages: + target_array.append(value) + target_array.sort() + return target_array diff --git a/src/nrp_devtools/commands/s3.py b/src/nrp_devtools/commands/s3.py new file mode 100644 index 0000000..9fac916 --- /dev/null +++ b/src/nrp_devtools/commands/s3.py @@ -0,0 +1,70 @@ +from urllib.parse import urlparse + +from minio import Minio + +from nrp_devtools.commands.check import check_failed +from nrp_devtools.commands.invenio import get_invenio_configuration, get_repository_info +from nrp_devtools.commands.utils import run_cmdline +from nrp_devtools.config import OARepoConfig + + +def check_s3_location_in_database(config: OARepoConfig, context=None, **kwargs): + s3_location_status = get_repository_info(config, context)["files"] + if s3_location_status == "default_location_missing": + check_failed( + f"S3 location is missing from the database.", + ) + + +DEFAULT_BUCKET_NAME = "default" + + +def fix_s3_location_in_database(config: OARepoConfig, context=None, **kwargs): + s3_location_status = get_repository_info(config, context)["files"] + if s3_location_status == "default_location_missing": + run_cmdline( + config.invenio_command, + "files", + "location", + "default", + f"s3://{DEFAULT_BUCKET_NAME}", + "--default", + ) + + # make the repository info reinitialize during the next check + context.pop("repository_info") + + +def check_s3_bucket_exists(config: OARepoConfig, context=None, **kwargs): + s3_location_status = get_repository_info(config, context)["files"] + if s3_location_status.startswith("bucket_does_not_exist:"): + check_failed( + f"S3 bucket {s3_location_status.split(':')[1]} does not exist.", + ) + + +def fix_s3_bucket_exists(config: OARepoConfig, context=None, **kwargs): + s3_endpoint_url, access_key, secret_key = get_invenio_configuration( + config, + context, + "S3_ENDPOINT_URL", + "S3_ACCESS_KEY_ID", + "S3_SECRET_ACCESS_KEY", + ) + parsed_s3_endpoint_url = urlparse(s3_endpoint_url) + host = parsed_s3_endpoint_url.hostname + port = parsed_s3_endpoint_url.port + + client = Minio( + f"{host}:{port}", + access_key=access_key, + secret_key=secret_key, + secure=False, + ) + + bucket_name = "default" + + client.make_bucket(bucket_name) + + # make the repository info reinitialize during the next check + context.pop("repository_info") diff --git a/src/nrp_devtools/commands/ui/__init__.py b/src/nrp_devtools/commands/ui/__init__.py new file mode 100644 index 0000000..90dd2da --- /dev/null +++ b/src/nrp_devtools/commands/ui/__init__.py @@ -0,0 +1,42 @@ +import json + +from ...config import OARepoConfig +from ..check import check_failed +from .assets import collect_assets, install_npm_packages +from .build import build_production_ui +from .less import register_less_components + + +def check_ui(config: OARepoConfig, **kwargs): + # check that there is a manifest.json there + manifest = config.invenio_instance_path / "static" / "dist" / "manifest.json" + if not manifest.exists(): + check_failed( + f"manifest.json file is missing.", + ) + + try: + json_data = json.loads(manifest.read_text()) + if json_data.get("status") != "done": + check_failed( + f"manifest.json file is not ready. " + f"Either a build is in progress or it failed.", + ) + except: # noqa + check_failed( + f"manifest.json file is not valid json file.", + ) + + +def fix_ui(config: OARepoConfig, **kwargs): + collect_assets(config) + install_npm_packages(config) + build_production_ui(config) + + +__all__ = [ + "register_less_components", + "build_production_ui", + "collect_assets", + "install_npm_packages", +] diff --git a/src/nrp_devtools/commands/ui/assets.py b/src/nrp_devtools/commands/ui/assets.py new file mode 100644 index 0000000..0092054 --- /dev/null +++ b/src/nrp_devtools/commands/ui/assets.py @@ -0,0 +1,56 @@ +import json +import shutil +from pathlib import Path + +from nrp_devtools.commands.ui import register_less_components +from nrp_devtools.commands.utils import run_cmdline + + +def load_watched_paths(paths_json, extra_paths): + watched_paths = {} + with open(paths_json) as f: + for target, paths in json.load(f).items(): + if target.startswith("@"): + continue + for pth in paths: + watched_paths[pth] = target + for e in extra_paths: + source, target = e.split("=", maxsplit=1) + watched_paths[source] = target + return watched_paths + + +def collect_assets(config): + invenio_instance_path = config.invenio_instance_path + shutil.rmtree(invenio_instance_path / "assets", ignore_errors=True) + shutil.rmtree(invenio_instance_path / "static", ignore_errors=True) + Path(invenio_instance_path / "assets").mkdir(parents=True) + Path(invenio_instance_path / "static").mkdir(parents=True) + register_less_components(config, invenio_instance_path) + run_cmdline( + config.invenio_command, + "oarepo", + "assets", + "collect", + f"{invenio_instance_path}/watch.list.json", + ) + run_cmdline( + config.invenio_command, + "webpack", + "clean", + "create", + ) + # shutil.copytree( + # config.site_dir / "assets", invenio_instance_path / "assets", dirs_exist_ok=True + # ) + # shutil.copytree( + # config.site_dir / "static", invenio_instance_path / "static", dirs_exist_ok=True + # ) + + +def install_npm_packages(config): + run_cmdline( + config.invenio_command, + "webpack", + "install", + ) diff --git a/src/nrp_devtools/commands/ui/build.py b/src/nrp_devtools/commands/ui/build.py new file mode 100644 index 0000000..1bff12c --- /dev/null +++ b/src/nrp_devtools/commands/ui/build.py @@ -0,0 +1,15 @@ +from nrp_devtools.commands.utils import run_cmdline +from nrp_devtools.config import OARepoConfig + + +def build_production_ui(config: OARepoConfig): + run_cmdline(config.invenio_command, "webpack", "build", "--production") + + # do not allow Clean plugin to remove files + webpack_config = ( + config.invenio_instance_path / "assets" / "build" / "webpack.config.js" + ).read_text() + webpack_config = webpack_config.replace("dry: false", "dry: true") + ( + config.invenio_instance_path / "assets" / "build" / "webpack.config.js" + ).write_text(webpack_config) diff --git a/src/nrp_devtools/commands/ui/create.py b/src/nrp_devtools/commands/ui/create.py new file mode 100644 index 0000000..8a1f8e1 --- /dev/null +++ b/src/nrp_devtools/commands/ui/create.py @@ -0,0 +1,86 @@ +from pathlib import Path + +import caseconverter + +from nrp_devtools.commands.pyproject import PyProject +from nrp_devtools.commands.utils import run_cookiecutter +from nrp_devtools.config import OARepoConfig + + +def create_page_ui(config: OARepoConfig, *, ui_name: str): + ui_config = config.get_ui(ui_name) + + if (config.ui_dir / ui_config.name).exists(): + return + + capitalized_name = caseconverter.camelcase(ui_config.name) + capitalized_name = capitalized_name[0].upper() + capitalized_name[1:] + + run_cookiecutter( + config.ui_dir, + template=Path(__file__).parent.parent.parent / "templates" / "ui_page", + extra_context={ + "name": ui_config.name, + "endpoint": ui_config.endpoint, + "capitalized_name": capitalized_name, + "template_name": capitalized_name + "Page", + }, + ) + + +def register_page_ui(config: OARepoConfig, *, ui_name: str): + ui_config = config.get_ui(ui_name) + + pyproject = PyProject(config.repository_dir / "pyproject.toml") + + pyproject.add_entry_point( + "invenio_base.blueprints", + ui_config.name, + f"{config.repository.ui_package}.{ui_config.name}:create_blueprint", + ) + + pyproject.save() + + +def create_model_ui(config: OARepoConfig, *, ui_name: str): + ui_config = config.get_ui(ui_name) + + if (config.ui_dir / ui_config.name).exists(): + return + + capitalized_name = caseconverter.camelcase(ui_config.name) + capitalized_name = capitalized_name[0].upper() + capitalized_name[1:] + + run_cookiecutter( + config.ui_dir, + template=Path(__file__).parent.parent.parent / "templates" / "ui_model", + extra_context={ + "name": ui_config.name, + "endpoint": ui_config.endpoint, + "capitalized_name": capitalized_name, + "resource": capitalized_name + "Resource", + "resource_config": capitalized_name + "ResourceConfig", + "ui_serializer_class": ui_config.ui_serializer_class, + "api_service": ui_config.api_service, + }, + ) + + +def register_model_ui(config: OARepoConfig, *, ui_name: str): + ui_config = config.get_ui(ui_name) + + pyproject = PyProject(config.repository_dir / "pyproject.toml") + + pyproject.add_entry_point( + "invenio_base.blueprints", + ui_config.name, + f"{config.repository.ui_package}.{ui_config.name}:create_blueprint", + ) + + pyproject.add_entry_point( + "invenio_assets.webpack", + ui_config.name, + f"{config.repository.ui_package}.{ui_config.name}.theme.webpack:theme", + ) + + pyproject.save() diff --git a/src/nrp_devtools/commands/ui/less.py b/src/nrp_devtools/commands/ui/less.py new file mode 100644 index 0000000..62c0bd0 --- /dev/null +++ b/src/nrp_devtools/commands/ui/less.py @@ -0,0 +1,30 @@ +import json +import re +from pathlib import Path + +from nrp_devtools.commands.utils import run_cmdline +from nrp_devtools.config import OARepoConfig + + +def register_less_components(config: OARepoConfig, invenio_instance_path): + run_cmdline( + config.invenio_command, + "oarepo", + "assets", + "less-components", + f"{invenio_instance_path}/less-components.json", + ) + data = json.loads(Path(f"{invenio_instance_path}/less-components.json").read_text()) + components = list(set(data["components"])) + theme_config_file = ( + config.ui_dir / "branding" / config.theme_dir_name / "less" / "theme.config" + ) + theme_data = theme_config_file.read_text() + for c in components: + match = re.search("^@" + c, theme_data, re.MULTILINE) + if not match: + match = theme_data.index("/* @my_custom_component : 'default'; */") + theme_data = ( + theme_data[:match] + f"\n@{c}: 'default';\n" + theme_data[match:] + ) + theme_config_file.write_text(theme_data) diff --git a/src/nrp_devtools/commands/ui/link_assets.py b/src/nrp_devtools/commands/ui/link_assets.py new file mode 100644 index 0000000..65d9675 --- /dev/null +++ b/src/nrp_devtools/commands/ui/link_assets.py @@ -0,0 +1,83 @@ +import sys +from pathlib import Path + +import click +from tqdm import tqdm + +from nrp_devtools.config import OARepoConfig + +from .assets import load_watched_paths + + +def link_assets(config: OARepoConfig): + assets = (config.site_dir / "assets").resolve() + static = (config.site_dir / "static").resolve() + + watched_paths = load_watched_paths( + config.invenio_instance_path / "watch.list.json", + [f"{assets}=assets", f"{static}=static"], + ) + kinds = {"assets", "static"} + + existing = {k: set() for k in kinds} + for kind, target in tqdm( + _list_files(kinds, config.invenio_instance_path), + desc="Enumerating existing paths", + ): + relative_path = target.relative_to(config.invenio_instance_path / kind) + if relative_path.parts[0] in ("node_modules", "patches", "build", "dist"): + continue + if len(relative_path.parts) == 1: + continue + existing[kind].add(target) + + linked = {k: {} for k in kinds} + + for kind, source_path, source_file in tqdm( + _list_source_files(watched_paths), desc="Checking paths" + ): + target_file = ( + config.invenio_instance_path / kind / source_file.relative_to(source_path) + ) + linked[kind][source_file] = target_file + if target_file in existing[kind]: + existing[kind].remove(target_file) + + for kind, existing_data in existing.items(): + if existing_data: + click.secho(f"Error: following {kind} are not linked:", fg="red") + for target in existing_data: + click.secho(f" {target}", fg="red") + sys.exit(1) + + for kind, source_file, target_file in tqdm( + _list_linked_files(linked), desc="Linking assets and statics" + ): + target_file.parent.mkdir(parents=True, exist_ok=True) + try: + target_file.unlink() + except FileNotFoundError: + pass + target_file.symlink_to(source_file) + + +def _list_files(kinds, base_path): + for kind in kinds: + for file_or_dir in Path(base_path / kind).glob("**/*"): + if file_or_dir.is_dir(): + continue + yield kind, file_or_dir + + +def _list_source_files(watched_paths): + for source_path, kind in watched_paths.items(): + for source_file in Path(source_path).glob("**/*"): + if source_file.is_dir(): + continue + yield kind, source_path, source_file + + +def _list_linked_files(linked): + for kind, linked_data in linked.items(): + for source_file, target_file in linked_data.items(): + yield kind, source_file, target_file diff --git a/src/nrp_devtools/commands/utils.py b/src/nrp_devtools/commands/utils.py new file mode 100644 index 0000000..27e042b --- /dev/null +++ b/src/nrp_devtools/commands/utils.py @@ -0,0 +1,187 @@ +import os +import subprocess +import sys +import traceback +from functools import wraps +from pathlib import Path +from typing import Callable, Union + +import click +from cookiecutter.main import cookiecutter + +from nrp_devtools.config import OARepoConfig + + +def run_cookiecutter( + output_dir, + template: Path, + extra_context=None, +): + cookiecutter( + str(template), + no_input=True, + extra_context=extra_context, + overwrite_if_exists=False, + output_dir=output_dir, + # keep_project_on_failure=False, + ) + + +def run_cmdline( + *cmdline, + cwd=".", + environ=None, + check_only=False, + grab_stdout=False, + grab_stderr=False, + discard_output=False, + raise_exception=False, + no_input=False, + no_environment=False, +): + if no_environment: + env = {} + else: + env = os.environ.copy() + + env.update(environ or {}) + cwd = Path(cwd).absolute() + cmdline = [str(x) for x in cmdline] + + click.secho(f"Running {' '.join(cmdline)}", fg="blue", err=True) + click.secho(f" inside {cwd}", fg="blue", err=True) + + try: + kwargs = {} + if no_input: + kwargs["stdin"] = subprocess.DEVNULL + if grab_stdout or grab_stderr or discard_output: + if grab_stdout or discard_output: + kwargs["stdout"] = subprocess.PIPE + if grab_stderr or discard_output: + kwargs["stderr"] = subprocess.PIPE + + ret = subprocess.run( + cmdline, + check=True, + cwd=cwd, + env=env, + **kwargs, + ) + ret = (ret.stdout or b"") + b"\n" + (ret.stderr or b"") + else: + ret = subprocess.call(cmdline, cwd=cwd, env=env, **kwargs) + if ret: + raise subprocess.CalledProcessError(ret, cmdline) + except subprocess.CalledProcessError as e: + if check_only: + return False + click.secho(f"Error running {' '.join(cmdline)}", fg="red", err=True) + if e.stdout: + click.secho(e.stdout.decode("utf-8"), fg="red", err=True) + if e.stderr: + click.secho(e.stderr.decode("utf-8"), fg="red", err=True) + if raise_exception: + raise + sys.exit(e.returncode) + + click.secho(f"Finished running {' '.join(cmdline)}", fg="green", err=True) + click.secho(f" inside {cwd}", fg="green", err=True) + + if grab_stdout: + return ret.decode("utf-8").strip() + + return True + + +def call_pip(venv_dir, *args, **kwargs): + return run_cmdline( + venv_dir / "bin" / "pip", + *args, + **{ + "raise_exception": True, + "no_environment": True, + **kwargs, + }, + ) + + +def install_python_modules(config, venv_dir, *modules): + run_cmdline( + os.environ.get("PYTHON", config.python), + "-m", + "venv", + str(venv_dir), + cwd=config.repository_dir, + raise_exception=True, + no_environment=True, + ) + + call_pip( + venv_dir, + "install", + "-U", + "--no-input", + *modules, + no_environment=True, + raise_exception=True, + ) + + +def run_steps(config, steps, step_commands, continue_on_errors=False): + steps = steps or [] + steps = [x.strip() for step in steps for x in step.split(",") if x.strip()] + errors = [] + for idx, fun in enumerate(step_commands): + function_name = fun.__name__ + if not steps or function_name in steps or str(idx + 1) in steps: + try: + fun(config=config) + except KeyboardInterrupt: + raise + except BaseException as e: + if continue_on_errors: + errors.append(e) + else: + raise + if errors: + raise errors[0] + + +def make_step( + fun, + *global_args, + _if: Union[bool, Callable[[OARepoConfig], bool]] = True, + **global_kwargs, +): + @wraps(fun) + def step(config, **kwargs): + should_call = _if if not callable(_if) else _if(config) + if should_call: + fun(config, *global_args, **global_kwargs, **kwargs) + + return step + + +def no_args(fun): + @wraps(fun) + def _no_args(*args, **kwargs): + fun() + + return _no_args + + +def run_fixup(check_function, fix_function, fix=True, **global_kwargs): + @wraps(check_function) + def _run_fixup(config, **kwargs): + try: + check_function(config, fast_fail=True, **kwargs, **global_kwargs) + except: + if global_kwargs.get("verbose"): + traceback.print_exc() + if not fix: + raise + fix_function(config, **kwargs, **global_kwargs) + check_function(config, fast_fail=False, **kwargs, **global_kwargs) + + return _run_fixup diff --git a/src/nrp_devtools/config/__init__.py b/src/nrp_devtools/config/__init__.py new file mode 100644 index 0000000..acad3f4 --- /dev/null +++ b/src/nrp_devtools/config/__init__.py @@ -0,0 +1,4 @@ +from .config import OARepoConfig +from .wizard import ask_for_configuration + +__all__ = ("OARepoConfig", "ask_for_configuration") diff --git a/src/nrp_devtools/config/config.py b/src/nrp_devtools/config/config.py new file mode 100644 index 0000000..6ba5f76 --- /dev/null +++ b/src/nrp_devtools/config/config.py @@ -0,0 +1,148 @@ +import dataclasses +from enum import Enum +from io import StringIO +from pathlib import Path +from typing import List, Optional, Set + +import dacite +import yaml +from yaml.representer import SafeRepresenter + +from .model_config import BaseModel, ModelConfig, ModelFeature +from .repository_config import RepositoryConfig +from .ui_config import UIConfig + +serialization_config = dacite.Config() +serialization_config.type_hooks = { + Path: lambda x: Path(x), + ModelFeature: lambda x: ModelFeature[x] if isinstance(x, str) else x, + BaseModel: lambda x: BaseModel[x] if isinstance(x, str) else x, + Set[ModelFeature]: lambda x: set(x), +} + + +def Enum_representer(dumper, data): + return dumper.represent_scalar("tag:yaml.org,2002:str", data.value) + + +def Set_representer(dumper, data): + return dumper.represent_sequence( + "tag:yaml.org,2002:seq", list(data), flow_style=True + ) + + +SafeRepresenter.add_multi_representer(Enum, Enum_representer) +SafeRepresenter.add_representer(set, Set_representer) + +UNKNOWN = ( + object() +) # marker for unknown default value in get_model which will emit KeyError + + +@dataclasses.dataclass +class OARepoConfig: + repository_dir: Path + repository: Optional[RepositoryConfig] = None + models: List[ModelConfig] = dataclasses.field(default_factory=list) + uis: List[UIConfig] = dataclasses.field(default_factory=list) + + python = "python3" + python_version = ">=3.9,<3.11" + + @property + def venv_dir(self): + return self.repository_dir / ".venv" + + @property + def pdm_dir(self): + return self.repository_dir / ".nrp/venv-pdm" + + @property + def ui_dir(self): + return self.repository_dir / self.repository.ui_package + + @property + def shared_dir(self): + return self.repository_dir / self.repository.shared_package + + @property + def models_dir(self): + return self.repository_dir / "models" + + @property + def invenio_instance_path(self): + return self.venv_dir / "var" / "instance" + + @property + def invenio_command(self): + return self.venv_dir / "bin" / "invenio" + + @property + def theme_dir_name(self): + return "semantic-ui" + + def add_model(self, model: ModelConfig): + self.models.append(model) + + def get_model(self, model_name: str, default=UNKNOWN) -> ModelConfig: + for model in self.models: + if model.model_name == model_name: + return model + if default is not UNKNOWN: + return default + known_models = ", ".join(sorted([model.model_name for model in self.models])) + raise KeyError( + f"Model {model_name} not found. Known models are: {known_models}" + ) + + def add_ui(self, ui: UIConfig): + self.uis.append(ui) + + def get_ui(self, ui_name: str, default=UNKNOWN) -> UIConfig: + for ui in self.uis: + if ui.name == ui_name: + return ui + if default is not UNKNOWN: + return default + known_uis = ", ".join(sorted([ui.name for ui in self.uis])) + raise KeyError(f"UI {ui_name} not found. Known UIs are: {known_uis}") + + @property + def config_file(self): + return self.repository_dir / "oarepo.yaml" + + def load(self, extra_config=None): + if extra_config: + config_file = extra_config + else: + config_file = self.config_file + if not config_file.exists(): + return + + with open(config_file) as f: + config_data = yaml.safe_load(f) + + loaded = dacite.from_dict( + type(self), + {"repository_dir": self.repository_dir, **config_data}, + serialization_config, + ) + + self.models = loaded.models + self.uis = loaded.uis + self.repository = loaded.repository + + def save(self): + if self.config_file.exists(): + previous_config_data = self.config_file.read_text().strip() + else: + previous_config_data = None + + io = StringIO() + dict_data = dataclasses.asdict(self) + dict_data.pop("repository_dir") + yaml.safe_dump(dict_data, io) + current_data = io.getvalue().strip() + + if previous_config_data != current_data: + self.config_file.write_text(current_data) diff --git a/src/nrp_devtools/config/enums.py b/src/nrp_devtools/config/enums.py new file mode 100644 index 0000000..5599079 --- /dev/null +++ b/src/nrp_devtools/config/enums.py @@ -0,0 +1,19 @@ +from enum import Enum + + +def enum(value, *, description): + return (value, description) + + +def enum_with_descriptions(clz) -> Enum: + clz_name = clz.__name__ + descriptions = {} + new_class_members = {} + for name, value in clz.__dict__.items(): + if isinstance(value, tuple): + new_class_members[name] = value[0] + descriptions[name] = value[1] + ret = Enum(clz_name, new_class_members) + for name, desc in descriptions.items(): + getattr(ret, name).description = desc + return ret diff --git a/src/nrp_devtools/config/model_config.py b/src/nrp_devtools/config/model_config.py new file mode 100644 index 0000000..0546aa6 --- /dev/null +++ b/src/nrp_devtools/config/model_config.py @@ -0,0 +1,140 @@ +import dataclasses +import re +from typing import Set + +from caseconverter import kebabcase, snakecase + +from .enums import enum, enum_with_descriptions + + +@enum_with_descriptions +class ModelFeature: + """ + Features of the repository model + """ + + files = enum("files", description="Files are used in the model") + + drafts = enum("drafts", description="Use drafts and published records") + + nr_vocabularies = enum( + "nr_vocabularies", + description="Use vocabularies from the Czech documents/data repository", + ) + + vocabularies = enum("vocabularies", description="Use plain invenio vocabularies") + + relations = enum("relations", description="Use relations to other models") + + tests = enum("tests", description="Generate basic tests for the model") + + custom_fields = enum( + "custom_fields", description="Make the model extendable via custom fields" + ) + + requests = enum( + "requests", description="Use requests for approving drafts and other actions" + ) + + communities = enum( + "communities", + description="Use communities for access control and other actions", + ) + + multilingual = enum( + "multilingual", + description="Use multilingual fields", + ) + + +@enum_with_descriptions +class BaseModel: + """ + Base model to use for the repository. + """ + + empty = enum( + "empty", description="Empty model, you will need to add all properties manually" + ) + + documents = enum( + "documents", description="Model suitable for documents (such as articles, ...)" + ) + + data = enum( + "data", description="Model suitable for data (such as datasets, images, ...)" + ) + + +@dataclasses.dataclass +class ModelConfig: + prompts = {} + options = {} + + base_model: BaseModel + prompts["base_model"] = "Base model to use for the repository" + + model_name: str + prompts["model_name"] = "Name of the model" + + model_description: str + prompts["model_description"] = "Description of the model" + + model_package: str + prompts["model_package"] = "Python package name of the model" + + api_prefix: str + prompts["api_prefix"] = "API prefix for the model (will be api/)" + + pid_type: str + prompts["pid_type"] = "PID type of the model. Must be up to 6 characters." + + features: Set[ModelFeature] + prompts["features"] = "Model features" + + @classmethod + def default_model_package(cls, config, values): + return snakecase(values["model_name"]) + + @classmethod + def default_api_prefix(cls, config, values): + return kebabcase(values["model_package"]) + + @classmethod + def default_pid_type(cls, config, values): + pid_base = values["model_package"] + pid_base = re.sub(r"[\s_-]", "", pid_base).lower() + if len(pid_base) > 6: + pid_base = re.sub(r"[AEIOU]", "", pid_base, flags=re.IGNORECASE) + if len(pid_base) > 6: + pid_base = pid_base[:3] + pid_base[len(pid_base) - 3 :] + return pid_base + + @classmethod + def default_features(cls, config, values): + return { + ModelFeature.nr_vocabularies, + ModelFeature.tests, + ModelFeature.custom_fields, + ModelFeature.requests, + ModelFeature.files, + ModelFeature.drafts, + } + + @property + def model_config_file(self): + return f"{self.model_name}.yaml" + + def after_user_input(self): + if self.base_model == BaseModel.documents or self.base_model == BaseModel.data: + self.features.add(ModelFeature.nr_vocabularies) + + if ModelFeature.nr_vocabularies in self.features: + self.features.add(ModelFeature.vocabularies) + + if ModelFeature.vocabularies in self.features: + self.features.add(ModelFeature.relations) + + if self.base_model == BaseModel.documents or self.base_model == BaseModel.data: + self.features.update(ModelFeature) + self.features.add(ModelFeature.multilingual) diff --git a/src/nrp_devtools/config/repository_config.py b/src/nrp_devtools/config/repository_config.py new file mode 100644 index 0000000..75d7ddb --- /dev/null +++ b/src/nrp_devtools/config/repository_config.py @@ -0,0 +1,53 @@ +import dataclasses +import typing +from pathlib import Path + +from caseconverter import snakecase + +if typing.TYPE_CHECKING: + from .config import OARepoConfig + + +@dataclasses.dataclass +class RepositoryConfig: + """Configuration of the repository""" + + prompts = {} + repository_human_name: str + prompts["repository_human_name"] = "Human name of the repository" + + repository_package: str + prompts["repository_package"] = "Python package name of the whole repository" + + oarepo_version: int + prompts["oarepo_version"] = "OARepo version to use" + + shared_package: str + prompts["shared_package"] = "Python package name of the shared code" + + ui_package: str + prompts["ui_package"] = "Python package name of the ui code" + + @classmethod + def default_shared_package(cls, config: "OARepoConfig", values): + """Returns the default site package name based on the project directory name""" + return f"shared" + + @classmethod + def default_ui_package(cls, config: "OARepoConfig", values): + """Returns the default site package name based on the project directory name""" + return f"ui" + + @classmethod + def default_repository_package(cls, config, values): + """Returns the default repository package name based on the project directory name""" + return snakecase(Path(config.repository_dir).name) + + @classmethod + def default_repository_human_name(cls, config, values): + """Returns the default repository package name based on the project directory name""" + return snakecase(Path(config.repository_dir).name).replace("_", " ").title() + + @classmethod + def default_oarepo_version(cls, config, values): + return 12 diff --git a/src/nrp_devtools/config/ui_config.py b/src/nrp_devtools/config/ui_config.py new file mode 100644 index 0000000..2a9a4dc --- /dev/null +++ b/src/nrp_devtools/config/ui_config.py @@ -0,0 +1,13 @@ +import dataclasses +from typing import Optional + + +@dataclasses.dataclass +class UIConfig: + name: str + endpoint: str + + # for model UIs + model: Optional[str] = None + api_service: Optional[str] = None + ui_serializer_class: Optional[str] = None diff --git a/src/nrp_devtools/config/wizard.py b/src/nrp_devtools/config/wizard.py new file mode 100644 index 0000000..24d6877 --- /dev/null +++ b/src/nrp_devtools/config/wizard.py @@ -0,0 +1,107 @@ +import dataclasses +import typing +from enum import Enum + +import click +from dacite import from_dict + +from nrp_devtools.config import OARepoConfig +from nrp_devtools.config.config import serialization_config + + +def ask_for_configuration(config: OARepoConfig, config_class, initial_values=None): + """ + Ask for configuration interactively. + """ + values = initial_values or {} + prompts = config_class.prompts + for field in dataclasses.fields(config_class): + if field.name not in prompts: + continue + + if field.name in values: + value = values[field.name] + elif hasattr(config_class, "default_" + field.name): + value = getattr(config_class, "default_" + field.name)(config, values) + elif not isinstance(field.default, dataclasses._MISSING_TYPE): + value = field.default or "" + else: + value = "" + + if typing.get_origin(field.type) == set: + # set of enums + value = prompt_set_choices( + typing.get_args(field.type)[0], prompts[field.name], default=value + ) + elif issubclass(field.type, Enum): + value = prompt_choices(field.type, prompts[field.name], default=value) + else: + value = click.prompt(prompts[field.name], default=value) + + values[field.name] = value + + ret = from_dict(config_class, values, serialization_config) + + if hasattr(ret, "after_user_input"): + ret.after_user_input() + + return ret + + +def prompt_choices(enum, prompt, default=None): + """ + Prompt user to choose from enum values. + """ + print(prompt) + + choices = {str(idx + 1): e for idx, e in enumerate(enum)} + + for idx, c in choices.items(): + print(f"{idx:5s} {c.description}") + + if default: + for idx, c in choices.items(): + if c.value == default: + default = idx + break + else: + default = None + + return choices[ + click.prompt( + "Your choice", + type=click.Choice(choices.keys()), + default=default, + ) + ].value + + +def prompt_set_choices(enum, prompt, default=None): + """ + Prompt user to choose from enum values. + """ + print(prompt) + + choices = {str(idx + 1): e for idx, e in enumerate(enum)} + + value = set(default or {}) + + while True: + for idx, c in choices.items(): + if c in value: + tick = "[x]" + else: + tick = "[ ]" + print(f"{idx:5s} {tick} {c.description}") + + inp = click.prompt( + "Enter number to toggle, c to continue", + type=click.Choice([*choices.keys(), ""]), + default="", + ) + if inp == "": + break + else: + value ^= {choices[inp].value} + + return value diff --git a/src/nrp_devtools/main.py b/src/nrp_devtools/main.py new file mode 100644 index 0000000..7c87190 --- /dev/null +++ b/src/nrp_devtools/main.py @@ -0,0 +1,4 @@ +from .cli import nrp_command + +if __name__ == "__main__": + nrp_command() diff --git a/src/nrp_devtools/templates/repository/cookiecutter.json b/src/nrp_devtools/templates/repository/cookiecutter.json new file mode 100644 index 0000000..edc2e43 --- /dev/null +++ b/src/nrp_devtools/templates/repository/cookiecutter.json @@ -0,0 +1,10 @@ +{ + "repository_name": "", + "repository_package": "", + "repository_human_name": "", + "shared_package": "", + "use_oarepo_vocabularies": true, + "site_package": "", + "ui_package": "", + "oarepo_version": "12" +} diff --git a/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/.dockerignore b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/.dockerignore new file mode 100644 index 0000000..a3d3e5a --- /dev/null +++ b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/.dockerignore @@ -0,0 +1,4 @@ +**/.venv* +**/__pycache__ +**/__pypackages__ +**/node_modules \ No newline at end of file diff --git a/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/.gitignore b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/.gitignore new file mode 100644 index 0000000..7ee2391 --- /dev/null +++ b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/.gitignore @@ -0,0 +1,5 @@ +.venv* +__pycache__ +__pypackages__ +node_modules +*.egg-info \ No newline at end of file diff --git a/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/README.md b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/README.md new file mode 100644 index 0000000..3eb58b3 --- /dev/null +++ b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/README.md @@ -0,0 +1,211 @@ +# {{cookiecutter.repository_human_name}} + + +## Repository layout + +The repository contains the following files and directories: + +- `oarepo.yaml` - the main configuration file for the repository +- `pyproject.toml` - python dependencies and plugins +- `site` - directory with the site sources, including styles, images, fonts, and docker files +- `ui` - directory containing the UI sources, such as title page, search page, record detail page, etc. + - `ui/branding` - branding information, including colors, logo, favicon etc. +- `models` - directory containing the metadata schemas +- `tests` - directory containing tests for the repository +- `shared` - directory with shared code, local implementation etc. +- `nrp` - the nrp command line tool + +The following files/directories are generated automatically +and should not be modified: + +- `` - one or more directories containing generated code for the models +- `.venv` - virtual environment for the repository +- `.venv-*` - additional virtual environments for tools + +## Basic commands + +### Checking requirements + +To check that the requirements are met, type: + +```bash +nrp check +``` + +This will check that all the requirements are met +and the repository can be run. If there are any errors, +they will be reported and the command will exit with +a non-zero exit code. + +To fix the problems, run the command with '--fix' option: + +```bash +nrp check --fix +``` + +### Running the repository in development mode + +To run the repository in development mode, type: + +```bash +nrp develop --extra-library +``` + +This will check the prerequisites, start the docker containers, +install the python dependencies, compile UI and start the development +server. The UI will be available at https://127.0.0.1:5000, the API +at https://127.0.0.1:5000/api + +If `extra-library` parameter is given, this library will be installed +in an editable mode to the repository's virtual environment. You can +repeat this parameter multiple times to install multiple libraries. + +Removal of extra libraries can be done by: +* calling `nrp build` or `nrp upgrade` commands +* removing the `.venv` directory and calling `nrp develop` again + +After the first run of `nrp develop`, you can speed up the subsequent +runs by adding `--skip-checks` commandline option. + +### Building the repository for production + +```bash +nrp build +``` + +This will build the repository for production. It will check that +the python dependencies are up to date (to skip the check, run +`nrp build --skip-checks`). It will also clear the virtual environment +and reinstall all the dependencies before building the repository. + + +### Running the repository in production mode + +To run the repository in production mode, type: + +```bash +nrp run +``` + +This will just run the repository, depending on it having been built +beforehand. If the repository has not been built, it will fail. + +In production mode, python/js sources are not watched for changes, +and the UI is build beforehand with minification and optimizations. + +### Creating production images + +To create a production image, type: + +```bash +nrp image +``` + +This will create a production image with the given name and tags. +The production image will be based on the `oarepo:oarepo-base-production:`. +The image will be tagged with the given tags and also with the +`:latest` tag. + +This steps expects that the repository has been built beforehand. +If not it will fail. + +Note: the image will not be pushed to the registry. To push the image +to the registry, use the `docker push` command. + +### Testing the repository + +To run test scenarios (integration API tests and UI tests), type: + +```bash +nrp test +``` + +This command will create new containers, run the API tests and UI tests +within the docker then destroy the database. If any of the tests fail, +it will report the failure and exit with a non-zero exit code. + +The command expects the repository to be built beforehand. If not, it +will fail. + +### Upgrading dependencies of the repository + +Run the following command to upgrade the dependencies of the repository: + +```bash +nrp upgrade +``` + +This will upgrade the dependencies of the repository to the latest +versions (python and node dependencies). After this it will run the +build via `nrp build --production` and `nrp test` to make sure that +the dependencies will build. + +## Handling models + +### Creating new models + +To create a new model, type: + +```bash +nrp model create +``` + +The command will ask a couple of questions and will create +`.yaml` file in the `models` directory. +Please edit the file to add the fields and other information +about the model. + +### Compiling and installing the model + +To compile the model, type: + +```bash +nrp model compile +``` + +This will compile the model and generate python code for it. +The generated sources and entrypoints are placed in the +`` directory and to `pyproject.toml` file. + +Alembic migrations will be generated (this requires that the containers +are running - run `nrp develop` or `nrp check` before running this command). + +After the model is compiled, run `nrp develop` and check that the +model is working correctly under the `/api` endpoint. + +## Handling UI + +### Creating UI pages for models + +To create UI pages for a model, type: + +```bash +nrp ui model create --model +``` + +The `ui-name` is optional, if not specified, it will be the same +as the `model-name`. The command will ask a couple of questions +and will create jinjax templates and react pages for displaying +a listing of the model, a detail page and a form for creating +and editing the model. + +### Creating UI pages for custom endpoints + +To create UI pages for a custom endpoint, type: + +```bash +nrp ui page create +``` + +The `page-endpoint` is the endpoint of the page, for example +`/about` or `/search`. The `page-name` is the name of the page, +for example `about` or `search`. + +If `page-endpoint` is not specified, it will be the same as +`page-name`. + +The command will create a jinjax template for the page and register +the page to the flask application. + +If you run the command with `--react` option, it will also create +react endpoint for the page and reference it from the jinjax template. \ No newline at end of file diff --git a/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/docker/Dockerfile.production b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/docker/Dockerfile.production new file mode 100644 index 0000000..b642ddd --- /dev/null +++ b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/docker/Dockerfile.production @@ -0,0 +1,62 @@ +ARG OAREPO_DEVELOPMENT_IMAGE=oarepo/oarepo-base-development:{{cookiecutter.oarepo_version}} +ARG OAREPO_PRODUCTION_IMAGE=oarepo/oarepo-base-production:{{cookiecutter.oarepo_version}} +ARG BUILDPLATFORM=linux/amd64 + +FROM --platform=$BUILDPLATFORM $OAREPO_DEVELOPMENT_IMAGE as production-build + +ARG REPOSITORY_SITE_NAME +ARG SITE_DIR=/repository/sites/${REPOSITORY_SITE_NAME} + +ENV SITE_DIR=$SITE_DIR +ENV REPOSITORY_SITE_NAME=${REPOSITORY_SITE_NAME} +ENV PATH=/pdm/bin:${PATH} + +RUN echo "Site dir is ${SITE_DIR}, repository site name is ${REPOSITORY_SITE_NAME}" + +COPY . /repository + +# DEBUG only +#COPY nrp-sources /nrp-sources +#RUN PYTHONPATH=/nrp-sources DOCKER_AROUND=1 /nrp/bin/nrp build --production --site $REPOSITORY_SITE_NAME --project-dir /repository + +# production +RUN DOCKER_AROUND=1 /nrp/bin/nrp build --production --site $REPOSITORY_SITE_NAME --project-dir /repository + + +FROM --platform=$BUILDPLATFORM ${OAREPO_PRODUCTION_IMAGE} as production + +ARG REPOSITORY_SITE_ORGANIZATION +ARG REPOSITORY_SITE_NAME +ARG REPOSITORY_IMAGE_URL +ARG REPOSITORY_AUTHOR +ARG REPOSITORY_GITHUB_URL +ARG REPOSITORY_URL +ARG REPOSITORY_DOCUMENTATION + +ARG SITE_DIR=/repository/sites/${REPOSITORY_SITE_NAME} + +LABEL maintainer="${REPOSITORY_SITE_ORGANIZATION}" \ + org.opencontainers.image.authors="${REPOSITORY_AUTHOR}" \ + org.opencontainers.image.title="MBDB production image for ${REPOSITORY_SITE_NAME}" \ + org.opencontainers.image.url="${REPOSITORY_IMAGE_URL}" \ + org.opencontainers.image.source="${REPOSITORY_GITHUB_URL}" \ + org.opencontainers.image.documentation="${REPOSITORY_DOCUMENTATION}" + + +# copy build +RUN mkdir -p /invenio/instance +COPY --from=production-build /invenio/instance/invenio.cfg /invenio/instance/invenio.cfg +COPY --from=production-build /invenio/instance/variables /invenio/instance/variables +COPY --from=production-build /invenio/instance/static /invenio/instance/static +COPY --from=production-build /invenio/venv /invenio/venv + +# copy sources (just for sure, should not be needed for production run) +COPY --from=production-build /repository /repository + +# copy uwsgi.ini - keep the path the same as in invenio +RUN mkdir -p /opt/invenio/src/uwsgi/ +COPY sites/${REPOSITORY_SITE_NAME}/docker/uwsgi/uwsgi.ini /opt/invenio/src/uwsgi/uwsgi.ini + +ENV PATH=${INVENIO_VENV}/bin:${PATH} + +ENTRYPOINT [ "sh", "-c" ] diff --git a/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/docker/docker-compose.yml b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/docker/docker-compose.yml new file mode 100644 index 0000000..35385d5 --- /dev/null +++ b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/docker/docker-compose.yml @@ -0,0 +1,65 @@ +version: "3.5" +name: "{{cookiecutter.repository_name}}" +services: + cache: + image: redis:7 + restart: "unless-stopped" + read_only: true + ports: + - "${INVENIO_REDIS_HOST}:${INVENIO_REDIS_PORT}:6379" + db: + image: postgres:15-alpine + restart: "unless-stopped" + environment: + - "POSTGRES_USER=${INVENIO_DATABASE_USER}" + - "POSTGRES_PASSWORD=${INVENIO_DATABASE_PASSWORD}" + - "POSTGRES_DB=${INVENIO_DATABASE_DBNAME}" + ports: + - "${INVENIO_DATABASE_HOST}:${INVENIO_DATABASE_PORT}:5432" + mq: + image: rabbitmq:3-management + restart: "unless-stopped" + environment: + RABBITMQ_DEFAULT_USER: "${INVENIO_RABBIT_USER}" + RABBITMQ_DEFAULT_PASS: "${INVENIO_RABBIT_PASSWORD}" + ports: + - "${INVENIO_RABBIT_HOST}:${INVENIO_RABBIT_ADMIN_PORT}:15672" + - "${INVENIO_RABBIT_HOST}:${INVENIO_RABBIT_PORT}:5672" + + search: + image: bitnami/opensearch:2 + restart: "unless-stopped" + environment: + # settings only for development. DO NOT use in production! + - bootstrap.memory_lock=true + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" + - "DISABLE_INSTALL_DEMO_CONFIG=true" + - "DISABLE_SECURITY_PLUGIN=true" + - "discovery.type=single-node" + - OPENSEARCH_PLUGINS=analysis-icu + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + mem_limit: 2g + ports: + - "${INVENIO_OPENSEARCH_HOST}:${INVENIO_OPENSEARCH_PORT}:9200" + - "${INVENIO_OPENSEARCH_HOST}:${INVENIO_OPENSEARCH_CLUSTER_PORT}:9600" + s3: + image: minio/minio:latest + restart: "unless-stopped" + environment: + MINIO_ROOT_USER: ${INVENIO_S3_ACCESS_KEY} + MINIO_ROOT_PASSWORD: ${INVENIO_S3_SECRET_KEY} + command: server /data --console-address :9001 + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 20s + retries: 3 + ports: + - "${INVENIO_S3_HOST}:${INVENIO_S3_PORT}:9000" + - "${INVENIO_S3_HOST}:${INVENIO_S3_PORT1}:9001" diff --git a/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/invenio.cfg b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/invenio.cfg new file mode 100644 index 0000000..a6f1c0b --- /dev/null +++ b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/invenio.cfg @@ -0,0 +1,303 @@ +from datetime import datetime +from flask_babelex import lazy_gettext as _ +from flask.config import Config +import os +from dotenv import dotenv_values +import json + +# Import the configuration from the local .env if it exists +# and overwrite it with environment variables +# Loading it this way so could interpolate values +def transform_value(x): + if not isinstance(x, str): + return x + if x == 'False': + return False + if x == 'True': + return False + try: + return json.loads(x) + except: + return x + +env = Config(os.path.dirname(__file__)) +bundled_env = os.path.join(os.path.dirname(__file__), 'variables') +if os.path.exists(bundled_env): + vals = dotenv_values(bundled_env) + env.from_mapping(vals) +if os.path.exists('.env'): + vals = dotenv_values('.env') + env.from_mapping(vals) +env.from_mapping({ + k:v for k, v in os.environ.items() if k.startswith('INVENIO_') +}) + +for k, v in env.items(): + setattr(env, k, transform_value(v)) + +def _(x): # needed to avoid start time failure with lazy strings + return x + +# Flask +# ===== +# See https://flask.palletsprojects.com/en/1.1.x/config/ + +# Define the value of the cache control header `max-age` returned by the server when serving +# public files. Files will be cached by the browser for the provided number of seconds. +# See flask documentation for more information: +# https://flask.palletsprojects.com/en/2.1.x/config/#SEND_FILE_MAX_AGE_DEFAULT +SEND_FILE_MAX_AGE_DEFAULT = 300 + + +# Since HAProxy and Nginx route all requests no matter the host header +# provided, the allowed hosts variable is set to localhost. In production it +# should be set to the correct host and it is strongly recommended to only +# route correct hosts to the application. +APP_ALLOWED_HOSTS = ['0.0.0.0', 'localhost', '127.0.0.1'] + + +# Flask-SQLAlchemy +# ================ +# See https://flask-sqlalchemy.palletsprojects.com/en/2.x/config/ + +SQLALCHEMY_DATABASE_URI=( + "postgresql+psycopg2://" + f"{env.INVENIO_DATABASE_USER}:{env.INVENIO_DATABASE_PASSWORD}" + f"@{env.INVENIO_DATABASE_HOST}:{env.INVENIO_DATABASE_PORT}" + f"/{env.INVENIO_DATABASE_DBNAME}" +) + + +# Invenio-App +# =========== +# See https://invenio-app.readthedocs.io/en/latest/configuration.html + +APP_DEFAULT_SECURE_HEADERS = { + 'content_security_policy': { + 'default-src': [ + "'self'", + 'data:', # for fonts + "'unsafe-inline'", # for inline scripts and styles + "blob:", # for pdf preview + # Add your own policies here (e.g. analytics) + ], + }, + 'content_security_policy_report_only': False, + 'content_security_policy_report_uri': None, + 'force_file_save': False, + 'force_https': True, + 'force_https_permanent': False, + 'frame_options': 'sameorigin', + 'frame_options_allow_from': None, + 'session_cookie_http_only': True, + 'session_cookie_secure': True, + 'strict_transport_security': True, + 'strict_transport_security_include_subdomains': True, + 'strict_transport_security_max_age': 31556926, # One year in seconds + 'strict_transport_security_preload': False, +} + + +# Flask-Babel +# =========== +# See https://python-babel.github.io/flask-babel/#configuration + +# Default locale (language) +BABEL_DEFAULT_LOCALE = 'en' +# Default time zone +BABEL_DEFAULT_TIMEZONE = 'Europe/Prague' + + +# Invenio-I18N +# ============ +# See https://invenio-i18n.readthedocs.io/en/latest/configuration.html + +# Other supported languages (do not include BABEL_DEFAULT_LOCALE in list). +I18N_LANGUAGES = [ + # ('de', _('German')), + # ('tr', _('Turkish')), +] + + +# Invenio-Theme +# ============= +# See https://invenio-theme.readthedocs.io/en/latest/configuration.html + +APP_THEME = ["semantic-ui"] + +INSTANCE_THEME_FILE = './less/theme.less' + + +# Invenio-Files-Rest +# ================== +FILES_REST_STORAGE_FACTORY='invenio_s3.s3fs_storage_factory' + + +# Invenio-S3 +# ========== +S3_ENDPOINT_URL=f"{env.INVENIO_S3_PROTOCOL}://{env.INVENIO_S3_HOST}:{env.INVENIO_S3_PORT}/" +S3_ACCESS_KEY_ID=env.INVENIO_S3_ACCESS_KEY +S3_SECRET_ACCESS_KEY=env.INVENIO_S3_SECRET_KEY + +# Allow S3 endpoint in the CSP rules +APP_DEFAULT_SECURE_HEADERS['content_security_policy']['default-src'].append( + S3_ENDPOINT_URL +) + +# Invenio-Records-Resources +# ========================= +# See https://github.com/inveniosoftware/invenio-records-resources/blob/master/invenio_records_resources/config.py + +SITE_UI_URL = f"https://{env.INVENIO_UI_HOST}:{env.INVENIO_UI_PORT}" +SITE_API_URL = f"https://{env.INVENIO_API_HOST}:{env.INVENIO_API_PORT}/api" + +# Authentication - Invenio-Accounts and Invenio-OAuthclient +# ========================================================= +# See: https://inveniordm.docs.cern.ch/customize/authentication/ + +# Invenio-Accounts +# ---------------- +# See https://github.com/inveniosoftware/invenio-accounts/blob/master/invenio_accounts/config.py +ACCOUNTS_LOCAL_LOGIN_ENABLED = env.INVENIO_ACCOUNTS_LOCAL_LOGIN_ENABLED # enable local login +SECURITY_REGISTERABLE = env.INVENIO_SECURITY_REGISTERABLE # local login: allow users to register +SECURITY_RECOVERABLE = env.INVENIO_SECURITY_RECOVERABLE # local login: allow users to reset the password +SECURITY_CHANGEABLE = env.INVENIO_SECURITY_CHANGEABLE # local login: allow users to change psw +SECURITY_CONFIRMABLE = env.INVENIO_SECURITY_CONFIRMABLE # local login: users can confirm e-mail address +SECURITY_LOGIN_WITHOUT_CONFIRMATION = env.INVENIO_SECURITY_LOGIN_WITHOUT_CONFIRMATION # require users to confirm email before being able to login + +# Invenio-OAuthclient +# ------------------- +# See https://github.com/inveniosoftware/invenio-oauthclient/blob/master/invenio_oauthclient/config.py + +OAUTHCLIENT_REMOTE_APPS = {} # configure external login providers + +from invenio_oauthclient.views.client import auto_redirect_login +ACCOUNTS_LOGIN_VIEW_FUNCTION = auto_redirect_login # autoredirect to external login if enabled +OAUTHCLIENT_AUTO_REDIRECT_TO_EXTERNAL_LOGIN = True # autoredirect to external login + +# Invenio-UserProfiles +# -------------------- +USERPROFILES_READ_ONLY = False # allow users to change profile info (name, email, etc...) + +# OAI-PMH +# ======= +# See https://github.com/inveniosoftware/invenio-oaiserver/blob/master/invenio_oaiserver/config.py + +OAISERVER_ID_PREFIX = SITE_UI_URL +"""The prefix that will be applied to the generated OAI-PMH ids.""" + +# Invenio-Search +# -------------- + +SEARCH_INDEX_PREFIX = env.INVENIO_SEARCH_INDEX_PREFIX + +SEARCH_HOSTS = [ + dict(host=env.INVENIO_OPENSEARCH_HOST, + port=env.INVENIO_OPENSEARCH_PORT), +] + +SEARCH_CLIENT_CONFIG = dict( + use_ssl = env.INVENIO_OPENSEARCH_USE_SSL, + verify_certs = env.INVENIO_OPENSEARCH_VERIFY_CERTS, + ssl_assert_hostname = env.INVENIO_OPENSEARCH_ASSERT_HOSTNAME, + ssl_show_warn = env.INVENIO_OPENSEARCH_SHOW_WARN, + ca_certs = env.get('INVENIO_OPENSEARCH_CA_CERTS_PATH', None) +) + +# Cache +# -------------- +INVENIO_CACHE_TYPE="redis" + +CACHE_REDIS_URL = ( + f'redis://{env.INVENIO_REDIS_HOST}:{env.INVENIO_REDIS_PORT}' + f'/{env.INVENIO_REDIS_CACHE_DB}' +) +ACCOUNTS_SESSION_REDIS_URL = ( + f'redis://{env.INVENIO_REDIS_HOST}:{env.INVENIO_REDIS_PORT}' + f'/{env.INVENIO_REDIS_SESSION_DB}' +) + + +# Local schema +# -------------- +RECORDS_REFRESOLVER_CLS = 'invenio_records.resolver.InvenioRefResolver' +RECORDS_REFRESOLVER_STORE = "invenio_jsonschemas.proxies.current_refresolver_store" +JSONSCHEMAS_HOST = SITE_UI_URL + + +{% if cookiecutter.use_oarepo_vocabularies %} + +# Extended vocabularies +# --------------------- + +from oarepo_vocabularies.services.config import VocabulariesConfig +from oarepo_vocabularies.resources.config import VocabulariesResourceConfig + +VOCABULARIES_SERVICE_CONFIG = VocabulariesConfig +VOCABULARIES_RESOURCE_CONFIG = VocabulariesResourceConfig + +{% endif %} + +# Files storage location +# --------------- +FILES_REST_STORAGE_CLASS_LIST = { + "L": "Local", + "F": "Fetch", + "R": "Remote", + } +FILES_REST_DEFAULT_STORAGE_CLASS = "L" + +# Redis port redirection +# --------------------- +CELERY_BROKER_URL = ( + f"amqp://{env.INVENIO_RABBIT_USER}:{env.INVENIO_RABBIT_PASSWORD}" + f"@{env.INVENIO_RABBIT_HOST}:{env.INVENIO_RABBIT_PORT}/" +) +BROKER_URL = CELERY_BROKER_URL +CELERY_RESULT_BACKEND = ( + f'redis://{env.INVENIO_REDIS_HOST}:{env.INVENIO_REDIS_PORT}' + f'/{env.INVENIO_REDIS_CELERY_RESULT_DB}' +) + +# Instance secret key, used to encrypt stuff (for example, access tokens) inside database +SECRET_KEY = env.INVENIO_SECRET_KEY + + +# Invenio hacks +# ------------- + +# Invenio has problems with order of loading templates. If invenio-userprofiles is loaded +# before invenio-theme, the userprofile page will not work because base settings page +# will be taken from userprofiles/semantic-ui/userprofiles/settings/base.html which is faulty. +# If invenio-theme is loaded first, SETTINGS_TEMPLATE is filled, then userprofiles will use +# it and the UI loads correctly. +# +# This line just makes sure that SETTINGS_TEMPLATE is always set up. +SETTINGS_TEMPLATE='invenio_theme/page_settings.html' + +# UI +# --- + +THEME_HEADER_TEMPLATE = "oarepo_ui/header.html" +THEME_FOOTER_TEMPLATE = "oarepo_ui/footer.html" +THEME_JAVASCRIPT_TEMPLATE = "oarepo_ui/javascript.html" +THEME_TRACKINGCODE_TEMPLATE = "oarepo_ui/trackingcode.html" + + +# remove when you create your own title page +THEME_FRONTPAGE = False + +# Header logo +THEME_LOGO = 'images/invenio-rdm.svg' + +THEME_SITENAME = _("{{ cookiecutter.repository_human_name }}") +THEME_FRONTPAGE_TITLE = "{{ cookiecutter.repository_human_name }}" +THEME_FRONTPAGE_TEMPLATE = "frontpage.html" +THEME_FRONTPAGE_LOGO = "images/repo_logo_eng_rgb.png" + + +# We set this to avoid bug: https://github.com/inveniosoftware/invenio-administration/issues/180 +THEME_HEADER_LOGIN_TEMPLATE = "header_login.html" + +RATELIMIT_GUEST_USER = "5000 per hour;500 per minute" +RATELIMIT_AUTHENTICATED_USER = "20000 per hour;2000 per minute" diff --git a/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/models/README.md b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/models/README.md new file mode 100644 index 0000000..cb09ce9 --- /dev/null +++ b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/models/README.md @@ -0,0 +1,50 @@ +# `models` folder + +Contains generated models. + +## Creating models + +```bash +nrp model add --config \ + --use [:] \ + --no-input +``` + +Will create a new model. You can provide your own oarepo.yaml +config for the model via the --config option (to get the format, +run the command without --config, answer all the questions +and then copy the model part of the oarepo.yaml to your own file) + +You can also include a custom model. The file will be copied +to the destination and referenced from the generated model file. +If no path is specified, it will be referenced from the root +of the file, with path the reference will be put there. + +Use `--no-input` to disable asking questions (and be sure to +run it with `--config`) + +## Compiling models + +```bash +nrp model compile +``` + +This command will compile your model into invenio sources + +## Installing models + +```bash +nrp model install [] +``` + +Will install the model into the given site. Site name +can be omitted if there is only one site in the monorepo. + +## Uninstalling models + +```bash +nrp model uninstall [] [--remove-directory] +``` + +Will uninstall the model from the given site. Site name +can be omitted if there is only one site in the monorepo. diff --git a/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/nrp b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/nrp new file mode 100755 index 0000000..a332069 --- /dev/null +++ b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/nrp @@ -0,0 +1,80 @@ +#!/bin/bash + +set -e + +# This script runs the NRP repository tools + +# Environment variables +# PYTHON: +# python executable to use for running the NRP tools +# LOCAL_NRP_TOOLS_LOCATION: +# location of the local NRP repository. +# If set, do not clone the NRP repository but use the local one. + + +# If there is a local environment file, source it. This is necessary on Mac OS X +# to set the correct environment variables for the python executable. +# An example file on mac os x is: +# +# ❯ cat ~/.envrc.local +# +# # dynamic libraries (such as cairo) +# export DYLD_FALLBACK_LIBRARY_PATH=/opt/homebrew/lib +# +if [ -f ~/.envrc.local ] ; then + source ~/.envrc.local +fi + + +SUPPORTED_PYTHON_VERSIONS=(3.10) + +if [ -z "$PYTHON" ] ; then + + # find a supported python + for version in "${SUPPORTED_PYTHON_VERSIONS[@]}"; do + if command -v python$version >/dev/null 2>&1; then + PYTHON=python$version + break + fi + done + + if [ -z "$PYTHON" ] ; then + echo "No supported python version found. Please install python 3.9 or higher + or set the PYTHON environment variable to the python executable." + exit 1 + fi +fi + +venv_directory=$(dirname "$0")/.venv + +if [ ! -d "$venv_directory" ] ; then + mkdir "$venv_directory" +fi + +devtools_cli_directory=$(dirname "$0")/.nrp/devtools + +# if there is a devtools directory and can not call nrp-devtools inside it, +# remove the directory +if [ -d "$devtools_cli_directory" ] ; then + if ! "$devtools_cli_directory"/bin/nrp-devtools --help >/dev/null 2>&1 ; then + rm -rf "$devtools_cli_directory" + fi +fi + +if [ ! -d "$devtools_cli_directory" ] ; then + # make parent directory if it does not exist + if [ ! -d "$(dirname "$devtools_cli_directory")" ] ; then + mkdir -p "$(dirname "$devtools_cli_directory")" + fi + $PYTHON -m venv "${devtools_cli_directory}" + "${devtools_cli_directory}"/bin/pip install -U setuptools pip wheel + if [ -z "$LOCAL_NRP_TOOLS_LOCATION" ] ; then + "${devtools_cli_directory}"/bin/pip install nrp-devtools + else + "${devtools_cli_directory}"/bin/pip install -e "$LOCAL_NRP_TOOLS_LOCATION" + fi +fi + +source "$devtools_cli_directory"/bin/activate + +"$devtools_cli_directory"/bin/nrp-devtools "$@" diff --git a/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/pyproject.toml b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/pyproject.toml new file mode 100644 index 0000000..67bfd1b --- /dev/null +++ b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/pyproject.toml @@ -0,0 +1,30 @@ +[project] +name = "repo" +version = "1.0.0" +description = "" +packages = [] +authors = [] +dependencies = [ + "json5", + "oarepo=={{cookiecutter.oarepo_version}}.*", + "oarepo-runtime", + "oarepo-ui", + "python-dotenv", + {% if cookiecutter.use_oarepo_vocabularies -%} + "oarepo-vocabularies", + {%- endif %} +] +requires-python = ">=3.9,<3.11" + + +[project.entry-points."invenio_assets.webpack"] +site = "{{cookiecutter.ui_package}}.branding.webpack:theme" + +[project.entry-points."invenio_base.blueprints"] +titlepage = "{{cookiecutter.ui_package}}.titlepage:create_blueprint" + +[build-system] +requires = [ + "pdm-backend", +] +build-backend = "pdm.backend" \ No newline at end of file diff --git a/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/translations/__init__.py b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/translations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/variables b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/variables new file mode 100644 index 0000000..65b1883 --- /dev/null +++ b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/variables @@ -0,0 +1,79 @@ +INVENIO_JSONSCHEMAS_HOST="" +INVENIO_WSGI_PROXIES=0 + + +INVENIO_UI_HOST=127.0.0.1 +INVENIO_UI_PORT=5000 + +INVENIO_API_HOST=127.0.0.1 +INVENIO_API_PORT=5000 + +INVENIO_REDIS_PORT=6579 +INVENIO_REDIS_HOST=127.0.0.1 +INVENIO_REDIS_CACHE_DB=0 +INVENIO_REDIS_SESSION_DB=1 +INVENIO_REDIS_CELERY_RESULT_DB=2 + + +INVENIO_RABBIT_HOST=127.0.0.1 +INVENIO_RABBIT_PORT=5872 +INVENIO_RABBIT_ADMIN_PORT=15872 +INVENIO_RABBIT_USER="{{cookiecutter.repository_package}}" +INVENIO_RABBIT_PASSWORD="{{cookiecutter.repository_package}}" + +INVENIO_OPENSEARCH_HOST=127.0.0.1 +INVENIO_OPENSEARCH_PORT=9400 +INVENIO_OPENSEARCH_CLUSTER_PORT=9600 +INVENIO_OPENSEARCH_DASHBOARD_HOST=127.0.0.1 +INVENIO_OPENSEARCH_DASHBOARD_PORT=5801 +INVENIO_SEARCH_INDEX_PREFIX="{{cookiecutter.repository_package}}-" +INVENIO_OPENSEARCH_USE_SSL=False +INVENIO_OPENSEARCH_VERIFY_CERTS=False +INVENIO_OPENSEARCH_ASSERT_HOSTNAME=False +INVENIO_OPENSEARCH_SHOW_WARN=False +#INVENIO_OPENSEARCH_CA_CERTS_PATH= + + +INVENIO_SECRET_KEY=changeme + +INVENIO_DATABASE_HOST=127.0.0.1 +INVENIO_DATABASE_PORT=5632 +INVENIO_DATABASE_USER="{{cookiecutter.repository_package}}" +INVENIO_DATABASE_PASSWORD="{{cookiecutter.repository_package}}" +INVENIO_DATABASE_DBNAME="{{cookiecutter.repository_package}}" + +INVENIO_S3_PROTOCOL=http +INVENIO_S3_HOST=127.0.0.1 +INVENIO_S3_PORT=9000 +INVENIO_S3_PORT1=9001 +# must be at least 3 characters +INVENIO_S3_ACCESS_KEY="aa-{{cookiecutter.repository_package}}-aa" + +# must be at least 8 characters +INVENIO_S3_SECRET_KEY="aaa-{{cookiecutter.repository_package}}-aaa" + + +# INVENIO_MAIL_SERVER=postfix-relay.mail.svc.cluster.local +# INVENIO_DOI_DATACITE_PASSWORD= +# INVENIO_DOI_DATACITE_PREFIX= +# INVENIO_DOI_DATACITE_TEST_URL= +# INVENIO_DOI_TEST_MODE= + + +INVENIO_ACCOUNTS_LOCAL_LOGIN_ENABLED = False +INVENIO_SECURITY_REGISTERABLE=False +INVENIO_SECURITY_RECOVERABLE=False +INVENIO_SECURITY_CONFIRMABLE=False +INVENIO_SECURITY_CHANGEABLE=False +INVENIO_SECURITY_LOGIN_WITHOUT_CONFIRMATION=False + +# pro perun +OPENIDC_KEY='' +OPENIDC_SECRET='' + +# default is INVENIO_SERVER_NAME +# INVENIO_JSONSCHEMAS_HOST + + + + diff --git a/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.shared_package}}/README.md b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.shared_package}}/README.md new file mode 100644 index 0000000..cbd4a97 --- /dev/null +++ b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.shared_package}}/README.md @@ -0,0 +1,17 @@ +# `shared` folder + +This folder contains your shared packages - packages that +are used by multiple models/uis in the monorepo. + +Feel free to add dependencies to `project.toml` and +any modules to the `{{cookiecutter.shared_package}}` folder. + +## Usage + +This folder is automatically installed to the running repository +when `nrp build` or `nrp develop` is called. You can then import +the modules in your models/uis as follows: + +```python +from {{cookiecutter.shared_package}} import my_module +``` \ No newline at end of file diff --git a/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.shared_package}}/__init__.py b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.shared_package}}/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.ui_package}}/README.md b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.ui_package}}/README.md new file mode 100644 index 0000000..f4037d0 --- /dev/null +++ b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.ui_package}}/README.md @@ -0,0 +1,33 @@ +# `ui` folder + +Contains user interface for your repository. + +## Creating UI + +```bash +nrp ui add --config \ + --no-input +``` + +Will create a template of a UI. + +Use `--no-input` to disable asking questions (and be sure to +run it with `--config`) + +## Installing UI + +```bash +nrp ui install [] +``` + +Will install the ui into the given site. Site name +can be omitted if there is only one site in the monorepo. + +## Uninstalling UI + +```bash +nrp ui uninstall [] [--remove-directory] +``` + +Will uninstall the ui from the given site. Site name +can be omitted if there is only one site in the monorepo. diff --git a/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.ui_package}}/__init__.py b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.ui_package}}/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.ui_package}}/branding/__init__.py b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.ui_package}}/branding/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.ui_package}}/branding/semantic-ui/less/globals/site.overrides b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.ui_package}}/branding/semantic-ui/less/globals/site.overrides new file mode 100644 index 0000000..e69de29 diff --git a/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.ui_package}}/branding/semantic-ui/less/globals/site.variables b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.ui_package}}/branding/semantic-ui/less/globals/site.variables new file mode 100644 index 0000000..e69de29 diff --git a/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.ui_package}}/branding/semantic-ui/less/theme.config b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.ui_package}}/branding/semantic-ui/less/theme.config new file mode 100644 index 0000000..efd6370 --- /dev/null +++ b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.ui_package}}/branding/semantic-ui/less/theme.config @@ -0,0 +1,107 @@ +/* + +████████╗██╗ ██╗███████╗███╗ ███╗███████╗███████╗ +╚══██╔══╝██║ ██║██╔════╝████╗ ████║██╔════╝██╔════╝ + ██║ ███████║█████╗ ██╔████╔██║█████╗ ███████╗ + ██║ ██╔══██║██╔══╝ ██║╚██╔╝██║██╔══╝ ╚════██║ + ██║ ██║ ██║███████╗██║ ╚═╝ ██║███████╗███████║ + ╚═╝ ╚═╝ ╚═╝╚══════╝╚═╝ ╚═╝╚══════╝╚══════╝ + +*/ + +/******************************* + Theme Selection +*******************************/ + +/* To override a theme for an individual element + specify theme name below +*/ + +/* Global */ +@site : 'default'; +@reset : 'default'; + +/* Elements */ +@button : 'default'; +@container : 'default'; +@divider : 'default'; +@flag : 'default'; +@header : 'default'; +@icon : 'default'; +@image : 'default'; +@input : 'default'; +@label : 'default'; +@list : 'default'; +@loader : 'default'; +@placeholder : 'default'; +@rail : 'default'; +@reveal : 'default'; +@segment : 'default'; +@step : 'default'; + +/* Collections */ +@breadcrumb : 'default'; +@form : 'default'; +@grid : 'default'; +@menu : 'default'; +@message : 'default'; +@table : 'default'; + +/* Modules */ +@accordion : 'default'; +@checkbox : 'default'; +@dimmer : 'default'; +@dropdown : 'default'; +@embed : 'default'; +@modal : 'default'; +@nag : 'default'; +@popup : 'default'; +@progress : 'default'; +@rating : 'default'; +@search : 'default'; +@shape : 'default'; +@sidebar : 'default'; +@sticky : 'default'; +@tab : 'default'; +@transition : 'default'; + +/* Views */ +@ad : 'default'; +@card : 'default'; +@comment : 'default'; +@feed : 'default'; +@item : 'default'; +@statistic : 'default'; + +{% if cookiecutter.use_oarepo_vocabularies == 'yes' %} + +@dl_table: 'default'; + +{% endif %} + +@datepicker : 'default'; + +/* Custom */ +/* @my_custom_component : 'default'; */ + +/******************************* + Folders +*******************************/ + +/* Path to theme packages */ +@themesFolder : '~semantic-ui-less/themes'; + +/* Path to site override folder */ +@siteFolder : '../../less'; +@imagesFolder : '../../images'; + + +/******************************* + Import Theme +*******************************/ + +@import (multiple) "./theme.less"; + +@fontPath : "../../../themes/@{theme}/assets/fonts"; + +/* End Config */ diff --git a/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.ui_package}}/branding/semantic-ui/less/theme.less b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.ui_package}}/branding/semantic-ui/less/theme.less new file mode 100644 index 0000000..6fd200d --- /dev/null +++ b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.ui_package}}/branding/semantic-ui/less/theme.less @@ -0,0 +1,93 @@ +/******************************* + Import Directives +*******************************/ + +/*------------------ + Theme +-------------------*/ + +@theme: @@element; + +/*-------------------- + Site Variables +---------------------*/ + +/* Default site.variables */ +@import '@{themesFolder}/default/globals/site.variables'; + +/* Packaged site.variables */ +@import '@{themesFolder}/@{siteFolder}/globals/site.variables'; + +/* Component's site.variables */ +@import (optional) '@{themesFolder}/@{theme}/globals/site.variables'; + +/* invenio-theme site.variables */ +@import (optional) 'themes/invenio/globals/site.variables'; + +/* oarepo theme global site.variables */ +@import (optional) 'themes/oarepo/globals/site.variables'; + +/* Site theme site.variables */ +@import (optional) '@{siteFolder}/globals/site.variables'; + +/*------------------- + Component Variables +---------------------*/ + +/* Default */ +@import (optional) '@{themesFolder}/default/@{type}s/@{element}.variables'; + +/* Packaged Theme */ +@import (optional, multiple) + '@{themesFolder}/@{theme}/@{type}s/@{element}.variables'; + +/* Your components +@import (optional, multiple) + '/${theme}/@{type}s/@{element}.variables'; + */ + + @import (optional, multiple) '{{cookiecutter.repository_package}}/theme/@{type}s/@{element}.variables'; + +/* Invenio-theme Theme */ +@import (optional, multiple) 'themes/invenio/@{type}s/@{element}.variables'; + +/* oarepo theme */ +@import (optional, multiple) 'themes/oarepo/@{type}s/@{element}.variables'; + +/* Site Theme */ +@import (optional, multiple) '@{siteFolder}/@{type}s/@{element}.variables'; + + +/******************************* + Mix-ins +*******************************/ + +/*------------------ + Fonts +-------------------*/ + +.loadFonts() when (@importGoogleFonts) { + @import (css) + url('@{googleProtocol}fonts.googleapis.com/css?family=@{googleFontRequest}'); +} + +/*------------------ + Overrides +-------------------*/ + +.loadUIOverrides() { + @import (optional, multiple) + '@{themesFolder}/@{theme}/@{type}s/@{element}.overrides'; + +/* Your components + @import (optional, multiple) + '/theme/@{type}s/@{element}.overrides'; + */ + +@import (optional, multiple) 'oarepo_ui/theme/@{type}s/@{element}.overrides'; +@import (optional, multiple) '{{cookiecutter.repository_package}}/theme/@{type}s/@{element}.overrides'; + + @import (optional, multiple) 'themes/invenio/@{type}s/@{element}.overrides'; + @import (optional, multiple) 'themes/oarepo/@{type}s/@{element}.overrides'; + @import (optional, multiple) '@{siteFolder}/@{type}s/@{element}.overrides'; +} diff --git a/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.ui_package}}/branding/semantic-ui/templates/.readme b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.ui_package}}/branding/semantic-ui/templates/.readme new file mode 100644 index 0000000..981e613 --- /dev/null +++ b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.ui_package}}/branding/semantic-ui/templates/.readme @@ -0,0 +1,3 @@ +The @templates alias is used for overwriting JSX templates. +The alias assumes that a folder "templates" exists in the +root of the var/instance/assets (root of webpack project). diff --git a/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.ui_package}}/branding/semantic-ui/templates/custom_fields/.readme b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.ui_package}}/branding/semantic-ui/templates/custom_fields/.readme new file mode 100644 index 0000000..a05bd04 --- /dev/null +++ b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.ui_package}}/branding/semantic-ui/templates/custom_fields/.readme @@ -0,0 +1 @@ +This directory is for community custom fields implementation diff --git a/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.ui_package}}/branding/webpack.py b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.ui_package}}/branding/webpack.py new file mode 100644 index 0000000..ff72803 --- /dev/null +++ b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.ui_package}}/branding/webpack.py @@ -0,0 +1,21 @@ +from invenio_assets.webpack import WebpackThemeBundle + +theme = WebpackThemeBundle( + __name__, + ".", + default="semantic-ui", + themes={ + "semantic-ui": { + "entry": {}, + "dependencies": { + "react-searchkit": "^2.0.0", + }, + "devDependencies": {}, + "aliases": { + "../../theme.config$": "less/theme.config", + "../../less/site": "less/site", + "../../less": "less", + }, + } + }, +) diff --git a/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.ui_package}}/titlepage/__init__.py b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.ui_package}}/titlepage/__init__.py new file mode 100644 index 0000000..0d57917 --- /dev/null +++ b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.ui_package}}/titlepage/__init__.py @@ -0,0 +1,16 @@ +from oarepo_ui.resources import UIResourceConfig +from oarepo_ui.resources.resource import TemplatePageUIResource + + +class TitlePageResourceConfig(UIResourceConfig): + url_prefix = "/" + blueprint_name = "titlepage" + template_folder = "templates" + pages = { + "": "TitlePage", + } + + +def create_blueprint(app): + """Register blueprint for this resource.""" + return TemplatePageUIResource(TitlePageResourceConfig()).as_blueprint() diff --git a/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.ui_package}}/titlepage/templates/TitlePage.jinja b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.ui_package}}/titlepage/templates/TitlePage.jinja new file mode 100644 index 0000000..ad2bdfe --- /dev/null +++ b/src/nrp_devtools/templates/repository/{{cookiecutter.repository_name}}/{{cookiecutter.ui_package}}/titlepage/templates/TitlePage.jinja @@ -0,0 +1,83 @@ +{% raw %} +{%- extends "invenio_theme/frontpage.html" %} + +{# TODO (UI): prepare a proper initial title page !!! #} + +{%- block page_header %} +{%- endblock page_header %} + +{%- block page_body %} + + {%- block intro_section %} + {%- endblock intro_section %} + + {%- block bypasslinks %} + {%- endblock bypasslinks %} + + {%- block top_banner %} + {%- endblock top_banner %} + + {%- block grid_section %} +
+
+
+ {% block main_column %} +
+ + {% block main_column_content %} + {%- block frontpage_search %} +
+
+
+ {%- block frontpage_title %} +
+
+ {%- if config.THEME_FRONTPAGE_LOGO %} + + {%- else %} + + {%- endif %} +
+
+

{{ _(config.THEME_FRONTPAGE_TITLE) }}

+
+
+ {%- endblock frontpage_title %} + {%- block frontpage_form %} +
+
+
+ +
+
+ + {%- endblock frontpage_form %} +
+
+
+ {%- endblock frontpage_search %} + {% endblock main_column_content %} + +
+ {% endblock main_column %} +
+
+
+ {%- endblock grid_section %} + +{%- block bottom_section %} +{%- endblock bottom_section %} + +{%- endblock page_body %} +{% endraw %} \ No newline at end of file diff --git a/src/nrp_devtools/templates/ui_model/cookiecutter.json b/src/nrp_devtools/templates/ui_model/cookiecutter.json new file mode 100644 index 0000000..3553212 --- /dev/null +++ b/src/nrp_devtools/templates/ui_model/cookiecutter.json @@ -0,0 +1,9 @@ +{ + "name": "", + "endpoint": "", + "capitalized_name": "", + "resource": "", + "resource_config": "", + "ui_serializer_class": "", + "api_service": "" +} diff --git a/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/__init__.py b/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/__init__.py new file mode 100644 index 0000000..ee8219d --- /dev/null +++ b/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/__init__.py @@ -0,0 +1,38 @@ +from oarepo_ui.resources import BabelComponent +from oarepo_ui.resources.config import RecordsUIResourceConfig +from oarepo_ui.resources.resource import RecordsUIResource + + +class {{cookiecutter.resource_config}}(RecordsUIResourceConfig): + template_folder = "../templates" + url_prefix = "{{cookiecutter.endpoint}}" + blueprint_name = "{{cookiecutter.name}}" + ui_serializer_class = "{{cookiecutter.ui_serializer_class}}" + api_service = "{{cookiecutter.api_service}}" + + components = [BabelComponent] + try: + from oarepo_vocabularies.ui.resources.components import ( + DepositVocabularyOptionsComponent, + ) + components.append(DepositVocabularyOptionsComponent) + except ImportError: + pass + + search_app_id="{{cookiecutter.name}}" + + templates = { + "detail": "{{cookiecutter.name}}.Detail", + "search": "{{cookiecutter.name}}.Search", + "edit": "{{cookiecutter.name}}.Deposit", + "create":"{{cookiecutter.name}}.Deposit", + } + + +class {{cookiecutter.resource}}(RecordsUIResource): + pass + + +def create_blueprint(app): + """Register blueprint for this resource.""" + return {{cookiecutter.resource}}({{cookiecutter.resource_config}}()).as_blueprint() diff --git a/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/templates/semantic-ui/{{cookiecutter.name}}/Deposit.jinja b/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/templates/semantic-ui/{{cookiecutter.name}}/Deposit.jinja new file mode 100644 index 0000000..3ce22a9 --- /dev/null +++ b/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/templates/semantic-ui/{{cookiecutter.name}}/Deposit.jinja @@ -0,0 +1,7 @@ +{% raw %}{% extends "oarepo_ui/form.html" %} + +{%- block javascript %} + {{ super() }} + {{ webpack['{% endraw %}{{cookiecutter.name}}{% raw %}_deposit_form.js'] }} +{%- endblock %} +{% endraw %} \ No newline at end of file diff --git a/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/templates/semantic-ui/{{cookiecutter.name}}/Detail.jinja b/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/templates/semantic-ui/{{cookiecutter.name}}/Detail.jinja new file mode 100644 index 0000000..a5c791b --- /dev/null +++ b/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/templates/semantic-ui/{{cookiecutter.name}}/Detail.jinja @@ -0,0 +1,14 @@ +{% raw %}{% extends "oarepo_ui/detail.html" %} + +{%- block head_links %} +{{ super() }} +{%- endblock %} + +{% block record_main_content %} + Add your main content here +{% endblock %} + +{% block record_sidebar %} + Add your sidebar content here +{% endblock record_sidebar %} +{% endraw %} \ No newline at end of file diff --git a/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/templates/semantic-ui/{{cookiecutter.name}}/Search.jinja b/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/templates/semantic-ui/{{cookiecutter.name}}/Search.jinja new file mode 100644 index 0000000..2bcaf0d --- /dev/null +++ b/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/templates/semantic-ui/{{cookiecutter.name}}/Search.jinja @@ -0,0 +1,14 @@ +{% raw %}{%- extends config.BASE_TEMPLATE %} +{%- set title = _("Search results") ~ " | " ~ _("{% endraw %}{{cookiecutter.name}}{% raw %}") %} + +{%- block javascript %} + {{ super() }} + {{ webpack['{% endraw %}{{cookiecutter.name}}{% raw %}_search.js'] }} +{%- endblock %} + +{%- block page_body %} +
+
+
+{%- endblock page_body %} +{% endraw %} diff --git a/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/__init__.py b/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/assets/semantic-ui/js/{{cookiecutter.name}}/forms/deposit/DepositForm.jsx b/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/assets/semantic-ui/js/{{cookiecutter.name}}/forms/deposit/DepositForm.jsx new file mode 100644 index 0000000..1d96f95 --- /dev/null +++ b/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/assets/semantic-ui/js/{{cookiecutter.name}}/forms/deposit/DepositForm.jsx @@ -0,0 +1,90 @@ +import React from "react"; +import _isEmpty from "lodash/isEmpty"; +import PropTypes from "prop-types"; +import { BaseForm, TextField, FieldLabel } from "react-invenio-forms"; +import { Container, Header, Message } from "semantic-ui-react"; +import { DepositValidationSchema } from "./DepositValidationSchema"; +import { + useFormConfig, + useOnSubmit, + submitContextType, +} from "@js/oarepo_ui"; + + +const CurrentRecord = (props) => { + const { record } = props; + return ( + + Current record state + 
{JSON.stringify(record)}
+ + ); +}; + +CurrentRecord.propTypes = { + record: PropTypes.object, +}; + +CurrentRecord.defaultProps = { + record: undefined, +}; + +const RecordPreviewer = ({record}) => + +RecordPreviewer.propTypes = { + record: PropTypes.object, +}; + +RecordPreviewer.defaultProps = { + record: undefined, +}; + +export const DepositForm = () => { + const { record, formConfig } = useFormConfig(); + const context = formConfig.createUrl + ? submitContextType.create + : submitContextType.update; + const { onSubmit } = useOnSubmit({ + apiUrl: formConfig.createUrl || formConfig.updateUrl, + context: context, + onSubmitSuccess: (result) => { + window.location.href = editMode + ? currentPath.replace("/edit", "") + : currentPath.replace("_new", result.id); + }, + onSubmitError: (error) => { + console.error('Sumbission failed', error) + } + }); + + return ( + + +
{{cookiecutter.name}} deposit form
+ } + placeholder="Enter a record ID" + required + className="id-field" + optimized + fluid + required + /> + 
Add more of your deposit form fields here 👇
+ + + + ); +}; diff --git a/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/assets/semantic-ui/js/{{cookiecutter.name}}/forms/deposit/DepositValidationSchema.jsx b/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/assets/semantic-ui/js/{{cookiecutter.name}}/forms/deposit/DepositValidationSchema.jsx new file mode 100644 index 0000000..a89a83b --- /dev/null +++ b/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/assets/semantic-ui/js/{{cookiecutter.name}}/forms/deposit/DepositValidationSchema.jsx @@ -0,0 +1,7 @@ +import * as Yup from "yup"; + +export const DepositValidationSchema = Yup.object().shape({ + id: Yup.string().required(), + // TODO: implement any yup form validations here + // https://github.com/jquense/yup +}); diff --git a/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/assets/semantic-ui/js/{{cookiecutter.name}}/forms/deposit/index.js b/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/assets/semantic-ui/js/{{cookiecutter.name}}/forms/deposit/index.js new file mode 100644 index 0000000..64aa575 --- /dev/null +++ b/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/assets/semantic-ui/js/{{cookiecutter.name}}/forms/deposit/index.js @@ -0,0 +1,8 @@ +import { createFormAppInit } from "@js/oarepo_ui"; +import { DepositForm } from "./DepositForm" + +export const overriddenComponents = { + "FormApp.layout": DepositForm, +}; + +createFormAppInit(overriddenComponents); diff --git a/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/assets/semantic-ui/js/{{cookiecutter.name}}/forms/index.js b/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/assets/semantic-ui/js/{{cookiecutter.name}}/forms/index.js new file mode 100644 index 0000000..59fd3ee --- /dev/null +++ b/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/assets/semantic-ui/js/{{cookiecutter.name}}/forms/index.js @@ -0,0 +1 @@ +export * from './deposit' \ No newline at end of file diff --git a/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/assets/semantic-ui/js/{{cookiecutter.name}}/search/components/EmptyResultsElement.jsx b/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/assets/semantic-ui/js/{{cookiecutter.name}}/search/components/EmptyResultsElement.jsx new file mode 100644 index 0000000..b90d11d --- /dev/null +++ b/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/assets/semantic-ui/js/{{cookiecutter.name}}/search/components/EmptyResultsElement.jsx @@ -0,0 +1,68 @@ +import React from "react"; +import PropTypes from "prop-types"; + +import { Grid, Header, Button, Icon, Segment } from "semantic-ui-react"; +import { i18next } from "@translations/i18next"; + +export const EmptyResultsElement = ({ + queryString, + searchPath, + resetQuery, +}) => { + return ( + + + +
+ {i18next.t("We couldn't find any matches for ")} + {(queryString && `'${queryString}'`) || i18next.t("your search")} +
+ + + + + + + + + + +
+ {i18next.t("ProTip")}! +
+

+ + metadata.publication_date:[2017-01-01 TO *] + {" "} + {i18next.t( + "will give you all the publications from 2017 until today." + )} +

+

+ {i18next.t("For more tips, check out our ")} + + {i18next.t("search guide")} + + {i18next.t(" for defining advanced search queries.")} +

+ + + + + ); +}; + +EmptyResultsElement.propTypes = { + queryString: PropTypes.string.isRequired, + resetQuery: PropTypes.func.isRequired, + searchPath: PropTypes.string, +}; + +EmptyResultsElement.defaultProps = { + searchPath: "", +}; diff --git a/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/assets/semantic-ui/js/{{cookiecutter.name}}/search/components/MultipleSearchBarElement.jsx b/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/assets/semantic-ui/js/{{cookiecutter.name}}/search/components/MultipleSearchBarElement.jsx new file mode 100644 index 0000000..30be050 --- /dev/null +++ b/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/assets/semantic-ui/js/{{cookiecutter.name}}/search/components/MultipleSearchBarElement.jsx @@ -0,0 +1,17 @@ +import React from "react"; +import { MultipleOptionsSearchBarRSK } from "@js/invenio_search_ui/components"; +import { i18next } from "@translations/i18next"; + +export const MultipleSearchBarElement = ({ queryString, onInputChange }) => { + const headerSearchbar = document.getElementById("header-search-bar"); + const searchbarOptions = JSON.parse(headerSearchbar.dataset.options); + + return ( + + ); +}; diff --git a/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/assets/semantic-ui/js/{{cookiecutter.name}}/search/components/ResultsGridItem.jsx b/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/assets/semantic-ui/js/{{cookiecutter.name}}/search/components/ResultsGridItem.jsx new file mode 100644 index 0000000..ec73bb6 --- /dev/null +++ b/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/assets/semantic-ui/js/{{cookiecutter.name}}/search/components/ResultsGridItem.jsx @@ -0,0 +1,38 @@ +import PropTypes from "prop-types"; +import { withState } from "react-searchkit"; + +// TODO: Update this according to the full List item template? +export const ResultsGridItem = ({ result }) => { + return ( + + + {result.metadata.record.title} + + + + + ); +}; + +ResultsGridItem.propTypes = { + result: PropTypes.object.isRequired, +}; + +export const ResultsGridItemWithState = withState( + ({ currentQueryState, result, appName }) => ( + + ) +); + +ResultsGridItemWithState.propTypes = { + currentQueryState: PropTypes.object, + result: PropTypes.object.isRequired, +}; + +ResultsGridItemWithState.defaultProps = { + currentQueryState: null, +}; diff --git a/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/assets/semantic-ui/js/{{cookiecutter.name}}/search/components/ResultsListItem.jsx b/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/assets/semantic-ui/js/{{cookiecutter.name}}/search/components/ResultsListItem.jsx new file mode 100644 index 0000000..3d9ea22 --- /dev/null +++ b/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/assets/semantic-ui/js/{{cookiecutter.name}}/search/components/ResultsListItem.jsx @@ -0,0 +1,127 @@ +import React, { useContext } from "react"; +import PropTypes from "prop-types"; +import Overridable from "react-overridable"; + +import _get from "lodash/get"; + +import { Grid, Item, Label, List, Icon } from "semantic-ui-react"; +import { withState, buildUID } from "react-searchkit"; +import { SearchConfigurationContext } from "@js/invenio_search_ui/components"; + +import { i18next } from "@translations/i18next"; + +const ItemHeader = ({ title, searchUrl, selfLink }) => { + const viewLink = new URL( + selfLink, + new URL(searchUrl, window.location.origin) + ); + return ( + + {title} + + ); +}; + +const ItemSubheader = ({ +}) => { + // just an example + return ( + <> + + + + + + + + + + ); +}; + +export const ResultsListItemComponent = ({ + currentQueryState, + result, + appName, + ...rest +}) => { + const searchAppConfig = useContext(SearchConfigurationContext); + + const title = _get(result, "metadata.title", '') + + return ( + + + + + + + + + + + + + + + + ); +}; + +ResultsListItemComponent.propTypes = { + currentQueryState: PropTypes.object, + result: PropTypes.object.isRequired, + appName: PropTypes.string, +}; + +ResultsListItemComponent.defaultProps = { + currentQueryState: null, + appName: "", +}; + +export const ResultsListItem = (props) => { + return ( + + + + ); +}; + +ResultsListItem.propTypes = { + currentQueryState: PropTypes.object, + result: PropTypes.object.isRequired, + appName: PropTypes.string, +}; + +ResultsListItem.defaultProps = { + currentQueryState: null, + appName: "", +}; + +export const ResultsListItemWithState = withState( + ({ currentQueryState, updateQueryState, result, appName }) => ( + + ) +); + +ResultsListItemWithState.propTypes = { + currentQueryState: PropTypes.object, + result: PropTypes.object.isRequired, +}; + +ResultsListItemWithState.defaultProps = { + currentQueryState: null, +}; diff --git a/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/assets/semantic-ui/js/{{cookiecutter.name}}/search/components/index.js b/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/assets/semantic-ui/js/{{cookiecutter.name}}/search/components/index.js new file mode 100644 index 0000000..74ce9e8 --- /dev/null +++ b/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/assets/semantic-ui/js/{{cookiecutter.name}}/search/components/index.js @@ -0,0 +1,4 @@ +export { EmptyResultsElement } from './EmptyResultsElement' +export { MultipleSearchBarElement } from './MultipleSearchBarElement' +export { ResultsGridItem, ResultsGridItemWithState } from "./ResultsGridItem"; +export { ResultsListItem, ResultsListItemWithState } from "./ResultsListItem"; \ No newline at end of file diff --git a/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/assets/semantic-ui/js/{{cookiecutter.name}}/search/index.js b/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/assets/semantic-ui/js/{{cookiecutter.name}}/search/index.js new file mode 100644 index 0000000..aee6293 --- /dev/null +++ b/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/assets/semantic-ui/js/{{cookiecutter.name}}/search/index.js @@ -0,0 +1,54 @@ +import { createSearchAppInit } from '@js/invenio_search_ui' +import { + ActiveFiltersElement, + BucketAggregationElement, + BucketAggregationValuesElement, + CountElement, + ErrorElement, + SearchAppFacets, + SearchAppLayout, + SearchAppResultOptions, + SearchAppSearchbarContainer, + SearchFiltersToggleElement, + SearchAppSort +} from '@js/oarepo_ui/search' +import { + EmptyResultsElement, + MultipleSearchBarElement, + ResultsGridItemWithState, + ResultsListItemWithState +} from './components' +import { parametrize, overrideStore } from 'react-overridable' + +const appName = '{{cookiecutter.name}}.Search' + +const SearchAppSearchbarContainerWithConfig = parametrize(SearchAppSearchbarContainer, { appName: appName }) +const ResultsListItemWithConfig = parametrize(ResultsListItemWithState, { appName: appName }) +const ResultsGridItemWithConfig = parametrize(ResultsGridItemWithState, { appName: appName }) + +export const defaultComponents = { + [`${appName}.ActiveFilters.element`]: ActiveFiltersElement, + [`${appName}.BucketAggregation.element`]: BucketAggregationElement, + [`${appName}.BucketAggregationValues.element`]: BucketAggregationValuesElement, + [`${appName}.Count.element`]: CountElement, + [`${appName}.EmptyResults.element`]: EmptyResultsElement, + [`${appName}.Error.element`]: ErrorElement, + [`${appName}.ResultsGrid.item`]: ResultsGridItemWithConfig, + [`${appName}.ResultsList.item`]: ResultsListItemWithConfig, + [`${appName}.SearchApp.facets`]: SearchAppFacets, + [`${appName}.SearchApp.layout`]: SearchAppLayout, + [`${appName}.SearchApp.searchbarContainer`]: SearchAppSearchbarContainerWithConfig, + [`${appName}.SearchApp.sort`]: SearchAppSort, + [`${appName}.SearchApp.resultOptions`]: SearchAppResultOptions, + [`${appName}.SearchFilters.Toggle.element`]: SearchFiltersToggleElement, + [`${appName}.SearchBar.element`]: MultipleSearchBarElement, +} + +const overriddenComponents = overrideStore.getAll() + +createSearchAppInit( + { ...defaultComponents, ...overriddenComponents }, + true, + 'invenio-search-config', + true, +) diff --git a/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/webpack.py b/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/webpack.py new file mode 100644 index 0000000..0e6c84d --- /dev/null +++ b/src/nrp_devtools/templates/ui_model/{{cookiecutter.name}}/theme/webpack.py @@ -0,0 +1,18 @@ +from invenio_assets.webpack import WebpackThemeBundle + +theme = WebpackThemeBundle( + __name__, + "assets", + default="semantic-ui", + themes={ + "semantic-ui": dict( + entry={ + "{{cookiecutter.name}}_search": "./js/{{cookiecutter.name}}/search/index.js", + "{{cookiecutter.name}}_deposit_form": "./js/{{cookiecutter.name}}/forms/deposit/index.js", + }, + dependencies={}, + devDependencies={}, + aliases={}, + ) + }, +) diff --git a/src/nrp_devtools/templates/ui_page/cookiecutter.json b/src/nrp_devtools/templates/ui_page/cookiecutter.json new file mode 100644 index 0000000..e1e8a7b --- /dev/null +++ b/src/nrp_devtools/templates/ui_page/cookiecutter.json @@ -0,0 +1,6 @@ +{ + "name": "", + "endpoint": "", + "capitalized_name": "", + "template_name": "" +} diff --git a/src/nrp_devtools/templates/ui_page/{{cookiecutter.name}}/__init__.py b/src/nrp_devtools/templates/ui_page/{{cookiecutter.name}}/__init__.py new file mode 100644 index 0000000..ba1cf80 --- /dev/null +++ b/src/nrp_devtools/templates/ui_page/{{cookiecutter.name}}/__init__.py @@ -0,0 +1,18 @@ +from oarepo_ui.resources.config import TemplatePageUIResourceConfig +from oarepo_ui.resources.resource import TemplatePageUIResource + + +class {{cookiecutter.capitalized_name}}PageResourceConfig(TemplatePageUIResourceConfig): + url_prefix = "/" + blueprint_name = "{{cookiecutter.name}}" + template_folder = "templates" + pages = { + "": "{{cookiecutter.template_name}}", + # add a new page here. The key is the URL path, the value is the name of the template + # then put .jinja into the templates folder + } + + +def create_blueprint(app): + """Register blueprint for this resource.""" + return TemplatePageUIResource({{cookiecutter.capitalized_name}}PageResourceConfig()).as_blueprint() diff --git a/src/nrp_devtools/templates/ui_page/{{cookiecutter.name}}/templates/{{cookiecutter.template_name}}.jinja b/src/nrp_devtools/templates/ui_page/{{cookiecutter.name}}/templates/{{cookiecutter.template_name}}.jinja new file mode 100644 index 0000000..4cf558a --- /dev/null +++ b/src/nrp_devtools/templates/ui_page/{{cookiecutter.name}}/templates/{{cookiecutter.template_name}}.jinja @@ -0,0 +1,7 @@ +{% raw -%} +{%- extends config.BASE_TEMPLATE %} + +{%- block page_body %} + Hello world, this is jinjax template! +{%- endblock page_body %} +{% endraw %} \ No newline at end of file diff --git a/src/nrp_devtools/x509.py b/src/nrp_devtools/x509.py new file mode 100644 index 0000000..52e1bd6 --- /dev/null +++ b/src/nrp_devtools/x509.py @@ -0,0 +1,86 @@ +# Copyright 2018 Simon Davy +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# WARNING: the code in the gist generates self-signed certs, for the purposes of testing in development. +# Do not use these certs in production, or You Will Have A Bad Time. +# +# Caveat emptor +# + +import ipaddress +from datetime import datetime, timedelta + +from cryptography import x509 +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.x509.oid import NameOID + + +def generate_selfsigned_cert(hostname, ip_addresses=None, key=None): + """Generates self signed certificate for a hostname, and optional IP addresses.""" + + # Generate our key + if key is None: + key = rsa.generate_private_key( + public_exponent=65537, + key_size=2048, + backend=default_backend(), + ) + + name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, hostname)]) + + # best practice seem to be to include the hostname in the SAN, which *SHOULD* mean COMMON_NAME is ignored. + alt_names = [x509.DNSName(hostname)] + + # allow addressing by IP, for when you don't have real DNS (common in most testing scenarios + if ip_addresses: + for addr in ip_addresses: + # openssl wants DNSnames for ips... + alt_names.append(x509.DNSName(addr)) + # ... whereas golang's crypto/tls is stricter, and needs IPAddresses + # note: older versions of cryptography do not understand ip_address objects + alt_names.append(x509.IPAddress(ipaddress.ip_address(addr))) + + san = x509.SubjectAlternativeName(alt_names) + + # path_len=0 means this cert can only sign itself, not other certs. + basic_contraints = x509.BasicConstraints(ca=True, path_length=0) + now = datetime.utcnow() + cert = ( + x509.CertificateBuilder() + .subject_name(name) + .issuer_name(name) + .public_key(key.public_key()) + .serial_number(1000) + .not_valid_before(now) + .not_valid_after(now + timedelta(days=10 * 365)) + .add_extension(basic_contraints, False) + .add_extension(san, False) + .sign(key, hashes.SHA256(), default_backend()) + ) + cert_pem = cert.public_bytes(encoding=serialization.Encoding.PEM) + key_pem = key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ) + + return cert_pem, key_pem diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cmdline.py b/tests/cmdline.py new file mode 100644 index 0000000..28cf3c1 --- /dev/null +++ b/tests/cmdline.py @@ -0,0 +1,129 @@ +import os +import select +import subprocess +import re +import sys +import time + +import click + +from nrp_cli.commands.pdm import remove_virtualenv_from_env + + +class CommandLineTester: + def __init__(self, command, *args, environment=None, cwd=None): + self.command = command + self.args = args + self.environment = environment or {} + self.cwd = cwd + self.process = None + + def start(self): + click.secho(f"Starting {self.command} {' '.join(self.args)} inside {self.cwd}", fg="yellow") + self.process = subprocess.Popen( + [self.command, *self.args], + env={**remove_virtualenv_from_env(), **self.environment}, + cwd=self.cwd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=subprocess.PIPE, + ) + os.set_blocking(self.process.stdout.fileno(), False) + os.set_blocking(self.process.stderr.fileno(), False) + return self + + def stop(self): + self.running = False + self.process.terminate() + self.process.wait(3) + if self.process.poll() is None: + self.process.kill() + return self + + def expect(self, regexp_line_str, timeout=5, stdout=True): + stream = self.process.stdout if stdout else self.process.stderr + other_stream = self.process.stderr if stdout else self.process.stdout + + streams = [stream, other_stream] + + start = time.time() + last_line = b"" + regexp_line = re.compile(regexp_line_str) + while True: + current = time.time() + + if current - start > timeout: + raise AssertionError(f"Expected line {regexp_line_str} not found.") + + selected = select.select(streams, [], [], 1) + + if stream in selected[0]: + # not very efficient, but we don't expect a lot of output in tests + while True: + c = stream.read(1) + if not c: + break + + if stdout: + sys.stdout.buffer.write(c) + sys.stdout.flush() + else: + sys.stderr.buffer.write(c) + sys.stderr.flush() + + if c == b"\n": + last_line = b"" + continue + + last_line += c + try: + ll = last_line.decode("utf-8") + if regexp_line.search(ll): + return True + except UnicodeDecodeError: + pass + + # output other stream + if other_stream in selected[0]: + if stdout: + sys.stderr.buffer.write(other_stream.read()) + sys.stderr.flush() + else: + sys.stdout.buffer.write(other_stream.read()) + sys.stdout.flush() + + def enter(self, line, eol=True): + sys.stdout.flush() + sys.stderr.flush() + print(line, flush=True) + + self.process.stdin.write(line.encode("utf-8")) + if eol: + self.process.stdin.write(b"\n") + + self.process.stdin.flush() + return self + + def __enter__(self): + self.start() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + print("Exiting") + self.stop() + + def wait_for_exit(self, timeout=5): + start = time.time() + while True: + current = time.time() + + if current - start > timeout: + raise AssertionError(f"Process did not exit in {timeout} seconds.") + + sys.stdout.buffer.write(self.process.stdout.read() or b"") + sys.stderr.buffer.write(self.process.stderr.read() or b"") + + if self.process.poll() is not None: + return + + time.sleep(0.1) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..9a8a411 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,19 @@ +import os +import shutil +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="session") +def test_repository_dir(): + """Create a temporary directory for testing.""" + test_dir = "temporary_test_repository" + if os.path.exists(test_dir) and not os.environ.get("NRP_TEST_KEEP_TEMPORARY_DIR"): + shutil.rmtree(test_dir) + yield test_dir + + +@pytest.fixture(scope="session") +def absolute_test_repository_dir(test_repository_dir): + yield Path(test_repository_dir).resolve() \ No newline at end of file diff --git a/tests/test_repository_from_scratch.py b/tests/test_repository_from_scratch.py new file mode 100644 index 0000000..b6846b2 --- /dev/null +++ b/tests/test_repository_from_scratch.py @@ -0,0 +1,73 @@ +import os +import subprocess + +from .cmdline import CommandLineTester +import requests + + +# +# Note: the following tests must be run in the order in which they +# are defined, because they depend on each other. +# + + +def test_initialization(test_repository_dir): + with CommandLineTester( + "./nrp-installer.sh", + test_repository_dir, + environment={"LOCAL_NRP_TOOLS_LOCATION": os.getcwd()}, + ) as tester: + tester.expect("Successfully installed pip") + tester.expect("Please answer a few questions", timeout=60) + + tester.expect("Human name of the repository") + tester.enter("Test Repository") + tester.expect("Python package name of the whole repository") + tester.enter("test_repository") + tester.expect("OARepo version to use") + tester.enter("") + tester.expect("Python package name of the shared code") + tester.enter("") + tester.expect("Python package name of the ui code") + tester.enter("") + tester.expect("Your repository is now initialized") + tester.wait_for_exit(timeout=10) + + +def test_build(absolute_test_repository_dir): + subprocess.call(["ls", "-la"], cwd=absolute_test_repository_dir) + + with CommandLineTester( + absolute_test_repository_dir / "nrp", + "build", + cwd=absolute_test_repository_dir, + ) as tester: + tester.expect("Building repository for production", + timeout=60) + tester.expect("Successfully built the repository", + timeout=2400) + + +def test_check_requirements(absolute_test_repository_dir): + with CommandLineTester( + absolute_test_repository_dir / "nrp", + "check", + cwd=absolute_test_repository_dir, + ) as tester: + tester.expect("Checking repository requirements", timeout=60) + tester.expect("Repository ready to be run", timeout=2400) + + +def test_ui_titlepage_running(absolute_test_repository_dir): + with CommandLineTester( + absolute_test_repository_dir / "nrp", + "run", + cwd=absolute_test_repository_dir, + ) as tester: + tester.expect("Starting python server", timeout=60) + tester.expect("Python server started", timeout=60) + data = requests.get('https://127.0.0.1:5000', allow_redirects=True, verify=False) + data.raise_for_status() + assert 'Test Repository' in data.text + assert 'main id="main"' in data.text + assert 'Powered by' in data.text