first commit

.gitignore

@@ -0,0 +1,73 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+
+# Idea software family
+.idea/
+# VSCode
+.vscode/
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Vim swapfiles
+.*.sw?
+venv/
+.envrc
+.venv/
+.venv-builder/
+node_modules/
+
+.DS_Store
+
+docker/.env
+
+temporary_test_repository/
+temporary_test_directory/

MANIFEST.in

@@ -0,0 +1,3 @@
+graft oarepo_tools
+global-exclude oarepo_tools/i18next/node_modules/**
+global-exclude **/__pycache__/**

README.md

@@ -0,0 +1,34 @@
+# NRP repository development tools
+
+This tool is used to setup and run an Invenio based NRP instance. 
+
+## Prerequisites
+
+- Python 3.10
+- node version 16 or greater, npm version 7, 8 or 10
+- imagemagick and development packages for imagemagick
+- standard build tools (gcc, make, ...), on ubuntu build-essential
+- Docker 20.10.10+ and docker-compose 1.17.0+ (or OrbStack on Mac)
+
+## Creating a new repository
+
+1. Download the installer
+
+```bash
+curl -sSL https://raw.githubusercontent.com/oarepo/nrp-tools/main/nrp-installer.sh
+```
+
+2. Run the installer with a directory where the repository will be created
+
+```bash 
+bash nrp-installer.sh my-repo
+```
+
+After asking a couple of questions, the installer will create the
+repository in the `my-repo` directory.
+
+It will also initialize git version control system and commit the initial
+sources.
+
+3. Go to the my-repo directory and see the README.md file there for further instructions
+   (or have a peek at [](src/nrp_devtools/...) )

nrp-installer.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+# This script installs the NRP repository tools
+
+# Environment variables
+# PYTHON:
+#              python executable to use for installation and running the NRP tools
+# NRP_GIT_URL:
+#              URL of the NRP git repository
+# NRP_GIT_BRANCH:
+#              branch of the NRP git repository to use
+# LOCAL_NRP_DEVTOOLS_LOCATION:
+#              location of the local NRP repository.
+#              If set, do not clone the NRP repository but use the local one.
+
+set -e
+
+NRP_GIT_URL=${NRP_GIT_URL:-https://github.com/oarepo/nrp-devtools.git}
+NRP_GIT_BRANCH=${NRP_GIT_BRANCH:-main}
+
+SUPPORTED_PYTHON_VERSIONS=(3.12 3.11 3.10 3.9)
+
+if [ -z "$PYTHON" ] ; then
+
+  # find a supported python
+  for version in "${SUPPORTED_PYTHON_VERSIONS[@]}"; do
+      if command -v python$version >/dev/null 2>&1; then
+          PYTHON=python$version
+          break
+      fi
+  done
+
+  if [ -z "$PYTHON" ] ; then
+    echo "No supported python version found. Please install python 3.9 or higher
+    or set the PYTHON environment variable to the python executable."
+    exit 1
+  fi
+fi
+
+# clone nrp tool to a temporary directory
+ACTUAL_DIR="$(pwd)"
+TMP_DIR=$(mktemp -d)
+
+trap 'rm -rf "$TMP_DIR"' EXIT
+
+echo "Installing temporary NRP CLI to $TMP_DIR, will clean it up on exit."
+
+
+$PYTHON -m venv "$TMP_DIR/.venv"
+source "$TMP_DIR/.venv/bin/activate"
+pip install -U setuptools pip wheel
+
+if [ -z "$LOCAL_NRP_DEVTOOLS_LOCATION" ] ; then
+  LOCAL_NRP_DEVTOOLS_LOCATION="$TMP_DIR/nrp-devtools"
+  git clone "$NRP_GIT_URL" --branch "$NRP_GIT_BRANCH" --depth 1 "$LOCAL_NRP_DEVTOOLS_LOCATION"
+fi
+pip install -e "$LOCAL_NRP_DEVTOOLS_LOCATION"
+
+"$TMP_DIR"/.venv/bin/nrp-devtools initialize "$@"

pyproject.toml

@@ -0,0 +1,60 @@
+[project]
+name = "nrp-devtools"
+version = "0.1.0"
+description = "NRP repository development tools"
+readme = "README.md"
+authors = [
+  {name = "Miroslav Simek", email = "[email protected]" }
+]
+dependencies = [
+  "setuptools",
+  "pip",
+  "wheel",
+  "click",
+  "aenum",
+
+  # config loading/writing
+  "PyYAML",
+  "dacite",
+  "ruamel.yaml",
+
+  # for code scaffolding
+  "case-converter",
+  "cookiecutter",
+  "cryptography",
+
+  # for managing requirements / installation
+  "pdm",
+  "tomli",
+  "tomli-w",
+  "requirements-parser",
+
+  # progress bar everywhere
+  "tqdm",
+
+  # for develop (webpack)
+  "watchdog",
+  "pytimedinput",
+  "psutil",
+
+  # nrp makemessages
+  "oarepo-tools",
+
+  # for checks
+  "minio",
+  "redis",
+  "psycopg[binary]",
+  "pika",
+  "opensearch-py",
+
+  # testing
+  "pytest",
+]
+
+[project.scripts]
+nrp-devtools = "nrp_devtools.main:nrp_command"
+
+
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"

setup.py

@@ -0,0 +1,3 @@
+import setuptools
+
+setuptools.setup()

src/nrp_devtools/cli/__init__.py

@@ -0,0 +1,21 @@
+from .base import nrp_command
+from .build import build_command
+from .check import check_command
+from .develop import develop_command
+from .initialize import initialize_command
+from .model import model_group
+from .run import run_command
+from .ui import ui_group
+from .upgrade import upgrade_command
+
+__all__ = [
+    "nrp_command",
+    "initialize_command",
+    "upgrade_command",
+    "ui_group",
+    "develop_command",
+    "check_command",
+    "build_command",
+    "run_command",
+    "model_group",
+]

src/nrp_devtools/cli/base.py

@@ -0,0 +1,97 @@
+"""
+This is the main entry point for the nrp devtools command line interface.
+"""
+import functools
+import sys
+from pathlib import Path
+from typing import Any, Callable, Dict, Union
+
+import click
+
+from nrp_devtools.commands.utils import run_steps
+from nrp_devtools.config import OARepoConfig
+
+
+@click.group()
+def nrp_command(**kwargs):
+    """NRP devtools command line interface."""
+
+
+def command_sequence(
+    repository_dir_must_exist=True,
+    repository_dir_as_argument=False,
+    continue_on_errors: Union[
+        bool, Callable[[OARepoConfig, Dict[str, Any]], bool]
+    ] = False,
+):
+    def wrapper(command):
+        command = click.option(
+            "--verbose", "-v", is_flag=True, help="Enables verbose mode."
+        )(command)
+        command = click.option("--step", help="Run only this step", multiple=True)(
+            command
+        )
+        command = click.option(
+            "--dry-run",
+            is_flag=True,
+            help="Show steps that would be run and exit.",
+        )(command)
+        command = click.option(
+            "--steps",
+            "show_steps",
+            is_flag=True,
+            help="Show steps that would be run and exit.",
+        )(command)
+        if repository_dir_as_argument:
+            command = click.argument(
+                "repository_dir",
+                type=click.Path(exists=repository_dir_must_exist),
+            )(command)
+        else:
+            command = click.option(
+                "--repository-dir",
+                "-d",
+                default=".",
+                help="Repository directory (default is the current directory).",
+                type=click.Path(exists=False),
+            )(command)
+
+        @functools.wraps(command)
+        def proxied(*args, **kwargs):
+            repository_dir = kwargs["repository_dir"]
+            steps = kwargs.pop("step", None)
+            dry_run = kwargs.pop("dry_run", False) or kwargs.pop("show_steps", False)
+            if repository_dir:
+                kwargs["repository_dir"] = repository_dir = Path(
+                    repository_dir
+                ).resolve()
+
+                if repository_dir_must_exist and not repository_dir.exists():
+                    click.secho("Project directory must exist", fg="red")
+                    sys.exit(1)
+
+            config = OARepoConfig(repository_dir)
+            config.load()
+
+            # run the command
+            step_commands = command(*args, config=config, **kwargs)
+            if dry_run:
+                click.secho("Steps that would be run:\n", fg="green")
+                for idx, step_cmd in enumerate(step_commands):
+                    click.secho(f"{idx+1:3d} {step_cmd.__name__}", fg="green")
+                return
+            elif step_commands:
+                _continue_on_errors = (
+                    continue_on_errors(config, kwargs)
+                    if callable(continue_on_errors)
+                    else continue_on_errors
+                )
+
+                run_steps(
+                    config, steps, step_commands, continue_on_errors=_continue_on_errors
+                )
+            config.save()
+
+        return proxied
+
+    return wrapper

src/nrp_devtools/cli/build.py

@@ -0,0 +1,36 @@
+from functools import partial
+
+import click
+
+from ..commands.invenio import install_invenio_cfg
+from ..commands.pdm import (
+    build_requirements,
+    check_requirements,
+    clean_previous_installation,
+    create_empty_venv,
+    install_python_repository,
+)
+from ..commands.ui import build_production_ui, collect_assets, install_npm_packages
+from ..commands.utils import make_step, no_args, run_fixup
+from ..config import OARepoConfig
+from .base import command_sequence, nrp_command
+
+
+@nrp_command.command(name="build")
+@command_sequence()
+def build_command(*, config: OARepoConfig, **kwargs):
+    """Builds the repository"""
+    return (
+        no_args(
+            partial(click.secho, "Building repository for production", fg="yellow")
+        ),
+        make_step(clean_previous_installation),
+        make_step(create_empty_venv),
+        run_fixup(check_requirements, build_requirements, fix=True),
+        install_python_repository,
+        install_invenio_cfg,
+        collect_assets,
+        install_npm_packages,
+        build_production_ui,
+        no_args(partial(click.secho, "Successfully built the repository", fg="green")),
+    )