Merge pull request #1 from oarepo/copy-instead-link

Copying instead of linking sources to assets

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "nrp-devtools"
-version = "0.1.10"
+version = "0.1.11"
 description = "NRP repository development tools"
 readme = "README.md"
 authors = [
@@ -34,7 +34,6 @@ dependencies = [
 
   # for develop (webpack)
   "watchdog",
-  "pytimedinput",
   "psutil",
 
   # nrp makemessages

src/nrp_devtools/cli/__init__.py

@@ -5,9 +5,9 @@
 from .initialize import initialize_command
 from .model import model_group
 from .run import run_command
+from .translations import translations_command
 from .ui import ui_group
 from .upgrade import upgrade_command
-from .translations import translations_command
 
 __all__ = [
     "nrp_command",
@@ -19,5 +19,5 @@
     "build_command",
     "run_command",
     "model_group",
-    "translations_command"
+    "translations_command",
 ]

src/nrp_devtools/cli/develop.py

@@ -2,7 +2,7 @@
 
 from ..commands.develop import Runner
 from ..commands.develop.controller import run_develop_controller
-from ..commands.ui.link_assets import link_assets
+from ..commands.ui.link_assets import copy_assets_to_webpack_build_dir
 from ..commands.utils import make_step
 from ..config import OARepoConfig
 from .base import command_sequence, nrp_command
@@ -31,7 +31,7 @@ def develop_command(
     context = {}
     return (
         *(check_commands(context, local_packages, fix=True) if checks else ()),
-        link_assets,
+        copy_assets_to_webpack_build_dir,
         make_step(
             lambda config=None, runner=None: runner.start_python_server(
                 development_mode=True

src/nrp_devtools/cli/model.py

@@ -4,11 +4,12 @@
 import click
 
 from ..commands.model.compile import (
+    add_model_to_i18n,
     add_requirements_and_entrypoints,
     compile_model_to_tempdir,
     copy_compiled_model,
     generate_alembic,
-    install_model_compiler, add_model_to_i18n,
+    install_model_compiler,
 )
 from ..commands.model.create import create_model
 from ..commands.pdm import install_python_repository

src/nrp_devtools/cli/translations.py

@@ -1,19 +1,21 @@
 import click
+from oarepo_tools.make_translations import main
+
 from ..config import OARepoConfig
 from .base import command_sequence, nrp_command
 
-from oarepo_tools.make_translations import main
-
 
 @nrp_command.command(name="translations")
 @command_sequence()
 @click.pass_context
-def translations_command(ctx, *, config: OARepoConfig, local_packages=None, checks=True, **kwargs):
+def translations_command(
+    ctx, *, config: OARepoConfig, local_packages=None, checks=True, **kwargs
+):
     """Create translations for the repository.
 
     This command will create source translation files inside the i18n/translations directory.
     Edit the .po files there and run nrp translations again to compile the translations.
 
     To change the translated languages, edit the oarepo.yaml file.
     """
-    ctx.invoke(main, setup_cfg=config.config_file)
+    ctx.invoke(main, setup_cfg=config.config_file)

src/nrp_devtools/commands/develop/controller.py

@@ -1,5 +1,9 @@
+import os
+import select
+import sys
+import time
+
 import click
-from pytimedinput import timedInput
 
 from nrp_devtools.config import OARepoConfig
 
@@ -26,6 +30,44 @@ def show_menu(server: bool, ui: bool, development_mode: bool):
     click.secho("")
 
 
+def timedInput(prompt, timeout):
+    try:
+        os.set_blocking(sys.stdin.fileno(), False)
+
+        start_time = time.time()
+        print(prompt, end="", flush=True)
+        choice = ""
+        while True:
+            remaining_time = timeout - (time.time() - start_time)
+            if remaining_time <= 0:
+                return (choice, True)
+
+            is_data = select.select([sys.stdin], [], [], remaining_time) == (
+                [sys.stdin],
+                [],
+                [],
+            )
+
+            while is_data:
+                for input_char in sys.stdin.read():
+                    if input_char in ("\n", "\r"):
+                        return (choice, False)
+                    else:
+                        choice += input_char
+
+                is_data = select.select([sys.stdin], [], [], 0.01) == (
+                    [sys.stdin],
+                    [],
+                    [],
+                )
+
+            time.sleep(
+                0.5
+            )  # sanity check, if the select returns immediately, sleep a bit
+    finally:
+        os.set_blocking(sys.stdin.fileno(), True)
+
+
 def run_develop_controller(
     config: OARepoConfig, runner: Runner, server=True, ui=True, development_mode=False
 ):

src/nrp_devtools/commands/develop/runner.py

@@ -1,23 +1,25 @@
 import os
+import shutil
 import subprocess
-import threading
 import time
 import traceback
-from threading import RLock
+from pathlib import Path
 from typing import Optional
 
 import click
 import psutil
+from watchdog.events import FileSystemEventHandler
+from watchdog.observers import Observer
 
-from nrp_devtools.commands.ui.link_assets import link_assets
+from nrp_devtools.commands.ui.assets import load_watched_paths
+from nrp_devtools.commands.ui.link_assets import copy_assets_to_webpack_build_dir
 from nrp_devtools.config import OARepoConfig
 
 
 class Runner:
     python_server_process: Optional[subprocess.Popen] = None
     webpack_server_process: Optional[subprocess.Popen] = None
-    file_watcher_thread: Optional[threading.Thread] = None
-    file_watcher_stopping = None
+    file_copier: Optional["FileCopier"] = None
 
     def __init__(self, config: OARepoConfig):
         self.config = config
@@ -90,17 +92,7 @@ def start_webpack_server(self):
 
     def start_file_watcher(self):
         click.secho("Starting file watcher", fg="yellow")
-
-        def watch_files():
-            while True:
-                if self.file_watcher_stopping.acquire(timeout=1):
-                    break
-
-        self.file_watcher_stopping = RLock()
-        self.file_watcher_stopping.acquire()
-
-        self.file_watcher_thread = threading.Thread(target=watch_files, daemon=True)
-        self.file_watcher_thread.start()
+        self.file_copier = FileCopier(self.config)
         click.secho("File watcher started", fg="green")
 
     def stop(self):
@@ -121,7 +113,7 @@ def restart_webpack_server(self):
             self.stop_file_watcher()
             # just for being sure, link assets
             # (they might have changed and were not registered before)
-            link_assets(self.config)
+            copy_assets_to_webpack_build_dir(self.config)
             self.start_file_watcher()
             self.start_webpack_server()
         except:
@@ -155,15 +147,124 @@ def stop_webpack_server(self):
 
     def stop_file_watcher(self):
         click.secho("Stopping file watcher", fg="yellow")
-        if self.file_watcher_thread:
-            self.file_watcher_stopping.release()
-            self.file_watcher_thread.join()
-            self.file_watcher_thread = None
-            self.file_watcher_stopping = None
+        if self.file_copier:
+            self.file_copier.join()
+            self.file_copier = None
 
     def _kill_process_tree(self, process_tree: subprocess.Popen):
         parent_pid = process_tree.pid
         parent = psutil.Process(parent_pid)
         for child in parent.children(recursive=True):
             child.kill()
         parent.kill()
+
+
+class FileCopier:
+    class Handler(FileSystemEventHandler):
+        def __init__(self, source_path: Path, target_path: Path, watcher):
+            self.source_root_path = source_path
+            self.target_root_path = target_path
+            self.watcher = watcher
+            print(f"Watching {source_path} -> {target_path}")
+
+        def on_closed(self, event):
+            if event.is_directory:
+                return
+
+            try:
+                time.sleep(0.01)
+                self.copy_file(event.src_path, self.make_target_path(event.src_path))
+            except:
+                traceback.print_exc()
+
+        def on_modified(self, event):
+            if event.is_directory:
+                return
+
+            try:
+                time.sleep(0.1)
+                self.copy_file(event.src_path, self.make_target_path(event.src_path))
+            except:
+                traceback.print_exc()
+
+        def on_moved(self, event):
+            try:
+                time.sleep(0.01)
+                self.remove_file(event.src_path, self.make_target_path(event.src_path))
+                self.copy_file(event.dest_path, self.make_target_path(event.dest_path))
+            except:
+                traceback.print_exc()
+
+        def on_created(self, event):
+            """When a new directory is created, add a watch for it"""
+            if event.is_directory:
+                self.watcher.schedule(
+                    type(self)(
+                        event.src_path,
+                        self.make_target_path(event.src_path),
+                        self.watcher,
+                    ),
+                    event.src_path,
+                    recursive=True,
+                )
+
+        def on_deleted(self, event):
+            try:
+                time.sleep(0.01)
+                self.remove_file(event.src_path, self.make_target_path(event.src_path))
+            except:
+                traceback.print_exc()
+
+        def make_target_path(self, source_path):
+            return self.target_root_path / Path(source_path).relative_to(
+                self.source_root_path
+            )
+
+        def copy_file(self, source_path, target_path):
+            if str(source_path).endswith("~"):
+                return
+            print(f"Copying {source_path} to {target_path}")
+            target_path.parent.mkdir(parents=True, exist_ok=True)
+            shutil.copy(source_path, target_path)
+
+        def remove_file(self, source_path, target_path):
+            print(f"Removing {target_path}")
+            if target_path.exists():
+                target_path.unlink()
+
+    def __init__(self, config):
+        self.config = config
+        static = (config.ui_dir / "static").resolve()
+
+        self.watched_paths = load_watched_paths(
+            config.invenio_instance_path / "watch.list.json",
+            [f"{static}=static"],
+        )
+        print(self.watched_paths)
+
+        self.watcher = Observer()
+        static_target_path = self.config.invenio_instance_path / "static"
+        assets_target_path = self.config.invenio_instance_path / "assets"
+
+        for path, kind in self.watched_paths.items():
+            path = Path(path).resolve()
+            if kind == "static":
+                self.watcher.schedule(
+                    self.Handler(path, static_target_path, self.watcher),
+                    str(path),
+                    recursive=True,
+                )
+            elif kind == "assets":
+                self.watcher.schedule(
+                    self.Handler(path, assets_target_path, self.watcher),
+                    str(path),
+                    recursive=True,
+                )
+        self.watcher.start()
+
+    def join(self):
+        try:
+            self.watcher.stop()
+            self.watcher.join(10)
+        except:
+            print("Could not stop watcher thread but continuing anyway")

src/nrp_devtools/commands/model/compile.py

@@ -393,4 +393,4 @@ def rewrite_revision_file(
 
 def add_model_to_i18n(config: OARepoConfig, *, model, **kwargs):
     i18n_config = config.i18n
-    i18n_config.babel_source_paths.append(model.model_package)
+    i18n_config.babel_source_paths.append(model.model_package)