From 9c6d8e930149f70590aaa85f8cd22eb01da9b1ad Mon Sep 17 00:00:00 2001 From: Mirek Simek Date: Tue, 23 Jan 2024 12:33:33 +0100 Subject: [PATCH 1/2] Copying instead of linking sources to assets --- pyproject.toml | 3 +- src/nrp_devtools/cli/__init__.py | 4 +- src/nrp_devtools/cli/develop.py | 4 +- src/nrp_devtools/cli/model.py | 3 +- src/nrp_devtools/cli/translations.py | 10 +- .../commands/develop/controller.py | 44 +++++- src/nrp_devtools/commands/develop/runner.py | 144 +++++++++++++++--- src/nrp_devtools/commands/model/compile.py | 2 +- src/nrp_devtools/commands/ui/link_assets.py | 21 +-- 9 files changed, 193 insertions(+), 42 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cfbdf46..82e58fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "nrp-devtools" -version = "0.1.10" +version = "0.1.11" description = "NRP repository development tools" readme = "README.md" authors = [ @@ -34,7 +34,6 @@ dependencies = [ # for develop (webpack) "watchdog", - "pytimedinput", "psutil", # nrp makemessages diff --git a/src/nrp_devtools/cli/__init__.py b/src/nrp_devtools/cli/__init__.py index 7dfe646..af6b9d3 100644 --- a/src/nrp_devtools/cli/__init__.py +++ b/src/nrp_devtools/cli/__init__.py @@ -5,9 +5,9 @@ from .initialize import initialize_command from .model import model_group from .run import run_command +from .translations import translations_command from .ui import ui_group from .upgrade import upgrade_command -from .translations import translations_command __all__ = [ "nrp_command", @@ -19,5 +19,5 @@ "build_command", "run_command", "model_group", - "translations_command" + "translations_command", ] diff --git a/src/nrp_devtools/cli/develop.py b/src/nrp_devtools/cli/develop.py index 2a5416d..0216171 100644 --- a/src/nrp_devtools/cli/develop.py +++ b/src/nrp_devtools/cli/develop.py @@ -2,7 +2,7 @@ from ..commands.develop import Runner from ..commands.develop.controller import run_develop_controller -from ..commands.ui.link_assets import link_assets +from ..commands.ui.link_assets import copy_assets_to_webpack_build_dir from ..commands.utils import make_step from ..config import OARepoConfig from .base import command_sequence, nrp_command @@ -31,7 +31,7 @@ def develop_command( context = {} return ( *(check_commands(context, local_packages, fix=True) if checks else ()), - link_assets, + copy_assets_to_webpack_build_dir, make_step( lambda config=None, runner=None: runner.start_python_server( development_mode=True diff --git a/src/nrp_devtools/cli/model.py b/src/nrp_devtools/cli/model.py index 4afb5f7..f5dd5e1 100644 --- a/src/nrp_devtools/cli/model.py +++ b/src/nrp_devtools/cli/model.py @@ -4,11 +4,12 @@ import click from ..commands.model.compile import ( + add_model_to_i18n, add_requirements_and_entrypoints, compile_model_to_tempdir, copy_compiled_model, generate_alembic, - install_model_compiler, add_model_to_i18n, + install_model_compiler, ) from ..commands.model.create import create_model from ..commands.pdm import install_python_repository diff --git a/src/nrp_devtools/cli/translations.py b/src/nrp_devtools/cli/translations.py index d2fbd65..91bc944 100644 --- a/src/nrp_devtools/cli/translations.py +++ b/src/nrp_devtools/cli/translations.py @@ -1,14 +1,16 @@ import click +from oarepo_tools.make_translations import main + from ..config import OARepoConfig from .base import command_sequence, nrp_command -from oarepo_tools.make_translations import main - @nrp_command.command(name="translations") @command_sequence() @click.pass_context -def translations_command(ctx, *, config: OARepoConfig, local_packages=None, checks=True, **kwargs): +def translations_command( + ctx, *, config: OARepoConfig, local_packages=None, checks=True, **kwargs +): """Create translations for the repository. This command will create source translation files inside the i18n/translations directory. @@ -16,4 +18,4 @@ def translations_command(ctx, *, config: OARepoConfig, local_packages=None, chec To change the translated languages, edit the oarepo.yaml file. """ - ctx.invoke(main, setup_cfg=config.config_file) \ No newline at end of file + ctx.invoke(main, setup_cfg=config.config_file) diff --git a/src/nrp_devtools/commands/develop/controller.py b/src/nrp_devtools/commands/develop/controller.py index 1f186bb..fa32756 100644 --- a/src/nrp_devtools/commands/develop/controller.py +++ b/src/nrp_devtools/commands/develop/controller.py @@ -1,5 +1,9 @@ +import os +import select +import sys +import time + import click -from pytimedinput import timedInput from nrp_devtools.config import OARepoConfig @@ -26,6 +30,44 @@ def show_menu(server: bool, ui: bool, development_mode: bool): click.secho("") +def timedInput(prompt, timeout): + try: + os.set_blocking(sys.stdin.fileno(), False) + + start_time = time.time() + print(prompt, end="", flush=True) + choice = "" + while True: + remaining_time = timeout - (time.time() - start_time) + if remaining_time <= 0: + return (choice, True) + + is_data = select.select([sys.stdin], [], [], remaining_time) == ( + [sys.stdin], + [], + [], + ) + + while is_data: + for input_char in sys.stdin.read(): + if input_char in ("\n", "\r"): + return (choice, False) + else: + choice += input_char + + is_data = select.select([sys.stdin], [], [], 0.01) == ( + [sys.stdin], + [], + [], + ) + + time.sleep( + 0.5 + ) # sanity check, if the select returns immediately, sleep a bit + finally: + os.set_blocking(sys.stdin.fileno(), True) + + def run_develop_controller( config: OARepoConfig, runner: Runner, server=True, ui=True, development_mode=False ): diff --git a/src/nrp_devtools/commands/develop/runner.py b/src/nrp_devtools/commands/develop/runner.py index b1857be..7fe12c1 100644 --- a/src/nrp_devtools/commands/develop/runner.py +++ b/src/nrp_devtools/commands/develop/runner.py @@ -1,23 +1,27 @@ import os +import shutil import subprocess import threading import time import traceback +from pathlib import Path from threading import RLock from typing import Optional import click import psutil +from watchdog.events import FileSystemEventHandler +from watchdog.observers import Observer -from nrp_devtools.commands.ui.link_assets import link_assets +from nrp_devtools.commands.ui.assets import load_watched_paths +from nrp_devtools.commands.ui.link_assets import copy_assets_to_webpack_build_dir from nrp_devtools.config import OARepoConfig class Runner: python_server_process: Optional[subprocess.Popen] = None webpack_server_process: Optional[subprocess.Popen] = None - file_watcher_thread: Optional[threading.Thread] = None - file_watcher_stopping = None + file_copier: Optional["FileCopier"] = None def __init__(self, config: OARepoConfig): self.config = config @@ -90,17 +94,7 @@ def start_webpack_server(self): def start_file_watcher(self): click.secho("Starting file watcher", fg="yellow") - - def watch_files(): - while True: - if self.file_watcher_stopping.acquire(timeout=1): - break - - self.file_watcher_stopping = RLock() - self.file_watcher_stopping.acquire() - - self.file_watcher_thread = threading.Thread(target=watch_files, daemon=True) - self.file_watcher_thread.start() + self.file_copier = FileCopier(self.config) click.secho("File watcher started", fg="green") def stop(self): @@ -121,7 +115,7 @@ def restart_webpack_server(self): self.stop_file_watcher() # just for being sure, link assets # (they might have changed and were not registered before) - link_assets(self.config) + copy_assets_to_webpack_build_dir(self.config) self.start_file_watcher() self.start_webpack_server() except: @@ -155,11 +149,9 @@ def stop_webpack_server(self): def stop_file_watcher(self): click.secho("Stopping file watcher", fg="yellow") - if self.file_watcher_thread: - self.file_watcher_stopping.release() - self.file_watcher_thread.join() - self.file_watcher_thread = None - self.file_watcher_stopping = None + if self.file_copier: + self.file_copier.join() + self.file_copier = None def _kill_process_tree(self, process_tree: subprocess.Popen): parent_pid = process_tree.pid @@ -167,3 +159,115 @@ def _kill_process_tree(self, process_tree: subprocess.Popen): for child in parent.children(recursive=True): child.kill() parent.kill() + + +class FileCopier: + class Handler(FileSystemEventHandler): + def __init__(self, source_path: Path, target_path: Path, watcher): + self.source_root_path = source_path + self.target_root_path = target_path + self.watcher = watcher + print(f"Watching {source_path} -> {target_path}") + + def on_closed(self, event): + if event.is_directory: + return + + try: + time.sleep(0.01) + self.copy_file(event.src_path, self.make_target_path(event.src_path)) + except: + traceback.print_exc() + + def on_modified(self, event): + if event.is_directory: + return + + try: + time.sleep(0.1) + self.copy_file(event.src_path, self.make_target_path(event.src_path)) + except: + traceback.print_exc() + + def on_moved(self, event): + try: + time.sleep(0.01) + self.remove_file(event.src_path, self.make_target_path(event.src_path)) + self.copy_file(event.dest_path, self.make_target_path(event.dest_path)) + except: + traceback.print_exc() + + def on_created(self, event): + """When a new directory is created, add a watch for it""" + if event.is_directory: + self.watcher.schedule( + type(self)( + event.src_path, + self.make_target_path(event.src_path), + self.watcher, + ), + event.src_path, + recursive=True, + ) + + def on_deleted(self, event): + try: + time.sleep(0.01) + self.remove_file(event.src_path, self.make_target_path(event.src_path)) + except: + traceback.print_exc() + + def make_target_path(self, source_path): + return self.target_root_path / Path(source_path).relative_to( + self.source_root_path + ) + + def copy_file(self, source_path, target_path): + if str(source_path).endswith('~'): + return + print(f"Copying {source_path} to {target_path}") + target_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(source_path, target_path) + + def remove_file(self, source_path, target_path): + print(f"Removing {target_path}") + if target_path.exists(): + target_path.unlink() + + def __init__(self, config): + self.config = config + static = (config.ui_dir / "static").resolve() + + self.watched_paths = load_watched_paths( + config.invenio_instance_path / "watch.list.json", + [f"{static}=static"], + ) + print(self.watched_paths) + + self.watcher = Observer() + static_target_path = self.config.invenio_instance_path / "static" + assets_target_path = self.config.invenio_instance_path / "assets" + + for path, kind in self.watched_paths.items(): + path = Path(path).resolve() + if kind == "static": + self.watcher.schedule( + self.Handler(path, static_target_path, self.watcher), + str(path), + recursive=True, + ) + elif kind == "assets": + self.watcher.schedule( + self.Handler(path, assets_target_path, self.watcher), + str(path), + recursive=True, + ) + self.watcher.start() + + def join(self): + try: + while self.watcher.isAlive(): + self.watcher.join(1) + finally: + self.watcher.stop() + self.watcher.join() diff --git a/src/nrp_devtools/commands/model/compile.py b/src/nrp_devtools/commands/model/compile.py index 3f20f1c..e7ebf0f 100644 --- a/src/nrp_devtools/commands/model/compile.py +++ b/src/nrp_devtools/commands/model/compile.py @@ -393,4 +393,4 @@ def rewrite_revision_file( def add_model_to_i18n(config: OARepoConfig, *, model, **kwargs): i18n_config = config.i18n - i18n_config.babel_source_paths.append(model.model_package) \ No newline at end of file + i18n_config.babel_source_paths.append(model.model_package) diff --git a/src/nrp_devtools/commands/ui/link_assets.py b/src/nrp_devtools/commands/ui/link_assets.py index 922efb1..7c47dd4 100644 --- a/src/nrp_devtools/commands/ui/link_assets.py +++ b/src/nrp_devtools/commands/ui/link_assets.py @@ -9,7 +9,7 @@ from .assets import load_watched_paths -def link_assets(config: OARepoConfig): +def copy_assets_to_webpack_build_dir(config: OARepoConfig): # assets = (config.site_dir / "assets").resolve() static = (config.ui_dir / "static").resolve() @@ -31,7 +31,7 @@ def link_assets(config: OARepoConfig): continue existing[kind].add(target) - linked = {k: {} for k in kinds} + copied = {k: {} for k in kinds} for kind, source_path, source_file in tqdm( _list_source_files(watched_paths), desc="Checking paths" @@ -39,17 +39,19 @@ def link_assets(config: OARepoConfig): target_file = ( config.invenio_instance_path / kind / source_file.relative_to(source_path) ) - linked[kind][source_file] = target_file + copied[kind][source_file] = target_file if target_file in existing[kind]: existing[kind].remove(target_file) for kind, existing_data in existing.items(): - if existing_data: + to_remove = [target for target in existing_data if target.exists()] + if to_remove: click.secho( - f"Error: following {kind} are not linked, will remove those from .venv assets", + f"Error: following {kind} are not in the source directories, " + "will remove those from .venv assets", fg="red", ) - for target in existing_data: + for target in to_remove: if target.exists(): click.secho(f" {target}", fg="red") if target.is_dir(): @@ -58,14 +60,15 @@ def link_assets(config: OARepoConfig): target.unlink() for kind, source_file, target_file in tqdm( - _list_linked_files(linked), desc="Linking assets and statics" + _list_copied_files(copied), desc="Linking assets and statics" ): target_file.parent.mkdir(parents=True, exist_ok=True) try: target_file.unlink() except FileNotFoundError: pass - target_file.symlink_to(source_file) + # copy source file to target file + shutil.copy(source_file, target_file) def _list_files(kinds, base_path): @@ -84,7 +87,7 @@ def _list_source_files(watched_paths): yield kind, source_path, source_file -def _list_linked_files(linked): +def _list_copied_files(linked): for kind, linked_data in linked.items(): for source_file, target_file in linked_data.items(): yield kind, source_file, target_file From 465d10c38a55ff1c1ab2ddd30ad64fc07b93c378 Mon Sep 17 00:00:00 2001 From: Mirek Simek Date: Tue, 23 Jan 2024 14:02:25 +0100 Subject: [PATCH 2/2] Removing stale subprocesses --- src/nrp_devtools/commands/develop/runner.py | 11 ++---- src/nrp_devtools/main.py | 41 +++++++++++++++++++++ 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/src/nrp_devtools/commands/develop/runner.py b/src/nrp_devtools/commands/develop/runner.py index 7fe12c1..0ac8ec2 100644 --- a/src/nrp_devtools/commands/develop/runner.py +++ b/src/nrp_devtools/commands/develop/runner.py @@ -1,11 +1,9 @@ import os import shutil import subprocess -import threading import time import traceback from pathlib import Path -from threading import RLock from typing import Optional import click @@ -223,7 +221,7 @@ def make_target_path(self, source_path): ) def copy_file(self, source_path, target_path): - if str(source_path).endswith('~'): + if str(source_path).endswith("~"): return print(f"Copying {source_path} to {target_path}") target_path.parent.mkdir(parents=True, exist_ok=True) @@ -266,8 +264,7 @@ def __init__(self, config): def join(self): try: - while self.watcher.isAlive(): - self.watcher.join(1) - finally: self.watcher.stop() - self.watcher.join() + self.watcher.join(10) + except: + print("Could not stop watcher thread but continuing anyway") diff --git a/src/nrp_devtools/main.py b/src/nrp_devtools/main.py index 7c87190..10c6ff6 100644 --- a/src/nrp_devtools/main.py +++ b/src/nrp_devtools/main.py @@ -1,4 +1,45 @@ +import atexit +import os + +import psutil + from .cli import nrp_command + +def kill_process(process): + if not process.is_running(): + return + try: + process.kill() + try: + process.wait(1) + except psutil.TimeoutExpired: + print(f"Can not kill process {process.pid} in time") + except psutil.NoSuchProcess: + pass + except: + print(f"Can not kill process {process.pid}, got an exception") + + +def kill_process_tree(process, except_process): + try: + sub_processes = process.children() + for sub_process in sub_processes: + kill_process_tree(sub_process, except_process) + if process != except_process: + kill_process(process) + except: + print(f"Can not kill process tree of {process.pid}, got an exception") + + +def exit_handler(*args, **kwargs): + print("Exit handler called, removing subprocesses ...") + this_process = psutil.Process(os.getpid()) + kill_process_tree(this_process, except_process=this_process) + + +atexit.register(exit_handler) + + if __name__ == "__main__": nrp_command()