From 2c5f304ebd06046dbbffe50c44bbfc8099389921 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Mon, 19 Nov 2018 14:03:14 +0100 Subject: [PATCH 01/14] Added test coverage with pytest-cov to run_tests.sh --- requirements-dev.txt | 3 ++- run_tests.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 8641e68..286ed9d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ pytest-mock pytest -testfixtures \ No newline at end of file +testfixtures +pytest-cov diff --git a/run_tests.sh b/run_tests.sh index 3c2bd6a..30cef2a 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,4 +1,4 @@ #!/bin/bash export PYTHONPATH=$PYTHONPATH:$(pwd)/modules -pytest --ignore ros2 +pytest --ignore ros2 --cov=modules test From 240ed4f25ac9db54c0f601c31f8e4178695b9edf Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Sun, 25 Nov 2018 01:18:27 +0100 Subject: [PATCH 02/14] Meta files, Docs adjustments for first PyPI release --- .gitignore | 7 +- MANIFEST.in | 1 + README.md | 86 ++++++++++++++----- config/hello_world.yml | 2 +- config/telegram_hello_world.yml | 2 +- deploy.sh | 4 + modules/ravestate/argparse.py | 6 +- modules/ravestate/property.py | 1 + .../__init__.py | 0 rasta | 12 +++ requirements-dev.txt | 1 + run.py | 11 --- run.sh | 4 - setup.py | 34 ++++++++ 14 files changed, 127 insertions(+), 44 deletions(-) create mode 100644 MANIFEST.in create mode 100755 deploy.sh rename modules/{hello_world => ravestate_hello_world}/__init__.py (100%) create mode 100644 rasta delete mode 100644 run.py delete mode 100755 run.sh create mode 100644 setup.py diff --git a/.gitignore b/.gitignore index af7bc7d..3dcb474 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,11 @@ *.key *.default +.coverage +.pytest_cache .idea __pycache__ tts.wav -keys.yml \ No newline at end of file +keys.yml +dist/* +build/* +*.egg-info \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..8fb96cf --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +recursive-include modules/ravestate_phrases_basic_en * \ No newline at end of file diff --git a/README.md b/README.md index b3a957b..dc5c07f 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,41 @@ - ``` -v0.1.0 __ __ - __ ____ _ ______ ______/ /_____/ /___ - / \/ __ \/ / / /__ \/ ___\, / __ \, /__ \ -/ /\/ /_/ /\ \/ / /_/ /\__, / / /_/ / / /_/ / -\/ \__/\/ \__/ ,___/\____/\/\__/\/\/ ,___/ - \____/ \____/ + _ __ _ __ + _ ___ ____ __ ______ ______/ /_____/ /___ + _ _ / \/ __ \/ / / /__ \/ ___\, / __ \, /__ \ +_ _ / /\/ /_/ /\ \/ / /_/ /\__, / / /_/ / / /_/ / + _ \/ _\__/\/ _\__/ ,___/\____/\/\__/\/\/ ,___/ + _____ _ _\____/ _ _\____/ +/_ _\ + 0> 0> +\__⊽__/ (C) Roboy 2018 + ⋂ ``` ## About Ravestate is a reactive library for real-time natural language dialog systems. -## Requirements: +## Dependencies + +### portaudio on macOS + +In order to install PyAudio with pip, you need to install portaudio first using: + +`brew install portaudio` -- Python >= 3.6 ## Installation -To install dependencies use: +### Via PIP + +The easiest way to install ravestate is through pip: + +`` +pip install ravestate +`` + +### For developers + +First, install dependencies: ```bash pip install -r requirements.txt @@ -26,24 +44,46 @@ pip install -r requirements.txt pip install -r requirements-dev.txt ``` -### Mac - -#### PyAudio +Then, you may open the repository in any IDE, and mark the +`modules` folder as a sources root. -In order to install PyAudio with pip, you need to install portaudio first using: - -`` -brew install portaudio -`` ## Running Hello World Ravestate applications are defined by a configuration, -that specifies the ravestate modules that should be loaded. +which specifies the ravestate modules that should be loaded. + +To run the basic hello world application, run ravestate +with a config file or command line arguments: -To run the basic hello world application, run ravestate with the proper config file: +### Running with command line spec + +You can easily run a combination of ravestate modules in a shared context, +by listing them as arguments to the `rasta` command, which is installed +with ravestate: ```bash -./run.sh -f config/hello_world.yml +rasta ravestate_conio ravestate_hello_world +``` + +Run `rasta -h` to see more options! + +### Running with config file(s) + +You may specify a series of config files to configure ravestate context, +when specifying everything through the command line becomes too laborious: + +```yaml +# In file hello_world.yml +module: core +config: + import: + - ravestate_conio + - ravestate_hello_world +``` +Then, run `rasta` with this config file: + +```bash +rasta -f hello_world.yml ``` ## Running tests @@ -51,6 +91,6 @@ To run the basic hello world application, run ravestate with the proper config f If you have installed the dependencies from ``requirements-dev.txt`` you may run the ravestate test suite as follows: -```bash +`` ./run_tests.sh -``` +`` diff --git a/config/hello_world.yml b/config/hello_world.yml index ea61c05..d78a1ed 100644 --- a/config/hello_world.yml +++ b/config/hello_world.yml @@ -2,4 +2,4 @@ module: core config: import: - ravestate_conio - - hello_world \ No newline at end of file + - ravestate_hello_world \ No newline at end of file diff --git a/config/telegram_hello_world.yml b/config/telegram_hello_world.yml index 8e75bf3..1562481 100644 --- a/config/telegram_hello_world.yml +++ b/config/telegram_hello_world.yml @@ -13,4 +13,4 @@ config: import: - ravestate_telegramio - ravestate_conio - - hello_world \ No newline at end of file + - ravestate_hello_world \ No newline at end of file diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 0000000..a6bf3ec --- /dev/null +++ b/deploy.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +python3 setup.py sdist bdist_wheel +twine upload dist/* \ No newline at end of file diff --git a/modules/ravestate/argparse.py b/modules/ravestate/argparse.py index 72cdffa..cf8d44c 100644 --- a/modules/ravestate/argparse.py +++ b/modules/ravestate/argparse.py @@ -27,11 +27,11 @@ def handle_args(*args) -> Tuple[List[str], List[Tuple[str, str, Any]], List[str] formatter_class=RawDescriptionHelpFormatter, epilog=""" usage: - > run.py ravestate_facerec hello_world + > rasta ravestate_facerec ravestate_hello_world Import two python modules and run a context. - > run.py \\ - -d core import ravestate_facerec hello_world \\ + > rasta \\ + -d core import ravestate_facerec ravestate_hello_world \\ -f generic.yml \\ -f user.yml Import two python modules and run a context, diff --git a/modules/ravestate/property.py b/modules/ravestate/property.py index 60048b8..7c0a357 100644 --- a/modules/ravestate/property.py +++ b/modules/ravestate/property.py @@ -3,6 +3,7 @@ from threading import Lock import logging + class PropertyBase: """ Base class for context properties. Controls read/write/push/pop/delete permissions, diff --git a/modules/hello_world/__init__.py b/modules/ravestate_hello_world/__init__.py similarity index 100% rename from modules/hello_world/__init__.py rename to modules/ravestate_hello_world/__init__.py diff --git a/rasta b/rasta new file mode 100644 index 0000000..1c76f0e --- /dev/null +++ b/rasta @@ -0,0 +1,12 @@ +#!python3 + +import sys + +from ravestate.context import Context + +ctx = Context(*sys.argv[1:]) +ctx.run() + +# TODO: Make sure that UI is cleanly integrated as a ravestate module. +# from ravestate_ui import service +# service.advertise(ctx=ctx) diff --git a/requirements-dev.txt b/requirements-dev.txt index 286ed9d..3f7418c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,3 +2,4 @@ pytest-mock pytest testfixtures pytest-cov +twine diff --git a/run.py b/run.py deleted file mode 100644 index 8332d41..0000000 --- a/run.py +++ /dev/null @@ -1,11 +0,0 @@ - -import sys - -from ravestate.context import Context -from ravestate_ui import service - - -ctx = Context(*sys.argv[1:]) -ctx.run() -service.advertise(ctx=ctx) -ctx.shutdown() diff --git a/run.sh b/run.sh deleted file mode 100755 index 112af39..0000000 --- a/run.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -export PYTHONPATH=$PYTHONPATH:$(pwd)/modules -python3 run.py "$@" \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..9ec91f4 --- /dev/null +++ b/setup.py @@ -0,0 +1,34 @@ +import setuptools + +with open("README.md", "r") as fh: + long_description = fh.read() + +with open("requirements.txt", "r") as freq: + required = freq.read().split() + +setuptools.setup( + name="ravestate", + version="0.1.post3", + url="https://github.com/roboy/ravestate", + author="Roboy", + author_email="info@roboy.org", + + description="Ravestate is a reactive library for real-time natural language dialog systems.", + long_description=long_description, + long_description_content_type="text/markdown", + + package_dir={'': 'modules'}, + packages=setuptools.find_packages("modules"), + include_package_data=True, + scripts=["rasta"], + + install_requires=required, + python_requires='>=3.6', + + # TODO: Add classifiers + # classifiers=[ + # "Programming Language :: Python :: 3", + # "License :: OSI Approved :: MIT License", + # "Operating System :: OS Independent", + # ], +) From 5f064d1f9a33875b932470205b9a76817ec39b96 Mon Sep 17 00:00:00 2001 From: Andreas Dolp Date: Fri, 30 Nov 2018 00:15:28 +0100 Subject: [PATCH 03/14] Implement constraints for triggers Configure rest of ravestate to use the new signals --- modules/hello_world/__init__.py | 4 +- modules/ravestate/activation.py | 17 +- modules/ravestate/constraint.py | 190 ++++++++++++++++++ modules/ravestate/context.py | 52 ++--- modules/ravestate/state.py | 17 +- modules/ravestate/wrappers.py | 4 +- modules/ravestate_conio/__init__.py | 3 +- modules/ravestate_facerec/__init__.py | 5 +- modules/ravestate_telegramio/telegram_bot.py | 3 +- test/modules/ravestate/test_activation.py | 16 +- test/modules/ravestate/test_constraint.py | 48 +++++ test/modules/ravestate/test_context.py | 5 +- test/modules/ravestate/test_state.py | 18 +- .../ravestate/test_wrappers_property.py | 3 +- 14 files changed, 328 insertions(+), 57 deletions(-) create mode 100644 modules/ravestate/constraint.py create mode 100644 test/modules/ravestate/test_constraint.py diff --git a/modules/hello_world/__init__.py b/modules/hello_world/__init__.py index 6c9312a..35a4e49 100644 --- a/modules/hello_world/__init__.py +++ b/modules/hello_world/__init__.py @@ -5,8 +5,10 @@ import ravestate_verbaliser import ravestate_phrases_basic_en +from ravestate.constraint import s -@state(triggers=":startup", write="verbaliser:intent") + +@state(triggers=s(":startup"), write="verbaliser:intent") def hello_world(ctx): ctx["verbaliser:intent"] = "greeting" diff --git a/modules/ravestate/activation.py b/modules/ravestate/activation.py index f10def9..524fc4c 100644 --- a/modules/ravestate/activation.py +++ b/modules/ravestate/activation.py @@ -1,16 +1,19 @@ # Ravestate class which encapsualtes the activation of a single state +from typing import List, Set from ravestate import state from ravestate import wrappers from ravestate import icontext from threading import Thread +from ravestate.constraint import Signal, Constraint, s + class StateActivation: def __init__(self, st: state.State, ctx: icontext.IContext): self.state_to_activate = st - self.unfulfilled = [set(trigger_clause) for trigger_clause in st.triggers] + self.unfulfilled: Constraint = st.triggers self.ctx = ctx self.args = [] self.kwargs = {} @@ -19,13 +22,9 @@ def specificity(self): # TODO: Calculate specificity properly return 1. - def notify_signal(self, signal_name: str): - for unfulfilled_trigger in self.unfulfilled: - if signal_name in unfulfilled_trigger: - unfulfilled_trigger.remove(signal_name) - if len(unfulfilled_trigger) == 0: - return 1 - return 0 + def notify_signal(self, signal: Signal): + self.unfulfilled.set_signal_true(signal) + return 1 if self.unfulfilled.evaluate() else 0 def run(self, args=(), kwargs={}): self.args = args @@ -36,6 +35,6 @@ def _run_private(self): context_wrapper = wrappers.ContextWrapper(self.ctx, self.state_to_activate) result = self.state_to_activate(context_wrapper, self.args, self.kwargs) if isinstance(result, state.Emit) and self.state_to_activate.signal: - self.ctx.emit(self.state_to_activate.signal_name()) + self.ctx.emit(s(self.state_to_activate.signal_name())) # TODO is the right signal emitted? if isinstance(result, state.Delete): self.ctx.rm_state(st=self.state_to_activate) diff --git a/modules/ravestate/constraint.py b/modules/ravestate/constraint.py new file mode 100644 index 0000000..d582a35 --- /dev/null +++ b/modules/ravestate/constraint.py @@ -0,0 +1,190 @@ +import logging +from typing import List, Set + + +def s(signalname: str): + """ + Alias to call Signal-constructor + + :param signalname: Name of the Signal + """ + return Signal(signalname) + + +class Constraint: + """ + Superclass for Signal, Conjunct and Disjunct + """ + + def get_all_signals(self) -> Set: + logging.error("Don't call this method on the super class Constraint") + return set() + + def set_signal_true(self, signal): + logging.error("Don't call this method on the super class Constraint") + pass + + def evaluate(self) -> bool: + logging.error("Don't call this method on the super class Constraint") + return False + + +class Signal(Constraint): + """ + Class that represents a Signal + """ + name: str = "" + fulfilled: bool = False + + def __init__(self, name: str): + self.name = name + + def __or__(self, other): + if isinstance(other, Signal): + return Disjunct(Conjunct(self), Conjunct(other)) + elif isinstance(other, Conjunct): + return Disjunct(Conjunct(self), other) + elif isinstance(other, Disjunct): + return Disjunct(self, *other) + + def __and__(self, other): + if isinstance(other, Signal): + return Conjunct(self, other) + elif isinstance(other, Conjunct): + return Conjunct(self, *other) + elif isinstance(other, Disjunct): + conjunct_list: List[Conjunct] = [] + for conjunct in other: + conjunct_list.append(Conjunct(*conjunct, self)) + return Disjunct(*conjunct_list) + + def __eq__(self, other): + return isinstance(other, Signal) and self.name == other.name + + def __hash__(self): + return hash(self.name) + + def get_all_signals(self) -> Set: + return {self} + + def set_signal_true(self, signal): + if self == signal: + self.fulfilled = True + + def evaluate(self) -> bool: + return self.fulfilled + + def __str__(self): + return self.name + + +class Conjunct(Constraint): + """ + Class that represents a Conjunction of Signals + """ + _signals: Set[Signal] = set() + + def __init__(self, *args): + for arg in args: + if not isinstance(arg, Signal): + logging.error("Conjunct can only be constructed with Signals.") + raise ValueError + self._signals = set(args) + + def __iter__(self): + for signal in self._signals: + yield signal + + def __or__(self, other): + if isinstance(other, Signal): + return Disjunct(self, Conjunct(other)) + elif isinstance(other, Conjunct): + return Disjunct(self, other) + elif isinstance(other, Disjunct): + return Disjunct(self, *other) + + def __and__(self, other): + if isinstance(other, Signal): + return Conjunct(other, *self) + elif isinstance(other, Conjunct): + return Conjunct(*self, *other) + elif isinstance(other, Disjunct): + conjunct_list: List[Conjunct] = [] + for conjunct in other: + conjunct_list.append(Conjunct(*conjunct, *self)) + return Disjunct(*conjunct_list) + + def get_all_signals(self) -> Set[Signal]: + return self._signals + + def set_signal_true(self, signal: Signal): + for si in self._signals: + si.set_signal_true(signal) + + def evaluate(self) -> bool: + return all(map(lambda si: si.evaluate(), self._signals)) + + def __str__(self): + ret_str = "" + for si in self._signals: + ret_str += f" {si.name} &" + ret_str = ret_str.strip('&') + return ret_str + + +class Disjunct(Constraint): + """ + Class that represents a Disjunction of Conjunctions + """ + _conjunctions: Set[Conjunct] = set() + + def __init__(self, *args): + for arg in args: + if not isinstance(arg, Conjunct): + logging.error("Disjunct can only be constructed with conjuncts.") + raise ValueError + self._conjunctions = set(args) + + def __iter__(self): + for conjunct in self._conjunctions: + yield conjunct + + def __or__(self, other): + if isinstance(other, Signal): + return Disjunct(*self, Conjunct(other)) + elif isinstance(other, Conjunct): + return Disjunct(*self, other) + elif isinstance(other, Disjunct): + return Disjunct(*self, *other) + + def __and__(self, other): + if isinstance(other, Signal): + conjunct_list: List[Conjunct] = [] + for conjunct in self: + conjunct_list.append(Conjunct(*conjunct, other)) + return Disjunct(*conjunct_list) + elif isinstance(other, Conjunct): + conjunct_list: List[Conjunct] = [] + for conjunct in self: + conjunct_list.append(Conjunct(*conjunct, *other)) + return Disjunct(*conjunct_list) + elif isinstance(other, Disjunct): + logging.error("Can't conjunct two disjunctions.") + raise ValueError("Can't conjunct two disjunctions.") + + def get_all_signals(self) -> Set[Signal]: + return {signal for conjunct in self._conjunctions for signal in conjunct._signals} + + def set_signal_true(self, signal: Signal): + for conjunct in self._conjunctions: + conjunct.set_signal_true(signal) + + def evaluate(self) -> bool: + return any(map(lambda si: si.evaluate(), self._conjunctions)) + + def __str__(self): + ret_str = "" + for conjunct in self._conjunctions: + ret_str += f"({conjunct}) |" + ret_str = ret_str.strip('|') + return ret_str diff --git a/modules/ravestate/context.py b/modules/ravestate/context.py index 15ba17f..d2142f0 100644 --- a/modules/ravestate/context.py +++ b/modules/ravestate/context.py @@ -3,7 +3,7 @@ import importlib from threading import Thread, Lock, Semaphore -from typing import Optional, Any +from typing import Optional, Any, Tuple, List, Set, Dict import logging from ravestate import icontext @@ -14,12 +14,13 @@ from ravestate import registry from ravestate import argparse from ravestate.config import Configuration +from ravestate.constraint import s, Signal class Context(icontext.IContext): - default_signals = (":startup", ":shutdown", ":idle") - default_property_signals = (":changed", ":pushed", ":popped", ":deleted") + default_signals: Tuple[Signal] = (s(":startup"), s(":shutdown"), s(":idle")) + default_property_signals: Tuple[Signal] = (s(":changed"), s(":pushed"), s(":popped"), s(":deleted")) core_module_name = "core" import_modules_config = "import" @@ -36,7 +37,7 @@ def __init__(self, *arguments): } self.config.add_conf(module.Module(name=self.core_module_name, config=self.core_config)) - self.signal_queue = [] + self.signal_queue: List[Signal] = [] self.signal_queue_lock = Lock() self.signal_queue_counter = Semaphore(0) self.run_task = None @@ -45,7 +46,7 @@ def __init__(self, *arguments): self.activation_candidates = dict() self.states = set() - self.states_per_signal = {signal_name: set() for signal_name in self.default_signals} + self.states_per_signal: Dict[Signal, Set] = {signal: set() for signal in self.default_signals} self.states_lock = Lock() # Set required config overrides @@ -55,14 +56,14 @@ def __init__(self, *arguments): for module_name in self.core_config[self.import_modules_config]+modules: self.add_module(module_name) - def emit(self, signal_name: str) -> None: + def emit(self, signal: Signal) -> None: """ Emit a signal to the signal processing loop. Note: The signal will only be processed if run() has been called! - :param signal_name: The name of the signal to be emitted. + :param signal: The signal to be emitted. """ with self.signal_queue_lock: - self.signal_queue.append(signal_name) + self.signal_queue.append(signal) self.signal_queue_counter.release() def run(self) -> None: @@ -74,7 +75,7 @@ def run(self) -> None: return self.run_task = Thread(target=self._run_private) self.run_task.start() - self.emit(":startup") + self.emit(s(":startup")) def shutting_down(self) -> bool: """ @@ -87,7 +88,7 @@ def shutdown(self) -> None: Sets the shutdown flag and waits for the signal processing thread to join. """ self.shutdown_flag = True - self.emit(":shutdown") + self.emit(s(":shutdown")) self.run_task.join() def add_module(self, module_name: str) -> None: @@ -122,13 +123,17 @@ def add_state(self, *, st: state.State) -> None: with self.states_lock: if st.signal: self.states_per_signal[st.signal] = set() + # check to recognize states using old signal implementation + if isinstance(st.triggers, str): + logging.error(f"Attempt to add state which depends on a signal `{st.triggers}` " + f"defined as a String and not Signal.") + # make sure that all of the state's depended-upon signals exist - for clause in st.triggers: - for signal in clause: - if signal in self.states_per_signal: - self.states_per_signal[signal].add(st) - else: - logging.error(f"Attempt to add state which depends on unknown signal `{signal}`!") + for signal in st.triggers.get_all_signals(): + if signal in self.states_per_signal: + self.states_per_signal[signal].add(st) + else: + logging.error(f"Attempt to add state which depends on unknown signal `{signal}`!") self.states.add(st) def rm_state(self, *, st: state.State) -> None: @@ -143,8 +148,7 @@ def rm_state(self, *, st: state.State) -> None: with self.states_lock: if st.signal: self.states_per_signal.pop(st.signal) - for clause in st.triggers: - for signal in clause: + for signal in st.triggers.get_all_signals(): self.states_per_signal[signal].remove(st) self.states.remove(st) @@ -162,7 +166,7 @@ def add_prop(self, *, prop: property.PropertyBase) -> None: # register all of the property's signals with self.states_lock: for signal in self.default_property_signals: - self.states_per_signal[prop.fullname()+signal] = set() + self.states_per_signal[s(prop.fullname()+signal.name)] = set() def get_prop(self, key: str) -> Optional[property.PropertyBase]: """ @@ -209,13 +213,13 @@ def _run_private(self): # TODO: Recognize and signal Idleness self.signal_queue_counter.acquire() with self.signal_queue_lock: - signal_name = self.signal_queue.pop(0) + signal = self.signal_queue.pop(0) # collect states which depend on the new signal, # and create state activation objects for them if necessary - logging.debug(f"Received {signal_name} ...") + logging.debug(f"Received {signal.name} ...") with self.states_lock: - for state in self.states_per_signal[signal_name]: + for state in self.states_per_signal[signal]: if state.name not in self.activation_candidates: self.activation_candidates[state.name] = activation.StateActivation(state, self) @@ -229,8 +233,8 @@ def _run_private(self): # go through candidates and remove those which want to be removed, # remember those which want to be remembered, forget those which want to be forgotten for state_name, act in current_activation_candidates.items(): - notify_return = act.notify_signal(signal_name) - logging.debug(f"-> {act.state_to_activate.name} returned {notify_return} on notify_signal {signal_name}") + notify_return = act.notify_signal(signal) + logging.debug(f"-> {act.state_to_activate.name} returned {notify_return} on notify_signal {signal.name}") if notify_return == 0: self.activation_candidates[state_name] = act elif notify_return > 0: diff --git a/modules/ravestate/state.py b/modules/ravestate/state.py index 2aa7870..879e5b4 100644 --- a/modules/ravestate/state.py +++ b/modules/ravestate/state.py @@ -3,6 +3,8 @@ import logging from typing import Callable, Optional, Any +from ravestate.constraint import Conjunct, Disjunct, s, Constraint + class StateActivationResult: """ @@ -34,7 +36,7 @@ def __init__(self, *, signal, write, read, triggers, action, is_receptor=False): self.name = action.__name__ # catch the insane case - if not len(read) and not len(triggers) and not is_receptor: + if not len(read) and not triggers and not is_receptor: logging.warning( f"The state `{self.name}` is not reading any properties, nor waiting for any triggers. " + "It will never be activated!") @@ -47,13 +49,8 @@ def __init__(self, *, signal, write, read, triggers, action, is_receptor=False): # listen to default changed-signals if no signals are given. # convert triggers to disjunctive normal form. - if not len(triggers): - triggers = tuple((f"{rprop_name}:changed",) for rprop_name in read) - else: - if isinstance(triggers, str): - triggers = (triggers,) - if isinstance(triggers[0], str): - triggers = (triggers,) + if not triggers and len(read) > 0: + triggers = Disjunct(*list(Conjunct(s(f"{rprop_name}:changed")) for rprop_name in read)) self.signal = signal self.write_props = write @@ -66,10 +63,10 @@ def __call__(self, context, args, kwargs) -> StateActivationResult: return self.action(context, *args, **kwargs) def signal_name(self): - return f"{self.module_name}:{self.signal}" + return f"{self.module_name}:{self.signal.signalname}" -def state(*, signal: str="", write: tuple=(), read: tuple=(), triggers: tuple=()): +def state(*, signal: str="", write: tuple=(), read: tuple=(), triggers: Constraint=None): """ Decorator to declare a new state, which may emit a certain signal, write to a certain set of properties (calling set, push, pop, delete), diff --git a/modules/ravestate/wrappers.py b/modules/ravestate/wrappers.py index b2b588e..15dcaa6 100644 --- a/modules/ravestate/wrappers.py +++ b/modules/ravestate/wrappers.py @@ -8,6 +8,8 @@ from ravestate import registry from typing import Any +from ravestate.constraint import s + class PropertyWrapper: """ @@ -55,7 +57,7 @@ def set(self, value): logging.error(f"Unauthorized write access in property-wrapper {self.prop.name}!") return False if self.prop.write(value): - self.ctx.emit(f"{self.prop.fullname()}:changed") + self.ctx.emit(s(f"{self.prop.fullname()}:changed")) class ContextWrapper: diff --git a/modules/ravestate_conio/__init__.py b/modules/ravestate_conio/__init__.py index c2aa8dd..6158581 100644 --- a/modules/ravestate_conio/__init__.py +++ b/modules/ravestate_conio/__init__.py @@ -1,5 +1,6 @@ from ravestate import registry +from ravestate.constraint import s from ravestate.state import state from ravestate.receptor import receptor import ravestate_rawio @@ -12,7 +13,7 @@ def console_shutdown(ctx): ctx.shutdown() -@state(triggers=":startup") +@state(triggers=s(":startup")) def console_input(ctx): @receptor(ctx_wrap=ctx, write="rawio:in") diff --git a/modules/ravestate_facerec/__init__.py b/modules/ravestate_facerec/__init__.py index eaef928..bea150f 100644 --- a/modules/ravestate_facerec/__init__.py +++ b/modules/ravestate_facerec/__init__.py @@ -1,6 +1,7 @@ import rclpy from ravestate import registry +from ravestate.constraint import s from ravestate.state import state from ravestate.receptor import receptor from ravestate.property import PropertyBase @@ -10,7 +11,7 @@ rclpy.init() node = rclpy.create_node("vision_node") -@state(triggers=":startup") +@state(triggers=s(":startup")) def facerec_run(ctx): @receptor(ctx_wrap=ctx, write="facerec:face") @@ -21,7 +22,7 @@ def face_recognition_callback(ctx, msg): rclpy.spin(node) -@state(triggers=":shutdown") +@state(triggers=s(":shutdown")) def facerec_shutdown(): node.destroy_node() rclpy.shutdown() diff --git a/modules/ravestate_telegramio/telegram_bot.py b/modules/ravestate_telegramio/telegram_bot.py index 53311db..0e42893 100644 --- a/modules/ravestate_telegramio/telegram_bot.py +++ b/modules/ravestate_telegramio/telegram_bot.py @@ -1,6 +1,7 @@ import os from typing import Set +from ravestate.constraint import s from ravestate.receptor import receptor from ravestate.state import state, Delete from ravestate.wrappers import ContextWrapper @@ -14,7 +15,7 @@ active_chats: Set[int] = set() -@state(triggers=":startup") +@state(triggers=s(":startup")) def telegram_run(ctx: ContextWrapper): """ Starts up the telegram bot and adds a handler to write incoming messages to rawio:in diff --git a/test/modules/ravestate/test_activation.py b/test/modules/ravestate/test_activation.py index 5142628..2784294 100644 --- a/test/modules/ravestate/test_activation.py +++ b/test/modules/ravestate/test_activation.py @@ -2,6 +2,7 @@ from threading import Thread from ravestate.activation import StateActivation +from ravestate.constraint import s from ravestate.state import State from ravestate.icontext import IContext @@ -9,7 +10,7 @@ @pytest.fixture def state_mock(mocker): state = mocker.Mock(name=State.__class__) - state.triggers = [['test']] + state.triggers = (s('test1') & s('test2')) | s('test3') return state @@ -38,13 +39,24 @@ def test_specifity(under_test): def test_notify_signal(under_test): - assert under_test.notify_signal('test') == 1 + assert under_test.notify_signal(s('test1')) == 0 + assert under_test.notify_signal(s('test2')) == 1 + + +def test_notify_signal_2(under_test): + assert under_test.notify_signal(s('test1')) == 0 + assert under_test.notify_signal(s('test3')) == 1 def test_notify_signal_mismatched(under_test): assert under_test.notify_signal('') == 0 +def test_notify_signal_mismatched_2(under_test): + assert under_test.notify_signal(s('test1')) == 0 + assert under_test.notify_signal(s('notest')) == 0 + + # TODO: Add tests for private run def test_run(under_test, default_args, default_kwargs): result = under_test.run(default_args, default_kwargs) diff --git a/test/modules/ravestate/test_constraint.py b/test/modules/ravestate/test_constraint.py new file mode 100644 index 0000000..7a472f5 --- /dev/null +++ b/test/modules/ravestate/test_constraint.py @@ -0,0 +1,48 @@ +import pytest +from ravestate.constraint import s + + +def test_signal(): + sig = s("mysig") + assert not sig.evaluate() + assert sig.get_all_signals() == {s("mysig")} + sig.set_signal_true(s("notmysig")) + assert not sig.evaluate() + sig.set_signal_true(s("mysig")) + assert sig.evaluate() + + sig_and_dis = s("sig") & (s("dis") | s("junct")) + assert not sig_and_dis.evaluate() + sig_and_dis.set_signal_true(s("sig")) + assert not sig_and_dis.evaluate() + sig_and_dis.set_signal_true(s("junct")) + assert sig_and_dis.evaluate() + + +def test_conjunct(): + conjunct = s("sig1") & s("sig2") & s("sig3") + assert not conjunct.evaluate() + assert conjunct.get_all_signals() == {s("sig1"), s("sig2"), s("sig3")} + conjunct.set_signal_true(s("sig1")) + assert not conjunct.evaluate() + conjunct.set_signal_true(s("sig2")) + assert not conjunct.evaluate() + conjunct.set_signal_true(s("sig2")) + assert not conjunct.evaluate() + conjunct.set_signal_true(s("sig3")) + assert conjunct.evaluate() + + +def test_disjunct(): + disjunct = (s("sig1") & s("sig2")) | s("sig3") + assert not disjunct.evaluate() + assert disjunct.get_all_signals() == {s("sig1"), s("sig2"), s("sig3")} + disjunct.set_signal_true(s("sig1")) + assert not disjunct.evaluate() + disjunct.set_signal_true(s("sig3")) + assert disjunct.evaluate() + + +def test_legal(): + with pytest.raises(ValueError): + v = (s("i") | s("am")) & (s("also") | s("illegal")) diff --git a/test/modules/ravestate/test_context.py b/test/modules/ravestate/test_context.py index ee71744..b5ea453 100644 --- a/test/modules/ravestate/test_context.py +++ b/test/modules/ravestate/test_context.py @@ -1,4 +1,5 @@ import pytest +from ravestate.constraint import s from testfixtures import LogCapture from ravestate.icontext import IContext @@ -38,7 +39,7 @@ def test_run(mocker, under_test): under_test.emit = mocker.stub() under_test.shutdown_flag = True under_test.run() - under_test.emit.assert_called_once_with(':startup') + under_test.emit.assert_called_once_with(s(':startup')) def test_run_error(under_test): @@ -66,7 +67,7 @@ def test_shutdown(mocker, under_test): under_test.emit = mocker.stub(name='emit') under_test.shutdown() - under_test.emit.assert_called_once_with(':shutdown') + under_test.emit.assert_called_once_with(s(':shutdown')) under_test.run_task.join.assert_called_once() diff --git a/test/modules/ravestate/test_state.py b/test/modules/ravestate/test_state.py index a9398d5..b56fd0b 100644 --- a/test/modules/ravestate/test_state.py +++ b/test/modules/ravestate/test_state.py @@ -1,11 +1,13 @@ import pytest +from ravestate.constraint import s from ravestate.state import State, state from ravestate.wrappers import ContextWrapper + @pytest.fixture def default_signal(): - return "test-signal" + return s("test-signal") @pytest.fixture @@ -20,7 +22,7 @@ def default_write(): @pytest.fixture def default_triggers(): - return ":idle" + return s(":idle") @pytest.fixture @@ -60,6 +62,16 @@ def test_state(_): assert (isinstance(test_state.action, type(under_test.action))) +def test_decorator_illegal_trigger(under_test, default_signal, default_read, default_write, default_action): + with pytest.raises(ValueError): + @state(signal=default_signal, + read=default_read, + write=default_write, + triggers=(s("rawio:in:changed") | s("facerec:face:changed")) & (s("sys:has-internet") | s("foo:poo"))) + def test_state(_): + return "Hello world!" + + def test_decorator_default(under_test): @state() def test_state(_): @@ -68,6 +80,6 @@ def test_state(_): assert (test_state.signal == "") assert (test_state.read_props == ()) assert (test_state.write_props == ()) - assert (test_state.triggers == ()) + assert (test_state.triggers is None) assert (test_state(default_context_wrapper, [], {}) == "Hello world!") assert (isinstance(test_state.action, type(under_test.action))) diff --git a/test/modules/ravestate/test_wrappers_property.py b/test/modules/ravestate/test_wrappers_property.py index 80c39e7..e09b16d 100644 --- a/test/modules/ravestate/test_wrappers_property.py +++ b/test/modules/ravestate/test_wrappers_property.py @@ -1,4 +1,5 @@ import pytest +from ravestate.constraint import s from testfixtures import LogCapture from ravestate.icontext import IContext @@ -80,4 +81,4 @@ def test_property_write(under_test_read_write: PropertyWrapper, default_property assert (default_property_base._lock.locked()) under_test_read_write.set(NEW_PROPERTY_VALUE) assert (under_test_read_write.get() == NEW_PROPERTY_VALUE) - context_mock.emit.assert_called_once_with(f"{under_test_read_write.prop.fullname()}:changed") + context_mock.emit.assert_called_once_with(s(f"{under_test_read_write.prop.fullname()}:changed")) From b58ddd47fa3cb6e6f70148d431ec70f5e94724f4 Mon Sep 17 00:00:00 2001 From: Andreas Dolp Date: Fri, 30 Nov 2018 12:06:41 +0100 Subject: [PATCH 04/14] add deepcopy to use activation multiple times, define default signals as strings, better __str__ implementation --- modules/ravestate/activation.py | 4 ++-- modules/ravestate/constraint.py | 12 ++---------- modules/ravestate/context.py | 10 +++++----- test/modules/ravestate/test_activation.py | 9 +++++++++ 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/modules/ravestate/activation.py b/modules/ravestate/activation.py index 524fc4c..a3d73ca 100644 --- a/modules/ravestate/activation.py +++ b/modules/ravestate/activation.py @@ -1,5 +1,5 @@ # Ravestate class which encapsualtes the activation of a single state -from typing import List, Set +import copy from ravestate import state from ravestate import wrappers @@ -13,7 +13,7 @@ class StateActivation: def __init__(self, st: state.State, ctx: icontext.IContext): self.state_to_activate = st - self.unfulfilled: Constraint = st.triggers + self.unfulfilled: Constraint = copy.deepcopy(st.triggers) self.ctx = ctx self.args = [] self.kwargs = {} diff --git a/modules/ravestate/constraint.py b/modules/ravestate/constraint.py index d582a35..7c682fa 100644 --- a/modules/ravestate/constraint.py +++ b/modules/ravestate/constraint.py @@ -125,11 +125,7 @@ def evaluate(self) -> bool: return all(map(lambda si: si.evaluate(), self._signals)) def __str__(self): - ret_str = "" - for si in self._signals: - ret_str += f" {si.name} &" - ret_str = ret_str.strip('&') - return ret_str + return "(" + " & ".join(map(lambda si: si.__str__(), self._signals)) + ")" class Disjunct(Constraint): @@ -183,8 +179,4 @@ def evaluate(self) -> bool: return any(map(lambda si: si.evaluate(), self._conjunctions)) def __str__(self): - ret_str = "" - for conjunct in self._conjunctions: - ret_str += f"({conjunct}) |" - ret_str = ret_str.strip('|') - return ret_str + return " | ".join(map(lambda conjunct: conjunct.__str__(), self._conjunctions)) diff --git a/modules/ravestate/context.py b/modules/ravestate/context.py index d2142f0..63fcb4c 100644 --- a/modules/ravestate/context.py +++ b/modules/ravestate/context.py @@ -19,8 +19,8 @@ class Context(icontext.IContext): - default_signals: Tuple[Signal] = (s(":startup"), s(":shutdown"), s(":idle")) - default_property_signals: Tuple[Signal] = (s(":changed"), s(":pushed"), s(":popped"), s(":deleted")) + default_signal_names: Tuple[str] = (":startup", ":shutdown", ":idle") + default_property_signal_names: Tuple[str] = (":changed", ":pushed", ":popped", ":deleted") core_module_name = "core" import_modules_config = "import" @@ -46,7 +46,7 @@ def __init__(self, *arguments): self.activation_candidates = dict() self.states = set() - self.states_per_signal: Dict[Signal, Set] = {signal: set() for signal in self.default_signals} + self.states_per_signal: Dict[Signal, Set] = {s(signal_name): set() for signal_name in self.default_signal_names} self.states_lock = Lock() # Set required config overrides @@ -165,8 +165,8 @@ def add_prop(self, *, prop: property.PropertyBase) -> None: self.properties[prop.fullname()] = prop # register all of the property's signals with self.states_lock: - for signal in self.default_property_signals: - self.states_per_signal[s(prop.fullname()+signal.name)] = set() + for signalname in self.default_property_signal_names: + self.states_per_signal[s(prop.fullname() + signalname)] = set() def get_prop(self, key: str) -> Optional[property.PropertyBase]: """ diff --git a/test/modules/ravestate/test_activation.py b/test/modules/ravestate/test_activation.py index 2784294..126255f 100644 --- a/test/modules/ravestate/test_activation.py +++ b/test/modules/ravestate/test_activation.py @@ -57,6 +57,15 @@ def test_notify_signal_mismatched_2(under_test): assert under_test.notify_signal(s('notest')) == 0 +def test_multiple_activation(state_mock, context_mock): + sa1 = StateActivation(state_mock, context_mock) + assert sa1.notify_signal(s('test1')) == 0 + assert sa1.notify_signal(s('test3')) == 1 + sa2 = StateActivation(state_mock, context_mock) + assert sa2.notify_signal(s('test1')) == 0 + assert sa2.notify_signal(s('test3')) == 1 + + # TODO: Add tests for private run def test_run(under_test, default_args, default_kwargs): result = under_test.run(default_args, default_kwargs) From 1d347a7c85bd551afd77f2579809456567c8fc3f Mon Sep 17 00:00:00 2001 From: Andreas Dolp Date: Fri, 30 Nov 2018 15:15:00 +0100 Subject: [PATCH 05/14] remove TODO --- modules/ravestate/activation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ravestate/activation.py b/modules/ravestate/activation.py index a3d73ca..ddcaf94 100644 --- a/modules/ravestate/activation.py +++ b/modules/ravestate/activation.py @@ -35,6 +35,6 @@ def _run_private(self): context_wrapper = wrappers.ContextWrapper(self.ctx, self.state_to_activate) result = self.state_to_activate(context_wrapper, self.args, self.kwargs) if isinstance(result, state.Emit) and self.state_to_activate.signal: - self.ctx.emit(s(self.state_to_activate.signal_name())) # TODO is the right signal emitted? + self.ctx.emit(s(self.state_to_activate.signal_name())) if isinstance(result, state.Delete): self.ctx.rm_state(st=self.state_to_activate) From 97e9a1f59aab1f9ab858316bfd8f43acfb0e3914 Mon Sep 17 00:00:00 2001 From: Emilka Date: Sat, 1 Dec 2018 16:08:42 +0100 Subject: [PATCH 06/14] Initial commit --- config/hello_world.yml | 2 +- modules/ravestate_nlp/__init__.py | 37 +++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 modules/ravestate_nlp/__init__.py diff --git a/config/hello_world.yml b/config/hello_world.yml index d78a1ed..8975ab3 100644 --- a/config/hello_world.yml +++ b/config/hello_world.yml @@ -2,4 +2,4 @@ module: core config: import: - ravestate_conio - - ravestate_hello_world \ No newline at end of file + - ravestate_nlp \ No newline at end of file diff --git a/modules/ravestate_nlp/__init__.py b/modules/ravestate_nlp/__init__.py new file mode 100644 index 0000000..1d6b37f --- /dev/null +++ b/modules/ravestate_nlp/__init__.py @@ -0,0 +1,37 @@ + +from ravestate import registry +from ravestate.constraint import s +from ravestate.property import PropertyBase +from ravestate.state import state +from ravestate.receptor import receptor + +import spacy + + +def init_model(): + global nlp + nlp = spacy.load('en_core_web_sm') + + +@state(read="rawio:in") +def nlp_preprocess(ctx): + + @receptor(ctx_wrap=ctx, write="rawio:out") + def write_tokens(ctx_input, nlp_doc): + ctx_input["rawio:out"] = [str(token) for token in nlp_doc] + + @receptor(ctx_wrap=ctx, write="rawio:out") + def write_postags(ctx_input, nlp_doc): + ctx_input["rawio:out"] = [str(token.pos_) for token in nlp_doc] + + doc = nlp(ctx["rawio:in"]) + write_tokens(doc) + write_postags(doc) + + +init_model() +registry.register( + name="nlp", + states=(nlp_preprocess,), + props=(PropertyBase(name="tokens", default=""), PropertyBase(name="pos_tags", default="")) +) From 2132ac84d3dd4947294cc98ba434af06e755406a Mon Sep 17 00:00:00 2001 From: Negin Date: Sun, 2 Dec 2018 00:56:54 +0100 Subject: [PATCH 07/14] spacy lemmas, named entities and rudimentary roboy detect --- modules/ravestate_nlp/__init__.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/modules/ravestate_nlp/__init__.py b/modules/ravestate_nlp/__init__.py index 1d6b37f..98dc7e8 100644 --- a/modules/ravestate_nlp/__init__.py +++ b/modules/ravestate_nlp/__init__.py @@ -24,9 +24,34 @@ def write_tokens(ctx_input, nlp_doc): def write_postags(ctx_input, nlp_doc): ctx_input["rawio:out"] = [str(token.pos_) for token in nlp_doc] + @receptor(ctx_wrap=ctx, write="rawio:out") + def write_lemmas(ctx_input, nlp_doc): + ctx_input["rawio:out"] = [str(token.lemma_) for token in nlp_doc] + + @receptor(ctx_wrap=ctx, write="rawio:out") + def write_tags(ctx_input, nlp_doc): + ctx_input["rawio:out"] = [str(token.tag_) for token in nlp_doc] + + @receptor(ctx_wrap=ctx, write="rawio:out") + def write_named_enities(ctx_input, nlp_doc): + ctx_input["rawio:out"] = [str(token.label_) for token in nlp_doc.ents] + + @receptor(ctx_wrap=ctx, write="rawio:out") + def write_roboy_detected(ctx_input, nlp_doc): + roboy = nlp("Roboy, roboy, Robot, robot, Roboboy") + similarity = 0.8 + for token in nlp_doc: + for roboy_token in roboy: + if token.similarity(roboy_token) > similarity: + ctx_input["rawio:out"] = ["ROBOY"] + doc = nlp(ctx["rawio:in"]) write_tokens(doc) write_postags(doc) + write_lemmas(doc) + write_tags(doc) + write_named_enities(doc) + write_roboy_detected(doc) init_model() From 58758f6364bcbb6a4b9836fc3c97d45db18c9633 Mon Sep 17 00:00:00 2001 From: Emilka Date: Sun, 2 Dec 2018 13:14:20 +0100 Subject: [PATCH 08/14] Added properties for nlp and testing states printing --- modules/ravestate_nlp/__init__.py | 73 +++++++++++++++++++++++-------- 1 file changed, 54 insertions(+), 19 deletions(-) diff --git a/modules/ravestate_nlp/__init__.py b/modules/ravestate_nlp/__init__.py index 98dc7e8..e4b7c1a 100644 --- a/modules/ravestate_nlp/__init__.py +++ b/modules/ravestate_nlp/__init__.py @@ -11,39 +11,37 @@ def init_model(): global nlp nlp = spacy.load('en_core_web_sm') + roboy_getter = lambda doc: any(roboy in doc.text.lower() for roboy in ('you', 'roboy', 'robot', 'roboboy')) + from spacy.tokens import Doc + Doc.set_extension('about_roboy', getter=roboy_getter) @state(read="rawio:in") def nlp_preprocess(ctx): - @receptor(ctx_wrap=ctx, write="rawio:out") + @receptor(ctx_wrap=ctx, write="nlp:tokens") def write_tokens(ctx_input, nlp_doc): - ctx_input["rawio:out"] = [str(token) for token in nlp_doc] + ctx_input["nlp:tokens"] = [str(token) for token in nlp_doc] - @receptor(ctx_wrap=ctx, write="rawio:out") + @receptor(ctx_wrap=ctx, write="nlp:postags") def write_postags(ctx_input, nlp_doc): - ctx_input["rawio:out"] = [str(token.pos_) for token in nlp_doc] + ctx_input["nlp:postags"] = [str(token.pos_) for token in nlp_doc] - @receptor(ctx_wrap=ctx, write="rawio:out") + @receptor(ctx_wrap=ctx, write="nlp:lemmas") def write_lemmas(ctx_input, nlp_doc): - ctx_input["rawio:out"] = [str(token.lemma_) for token in nlp_doc] + ctx_input["nlp:lemmas"] = [str(token.lemma_) for token in nlp_doc] - @receptor(ctx_wrap=ctx, write="rawio:out") + @receptor(ctx_wrap=ctx, write="nlp:tags") def write_tags(ctx_input, nlp_doc): - ctx_input["rawio:out"] = [str(token.tag_) for token in nlp_doc] + ctx_input["nlp:tags"] = [str(token.tag_) for token in nlp_doc] - @receptor(ctx_wrap=ctx, write="rawio:out") + @receptor(ctx_wrap=ctx, write="nlp:ner") def write_named_enities(ctx_input, nlp_doc): - ctx_input["rawio:out"] = [str(token.label_) for token in nlp_doc.ents] + ctx_input["nlp:ner"] = [(str(ents.text), str(ents.label_)) for ents in nlp_doc.ents] - @receptor(ctx_wrap=ctx, write="rawio:out") + @receptor(ctx_wrap=ctx, write="nlp:roboy") def write_roboy_detected(ctx_input, nlp_doc): - roboy = nlp("Roboy, roboy, Robot, robot, Roboboy") - similarity = 0.8 - for token in nlp_doc: - for roboy_token in roboy: - if token.similarity(roboy_token) > similarity: - ctx_input["rawio:out"] = ["ROBOY"] + ctx_input["nlp:roboy"] = nlp_doc._.about_roboy doc = nlp(ctx["rawio:in"]) write_tokens(doc) @@ -54,9 +52,46 @@ def write_roboy_detected(ctx_input, nlp_doc): write_roboy_detected(doc) +@state(read="nlp:tokens") +def tokens_output(ctx): + print('[NLP:tokens]:', ctx["nlp:tokens"]) + + +@state(read="nlp:postags") +def postags_output(ctx): + print('[NLP:postags]:', ctx["nlp:postags"]) + + +@state(read="nlp:lemmas") +def lemmas_output(ctx): + print('[NLP:lemmas]:', ctx["nlp:lemmas"]) + + +@state(read="nlp:ner") +def ner_output(ctx): + print('[NLP:ner]:', ctx["nlp:ner"]) + + +@state(read="nlp:tags") +def tags_output(ctx): + print('[NLP:tags]:', ctx["nlp:tags"]) + + +@state(read="nlp:roboy") +def roboy_output(ctx): + print('[NLP:roboy]:', ctx["nlp:roboy"]) + + init_model() registry.register( name="nlp", - states=(nlp_preprocess,), - props=(PropertyBase(name="tokens", default=""), PropertyBase(name="pos_tags", default="")) + states=(nlp_preprocess, tokens_output, postags_output, lemmas_output, ner_output, tags_output, roboy_output), + props=( + PropertyBase(name="tokens", default=""), + PropertyBase(name="postags", default=""), + PropertyBase(name="lemmas", default=""), + PropertyBase(name="tags", default=""), + PropertyBase(name="ner", default=""), + PropertyBase(name="roboy", default="") + ) ) From 0531a60dd75413a687d2756e7282c4fe229af1d8 Mon Sep 17 00:00:00 2001 From: Negin Date: Sun, 2 Dec 2018 17:23:58 +0100 Subject: [PATCH 09/14] rafactoring of nlp and bug fix for roboy recognition --- modules/ravestate/property.py | 6 ++- modules/ravestate_nlp/__init__.py | 70 +++++++++++-------------------- 2 files changed, 28 insertions(+), 48 deletions(-) diff --git a/modules/ravestate/property.py b/modules/ravestate/property.py index 7c0a357..8b474a1 100644 --- a/modules/ravestate/property.py +++ b/modules/ravestate/property.py @@ -18,7 +18,8 @@ def __init__( allow_push=True, allow_pop=True, allow_delete=True, - default=None): + default=None, + always_signal_changed=False): self.name = name self.allow_read = allow_read @@ -29,6 +30,7 @@ def __init__( self.value = default self._lock = Lock() self.module_name = "" + self.always_signal_changed = always_signal_changed def fullname(self): return f"{self.module_name}:{self.name}" @@ -57,7 +59,7 @@ def write(self, value): if not self.allow_write: logging.error(f"Unauthorized write access in property {self.name}!") return False - if self.value != value: + if self.always_signal_changed or self.value != value: self.value = value return True else: diff --git a/modules/ravestate_nlp/__init__.py b/modules/ravestate_nlp/__init__.py index e4b7c1a..4ca9c33 100644 --- a/modules/ravestate_nlp/__init__.py +++ b/modules/ravestate_nlp/__init__.py @@ -6,80 +6,58 @@ from ravestate.receptor import receptor import spacy +import logging +logger = logging.getLogger(__name__) def init_model(): global nlp nlp = spacy.load('en_core_web_sm') - roboy_getter = lambda doc: any(roboy in doc.text.lower() for roboy in ('you', 'roboy', 'robot', 'roboboy')) + roboy_names = ('you', 'roboy', 'robot', 'roboboy') + roboy_getter = lambda doc: any(roboy in doc.text.lower() for roboy in roboy_names) from spacy.tokens import Doc Doc.set_extension('about_roboy', getter=roboy_getter) -@state(read="rawio:in") +@state(read="rawio:in", write=("nlp:tokens", "nlp:postags", "nlp:lemmas", "nlp:tags", "nlp:ner", "nlp:roboy")) def nlp_preprocess(ctx): - - @receptor(ctx_wrap=ctx, write="nlp:tokens") - def write_tokens(ctx_input, nlp_doc): - ctx_input["nlp:tokens"] = [str(token) for token in nlp_doc] - - @receptor(ctx_wrap=ctx, write="nlp:postags") - def write_postags(ctx_input, nlp_doc): - ctx_input["nlp:postags"] = [str(token.pos_) for token in nlp_doc] - - @receptor(ctx_wrap=ctx, write="nlp:lemmas") - def write_lemmas(ctx_input, nlp_doc): - ctx_input["nlp:lemmas"] = [str(token.lemma_) for token in nlp_doc] - - @receptor(ctx_wrap=ctx, write="nlp:tags") - def write_tags(ctx_input, nlp_doc): - ctx_input["nlp:tags"] = [str(token.tag_) for token in nlp_doc] - - @receptor(ctx_wrap=ctx, write="nlp:ner") - def write_named_enities(ctx_input, nlp_doc): - ctx_input["nlp:ner"] = [(str(ents.text), str(ents.label_)) for ents in nlp_doc.ents] - - @receptor(ctx_wrap=ctx, write="nlp:roboy") - def write_roboy_detected(ctx_input, nlp_doc): - ctx_input["nlp:roboy"] = nlp_doc._.about_roboy - - doc = nlp(ctx["rawio:in"]) - write_tokens(doc) - write_postags(doc) - write_lemmas(doc) - write_tags(doc) - write_named_enities(doc) - write_roboy_detected(doc) + nlp_doc = nlp(ctx["rawio:in"]) + ctx["nlp:tokens"] = tuple(str(token) for token in nlp_doc) + ctx["nlp:postags"] = tuple(str(token.pos_) for token in nlp_doc) + ctx["nlp:lemmas"] = tuple(str(token.lemma_) for token in nlp_doc) + ctx["nlp:tags"] = tuple(str(token.tag_) for token in nlp_doc) + ctx["nlp:ner"] = tuple((str(ents.text), str(ents.label_)) for ents in nlp_doc.ents) + ctx["nlp:roboy"] = nlp_doc._.about_roboy @state(read="nlp:tokens") def tokens_output(ctx): - print('[NLP:tokens]:', ctx["nlp:tokens"]) + logger.info('[NLP:tokens]:', ctx["nlp:tokens"]) @state(read="nlp:postags") def postags_output(ctx): - print('[NLP:postags]:', ctx["nlp:postags"]) + logger.info('[NLP:postags]:', ctx["nlp:postags"]) @state(read="nlp:lemmas") def lemmas_output(ctx): - print('[NLP:lemmas]:', ctx["nlp:lemmas"]) + logger.info('[NLP:lemmas]:', ctx["nlp:lemmas"]) @state(read="nlp:ner") def ner_output(ctx): - print('[NLP:ner]:', ctx["nlp:ner"]) + logger.info('[NLP:ner]:', ctx["nlp:ner"]) @state(read="nlp:tags") def tags_output(ctx): - print('[NLP:tags]:', ctx["nlp:tags"]) + logger.info('[NLP:tags]:', ctx["nlp:tags"]) @state(read="nlp:roboy") def roboy_output(ctx): - print('[NLP:roboy]:', ctx["nlp:roboy"]) + logger.info('[NLP:roboy]:', ctx["nlp:roboy"]) init_model() @@ -87,11 +65,11 @@ def roboy_output(ctx): name="nlp", states=(nlp_preprocess, tokens_output, postags_output, lemmas_output, ner_output, tags_output, roboy_output), props=( - PropertyBase(name="tokens", default=""), - PropertyBase(name="postags", default=""), - PropertyBase(name="lemmas", default=""), - PropertyBase(name="tags", default=""), - PropertyBase(name="ner", default=""), - PropertyBase(name="roboy", default="") + PropertyBase(name="tokens", default="", always_signal_changed=True), + PropertyBase(name="postags", default="", always_signal_changed=True), + PropertyBase(name="lemmas", default="", always_signal_changed=True), + PropertyBase(name="tags", default="", always_signal_changed=True), + PropertyBase(name="ner", default="", always_signal_changed=True), + PropertyBase(name="roboy", default="", always_signal_changed=True) ) ) From 4cd057ad48efb368090d5a4049bb9a3b86fbe18a Mon Sep 17 00:00:00 2001 From: Negin Date: Sun, 2 Dec 2018 17:33:44 +0100 Subject: [PATCH 10/14] requirements for nlp added --- requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0856c6c..8999e63 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,6 @@ watson-developer-cloud>=2.2.6 PyAudio Flask python-telegram-bot -pyyaml-include \ No newline at end of file +pyyaml-include +spacy +https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.0.0/en_core_web_sm-2.0.0.tar.gz \ No newline at end of file From 8b7bbce297f20233840bd583d6d6ee905f707eef Mon Sep 17 00:00:00 2001 From: Negin Date: Sun, 2 Dec 2018 18:08:06 +0100 Subject: [PATCH 11/14] Refactor of logging in nlp --- config/telegram_hello_world.yml | 3 ++- modules/ravestate_nlp/__init__.py | 37 +++++-------------------------- 2 files changed, 8 insertions(+), 32 deletions(-) diff --git a/config/telegram_hello_world.yml b/config/telegram_hello_world.yml index 1562481..0476e79 100644 --- a/config/telegram_hello_world.yml +++ b/config/telegram_hello_world.yml @@ -13,4 +13,5 @@ config: import: - ravestate_telegramio - ravestate_conio - - ravestate_hello_world \ No newline at end of file + - ravestate_hello_world + - ravestate_nlp \ No newline at end of file diff --git a/modules/ravestate_nlp/__init__.py b/modules/ravestate_nlp/__init__.py index 4ca9c33..a1cda45 100644 --- a/modules/ravestate_nlp/__init__.py +++ b/modules/ravestate_nlp/__init__.py @@ -8,8 +8,6 @@ import spacy import logging -logger = logging.getLogger(__name__) - def init_model(): global nlp nlp = spacy.load('en_core_web_sm') @@ -29,35 +27,12 @@ def nlp_preprocess(ctx): ctx["nlp:ner"] = tuple((str(ents.text), str(ents.label_)) for ents in nlp_doc.ents) ctx["nlp:roboy"] = nlp_doc._.about_roboy - -@state(read="nlp:tokens") -def tokens_output(ctx): - logger.info('[NLP:tokens]:', ctx["nlp:tokens"]) - - -@state(read="nlp:postags") -def postags_output(ctx): - logger.info('[NLP:postags]:', ctx["nlp:postags"]) - - -@state(read="nlp:lemmas") -def lemmas_output(ctx): - logger.info('[NLP:lemmas]:', ctx["nlp:lemmas"]) - - -@state(read="nlp:ner") -def ner_output(ctx): - logger.info('[NLP:ner]:', ctx["nlp:ner"]) - - -@state(read="nlp:tags") -def tags_output(ctx): - logger.info('[NLP:tags]:', ctx["nlp:tags"]) - - -@state(read="nlp:roboy") -def roboy_output(ctx): - logger.info('[NLP:roboy]:', ctx["nlp:roboy"]) + logging.info('[NLP:tokens]:', ctx["nlp:tokens"]) + logging.info('[NLP:postags]:', ctx["nlp:postags"]) + logging.info('[NLP:lemmas]:', ctx["nlp:lemmas"]) + logging.info('[NLP:ner]:', ctx["nlp:ner"]) + logging.info('[NLP:tags]:', ctx["nlp:tags"]) + logging.info('[NLP:roboy]:', ctx["nlp:roboy"]) init_model() From 52f8b33eac6eaf9954999aa7bce0dd77ec653d21 Mon Sep 17 00:00:00 2001 From: Negin Date: Sun, 2 Dec 2018 18:11:02 +0100 Subject: [PATCH 12/14] added ravestate_hello_word to hello_word.yml --- config/hello_world.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/config/hello_world.yml b/config/hello_world.yml index 8975ab3..0e12ef6 100644 --- a/config/hello_world.yml +++ b/config/hello_world.yml @@ -2,4 +2,5 @@ module: core config: import: - ravestate_conio + - ravestate_hello_world - ravestate_nlp \ No newline at end of file From 9a1f0d141a85048dfc16382e0f505134d82e69a9 Mon Sep 17 00:00:00 2001 From: Negin Date: Mon, 3 Dec 2018 01:04:04 +0100 Subject: [PATCH 13/14] bug fix in nlp --- modules/ravestate_nlp/__init__.py | 37 ++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/modules/ravestate_nlp/__init__.py b/modules/ravestate_nlp/__init__.py index a1cda45..d9cf711 100644 --- a/modules/ravestate_nlp/__init__.py +++ b/modules/ravestate_nlp/__init__.py @@ -20,25 +20,36 @@ def init_model(): @state(read="rawio:in", write=("nlp:tokens", "nlp:postags", "nlp:lemmas", "nlp:tags", "nlp:ner", "nlp:roboy")) def nlp_preprocess(ctx): nlp_doc = nlp(ctx["rawio:in"]) - ctx["nlp:tokens"] = tuple(str(token) for token in nlp_doc) - ctx["nlp:postags"] = tuple(str(token.pos_) for token in nlp_doc) - ctx["nlp:lemmas"] = tuple(str(token.lemma_) for token in nlp_doc) - ctx["nlp:tags"] = tuple(str(token.tag_) for token in nlp_doc) - ctx["nlp:ner"] = tuple((str(ents.text), str(ents.label_)) for ents in nlp_doc.ents) - ctx["nlp:roboy"] = nlp_doc._.about_roboy + + nlp_tokens = tuple(str(token) for token in nlp_doc) + ctx["nlp:tokens"] = nlp_tokens + logging.info(f"[NLP:tokens]: {nlp_tokens}") - logging.info('[NLP:tokens]:', ctx["nlp:tokens"]) - logging.info('[NLP:postags]:', ctx["nlp:postags"]) - logging.info('[NLP:lemmas]:', ctx["nlp:lemmas"]) - logging.info('[NLP:ner]:', ctx["nlp:ner"]) - logging.info('[NLP:tags]:', ctx["nlp:tags"]) - logging.info('[NLP:roboy]:', ctx["nlp:roboy"]) + nlp_postags = tuple(str(token.pos_) for token in nlp_doc) + ctx["nlp:postags"] = nlp_postags + logging.info(f"[NLP:postags]: {nlp_postags}") + + nlp_lemmas = tuple(str(token.lemma_) for token in nlp_doc) + ctx["nlp:lemmas"] = nlp_lemmas + logging.info(f"[NLP:lemmas]: {nlp_lemmas}") + + nlp_tags = tuple(str(token.tag_) for token in nlp_doc) + ctx["nlp:tags"] = nlp_tags + logging.info(f"[NLP:tags]: {nlp_tags}") + + nlp_ner = tuple((str(ents.text), str(ents.label_)) for ents in nlp_doc.ents) + ctx["nlp:ner"] = nlp_ner + logging.info(f"[NLP:ner]: {nlp_ner}") + + nlp_roboy = nlp_doc._.about_roboy + ctx["nlp:roboy"] = nlp_roboy + logging.info(f"[NLP:roboy]: {nlp_roboy}") init_model() registry.register( name="nlp", - states=(nlp_preprocess, tokens_output, postags_output, lemmas_output, ner_output, tags_output, roboy_output), + states=(nlp_preprocess,), props=( PropertyBase(name="tokens", default="", always_signal_changed=True), PropertyBase(name="postags", default="", always_signal_changed=True), From db7525afe4a94623c39734dfe072094dd3b18475 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Mon, 3 Dec 2018 01:14:03 +0100 Subject: [PATCH 14/14] Added ravestate_wildtalk. --- .gitignore | 5 ++++- config/hello_world.yml | 2 +- config/telegram_hello_world.yml | 2 +- modules/ravestate_wildtalk/__init__.py | 14 ++++++++++++++ requirements.txt | 1 + 5 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 modules/ravestate_wildtalk/__init__.py diff --git a/.gitignore b/.gitignore index 3dcb474..eeeb8eb 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,7 @@ tts.wav keys.yml dist/* build/* -*.egg-info \ No newline at end of file +*.egg-info +ros2/build +ros2/install +ros2/log diff --git a/config/hello_world.yml b/config/hello_world.yml index 0e12ef6..74351fc 100644 --- a/config/hello_world.yml +++ b/config/hello_world.yml @@ -2,5 +2,5 @@ module: core config: import: - ravestate_conio - - ravestate_hello_world + - ravestate_wildtalk - ravestate_nlp \ No newline at end of file diff --git a/config/telegram_hello_world.yml b/config/telegram_hello_world.yml index 0476e79..f816d58 100644 --- a/config/telegram_hello_world.yml +++ b/config/telegram_hello_world.yml @@ -13,5 +13,5 @@ config: import: - ravestate_telegramio - ravestate_conio - - ravestate_hello_world + - ravestate_wildtalk - ravestate_nlp \ No newline at end of file diff --git a/modules/ravestate_wildtalk/__init__.py b/modules/ravestate_wildtalk/__init__.py new file mode 100644 index 0000000..eaf04b3 --- /dev/null +++ b/modules/ravestate_wildtalk/__init__.py @@ -0,0 +1,14 @@ + +from ravestate.state import state +from ravestate import registry + +from roboy_parlai import wildtalk +import ravestate_rawio + + +@state(read="rawio:in", write="rawio:out") +def wildtalk_state(ctx): + ctx["rawio:out"] = wildtalk(ctx["rawio:in"]) + + +registry.register(name="wildtalk", states=(wildtalk_state,)) diff --git a/requirements.txt b/requirements.txt index 8999e63..7ea6fb4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,5 +6,6 @@ PyAudio Flask python-telegram-bot pyyaml-include +roboy_parlai>=0.1.post3 spacy https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.0.0/en_core_web_sm-2.0.0.tar.gz \ No newline at end of file