From 8d59c424d7a9249405c9f4c59c5e1784773fff8a Mon Sep 17 00:00:00 2001 From: Eric Callahan Date: Mon, 6 Nov 2023 07:40:01 -0500 Subject: [PATCH 1/9] server: convert history a core component The history component requires no specific configuration, is generally always used, and has no dependencies that prevent loading at startup. Convert history to a core component, making it eligible for lookup/use in optional components. Signed-off-by: Eric Callahan --- moonraker/server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moonraker/server.py b/moonraker/server.py index 1212e70a..d26acc9d 100755 --- a/moonraker/server.py +++ b/moonraker/server.py @@ -61,8 +61,8 @@ CORE_COMPONENTS = [ 'dbus_manager', 'database', 'file_manager', 'authorization', 'klippy_apis', 'machine', 'data_store', 'shell_command', - 'proc_stats', 'job_state', 'job_queue', 'http_client', - 'announcements', 'webcam', 'extensions' + 'proc_stats', 'job_state', 'job_queue', 'history', + 'http_client', 'announcements', 'webcam', 'extensions' ] From 4b1a3b87926d080c240e146c0ffc32b9b1f6a37d Mon Sep 17 00:00:00 2001 From: Eric Callahan Date: Wed, 20 Dec 2023 18:17:25 -0500 Subject: [PATCH 2/9] sensor: minor refactoring Remove dataclasses, refactor for consistency with other components. Signed-off-by: Eric Callahan --- moonraker/components/sensor.py | 70 ++++++++++------------------------ 1 file changed, 20 insertions(+), 50 deletions(-) diff --git a/moonraker/components/sensor.py b/moonraker/components/sensor.py index 5115bd2e..71d7dd2e 100644 --- a/moonraker/components/sensor.py +++ b/moonraker/components/sensor.py @@ -10,7 +10,6 @@ import logging from collections import defaultdict, deque -from dataclasses import dataclass, replace from functools import partial from ..common import RequestType @@ -35,15 +34,6 @@ SENSOR_UPDATE_TIME = 1.0 SENSOR_EVENT_NAME = "sensors:sensor_update" - -@dataclass(frozen=True) -class SensorConfiguration: - id: str - name: str - type: str - source: str = "" - - def _set_result( name: str, value: Union[int, float], store: Dict[str, Union[int, float]] ) -> None: @@ -53,24 +43,16 @@ def _set_result( store[name] = value -@dataclass(frozen=True) -class Sensor: - config: SensorConfiguration - values: Dict[str, Deque[Union[int, float]]] - - class BaseSensor: - def __init__(self, name: str, cfg: ConfigHelper, store_size: int = 1200) -> None: - self.server = cfg.get_server() + def __init__(self, config: ConfigHelper) -> None: + self.server = config.get_server() self.error_state: Optional[str] = None - - self.config = SensorConfiguration( - id=name, - type=cfg.get("type"), - name=cfg.get("name", name), - ) + self.id = config.get_name().split(maxsplit=1)[-1] + self.type = config.get("type") + self.name = config.get("name", self.id) self.last_measurements: Dict[str, Union[int, float]] = {} self.last_value: Dict[str, Union[int, float]] = {} + store_size = config.getint("sensor_store_size", 1200) self.values: DefaultDict[str, Deque[Union[int, float]]] = defaultdict( lambda: deque(maxlen=store_size) ) @@ -89,14 +71,14 @@ async def initialize(self) -> bool: """ Sensor initialization executed on Moonraker startup. """ - logging.info("Registered sensor '%s'", self.config.name) + logging.info("Registered sensor '%s'", self.name) return True def get_sensor_info(self) -> Dict[str, Any]: return { - "id": self.config.id, - "friendly_name": self.config.name, - "type": self.config.type, + "id": self.id, + "friendly_name": self.name, + "type": self.type, "values": self.last_measurements, } @@ -104,22 +86,19 @@ def get_sensor_measurements(self) -> Dict[str, List[Union[int, float]]]: return {key: list(values) for key, values in self.values.items()} def get_name(self) -> str: - return self.config.name + return self.name def close(self) -> None: pass class MQTTSensor(BaseSensor): - def __init__(self, name: str, cfg: ConfigHelper, store_size: int = 1200): - super().__init__(name=name, cfg=cfg) - self.mqtt: MQTTClient = self.server.load_component(cfg, "mqtt") - - self.state_topic: str = cfg.get("state_topic") - self.state_response = cfg.gettemplate("state_response_template") - self.config = replace(self.config, source=self.state_topic) - self.qos: Optional[int] = cfg.getint("qos", None, minval=0, maxval=2) - + def __init__(self, config: ConfigHelper) -> None: + super().__init__(config=config) + self.mqtt: MQTTClient = self.server.load_component(config, "mqtt") + self.state_topic: str = config.get("state_topic") + self.state_response = config.gettemplate("state_response_template") + self.qos: Optional[int] = config.getint("qos", None, minval=0, maxval=2) self.server.register_event_handler( "mqtt:disconnected", self._on_mqtt_disconnected ) @@ -141,7 +120,7 @@ def _on_state_update(self, payload: bytes) -> None: self.last_measurements = measurements logging.debug( "Received updated sensor value for %s: %s", - self.config.name, + self.name, self.last_measurements, ) @@ -169,8 +148,6 @@ class Sensors: def __init__(self, config: ConfigHelper) -> None: self.server = config.get_server() - self.store_size = config.getint("sensor_store_size", 1200) - prefix_sections = config.get_prefix_sections("sensor") self.sensors: Dict[str, BaseSensor] = {} # Register timer to update sensor values in store @@ -197,18 +174,15 @@ def __init__(self, config: ConfigHelper) -> None: # Register notifications self.server.register_notification(SENSOR_EVENT_NAME) - + prefix_sections = config.get_prefix_sections("sensor ") for section in prefix_sections: cfg = config[section] - try: try: _, name = cfg.get_name().split(maxsplit=1) except ValueError: raise cfg.error(f"Invalid section name: {cfg.get_name()}") - logging.info(f"Configuring sensor: {name}") - sensor_type: str = cfg.get("type") sensor_class: Optional[Type[BaseSensor]] = self.__sensor_types.get( sensor_type.upper(), None @@ -216,11 +190,7 @@ def __init__(self, config: ConfigHelper) -> None: if sensor_class is None: raise config.error(f"Unsupported sensor type: {sensor_type}") - self.sensors[name] = sensor_class( - name=name, - cfg=cfg, - store_size=self.store_size, - ) + self.sensors[name] = sensor_class(cfg) except Exception as e: # Ensures that configuration errors are shown to the user self.server.add_warning( From e7d3f3e961cab9d809c6dfa69e1979fbf5069265 Mon Sep 17 00:00:00 2001 From: Eric Callahan Date: Wed, 24 Apr 2024 08:26:04 -0400 Subject: [PATCH 3/9] job_state: track and store most recent Job Event Signed-off-by: Eric Callahan --- moonraker/components/job_state.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/moonraker/components/job_state.py b/moonraker/components/job_state.py index 0ac9f0dd..098382f8 100644 --- a/moonraker/components/job_state.py +++ b/moonraker/components/job_state.py @@ -24,6 +24,7 @@ class JobState: def __init__(self, config: ConfigHelper) -> None: self.server = config.get_server() self.last_print_stats: Dict[str, Any] = {} + self.last_event: JobEvent = JobEvent.STANDBY self.server.register_event_handler( "server:klippy_started", self._handle_started ) @@ -36,6 +37,7 @@ def _handle_disconnect(self): if state in ("printing", "paused"): # set error state self.last_print_stats["state"] = "error" + self.last_event = JobEvent.ERROR async def _handle_started(self, state: KlippyState) -> None: if state != KlippyState.READY: @@ -80,9 +82,10 @@ async def _status_update(self, data: Dict[str, Any], _: float) -> None: # should register handlers for "job_state: status_changed" and # match against the JobEvent object provided. self.server.send_event(f"job_state:{new_state}", prev_ps, new_ps) + self.last_event = JobEvent.from_string(new_state) self.server.send_event( "job_state:state_changed", - JobEvent.from_string(new_state), + self.last_event, prev_ps, new_ps ) @@ -108,5 +111,8 @@ def _check_resumed(self, def get_last_stats(self) -> Dict[str, Any]: return dict(self.last_print_stats) + def get_last_job_event(self) -> JobEvent: + return self.last_event + def load_component(config: ConfigHelper) -> JobState: return JobState(config) From b6896c7c0aedab2f3bc91a198bd9e240e2f77792 Mon Sep 17 00:00:00 2001 From: Eric Callahan Date: Wed, 24 Apr 2024 17:01:04 -0400 Subject: [PATCH 4/9] common: add common history tracking implementation The HistoryDataField class can be instantiated by any component. It can then be used to register additional fields tracked in the job history. Signed-off-by: Eric Callahan --- moonraker/common.py | 364 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 359 insertions(+), 5 deletions(-) diff --git a/moonraker/common.py b/moonraker/common.py index c8460389..3ebb421d 100644 --- a/moonraker/common.py +++ b/moonraker/common.py @@ -29,16 +29,17 @@ List, Awaitable, ClassVar, - Tuple + Tuple, + Generic ) if TYPE_CHECKING: from .server import Server from .components.websockets import WebsocketManager from .components.authorization import Authorization + from .components.history import History from .utils import IPAddress from asyncio import Future - _T = TypeVar("_T") _C = TypeVar("_C", str, bool, float, int) _F = TypeVar("_F", bound="ExtendedFlag") ConvType = Union[str, bool, float, int] @@ -46,6 +47,7 @@ RPCCallback = Callable[..., Coroutine] AuthComp = Optional[Authorization] +_T = TypeVar("_T") ENDPOINT_PREFIXES = ["printer", "server", "machine", "access", "api", "debug"] class ExtendedFlag(Flag): @@ -254,6 +256,8 @@ def create( name_parts = http_path[1:].split('/') if len(request_types) > 1: for rtype in request_types: + if rtype.name is None: + continue func_name = rtype.name.lower() + "_" + name_parts[-1] rpc_methods.append(".".join(name_parts[:-1] + [func_name])) else: @@ -384,9 +388,9 @@ async def _process_message(self, message: str) -> None: logging.exception("Websocket Command Error") def queue_message(self, message: Union[bytes, str, Dict[str, Any]]): - if isinstance(message, dict): - message = jsonw.dumps(message) - self.message_buf.append(message) + self.message_buf.append( + jsonw.dumps(message) if isinstance(message, dict) else message + ) if self.queue_busy: return self.queue_busy = True @@ -866,3 +870,353 @@ def build_error( 'error': {'code': code, 'message': msg}, 'id': req_id } + + +# *** Job History Common Clases *** + +class FieldTracker(Generic[_T]): + history: History = None # type: ignore + def __init__( + self, + value: _T = None, # type: ignore + reset_callback: Optional[Callable[[], _T]] = None, + exclude_paused: bool = False, + ) -> None: + self.tracked_value = value + self.exclude_paused = exclude_paused + self.reset_callback: Optional[Callable[[], _T]] = reset_callback + + def set_reset_callback(self, cb: Optional[Callable[[], _T]]) -> None: + self.reset_callback = cb + + def set_exclude_paused(self, exclude: bool) -> None: + self.exclude_paused = exclude + + def reset(self) -> None: + raise NotImplementedError() + + def update(self, value: _T) -> None: + raise NotImplementedError() + + def get_tracked_value(self) -> _T: + return self.tracked_value + + def has_totals(self) -> bool: + return False + + @classmethod + def class_init(cls, history: History) -> None: + cls.history = history + + +class BasicTracker(FieldTracker[Any]): + def __init__( + self, + value: Any = None, + reset_callback: Optional[Callable[[], Any]] = None, + exclude_paused: bool = False + ) -> None: + super().__init__(value, reset_callback, exclude_paused) + + def reset(self) -> None: + if self.reset_callback is not None: + self.tracked_value = self.reset_callback() + + def update(self, value: Any) -> None: + if self.history.tracking_enabled(self.exclude_paused): + self.tracked_value = value + + def has_totals(self) -> bool: + return isinstance(self.tracked_value, (int, float)) + + +class DeltaTracker(FieldTracker[Union[int, float]]): + def __init__( + self, + value: Union[int, float] = 0, + reset_callback: Optional[Callable[[], Union[float, int]]] = None, + exclude_paused: bool = False + ) -> None: + super().__init__(value, reset_callback, exclude_paused) + self.last_value: Union[float, int, None] = None + + def reset(self) -> None: + self.tracked_value = 0 + self.last_value = None + if self.reset_callback is not None: + self.last_value = self.reset_callback() + if not isinstance(self.last_value, (float, int)): + logging.info("DeltaTracker reset to invalid type") + self.last_value = None + + def update(self, value: Union[int, float]) -> None: + if not isinstance(value, (int, float)): + return + if self.history.tracking_enabled(self.exclude_paused): + if self.last_value is not None: + self.tracked_value += value - self.last_value + self.last_value = value + + def has_totals(self) -> bool: + return True + + +class CumulativeTracker(FieldTracker[Union[int, float]]): + def __init__( + self, + value: Union[int, float] = 0, + reset_callback: Optional[Callable[[], Union[float, int]]] = None, + exclude_paused: bool = False + ) -> None: + super().__init__(value, reset_callback, exclude_paused) + + def reset(self) -> None: + if self.reset_callback is not None: + self.tracked_value = self.reset_callback() + if not isinstance(self.tracked_value, (float, int)): + logging.info(f"{self.__class__.__name__} reset to invalid type") + self.tracked_value = 0 + else: + self.tracked_value = 0 + + def update(self, value: Union[int, float]) -> None: + if not isinstance(value, (int, float)): + return + if self.history.tracking_enabled(self.exclude_paused): + self.tracked_value += value + + def has_totals(self) -> bool: + return True + +class AveragingTracker(CumulativeTracker): + def __init__( + self, + value: Union[int, float] = 0, + reset_callback: Optional[Callable[[], Union[float, int]]] = None, + exclude_paused: bool = False + ) -> None: + super().__init__(value, reset_callback, exclude_paused) + self.count = 0 + + def reset(self) -> None: + super().reset() + self.count = 0 + + def update(self, value: Union[int, float]) -> None: + if not isinstance(value, (int, float)): + return + if self.history.tracking_enabled(self.exclude_paused): + lv = self.tracked_value + self.count += 1 + self.tracked_value = (lv * (self.count - 1) + value) / self.count + + +class MaximumTracker(CumulativeTracker): + def __init__( + self, + value: Union[int, float] = 0, + reset_callback: Optional[Callable[[], Union[float, int]]] = None, + exclude_paused: bool = False + ) -> None: + super().__init__(value, reset_callback, exclude_paused) + self.initialized = False + + def reset(self) -> None: + self.initialized = False + if self.reset_callback is not None: + self.tracked_value = self.reset_callback() + if not isinstance(self.tracked_value, (int, float)): + self.tracked_value = 0 + logging.info("MaximumTracker reset to invalid type") + else: + self.initialized = True + else: + self.tracked_value = 0 + + def update(self, value: Union[float, int]) -> None: + if not isinstance(value, (int, float)): + return + if self.history.tracking_enabled(self.exclude_paused): + if not self.initialized: + self.tracked_value = value + self.initialized = True + else: + self.tracked_value = max(self.tracked_value, value) + +class MinimumTracker(CumulativeTracker): + def __init__( + self, + value: Union[int, float] = 0, + reset_callback: Optional[Callable[[], Union[float, int]]] = None, + exclude_paused: bool = False + ) -> None: + super().__init__(value, reset_callback, exclude_paused) + self.initialized = False + + def reset(self) -> None: + self.initialized = False + if self.reset_callback is not None: + self.tracked_value = self.reset_callback() + if not isinstance(self.tracked_value, (int, float)): + self.tracked_value = 0 + logging.info("MinimumTracker reset to invalid type") + else: + self.initialized = True + else: + self.tracked_value = 0 + + def update(self, value: Union[float, int]) -> None: + if not isinstance(value, (int, float)): + return + if self.history.tracking_enabled(self.exclude_paused): + if not self.initialized: + self.tracked_value = value + self.initialized = True + else: + self.tracked_value = min(self.tracked_value, value) + +class CollectionTracker(FieldTracker[List[Any]]): + MAX_SIZE = 100 + def __init__( + self, + value: List[Any] = [], + reset_callback: Optional[Callable[[], List[Any]]] = None, + exclude_paused: bool = False + ) -> None: + super().__init__(list(value), reset_callback, exclude_paused) + + def reset(self) -> None: + if self.reset_callback is not None: + self.tracked_value = self.reset_callback() + if not isinstance(self.tracked_value, list): + logging.info("CollectionTracker reset to invalid type") + self.tracked_value = [] + else: + self.tracked_value.clear() + + def update(self, value: Any) -> None: + if value in self.tracked_value: + return + if self.history.tracking_enabled(self.exclude_paused): + self.tracked_value.append(value) + if len(self.tracked_value) > self.MAX_SIZE: + self.tracked_value.pop(0) + + def has_totals(self) -> bool: + return False + + +class TrackingStrategy(ExtendedEnum): + BASIC = 1 + DELTA = 2 + ACCUMULATE = 3 + AVERAGE = 4 + MAXIMUM = 5 + MINIMUM = 6 + COLLECT = 7 + + def get_tracker(self, **kwargs) -> FieldTracker: + trackers: Dict[TrackingStrategy, Type[FieldTracker]] = { + TrackingStrategy.BASIC: BasicTracker, + TrackingStrategy.DELTA: DeltaTracker, + TrackingStrategy.ACCUMULATE: CumulativeTracker, + TrackingStrategy.AVERAGE: AveragingTracker, + TrackingStrategy.MAXIMUM: MaximumTracker, + TrackingStrategy.MINIMUM: MinimumTracker, + TrackingStrategy.COLLECT: CollectionTracker + } + return trackers[self](**kwargs) + + +class HistoryFieldData: + def __init__( + self, + field_name: str, + provider: str, + desc: str, + strategy: str, + units: Optional[str] = None, + reset_callback: Optional[Callable[[], _T]] = None, + exclude_paused: bool = False, + report_total: bool = False, + report_maximum: bool = False, + precision: Optional[int] = None + ) -> None: + self._name = field_name + self._provider = provider + self._desc = desc + self._strategy = TrackingStrategy.from_string(strategy) + self._units = units + self._tracker = self._strategy.get_tracker( + reset_callback=reset_callback, + exclude_paused=exclude_paused + ) + self._report_total = report_total + self._report_maximum = report_maximum + self._precision = precision + + @property + def name(self) -> str: + return self._name + + @property + def provider(self) -> str: + return self._provider + + @property + def tracker(self) -> FieldTracker: + return self._tracker + + def __eq__(self, value: object) -> bool: + if isinstance(value, HistoryFieldData): + return value._provider == self._provider and value._name == self._name + raise ValueError("Invalid type for comparison") + + def as_dict(self) -> Dict[str, Any]: + val = self._tracker.get_tracked_value() + if self._precision is not None and isinstance(val, float): + val = round(val, self._precision) + return { + "provider": self._provider, + "name": self.name, + "value": val, + "description": self._desc, + "units": self._units + } + + def has_totals(self) -> bool: + return ( + self._tracker.has_totals() and + (self._report_total or self._report_maximum) + ) + + def get_totals( + self, last_totals: List[Dict[str, Any]], reset: bool = False + ) -> Dict[str, Any]: + if not self.has_totals(): + return {} + if reset: + maximum: Optional[float] = 0 if self._report_maximum else None + total: Optional[float] = 0 if self._report_total else None + else: + cur_val: Union[float, int] = self._tracker.get_tracked_value() + maximum = cur_val if self._report_maximum else None + total = cur_val if self._report_total else None + for obj in last_totals: + if obj["provider"] == self._provider and obj["field"] == self._name: + if maximum is not None: + maximum = max(cur_val, obj["maximum"] or 0) + if total is not None: + total = cur_val + (obj["total"] or 0) + break + if self._precision is not None: + if maximum is not None: + maximum = round(maximum, self._precision) + if total is not None: + total = round(total, self._precision) + return { + "provider": self._provider, + "field": self._name, + "maximum": maximum, + "total": total + } From 1dfbffb42203cc0639d6445703b7ef0f3f3f2e30 Mon Sep 17 00:00:00 2001 From: Eric Callahan Date: Mon, 6 Nov 2023 18:06:14 -0500 Subject: [PATCH 5/9] history: add support for auxiliary fields Allow other components to register custom fields tracked and reported in job history. Signed-off-by: Eric Callahan --- moonraker/components/history.py | 137 ++++++++++++++++++++++---------- 1 file changed, 94 insertions(+), 43 deletions(-) diff --git a/moonraker/components/history.py b/moonraker/components/history.py index cf62fcad..cd2319ae 100644 --- a/moonraker/components/history.py +++ b/moonraker/components/history.py @@ -6,7 +6,12 @@ import time import logging from asyncio import Lock -from ..common import JobEvent, RequestType +from ..common import ( + JobEvent, + RequestType, + HistoryFieldData, + FieldTracker +) # Annotation imports from typing import ( @@ -15,18 +20,30 @@ Union, Optional, Dict, - List, + List ) + if TYPE_CHECKING: from ..confighelper import ConfigHelper from ..common import WebRequest from .database import MoonrakerDatabase as DBComp from .job_state import JobState from .file_manager.file_manager import FileManager + Totals = Dict[str, Union[float, int]] + AuxTotals = List[Dict[str, Any]] + HIST_NAMESPACE = "history" HIST_VERSION = 1 MAX_JOBS = 10000 +BASE_TOTALS = { + "total_jobs": 0, + "total_time": 0., + "total_print_time": 0., + "total_filament_used": 0., + "longest_job": 0., + "longest_print": 0. +} class History: def __init__(self, config: ConfigHelper) -> None: @@ -34,17 +51,13 @@ def __init__(self, config: ConfigHelper) -> None: self.file_manager: FileManager = self.server.lookup_component( 'file_manager') self.request_lock = Lock() + FieldTracker.class_init(self) + self.auxiliary_fields: List[HistoryFieldData] = [] database: DBComp = self.server.lookup_component("database") - self.job_totals: Dict[str, float] = database.get_item( - "moonraker", "history.job_totals", - { - 'total_jobs': 0, - 'total_time': 0., - 'total_print_time': 0., - 'total_filament_used': 0., - 'longest_job': 0., - 'longest_print': 0. - }).result() + hist_info: Dict[str, Any] + hist_info = database.get_item("moonraker", "history", {}).result() + self.job_totals: Totals = hist_info.get("job_totals", dict(BASE_TOTALS)) + self.aux_totals: AuxTotals = hist_info.get("aux_totals", []) self.server.register_event_handler( "server:klippy_disconnect", self._handle_disconnect) @@ -75,6 +88,7 @@ def __init__(self, config: ConfigHelper) -> None: self.current_job: Optional[PrinterJob] = None self.current_job_id: Optional[str] = None + self.job_paused: bool = False self.next_job_id: int = 0 self.cached_job_ids = self.history_ns.keys().result() if self.cached_job_ids: @@ -189,30 +203,30 @@ async def _handle_jobs_list(self, return {"count": count, "jobs": jobs} - async def _handle_job_totals(self, - web_request: WebRequest - ) -> Dict[str, Dict[str, float]]: - return {'job_totals': self.job_totals} + async def _handle_job_totals( + self, web_request: WebRequest + ) -> Dict[str, Union[Totals, AuxTotals]]: + return { + "job_totals": self.job_totals, + "auxiliary_totals": self.aux_totals + } - async def _handle_job_total_reset(self, - web_request: WebRequest, - ) -> Dict[str, Dict[str, float]]: + async def _handle_job_total_reset( + self, web_request: WebRequest + ) -> Dict[str, Union[Totals, AuxTotals]]: if self.current_job is not None: - raise self.server.error( - "Job in progress, cannot reset totals") - last_totals = dict(self.job_totals) - self.job_totals = { - 'total_jobs': 0, - 'total_time': 0., - 'total_print_time': 0., - 'total_filament_used': 0., - 'longest_job': 0., - 'longest_print': 0. - } + raise self.server.error("Job in progress, cannot reset totals") + last_totals = self.job_totals + self.job_totals = dict(BASE_TOTALS) + last_aux_totals = self.aux_totals + self._update_aux_totals(reset=True) database: DBComp = self.server.lookup_component("database") - await database.insert_item( - "moonraker", "history.job_totals", self.job_totals) - return {'last_totals': last_totals} + await database.insert_item("moonraker", "history.job_totals", self.job_totals) + await database.insert_item("moonraker", "history.aux_totals", self.aux_totals) + return { + "last_totals": last_totals, + "last_auxiliary_totals": last_aux_totals + } def _on_job_state_changed( self, @@ -220,6 +234,7 @@ def _on_job_state_changed( prev_stats: Dict[str, Any], new_stats: Dict[str, Any] ) -> None: + self.job_paused = job_event == JobEvent.PAUSED if job_event == JobEvent.STARTED: if self.current_job is not None: # Finish with the previous state @@ -251,6 +266,9 @@ def add_job(self, job: PrinterJob) -> None: self.current_job = job self.current_job_id = job_id self.grab_job_metadata() + for field in self.auxiliary_fields: + field.tracker.reset() + self.current_job.set_aux_data(self.auxiliary_fields) self.history_ns[job_id] = job.get_stats() self.cached_job_ids.append(job_id) self.next_job_id += 1 @@ -281,6 +299,7 @@ def finish_job(self, status: str, pstats: Dict[str, Any]) -> None: self.current_job.finish(status, pstats) # Regrab metadata incase metadata wasn't parsed yet due to file upload self.grab_job_metadata() + self.current_job.set_aux_data(self.auxiliary_fields) self.save_current_job() self._update_job_totals() logging.debug( @@ -332,17 +351,30 @@ def _update_job_totals(self) -> None: if self.current_job is None: return job = self.current_job - self.job_totals['total_jobs'] += 1 - self.job_totals['total_time'] += job.get('total_duration') - self.job_totals['total_print_time'] += job.get('print_duration') - self.job_totals['total_filament_used'] += job.get('filament_used') - self.job_totals['longest_job'] = max( - self.job_totals['longest_job'], job.get('total_duration')) - self.job_totals['longest_print'] = max( - self.job_totals['longest_print'], job.get('print_duration')) + self._accumulate_total("total_jobs", 1) + self._accumulate_total("total_time", job.total_duration) + self._accumulate_total("total_print_time", job.print_duration) + self._accumulate_total("total_filament_used", job.filament_used) + self._maximize_total("longest_job", job.total_duration) + self._maximize_total("longest_print", job.print_duration) + self._update_aux_totals() database: DBComp = self.server.lookup_component("database") - database.insert_item( - "moonraker", "history.job_totals", self.job_totals) + database.insert_item("moonraker", "history.job_totals", self.job_totals) + database.insert_item("moonraker", "history.aux_totals", self.aux_totals) + + def _accumulate_total(self, field: str, val: Union[int, float]) -> None: + self.job_totals[field] += val + + def _maximize_total(self, field: str, val: Union[int, float]) -> None: + self.job_totals[field] = max(self.job_totals[field], val) + + def _update_aux_totals(self, reset: bool = False) -> None: + last_totals = self.aux_totals + self.aux_totals = [ + field.get_totals(last_totals, reset) + for field in self.auxiliary_fields + if field.has_totals() + ] def send_history_event(self, evt_action: str) -> None: if self.current_job is None or self.current_job_id is None: @@ -363,6 +395,20 @@ def _prep_requested_job(self, ) return job + def register_auxiliary_field(self, new_field: HistoryFieldData) -> None: + for field in self.auxiliary_fields: + if field == new_field: + raise self.server.error( + f"Field {field.name} already registered by " + f"provider {field.provider}." + ) + self.auxiliary_fields.append(new_field) + + def tracking_enabled(self, check_paused: bool) -> bool: + if self.current_job is None: + return False + return not self.job_paused if check_paused else True + def on_exit(self) -> None: jstate: JobState = self.server.lookup_component("job_state") last_ps = jstate.get_last_stats() @@ -378,6 +424,7 @@ def __init__(self, data: Dict[str, Any] = {}) -> None: self.status: str = "in_progress" self.start_time = time.time() self.total_duration: float = 0. + self.auxiliary_data: List[Dict[str, Any]] = [] self.update_from_ps(data) def finish(self, @@ -401,10 +448,14 @@ def set(self, name: str, val: Any) -> None: return setattr(self, name, val) + def set_aux_data(self, fields: List[HistoryFieldData]) -> None: + self.auxiliary_data = [field.as_dict() for field in fields] + def update_from_ps(self, data: Dict[str, Any]) -> None: for i in data: if hasattr(self, i): setattr(self, i, data[i]) + def load_component(config: ConfigHelper) -> History: return History(config) From 531028ef4fa75c268be197e594fb21bb58ece634 Mon Sep 17 00:00:00 2001 From: Eric Callahan Date: Thu, 25 Apr 2024 11:52:32 -0400 Subject: [PATCH 6/9] history: report user in job history When possible record the name of the user that requested the job. The klippy_api's component now takes an optional user argument in its "start_print" method. This user is broadcast in an event after a print request successfully returns. Signed-off-by: Eric Callahan --- moonraker/components/application.py | 4 +++- .../components/file_manager/file_manager.py | 8 +++++--- moonraker/components/history.py | 13 ++++++++++++ moonraker/components/job_queue.py | 20 ++++++++++++++----- moonraker/components/klippy_apis.py | 12 ++++++++--- moonraker/components/octoprint_compat.py | 5 +++-- moonraker/components/simplyprint.py | 2 +- 7 files changed, 49 insertions(+), 15 deletions(-) diff --git a/moonraker/components/application.py b/moonraker/components/application.py index 6ee776fd..dc69e010 100644 --- a/moonraker/components/application.py +++ b/moonraker/components/application.py @@ -65,10 +65,11 @@ from io import BufferedReader from .authorization import Authorization from .template import TemplateFactory, JinjaTemplate - MessageDelgate = Optional[tornado.httputil.HTTPMessageDelegate] + MessageDelgate = Optional[HTTPMessageDelegate] AuthComp = Optional[Authorization] APICallback = Callable[[WebRequest], Coroutine] +# mypy: disable-error-code="attr-defined,name-defined" # 50 MiB Max Standard Body Size MAX_BODY_SIZE = 50 * 1024 * 1024 @@ -1010,6 +1011,7 @@ async def post(self) -> None: for name, value in form_args.items(): debug_msg += f"\n{name}: {value}" debug_msg += f"\nChecksum: {calc_chksum}" + form_args["current_user"] = self.current_user logging.debug(debug_msg) logging.info(f"Processing Uploaded File: {self._file.multipart_filename}") try: diff --git a/moonraker/components/file_manager/file_manager.py b/moonraker/components/file_manager/file_manager.py index fada4e1d..de336020 100644 --- a/moonraker/components/file_manager/file_manager.py +++ b/moonraker/components/file_manager/file_manager.py @@ -880,7 +880,8 @@ def _parse_upload_args(self, 'start_print': start_print, 'unzip_ufp': unzip_ufp, 'ext': f_ext, - "is_link": os.path.islink(dest_path) + "is_link": os.path.islink(dest_path), + "user": upload_args.get("current_user") } async def _finish_gcode_upload( @@ -901,10 +902,11 @@ async def _finish_gcode_upload( started: bool = False queued: bool = False if upload_info['start_print']: + user: Optional[Dict[str, Any]] = upload_info.get("user") if can_start: kapis: APIComp = self.server.lookup_component('klippy_apis') try: - await kapis.start_print(upload_info['filename']) + await kapis.start_print(upload_info['filename'], user=user) except self.server.error: # Attempt to start print failed pass @@ -913,7 +915,7 @@ async def _finish_gcode_upload( if self.queue_gcodes and not started: job_queue: JobQueue = self.server.lookup_component('job_queue') await job_queue.queue_job( - upload_info['filename'], check_exists=False) + upload_info['filename'], check_exists=False, user=user) queued = True self.fs_observer.on_item_create("gcodes", upload_info["dest_path"]) result = dict(self._sched_changed_event( diff --git a/moonraker/components/history.py b/moonraker/components/history.py index cd2319ae..c0e8680f 100644 --- a/moonraker/components/history.py +++ b/moonraker/components/history.py @@ -65,6 +65,8 @@ def __init__(self, config: ConfigHelper) -> None: "server:klippy_shutdown", self._handle_shutdown) self.server.register_event_handler( "job_state:state_changed", self._on_job_state_changed) + self.server.register_event_handler( + "klippy_apis:job_start_complete", self._on_job_requested) self.server.register_notification("history:history_changed") self.server.register_endpoint( @@ -88,6 +90,7 @@ def __init__(self, config: ConfigHelper) -> None: self.current_job: Optional[PrinterJob] = None self.current_job_id: Optional[str] = None + self.job_user: str = "No User" self.job_paused: bool = False self.next_job_id: int = 0 self.cached_job_ids = self.history_ns.keys().result() @@ -249,6 +252,12 @@ def _on_job_state_changed( # `CLEAR_PAUSE/SDCARD_RESET_FILE` workflow self.finish_job("cancelled", prev_stats) + def _on_job_requested(self, user: Optional[Dict[str, Any]]) -> None: + username = (user or {}).get("username", "No User") + self.job_user = username + if self.current_job is not None: + self.current_job.user = username + def _handle_shutdown(self) -> None: jstate: JobState = self.server.lookup_component("job_state") last_ps = jstate.get_last_stats() @@ -265,6 +274,7 @@ def add_job(self, job: PrinterJob) -> None: job_id = f"{self.next_job_id:06X}" self.current_job = job self.current_job_id = job_id + self.current_job.user = self.job_user self.grab_job_metadata() for field in self.auxiliary_fields: field.tracker.reset() @@ -296,6 +306,7 @@ def finish_job(self, status: str, pstats: Dict[str, Any]) -> None: # Print stats have been reset, do not update this job with them pstats = {} + self.current_job.user = self.job_user self.current_job.finish(status, pstats) # Regrab metadata incase metadata wasn't parsed yet due to file upload self.grab_job_metadata() @@ -310,6 +321,7 @@ def finish_job(self, status: str, pstats: Dict[str, Any]) -> None: self.send_history_event("finished") self.current_job = None self.current_job_id = None + self.job_user = "No User" async def get_job(self, job_id: Union[int, str] @@ -425,6 +437,7 @@ def __init__(self, data: Dict[str, Any] = {}) -> None: self.start_time = time.time() self.total_duration: float = 0. self.auxiliary_data: List[Dict[str, Any]] = [] + self.user: str = "No User" self.update_from_ps(data) def finish(self, diff --git a/moonraker/components/job_queue.py b/moonraker/components/job_queue.py index 279b3cac..a5ae53b7 100644 --- a/moonraker/components/job_queue.py +++ b/moonraker/components/job_queue.py @@ -135,7 +135,9 @@ async def _pop_job(self, need_transition: bool = True) -> None: raise self.server.error( "Queue State Changed during Transition Gcode") self._set_queue_state("starting") - await kapis.start_print(filename, wait_klippy_started=True) + await kapis.start_print( + filename, wait_klippy_started=True, user=job.user + ) except self.server.error: logging.exception(f"Error Loading print: {filename}") self._set_queue_state("paused") @@ -165,7 +167,8 @@ async def _check_can_print(self) -> bool: async def queue_job(self, filenames: Union[str, List[str]], check_exists: bool = True, - reset: bool = False + reset: bool = False, + user: Optional[Dict[str, Any]] = None ) -> None: async with self.lock: # Make sure that the file exists @@ -178,7 +181,7 @@ async def queue_job(self, if reset: self.queued_jobs.clear() for fname in filenames: - queued_job = QueuedJob(fname) + queued_job = QueuedJob(fname, user) self.queued_jobs[queued_job.job_id] = queued_job self._send_queue_event(action="jobs_added") @@ -224,6 +227,7 @@ async def start_queue(self) -> None: else: qs = "ready" if self.automatic else "paused" self._set_queue_state(qs) + def _job_map_to_list(self) -> List[Dict[str, Any]]: cur_time = time.time() return [job.as_dict(cur_time) for @@ -261,7 +265,8 @@ async def _handle_job_request( files = web_request.get_list('filenames') reset = web_request.get_boolean("reset", False) # Validate that all files exist before queueing - await self.queue_job(files, reset=reset) + user = web_request.get_current_user() + await self.queue_job(files, reset=reset, user=user) elif req_type == RequestType.DELETE: if web_request.get_boolean("all", False): await self.delete_job([], all=True) @@ -319,14 +324,19 @@ async def close(self): await self.pause_queue() class QueuedJob: - def __init__(self, filename: str) -> None: + def __init__(self, filename: str, user: Optional[Dict[str, Any]] = None) -> None: self.filename = filename self.job_id = f"{id(self):016X}" self.time_added = time.time() + self._user = user def __str__(self) -> str: return self.filename + @property + def user(self) -> Optional[Dict[str, Any]]: + return self._user + def as_dict(self, cur_time: float) -> Dict[str, Any]: return { 'filename': self.filename, diff --git a/moonraker/components/klippy_apis.py b/moonraker/components/klippy_apis.py index b24e65b9..c11c324e 100644 --- a/moonraker/components/klippy_apis.py +++ b/moonraker/components/klippy_apis.py @@ -89,7 +89,8 @@ async def _gcode_cancel(self, web_request: WebRequest) -> str: async def _gcode_start_print(self, web_request: WebRequest) -> str: filename: str = web_request.get_str('filename') - return await self.start_print(filename) + user = web_request.get_current_user() + return await self.start_print(filename, user=user) async def _gcode_restart(self, web_request: WebRequest) -> str: return await self.do_restart("RESTART") @@ -123,7 +124,10 @@ async def run_gcode(self, return result async def start_print( - self, filename: str, wait_klippy_started: bool = False + self, + filename: str, + wait_klippy_started: bool = False, + user: Optional[Dict[str, Any]] = None ) -> str: # WARNING: Do not call this method from within the following # event handlers when "wait_klippy_started" is set to True: @@ -139,7 +143,9 @@ async def start_print( if wait_klippy_started: await self.klippy.wait_started() logging.info(f"Requesting Job Start, filename = {filename}") - return await self.run_gcode(script) + ret = await self.run_gcode(script) + self.server.send_event("klippy_apis:job_start_complete", user) + return ret async def pause_print( self, default: Union[Sentinel, _T] = Sentinel.MISSING diff --git a/moonraker/components/octoprint_compat.py b/moonraker/components/octoprint_compat.py index fca83c74..bd68e0a4 100644 --- a/moonraker/components/octoprint_compat.py +++ b/moonraker/components/octoprint_compat.py @@ -388,9 +388,10 @@ async def _select_file(self, except self.server.error: pstate = "not_avail" started: bool = False + user = web_request.get_current_user() if pstate not in ["printing", "paused", "not_avail"]: try: - await self.klippy_apis.start_print(filename) + await self.klippy_apis.start_print(filename, user=user) except self.server.error: started = False else: @@ -400,7 +401,7 @@ async def _select_file(self, if fmgr.upload_queue_enabled(): job_queue: JobQueue = self.server.lookup_component( 'job_queue') - await job_queue.queue_job(filename, check_exists=False) + await job_queue.queue_job(filename, check_exists=False, user=user) logging.debug(f"Job '{filename}' queued via OctoPrint API") else: raise self.server.error("Conflict", 409) diff --git a/moonraker/components/simplyprint.py b/moonraker/components/simplyprint.py index 6d7d91cc..e2c3d3c6 100644 --- a/moonraker/components/simplyprint.py +++ b/moonraker/components/simplyprint.py @@ -1598,7 +1598,7 @@ async def start_print(self) -> None: kapi: KlippyAPI = self.server.lookup_component("klippy_apis") data = {"state": "started"} try: - await kapi.start_print(pending) + await kapi.start_print(pending, user={"username": "SimplyPrint"}) except Exception: logging.exception("Print Failed to start") data["state"] = "error" From b60e6dc31189635bea33288db4e5e82647d74e95 Mon Sep 17 00:00:00 2001 From: Eric Callahan Date: Thu, 25 Apr 2024 12:24:07 -0400 Subject: [PATCH 7/9] spoolman: add history field tracking spool ids Signed-off-by: Eric Callahan --- moonraker/components/spoolman.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/moonraker/components/spoolman.py b/moonraker/components/spoolman.py index f545b914..d8a3e1d0 100644 --- a/moonraker/components/spoolman.py +++ b/moonraker/components/spoolman.py @@ -10,7 +10,7 @@ import re import contextlib import tornado.websocket as tornado_ws -from ..common import RequestType +from ..common import RequestType, HistoryFieldData from ..utils import json_wrapper as jsonw from typing import ( TYPE_CHECKING, @@ -29,6 +29,7 @@ from .database import MoonrakerDatabase from .announcements import Announcements from .klippy_apis import KlippyAPI as APIComp + from .history import History from tornado.websocket import WebSocketClientConnection DB_NAMESPACE = "moonraker" @@ -52,6 +53,12 @@ def __init__(self, config: ConfigHelper): self._error_logged: bool = False self._highest_epos: float = 0 self._current_extruder: str = "extruder" + self.spool_history = HistoryFieldData( + "spool_ids", "spoolman", "Spool IDs used", "collect", + reset_callback=self._on_history_reset + ) + history: History = self.server.lookup_component("history") + history.register_auxiliary_field(self.spool_history) self.klippy_apis: APIComp = self.server.lookup_component("klippy_apis") self.http_client: HttpClient = self.server.lookup_component("http_client") self.database: MoonrakerDatabase = self.server.lookup_component("database") @@ -103,6 +110,11 @@ def _register_endpoints(self): self._handle_status_request, ) + def _on_history_reset(self) -> List[int]: + if self.spool_id is None: + return [] + return [self.spool_id] + async def component_init(self) -> None: self.spool_id = await self.database.get_item( DB_NAMESPACE, ACTIVE_SPOOL_KEY, None @@ -270,6 +282,7 @@ def set_active_spool(self, spool_id: Union[int, None]) -> None: if self.spool_id == spool_id: logging.info(f"Spool ID already set to: {spool_id}") return + self.spool_history.tracker.update(spool_id) self.spool_id = spool_id self.database.insert_item(DB_NAMESPACE, ACTIVE_SPOOL_KEY, spool_id) self.server.send_event( From 81899e04fdc3c711937e593080351f72b35c4fc6 Mon Sep 17 00:00:00 2001 From: Eric Callahan Date: Fri, 26 Apr 2024 10:47:54 -0400 Subject: [PATCH 8/9] sensor: add history field options Signed-off-by: Eric Callahan --- moonraker/components/sensor.py | 67 +++++++++++++++++++++++++++++++--- 1 file changed, 61 insertions(+), 6 deletions(-) diff --git a/moonraker/components/sensor.py b/moonraker/components/sensor.py index 71d7dd2e..4c91ad8e 100644 --- a/moonraker/components/sensor.py +++ b/moonraker/components/sensor.py @@ -11,7 +11,7 @@ import logging from collections import defaultdict, deque from functools import partial -from ..common import RequestType +from ..common import RequestType, HistoryFieldData # Annotation imports from typing import ( @@ -24,12 +24,14 @@ Type, TYPE_CHECKING, Union, + Callable ) if TYPE_CHECKING: from ..confighelper import ConfigHelper from ..common import WebRequest from .mqtt import MQTTClient + from .history import History SENSOR_UPDATE_TIME = 1.0 SENSOR_EVENT_NAME = "sensors:sensor_update" @@ -56,6 +58,57 @@ def __init__(self, config: ConfigHelper) -> None: self.values: DefaultDict[str, Deque[Union[int, float]]] = defaultdict( lambda: deque(maxlen=store_size) ) + history: History = self.server.lookup_component("history") + self.field_info: Dict[str, List[HistoryFieldData]] = {} + all_opts = list(config.get_options().keys()) + cfg_name = config.get_name() + hist_field_prefix = "history_field_" + for opt in all_opts: + if not opt.startswith(hist_field_prefix): + continue + name = opt[len(hist_field_prefix):] + field_cfg: Dict[str, str] = config.getdict(opt) + ident: Optional[str] = field_cfg.pop("parameter", None) + if ident is None: + raise config.error( + f"[{cfg_name}]: option '{opt}', key 'parameter' must be" + f"specified" + ) + do_init: str = field_cfg.pop("init_tracker", "false").lower() + reset_cb = self._gen_reset_callback(ident) if do_init == "true" else None + excl_paused: str = field_cfg.pop("exclude_paused", "false").lower() + report_total: str = field_cfg.pop("report_total", "false").lower() + report_max: str = field_cfg.pop("report_maximum", "false").lower() + precision: Optional[str] = field_cfg.pop("precision", None) + try: + fdata = HistoryFieldData( + name, + cfg_name, + field_cfg.pop("desc", f"{ident} tracker"), + field_cfg.pop("strategy", "basic"), + units=field_cfg.pop("units", None), + reset_callback=reset_cb, + exclude_paused=excl_paused == "true", + report_total=report_total == "true", + report_maximum=report_max == "true", + precision=int(precision) if precision is not None else None, + ) + except Exception as e: + raise config.error( + f"[{cfg_name}]: option '{opt}', error encountered during " + f"sensor field configuration: {e}" + ) from e + for key in field_cfg.keys(): + self.server.add_warning( + f"[{cfg_name}]: Option '{opt}' contains invalid key '{key}'" + ) + self.field_info.setdefault(ident, []).append(fdata) + history.register_auxiliary_field(fdata) + + def _gen_reset_callback(self, param_name: str) -> Callable[[], float]: + def on_reset() -> float: + return self.last_measurements.get(param_name, 0) + return on_reset def _update_sensor_value(self, eventtime: float) -> None: """ @@ -108,6 +161,7 @@ def _on_state_update(self, payload: bytes) -> None: context = { "payload": payload.decode(), "set_result": partial(_set_result, store=measurements), + "log_debug": logging.debug } try: @@ -118,11 +172,12 @@ def _on_state_update(self, payload: bytes) -> None: else: self.error_state = None self.last_measurements = measurements - logging.debug( - "Received updated sensor value for %s: %s", - self.name, - self.last_measurements, - ) + for name, value in measurements.items(): + fdata_list = self.field_info.get(name) + if fdata_list is None: + continue + for fdata in fdata_list: + fdata.tracker.update(value) async def _on_mqtt_disconnected(self): self.error_state = "MQTT Disconnected" From 326d23a509b69d90d8e1ad3670641a8410071fde Mon Sep 17 00:00:00 2001 From: Eric Callahan Date: Fri, 3 May 2024 08:16:50 -0400 Subject: [PATCH 9/9] docs: document history additions Signed-off-by: Eric Callahan --- docs/changelog.md | 4 + docs/configuration.md | 152 +++++++++++++++++++++++++++++++++++-- docs/web_api.md | 172 +++++++++++++++++++++++++++++++++++++++++- 3 files changed, 320 insertions(+), 8 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 708e3433..f45e555f 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -35,6 +35,10 @@ The format is based on [Keep a Changelog]. - **machine**: Add support for system peripheral queries - **mqtt**: Added the `status_interval` option to support rate limiting - **mqtt**: Added the `enable_tls` option to support ssl/tls connections +- **history**: Added `user` field to job history data +- **history**: Added support for auxiliary history fields +- **spoolman**: Report spool ids set during a print in history auxiliary data +- **sensor**: Added support for history fields reported in auxiliary data ### Fixed diff --git a/docs/configuration.md b/docs/configuration.md index 0aab6f76..6645a167 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -41,7 +41,7 @@ opt_four: Once again\# not a comment resulting `#` is stored in the value. - Option `opt_three` resolves to a value of `This is the value`. The comment specifier is preceded by whitespace, thus the remainder of the line is - evaluted as a comment and removed from the option. + evaluated as a comment and removed from the option. - Option `opt_four` resolves to a value of `Once again\# not a comment`. The `\` character is not preceded by whitespace and not evaluated as an escape sequence, thus the escape character is not removed from the value. @@ -128,7 +128,7 @@ check_klipper_config_path: True # By default Moonraker will validate that Klipper's configuration file exists # within the data path's "config" folder, as this is a requirement for # Moonraker to write to the configuration. If this validation check fails -# Moonaker will warn the user. Installations that do not wish to use Moonraker +# Moonraker will warn the user. Installations that do not wish to use Moonraker # to manage Klipper's configuration may set this option to False to bypass the # location check. The default is True. enable_object_processing: False @@ -1324,9 +1324,8 @@ address: # A valid ip address or hostname of the Philips Hue Bridge. This # parameter must be provided. port: -# A port number if an alternative Zigbee bridge is used on a HTTP port +# A port number if an alternative Zigbee bridge is used on a HTTP port # different from the default 80/443 -# user: # The api key used for request authorization. This option accepts # Jinja2 Templates, see the [secrets] section for details. @@ -2814,7 +2813,101 @@ type: name: # The friendly display name of the sensor. # The default is the sensor source name. -``` +history_field_{field_name}: +# Optional history field description. When provided the named +# field will be tracked in Moonraker's Job History component. +# The "field_name" portion of the option is the identifier used +# when reported in the history. Multiple history fields may be +# added and tracked for a sensor. See the "History Fields" note +# for a detailed explanation of this option. +``` + +!!! note "History Fields" + A `history_field_{name}` option must contain a series of key-value pairs. + The key and value must be separated by an equal sign (=), and each + pair must be separated by a newline. The following keys are + available: + + - `parameter`: The name of the sensor parameter which is used to + provide values for this field. This name must match a field name + set in the specific sensor implementation (ie: see the + "state_response_template" option for the MQTT type.) This must + be provided. + - `desc`: A brief description of the field. + - `strategy`: The tracking strategy used to calculate the value + stored in the history. See below for available strategies. + The default is "basic". + - `units`: An optional unit specifier for the value + - `init_tracker`: When set to true the tracked value will be initialized + to the last sensor measurement when a job starts. The "delta" + strategy will initialize its "last value", setting this measurement + as the reference rather than the first received after the print starts. + Default is false. + - `exclude_paused`: When set to true the values received when + a job is paused will be ignored. Default is false. + - `report_total`: When set to true the value reported for all + jobs will be accumulated and reported in the history totals. + Default is false. + - `report_maximum`: When set to true maximum value for all jobs + will be reported in the history totals. Default is false. + - `precision`: An integer value indicating the precision to use when + reporting decimal values. This precision applies to both job history + AND job totals. The default is no precision, ie: no rounding will + occur. + + Note that job totals for history fields only persist for a currently + configured sensor and history field name. If the name of the sensor + changes, the name of the field changes, or if either are removed + from the configuration, then their totals will be discarded. This + prevents the accumulation of stale totals. + + Moonraker provides several history tracking strategies that can be used + accommodate how values should be tracked and stored in the job history: + + - `basic`: This strategy should be used if the value should be stored + in history directly as it is received. Simply put, the last value + received before a job completes wiill the the value stored in the job + history. + - `accumulate`: When a job starts, the tracked value initialized to 0 or + the last received measurement. New measurements will be added to the + tracked value as they are received. The total cumulative value will be + reported when the job ends. + - `delta`: When a job starts the tracked value is 0. The total value + will be the delta between the final measurement received before the job + ends and the first measurement received when after job begins. Note that + if `exclude_paused` is set then the tracker will accumulate deltas + between pauses. If the measurement does not update frequently this could + significantly alter the final result. + - `average`: Reports an average of all measurements received during the job. + - `maximum`: Reports the maximum value of all measurements received during + the job. + - `minimum`: Reports the minimum value of all measurements received during + the job. + - `collect`: Measurements are stored in a list as they are received. + Duplicate measurements are discarded. A maximum of 100 entries may + be stored, the oldest measurements will be discarded when this limit + is exceeded. This strategy is useful for a sensor that reports some + data infrequently and its desirable to include all measurements in the + job history. For example, the `spoolman` component uses this strategy + to report all spool IDs set during a job. When this strategy is enabled + the `track_total` and `track_maximum` options are ignored, as it is not + possible to report totals for a collection. + + Example: + + ``` + history_field_total_energy: + parameter=energy + desc=Printer power consumption + strategy=delta + units=kWh + init_tracker=false + exclude_paused=false + report_total=true + report_maximum=true + precision=6 + ``` + #### MQTT Sensor Configuration @@ -2881,6 +2974,55 @@ state_response_template: {set_result("energy", notification["aenergy"]["by_minute"][0]|float * 0.000001)} ``` +Tasmota Example: + +!!! Note + It may be necessary to set Tasmota's Telemetry Period to a low value + to acheive a decent response. This can be done in the with the + `TelePeriod` command via the console. For example, the command + to set the telemetry period to 10 seconds is: + + `cmnd/%device_name%/TelePeriod` with a payload of `10`. + +```ini +[sensor tasmota_power] +type: mqtt +state_topic: tele/tasmota_switch/SENSOR +state_response_template: + {% set resp = payload|fromjson %} + {% set edata = resp["ENERGY"] %} + {set_result("energy", edata["Total"])} + {set_result("voltage", edata["Voltage"])} + {set_result("power", edata["Power"])} + {set_result("current", edata["Current"])} +history_field_energy_consumption: + parameter=energy + desc=Printer energy consumption + strategy=delta + units=kWh + init_tracker=true + precision=6 + exclude_paused=false + report_total=true + report_maximum=true +history_field_average_current: + parameter=current + desc=Average current draw + strategy=average + units=A + report_total=false + report_maximum=true +# Mulitple history fields may track the same sensor parameter: +history_field_max_current: + parameter=current + desc=Maximum current draw + strategy=maximum + units=A + init_tracker=true + report_total=false + report_maximum=false +``` + ### `[spoolman]` Enables integration with the [Spoolman](https://github.com/Donkie/Spoolman) diff --git a/docs/web_api.md b/docs/web_api.md index abffcd7d..a0697dc8 100644 --- a/docs/web_api.md +++ b/docs/web_api.md @@ -1939,6 +1939,7 @@ GET /machine/peripherals/video "camera_name": "unicam", "driver_name": "unicam", "hardware_bus": "platform:3f801000.csi", + "modes": [], "capabilities": [ "VIDEO_CAPTURE", "EXT_PIX_FORMAT", @@ -1958,6 +1959,62 @@ GET /machine/peripherals/video "camera_name": "UVC Camera (046d:0825)", "driver_name": "uvcvideo", "hardware_bus": "usb-3f980000.usb-1.1", + "modes": [ + { + "format": "YUYV", + "description": "YUYV 4:2:2", + "flags": [], + "resolutions": [ + "640x480", + "160x120", + "176x144", + "320x176", + "320x240", + "352x288", + "432x240", + "544x288", + "640x360", + "752x416", + "800x448", + "800x600", + "864x480", + "960x544", + "960x720", + "1024x576", + "1184x656", + "1280x720", + "1280x960" + ] + }, + { + "format": "MJPG", + "description": "Motion-JPEG", + "flags": [ + "COMPRESSED" + ], + "resolutions": [ + "640x480", + "160x120", + "176x144", + "320x176", + "320x240", + "352x288", + "432x240", + "544x288", + "640x360", + "752x416", + "800x448", + "800x600", + "864x480", + "960x544", + "960x720", + "1024x576", + "1184x656", + "1280x720", + "1280x960" + ] + } + ], "capabilities": [ "VIDEO_CAPTURE", "EXT_PIX_FORMAT", @@ -1975,6 +2032,7 @@ GET /machine/peripherals/video "camera_name": "bcm2835-isp", "driver_name": "bcm2835-isp", "hardware_bus": "platform:bcm2835-isp", + "modes": [], "capabilities": [ "VIDEO_CAPTURE", "EXT_PIX_FORMAT", @@ -1992,6 +2050,7 @@ GET /machine/peripherals/video "camera_name": "bcm2835-isp", "driver_name": "bcm2835-isp", "hardware_bus": "platform:bcm2835-isp", + "modes": [], "capabilities": [ "VIDEO_CAPTURE", "EXT_PIX_FORMAT", @@ -2009,6 +2068,7 @@ GET /machine/peripherals/video "camera_name": "bcm2835-isp", "driver_name": "bcm2835-isp", "hardware_bus": "platform:bcm2835-isp", + "modes": [], "capabilities": [ "VIDEO_CAPTURE", "EXT_PIX_FORMAT", @@ -2026,6 +2086,7 @@ GET /machine/peripherals/video "camera_name": "bcm2835-isp", "driver_name": "bcm2835-isp", "hardware_bus": "platform:bcm2835-isp", + "modes": [], "capabilities": [ "VIDEO_CAPTURE", "EXT_PIX_FORMAT", @@ -2134,6 +2195,7 @@ V4L2 Device | `alt_name` | string? | An alternative device name optionally reported by | | | | sysfs. Will be `null` if the name file does not exist. |^ | `hardware_bus` | string | A description of the hardware location of the device | +| `modes` | array | An array of V4L2 mode objects. | | `capabilities` | array | An array of strings indicating the capabilities the | | | | device supports as reported by V4L2. |^ | `version` | string | The device version as reported by V4L2. | @@ -2145,6 +2207,16 @@ V4L2 Device | `usb_location` | string? | An identifier derived from the reported usb bus and | | | | device numbers. Will be `null` for non-usb devices. |^ +V4L2 Mode + +| Field | Type | Description | +| ------------- | :----: | ------------------------------------------------------------ | +| `description` | string | The description of the mode reported by the V4L2 driver. | +| `flags` | array | An array of strings describing flags reported by the driver. | +| `format` | string | The pixel format of the mode. | +| `resolutions` | array | An array of strings describing the resolutions supported by | +| | | the mode. Each entry is reported as `x` |^ + Libcamera Device | Field | Type | Description | @@ -6374,7 +6446,61 @@ An array of requested historical jobs: "print_duration": 18.37201827496756, "status": "completed", "start_time": 1615764496.622146, - "total_duration": 18.37201827496756 + "total_duration": 18.37201827496756, + "user": "testuser", + "auxiliary_data": [ + { + "provider": "sensor hist_test", + "name": "power_consumption", + "value": 4.119977, + "description": "Printer Power Consumption", + "units": "kWh" + }, + { + "provider": "sensor hist_test", + "name": "max_current", + "value": 2.768851, + "description": "Maximum current draw", + "units": "A" + }, + { + "provider": "sensor hist_test", + "name": "min_current", + "value": 0.426725, + "description": "Minmum current draw", + "units": "A" + }, + { + "provider": "sensor hist_test", + "name": "avg_current", + "value": 1.706872, + "description": "Average current draw", + "units": "A" + }, + { + "provider": "sensor hist_test", + "name": "status", + "value": 2, + "description": "Power Switch Status", + "units": null + }, + { + "provider": "sensor hist_test", + "name": "filament", + "value": 19.08058495194607, + "description": "filament usage tracker", + "units": "mm" + }, + { + "provider": "spoolman", + "name": "spool_ids", + "value": [ + 1 + ], + "description": "Spool IDs used", + "units": null + } + ] } ] } @@ -6406,7 +6532,27 @@ An object containing the following total job statistics: "total_filament_used": 11615.718840001999, "longest_job": 11665.191012736992, "longest_print": 11348.794790096988 - } + }, + "auxiliary_totals": [ + { + "provider": "sensor hist_test", + "field": "power_consumption", + "maximum": 4.119977, + "total": 4.119977 + }, + { + "provider": "sensor hist_test", + "field": "avg_current", + "maximum": 1.706872, + "total": null + }, + { + "provider": "sensor hist_test", + "field": "filament", + "maximum": 19.08058495194607, + "total": 19.08058495194607 + } + ] } ``` @@ -6439,7 +6585,27 @@ The totals prior to the reset: "total_filament_used": 11615.718840001999, "longest_job": 11665.191012736992, "longest_print": 11348.794790096988 - } + }, + "last_auxiliary_totals": [ + { + "provider": "sensor hist_test", + "field": "power_consumption", + "maximum": 4.119977, + "total": 4.119977 + }, + { + "provider": "sensor hist_test", + "field": "avg_current", + "maximum": 1.706872, + "total": null + }, + { + "provider": "sensor hist_test", + "field": "filament", + "maximum": 19.08058495194607, + "total": 19.08058495194607 + } + ] } ```