Support loading gcode_macro templates from files

Currently, writing a python macro is somewhat painful, requiring preficing
each line with a single `!` to make preserve indentation. This also heavily
limits editor support.

This PR adds support for `!!include path/to/macro.py` (and technically
`!!include macro.notpy` resolves as jinja2 gcode templates).

For python macros, I've also added type stubs in `klippy/macro.pyi` and
added `TYPE_CHECKING` to the environment. This allows for basic typing
support in macro definitions. In the future I would love to be able to
generate a bespoke type stub including Printer objects, but thats long
term.

```
if TYPE_CHECKING:
  from klippy.macro import *
```

README.md

@@ -124,6 +124,8 @@ See the [Danger Features document](https://docs.kalico.gg/Danger_Features.html)
 
 - [gcode_macros: !python templates](https://github.com/KalicoCrew/kalico/pull/360)
 
+- [gcode_macros: !!include macros/my_macro.py](https://github.com/KalicoCrew/kalico/pull/578)
+
 - [core: action_log](https://github.com/KalicoCrew/kalico/pull/367)
 
 - [danger_options: configurable homing constants](https://github.com/KalicoCrew/kalico/pull/378)

docs/Command_Templates.md

@@ -199,6 +199,26 @@ gcode:
   !    emit(f"G0 X{coordinate[0]} Y{coordinate[1] + 0.25 * wipe} Z9.7 F12000")
 ```
 
+For ease of writing python macros, they may be read from a `.py` file. Python type stubs for macros are also available under `klippy.macro`.
+
+```
+## printer.cfg
+
+[gcode_macro clean_nozzle]
+gcode: !!include my_macros/clean_nozzle.py
+
+## my_macros/clean_nozzle.py
+
+if TYPE_CHECKING:
+  from klippy.macro import *
+
+wipe_count = 8
+emit("G90")
+emit("G0 Z15 F300")
+...
+
+```
+
 #### Python: Rawparams
 
 ```

docs/Kalico_Additions.md

@@ -66,6 +66,7 @@
 - The python [`math`](https://docs.python.org/3/library/math.html) library is available to macros. `{math.sin(math.pi * variable)}` and more!
 - New [`RELOAD_GCODE_MACROS`](./G-Codes.md#reload_gcode_macros) G-Code command to reload `[gcode_macro]` templates without requiring a restart.
 - G-Code Macros can be written in Python. Read more [here](./Command_Templates.md)
+  - Macros may also be loaded from other files, using `!!include path/to/file.py`
 
 ## Plugins
 

klippy/configfile.py

@@ -4,6 +4,7 @@
 #
 # This file may be distributed under the terms of the GNU GPLv3 license.
 import sys, os, glob, re, time, logging, configparser, io
+import pathlib
 from .extras.danger_options import get_danger_options
 from . import mathutil
 
@@ -15,6 +16,17 @@ class sentinel:
     pass
 
 
+PYTHON_SCRIPT_PREFIX = "!"
+_INCLUDERE = re.compile(r"!!include (?P<file>.*)")
+
+
+def _fix_include_path(source_file: str, match: re.Match) -> pathlib.Path:
+    new_path = pathlib.Path(source_file).parent.absolute() / match.group("file")
+    if not new_path.is_file():
+        raise error(f"Attempted to include non-existent file {new_path}")
+    return f"!!include {new_path}"
+
+
 class SectionInterpolation(configparser.Interpolation):
     """
     variable interpolation replacing ${[section.]option}
@@ -125,6 +137,28 @@ def get(self, option, default=sentinel, note_valid=True):
             self.fileconfig.get, option, default, note_valid=note_valid
         )
 
+    def getscript(self, option, default=sentinel, note_valid=True):
+        value: str = self.get(option, default, note_valid).strip()
+
+        match = _INCLUDERE.search(value)
+        if match:
+            file_path = pathlib.Path(match.group("file"))
+            if file_path.suffix.lower() == ".py":
+                return ("python", file_path.read_text())
+            else:
+                return ("gcode", file_path.read_text())
+
+        elif value.startswith(PYTHON_SCRIPT_PREFIX):
+            return (
+                "python",
+                "\n".join(
+                    line.removeprefix(PYTHON_SCRIPT_PREFIX)
+                    for line in value.splitlines()
+                ),
+            )
+
+        return ("gcode", value)
+
     def getint(
         self,
         option,
@@ -445,6 +479,10 @@ def _parse_config(self, data, filename, fileconfig, visited):
                     filename, include_spec, fileconfig, visited
                 )
             else:
+                line = _INCLUDERE.sub(
+                    lambda match: _fix_include_path(filename, match),
+                    line,
+                )
                 buffer.append(line)
         self._parse_config_buffer(buffer, filename, fileconfig)
         visited.remove(path)

klippy/extras/gcode_macro.py

@@ -5,9 +5,10 @@
 # This file may be distributed under the terms of the GNU GPLv3 license.
 import traceback, logging, ast, copy, json, threading
 import jinja2, math
+import typing
+
 from klippy import configfile
 
-PYTHON_SCRIPT_PREFIX = "!"
 
 ######################################################################
 # Template handling
@@ -49,7 +50,6 @@ def __iter__(self):
 class GetStatusWrapperPython:
     def __init__(self, printer):
         self.printer = printer
-        self.cache = {}
 
     def __getitem__(self, val):
         sval = str(val).strip()
@@ -74,6 +74,14 @@ def __iter__(self):
             if self.__contains__(name):
                 yield name
 
+    def get(self, key: str, default: configfile.sentinel):
+        try:
+            return self[key]
+        except KeyError:
+            if default is not configfile.sentinel:
+                return default
+            raise
+
 
 # Wrapper around a Jinja2 template
 class TemplateWrapperJinja:
@@ -123,9 +131,6 @@ def __init__(self, printer, env, name, script):
         self.vars = None
 
         try:
-            script = "\n".join(
-                map(lambda l: l.removeprefix("!"), script.split("\n"))
-            )
             self.func = compile(script, name, "exec")
         except SyntaxError as e:
             msg = "Error compiling expression '%s': %s at line %d column %d" % (
@@ -139,6 +144,7 @@ def __init__(self, printer, env, name, script):
 
     def run_gcode_from_command(self, context=None):
         helpers = {
+            "TYPE_CHECKING": False,
             "printer": GetStatusWrapperPython(self.printer),
             "emit": self._action_emit,
             "wait_while": self._action_wait_while,
@@ -262,21 +268,24 @@ def __iter__(self):
 
 
 class Template:
-    def __init__(self, printer, env, name, script) -> None:
+    def __init__(self, printer, env, name, script, script_type="gcode") -> None:
         self.name = name
         self.printer = printer
         self.env = env
-        self.reload(script)
+        self.reload(script_type, script)
 
     def __call__(self, context=None):
         return self.function(context)
 
     def __getattr__(self, name):
         return getattr(self.function, name)
 
-    def reload(self, script):
-        if script.startswith(PYTHON_SCRIPT_PREFIX):
-            script = script[len(PYTHON_SCRIPT_PREFIX) :]
+    def reload(
+        self,
+        script_type: typing.Literal["python", "gcode"],
+        script: str,
+    ):
+        if script_type == "python":
             self.function = TemplateWrapperPython(
                 self.printer, self.env, self.name, script
             )
@@ -309,11 +318,12 @@ def __init__(self, config):
     def load_template(self, config, option, default=None):
         name = "%s:%s" % (config.get_name(), option)
         if default is None:
-            script = config.get(option)
+            script_type, script = config.getscript(option)
         else:
-            script = config.get(option, default)
-        script = script.strip()
-        return Template(self.printer, self.env, name, script)
+            script_type, script = config.getscript(option, default)
+        return Template(
+            self.printer, self.env, name, script, script_type=script_type
+        )
 
     def _action_emergency_stop(self, msg="action_emergency_stop"):
         self.printer.invoke_shutdown("Shutdown due to %s" % (msg,))
@@ -363,8 +373,8 @@ def cmd_RELOAD_GCODE_MACROS(self, gcmd):
                 for s in new_config.get_prefix_sections("gcode_macro")
             ]:
                 template = obj.template
-                new_script = new_section.get("gcode").strip()
-                template.reload(new_script)
+                script_type, new_script = new_section.getscript("gcode")
+                template.reload(script_type, new_script)
 
 
 def load_config(config):

klippy/macro.pyi

@@ -0,0 +1,78 @@
+import typing
+import math
+
+rawparams: str
+params: dict[str, str]
+own_vars: dict[str, typing.Any]
+
+printer: dict[str, dict[str, typing.Any]]
+
+def emit(gcode: str) -> None:
+    "Run a G-Code"
+
+def wait_while(condition: typing.Callable[[], bool]) -> None:
+    "Wait while a condition is True"
+
+def wait_until(condition: typing.Callable[[], bool]) -> None:
+    "Wait until a condition is True"
+
+def wait_moves() -> None:
+    "Wait until all moves are completed"
+
+def blocking(function: typing.Callable[[], typing._T]) -> typing._T:
+    "Run a blocking task in a thread, waiting for the result"
+
+def sleep(timeout: float) -> None:
+    "Wait a given number of seconds"
+
+def set_gcode_variable(macro: str, variable: str, value: typing.Any) -> None:
+    "Save a variable to a gcode_macro"
+
+def action_log(msg: str) -> typing.Literal[""]:
+    "Log a message to klippy.log"
+
+def action_emergency_stop(
+    msg: str = "action_emergency_stop",
+) -> typing.Literal[""]:
+    "Immediately shutdown Kalico"
+
+def action_respond_info(msg: str) -> typing.Literal[""]:
+    "Send a message to the console"
+
+def action_raise_error(msg) -> None:
+    "Raise a G-Code command error"
+
+def action_call_remote_method(method: str, **kwargs) -> typing.Literal[""]:
+    "Call a Kalico webhooks method"
+
+emergency_stop = action_emergency_stop
+respond_info = action_respond_info
+raise_error = action_raise_error
+call_remote_method = action_call_remote_method
+
+TYPE_CHECKING: False
+
+__all__ = (
+    "params",
+    "rawparams",
+    "own_vars",
+    "printer",
+    "emit",
+    "wait_while",
+    "wait_until",
+    "wait_moves",
+    "blocking",
+    "sleep",
+    "set_gcode_variable",
+    "emergency_stop",
+    "respond_info",
+    "raise_error",
+    "call_remote_method",
+    "action_call_remote_method",
+    "action_emergency_stop",
+    "action_log",
+    "action_raise_error",
+    "action_respond_info",
+    "math",
+    "TYPE_CHECKING",
+)

test/klippy/macro_loading.cfg

@@ -0,0 +1,85 @@
+# Test config for gcode python
+[stepper_x]
+step_pin: PF0
+dir_pin: PF1
+enable_pin: !PD7
+microsteps: 16
+rotation_distance: 40
+endstop_pin: ^PE5
+position_endstop: 0
+position_max: 200
+homing_speed: 50
+
+[stepper_y]
+step_pin: PF6
+dir_pin: !PF7
+enable_pin: !PF2
+microsteps: 16
+rotation_distance: 40
+endstop_pin: ^PJ1
+position_endstop: 0
+position_max: 200
+homing_speed: 50
+
+[stepper_z]
+step_pin: PL3
+dir_pin: PL1
+enable_pin: !PK0
+microsteps: 16
+rotation_distance: 8
+endstop_pin: probe:z_virtual_endstop
+position_max: 200
+
+[extruder]
+step_pin: PA4
+dir_pin: PA6
+enable_pin: !PA2
+microsteps: 16
+rotation_distance: 33.5
+nozzle_diameter: 0.400
+filament_diameter: 1.750
+heater_pin: PB4
+sensor_type: EPCOS 100K B57560G104F
+sensor_pin: PK5
+control: pid
+pid_Kp: 22.2
+pid_Ki: 1.08
+pid_Kd: 114
+min_temp: 0
+max_temp: 250
+
+[heater_bed]
+heater_pin: PH5
+sensor_type: EPCOS 100K B57560G104F
+sensor_pin: PK6
+control: watermark
+min_temp: 0
+max_temp: 130
+
+[probe]
+pin: PH6
+z_offset: 1.15
+drop_first_result: true
+
+[bed_mesh]
+mesh_min: 10,10
+mesh_max: 180,180
+
+[mcu]
+serial: /dev/ttyACM0
+
+[printer]
+kinematics: cartesian
+max_velocity: 300
+max_accel: 3000
+max_z_velocity: 5
+max_z_accel: 100
+
+[gcode_macro EXTERNAL_GCODE]
+gcode: !!include macro_loading/macro.gcode.j2
+
+[gcode_macro EXTERNAL_PYTHON]
+gcode: !!include macro_loading/macro.py
+
+# Use the include here to test the include-relative imports work
+[include macro_loading/include.cfg]

test/klippy/macro_loading.test

@@ -0,0 +1,8 @@
+# Test case for python gcode
+CONFIG macro_loading.cfg
+DICTIONARY atmega2560.dict
+
+EXTERNAL_GCODE
+EXTERNAL_PYTHON
+INCLUDE_GCODE
+INCLUDE_PYTHON