New Computer Update (added debug mode to cursor operations)

interpreter/core/computer/computer.py

@@ -15,6 +15,7 @@ def __init__(self):
         self.terminal = Terminal()
 
         self.offline = False  # Soon, inherit this, and many other settings on import
+        self.debug_mode = False
 
         # OS mode
         try:

interpreter/core/computer/mouse/mouse.py

@@ -12,6 +12,7 @@ def __init__(self, computer):
         self.computer = computer
 
     def move(self, *args, x=None, y=None, icon=None):
+        screenshot = None
         if len(args) > 1:
             raise ValueError(
                 "Too many positional arguments provided: click(*args, x=None, y=None, show=True, index=None)\n\nPlease take a computer.screenshot() to find text/icons to click, then use computer.mouse.click(text) or computer.mouse.click(icon=description_of_icon) if at all possible. This is significantly more accurate."
@@ -82,6 +83,27 @@ def move(self, *args, x=None, y=None, icon=None):
         else:
             raise ValueError("Either text, icon, or both x and y must be provided")
 
+        if self.computer.debug_mode:
+            if not screenshot:
+                screenshot = self.computer.display.screenshot(show=False)
+            # Convert the screenshot to a numpy array for drawing
+            img_array = np.array(screenshot)
+            gray = cv2.cvtColor(img_array, cv2.COLOR_BGR2GRAY)
+            img_draw = cv2.cvtColor(gray, cv2.COLOR_GRAY2RGB)
+
+            # Draw a solid blue circle around the place we're clicking
+            cv2.circle(img_draw, (x, y), 20, (0, 0, 255), -1)
+
+            # Convert the drawn image back to a PIL image
+            img_pil = Image.fromarray(img_draw)
+            # Show the image
+            img_pil.show()
+
+            time.sleep(4)
+
+            # ^ This should ideally show it on the users computer but not show it to the LLM
+            # because we're in debug mode, trying to do debug
+
         pyautogui.moveTo(x, y, duration=0.5)
 
     def click(self, *args, button="left", clicks=1, interval=0.1, **kwargs):

interpreter/core/respond.py

@@ -168,6 +168,9 @@ def respond(interpreter):
                         r"from computer import (\w+)", r"\1 = computer.\1", code
                     )
 
+                # sync up debug mode (is this how we want to do this?)
+                interpreter.computer.debug_mode = interpreter.debug_mode
+
                 # yield each line
                 for line in interpreter.computer.run(language, code):
                     yield {"role": "computer", **line}