Merge branches 'dev-x-bot' and 'dev-x-bot' of github.com:myshell-ai/S…

…hellAgent into dev-x-bot

proconfig/core/block.py

@@ -11,7 +11,7 @@
 class Block(BaseModel):
     type: BlockType = Field(..., description="type of the block")
     name: str = Field("", description="name of the block")
-    display_name: str = Field(None, description="the display name of the block")
+    display_name: str = Field("", description="the display name of the block")
     properties: Dict[str, Any] = Field({}, description="to specify other properties")
     inputs: Dict[CustomKey, Union[Input, Value]] = {}
     outputs: Dict[Union[CustomKey, ContextCustomKey], Union[Variable, Value]] = {}
@@ -64,7 +64,7 @@ def sanity_check(self):
 
 
 class BaseProp(BaseModel):
-    cache: bool = None 
+    cache: bool = False
 
 class TaskBase(Block):
     type: Literal["task"]

proconfig/core/common.py

@@ -17,7 +17,7 @@
     'transitions',
     'states',
     'context',
-    'payload',
+    # 'payload',
     'condition',
     'blocks',
     'buttons',

proconfig/core/render.py

@@ -1,8 +1,8 @@
 from typing import Union, Dict, List, Optional
 from pydantic import BaseModel, Field
 
-from .common import CustomKey, CustomEventName
-from .variables import URLString, Expression, Value
+from proconfig.core.common import CustomKey, CustomEventName
+from proconfig.core.variables import URLString, Expression, Value
 
 
 class EventPayload(BaseModel):
@@ -14,7 +14,7 @@ class Button(BaseModel):
     content: str = ""
     description: Optional[str] = Field(default=None, description="Tooltip when hovering button")
     on_click: Union[CustomEventName, EventPayload] = Field(..., description="event name triggered")
-    style: Dict[str, str] = None # TODO
+    style: Dict[str, str] | None = None # TODO
 
 
 class RenderConfig(BaseModel):

proconfig/core/state.py

@@ -12,7 +12,7 @@ class StateProp(BaseProp):
     is_final: bool = False 
 
 class State(Block):
-    type: Literal["state"] = "state"
+    type: Literal["state", "dispatcher"] = "state"
     properties: StateProp = StateProp()
     blocks: BlockChildren[Union[Task, Workflow]] = Field({})
     render: RenderConfig = RenderConfig()
@@ -38,6 +38,7 @@ def sanity_check(self):
     def check_input_IM(self):
         IM_count = 0
         for k, v in self.inputs.items():
+            print(k, v)
             if v.type in ["text", "string"] and v.source == "IM":
                 IM_count += 1
         if IM_count > 1:

proconfig/core/variables.py

@@ -13,7 +13,7 @@
 
 class VariableBase(BaseModel):
     type: str
-    name: str = None
+    name: str = ""
     value: Any = None
 
 
@@ -73,7 +73,7 @@ class InputVariableBase(VariableBase, Generic[T]):
     default_value: Union[T, Expression] | None = None
     value: Union[T, Expression] | None = None
     user_input: bool = False
-    source: Literal["IM", "form"] = None
+    source: Literal["IM", "form"] = "form"
 
 class InputTextVar(TextVar, InputVariableBase[str], ChoicesBase[str]):
     type: Literal["text", "string"]

proconfig/runners/runner.py

@@ -220,20 +220,27 @@ def run_block_task(self, task, environ, local_vars):
         local_vars[task.name] = outputs
 
     def process_comfy_extra_inputs(self, task):
-
         comfy_extra_inputs = {
             "api": task.api,
             "comfy_workflow_id": task.comfy_workflow_id,
             "location": task.location,
         }
         task.inputs["comfy_extra_inputs"] = comfy_extra_inputs
+
+    def process_myshell_extra_inputs(self, task):
+        task.inputs = {
+            "widget_id": task.widget_name.split("/")[1],
+            "inputs": task.inputs
+        }
 
     def run_task(self, container, task, environ, local_vars):
 
         # block names: names of the (task) blocks of the same level
         if task.widget_class_name == "ComfyUIWidget": # hasattr(task, "comfy_workflow_id")
             assert hasattr(task, "comfy_workflow_id"), "no comfy_workflow_id is founded"
             self.process_comfy_extra_inputs(task)
+        elif task.widget_class_name == "MyShellAnyWidgetCallerWidget":
+            self.process_myshell_extra_inputs(task)
 
         if task.mode in ["widget", "comfy_workflow"]:
             return self.run_widget_task(container, task, environ, local_vars)
@@ -564,27 +571,40 @@ def run_automata(self, automata: Automata, sess_state: SessionState, payload: di
         local_transitions = automata.blocks[current_state_name].transitions or {}
         global_transitions = automata.transitions or {}
 
-        events = []
+        events = {}
         for button_id, button in enumerate(render.get("buttons", [])):
             event_name = button["on_click"]
             if isinstance(event_name, dict):
                 event_payload = event_name.get("payload", {})
                 event_name = event_name["event"]
             else:
                 event_payload = {}
-            events.append({
-                "event_key": f"BUTTON_{button_id}",
+            event_key = f"BUTTON_{button_id}"
+            events[event_key] = {
                 "event_name": event_name,
-                "payload": event_payload
-            })
+                "payload": event_payload,
+                "is_button": True
+            }
 
-        events.append({
-                "event_key": "CHAT",
-                "event_name": "CHAT",
-                "payload": {}
-        })
+        events["CHAT"] = {
+            "event_name": "CHAT",
+            "payload": {}
+        }
+
+        skip_evaluate_events = ["X.MENTIONED", "X.PINNED.REPLIED"]
+
+        # final event mapping, not target_inputs
+        for key, transition in local_transitions.items():
+            # evaluate the transitions
+            if key not in events:
+                events[key] = {
+                    "event_name": key,
+                    "payload": {}
+                }
+                if key in skip_evaluate_events:
+                    events[key]["skip_evaluate_target_inputs"] = True
 
-        for event_item in events:
+        for event_key, event_item in events.items():
             event_name = event_item["event_name"]
             event_payload = event_item["payload"]
             for payload_key, payload_value in event_payload.items():
@@ -609,7 +629,10 @@ def run_automata(self, automata: Automata, sess_state: SessionState, payload: di
             # get the target_inputs from transition
             target_inputs_transition = {}
             for k, v in transition_case.target_inputs.items():
-                target_inputs_transition[k] = calc_expression(v, {'payload': event_payload, **local_vars}) # the target_inputs defined by the transition
+                if event_item.get("skip_evaluate_target_inputs", False):
+                    target_inputs_transition[k] = v
+                else:
+                    target_inputs_transition[k] = calc_expression(v, {'payload': event_payload, **local_vars}) # the target_inputs defined by the transition
 
             # get the target inputs
             target_inputs = automata.blocks[target_state].inputs
@@ -625,14 +648,16 @@ def run_automata(self, automata: Automata, sess_state: SessionState, payload: di
                     setattr(input_var, k, tree_map(lambda x: calc_expression(x, visible_variables), getattr(input_var, k)))
 
             # target_inputs = tree_map(lambda x: calc_expression(x, local_vars), target_inputs)
-            event_mapping[event_item["event_key"]] = EventItem(**{
+            event_mapping[event_key] = EventItem(**{
                 "target_state": target_state,
                 "target_inputs": target_inputs,
                 "target_inputs_transition": target_inputs_transition, # the target_inputs from the transition
             })
+
+
 
         message_count = sess_state.message_count
-        event_mapping = {f'MESSAGE_{message_count}_{k}' if k != "CHAT" else k: v for k, v in event_mapping.items()}
+        event_mapping = {f'MESSAGE_{message_count}_{k}' if events[k].get("is_button", False) else k: v for k, v in event_mapping.items()}
         sess_state.event_mapping.update(event_mapping)
 
         return sess_state, render

proconfig/utils/x_bot.py

@@ -0,0 +1,141 @@
+import json
+from pydantic import BaseModel
+from typing import List, Literal
+from proconfig.core import Automata, State
+from proconfig.core.automata import StateWithTransitions
+from proconfig.core.variables import InputTextVar
+from proconfig.core.common import TransitionCase
+from proconfig.core.render import Button
+from proconfig.utils.misc import tree_map
+from datetime import date
+
+
+class TwitterUser(BaseModel):
+    creation_date: str = str(date.today())
+    user_id: str = "user_id_123"
+    username: str = "username_default"
+    name: str = "Default Name"
+    is_private: bool = False
+    is_verified: bool = False
+    is_blue_verified: bool = False
+    bot: bool = False
+    verified_type: Literal['blue', 'business', 'government'] = 'blue'
+
+
+class TweetDetail(BaseModel):
+    tweet_id: str = "tweet_id_123"
+    creation_date: str = str(date.today())
+    text: str = "This is an example text"
+    user: TwitterUser = TwitterUser()
+    media_url: List[str] = []
+
+
+x_special_events = {
+    "X.MENTIONED": {
+        "button_name": "Reply to mentioned",
+    },
+    "X.PINNED.REPLIED": {
+        "button_name": "Reply to pinned tweet"
+    },
+}
+
+    # content: str = ""
+    # description: Optional[str] = Field(default=None, description="Tooltip when hovering button")
+    # on_click: Union[CustomEventName, EventPayload] = Field(..., description="event name triggered")
+    # style: Dict[str, str] = None # TODO
+
+def replace_payload_to_payload1(x):
+    if type(x) == str:
+        x = x.replace("payload.", "payload1.")
+    return x
+
+def convert_x_bot_to_automata(automata_x):
+    # build default_payload
+    default_payload = TweetDetail().model_dump_json(indent=2)
+    # user = default_payload["user"]
+    # default_payload["user"] = f"{{{{{user}}}}}"
+
+    # automata_x = Automata.model_validate(automata_x)
+
+
+    new_states = {}
+    automata_x["blocks"] = automata_x.get("blocks", {})
+    for state_name, state in automata_x["blocks"].items():
+        timers = {}
+        state["properties"] = state.get("properties", {})
+        if "timers" in state["properties"]:
+            for timer_key, timer_info in state["properties"]["timers"].items():
+                timers[timer_info["event"]] = timer_key
+
+        state["render"] = state.get("render", {})
+        buttons = state["render"].get("buttons", [])
+        # buttons = state.render.buttons
+        # print("buttons:", buttons)
+        # first, handle the special events
+        count = 0
+        state["transitions"] = state.get("transitions") or {}
+        for event_name, transition_cases in state["transitions"].items():
+            if event_name in x_special_events:
+                new_button = {
+                    "content": x_special_events[event_name]["button_name"],
+                    "on_click": {
+                        "event": event_name,
+                        "payload": {}
+                    }
+                }
+                # add a pseudo state
+                new_state = {
+                    "display_name": "",
+                    "inputs": {
+                        "payload_str": {
+                            "type": "text",
+                            "name": "payload_str",
+                            "default_value": default_payload,
+                            "user_input": True,
+                            "source": "form"
+                        }
+                    },
+                    "outputs": {
+                        "payload1": "{{json.loads(payload_str)}}"
+                    },
+                    "transitions": {
+                        "ALWAYS": tree_map(replace_payload_to_payload1, transition_cases)
+                    }
+                }
+                new_state_name = f"{state_name}_event_{count}"
+                # state.transitions[event_name] = TransitionCase(target=new_state_name)
+                # new_states[new_state_name] = StateWithTransitions.model_validate(new_state)
+
+                state["transitions"][event_name] = {
+                    "target": new_state_name
+                }
+                new_states[new_state_name] = new_state
+            elif event_name in timers:
+                new_button = {
+                    "content": f"Timer [{timers[event_name]}] triggered",
+                    "on_click": {
+                        "event": event_name,
+                        "payload": {}
+                    }
+                }
+            else:
+                continue
+            count += 1
+            buttons.append(new_button)
+            # buttons.append(Button.model_validate(new_button))
+        state["render"]["buttons"] = buttons
+    automata_x["blocks"].update(new_states)
+    return automata_x
+    return automata_x.model_dump()
+
+
+if __name__ == "__main__":
+    automata_x_hello = json.load(open("temp/automata_x.json"))
+    automata_hello = convert_x_bot_to_automata(automata_x_hello)
+    # import pdb; pdb.set_trace()
+    # InputTextVar.model_validate(automata_hello["blocks"]["idle_event_0"]["inputs"]["payload_str"])
+    # automata_hello_load = Automata.model_validate(automata_hello)
+    json.dump(automata_hello, open("temp/automata_x_converted.json", "w"), indent=2)
+
+
+    import pdb; pdb.set_trace()

proconfig/widgets/imagen_widgets/comfyui_widget.py

@@ -383,7 +383,16 @@ def comfyui_run_myshell(workflow_id, inputs, extra_headers):
     response = requests.post(url, headers=headers, json=data)
     # Parse the JSON response
     if response.status_code == 200:
-        return json.loads(response.json()['result'])
+        response_json = response.json()
+        if response_json["success "]:
+            return json.loads(response.json()['result'])
+        else:
+            error = {
+                'error_code': 'SHELL-1109',
+                'error_head': 'HTTP Request Error',
+                'msg': response_json["error_message"],
+            }
+            raise ShellException(**error)
     else:
         try:
             response_content = response.json()

proconfig/widgets/language_models/claude_widgets.py

@@ -45,6 +45,7 @@ class MemoryItem(BaseModel):
 class ClaudeWidget(BaseWidget):
     NAME = "Claude 3.5 Sonnet"
     CATEGORY = "Large Language Model/Claude"
+    MYSHELL_WIDGET_NAME = "@myshell_llm/1744218088699596812"
 
     class InputsSchema(BaseWidget.InputsSchema):
         system_prompt: str = ""

proconfig/widgets/myshell_widgets/__init__.py

@@ -1,4 +1,5 @@
 from proconfig.widgets.myshell_widgets.tools.image_text_fuser import ImageTextFuserWidget
 from proconfig.widgets.myshell_widgets.tools.image_canvas import ImageCanvasWidget
 from proconfig.widgets.myshell_widgets.tools.twitter_search import XWidget
-from proconfig.widgets.myshell_widgets.tools.html2img import Html2ImgWidget
+from proconfig.widgets.myshell_widgets.tools.html2img import Html2ImgWidget
+from proconfig.widgets.myshell_widgets.myshell_widget_caller import MyShellAnyWidgetCallerWidget