fix: bug of insight visualizer

packages/visualizer/src/component/detail-side.tsx

@@ -320,6 +320,7 @@ const DetailSide = (): JSX.Element => {
       content={<pre>{JSON.stringify(dump.data, undefined, 2)}</pre>}
     ></Card>
   ) : null;
+  console.log('dump is', dump);
 
   const plans = (task as ExecutionTaskPlanning)?.output?.plans;
   let timelineData: TimelineItemProps[] = [];

packages/visualizer/src/component/sidebar.tsx

@@ -8,7 +8,7 @@ import {
   LogoutOutlined,
   MinusOutlined,
 } from '@ant-design/icons';
-import { ExecutionTask } from '@midscene/core';
+import { ExecutionTask, ExecutionTaskInsightQuery } from '@midscene/core';
 import { Button } from 'antd';
 import PanelTitle from './panel-title';
 import { timeCostStrElement } from './misc';
@@ -43,8 +43,17 @@ const SideItem = (props: {
     statusText = timeCostStrElement(task.timing.cost);
   }
 
-  const contentRow =
-    task.type === 'Planning' ? <div className="side-item-content">{task.param?.userPrompt} </div> : null;
+  let contentRow: JSX.Element | undefined;
+  if (task.type === 'Planning') {
+    contentRow = <div className="side-item-content">{task.param?.userPrompt}</div>;
+  } else if (task.type === 'Insight' && task.subType === 'Query') {
+    // debugger;
+    const demand = (task as ExecutionTaskInsightQuery).param?.dataDemand;
+    const contentToShow = typeof demand === 'string' ? demand : JSON.stringify(demand);
+    contentRow = <div className="side-item-content">{contentToShow}</div>;
+  } else {
+    // debugger;
+  }
   // add hover listener
   return (
     <div

packages/visualizer/src/component/store.tsx

@@ -5,6 +5,7 @@ import {
   ExecutionTaskInsightLocate,
   ExecutionTask,
   GroupedActionDump,
+  ExecutionTaskInsightQuery,
 } from '../../../midscene/dist/types';
 
 export const useBlackboardPreference = create<{
@@ -33,7 +34,7 @@ export const useExecutionDump = create<{
   hoverPreviewConfig: { x: number; y: number } | null;
   setHoverPreviewConfig: (config: { x: number; y: number } | null) => void;
   reset: () => void;
-}>((set) => {
+}>((set, get) => {
   const initData = {
     dump: null,
     activeTask: null,
@@ -62,14 +63,15 @@ export const useExecutionDump = create<{
       // set the first one as selected
       for (const item of dump) {
         if (item.executions.length > 0 && item.executions[0].tasks.length > 0) {
-          set({ activeTask: item.executions[0].tasks[0] });
+          get().setActiveTask(item.executions[0].tasks[0]);
           break;
         }
       }
     },
     setActiveTask(task: ExecutionTask) {
       set({ activeTask: task });
-      if ((task as ExecutionTaskInsightLocate).log?.dump?.matchedElement) {
+      console.log('task set', task);
+      if (task.type === 'Insight') {
         syncToInsightDump((task as ExecutionTaskInsightLocate).log!.dump!);
       } else {
         resetInsightDump();

packages/visualizer/src/index.tsx

@@ -148,7 +148,7 @@ const Index = (props: { hideLogo?: boolean }): JSX.Element => {
           }
         }}
       >
-        <Panel maxSize={95}>
+        <Panel maxSize={95} defaultSize={20}>
           <Sidebar hideLogo={props?.hideLogo} />
         </Panel>
         <PanelResizeHandle

packages/web-integration/src/common/agent.ts

@@ -29,7 +29,7 @@ export class PageAgent {
     writeDumpFile(`playwright-${process.pid}`, groupedActionDumpFileExt, JSON.stringify(this.dumps));
   }
 
-  async aiAction(taskPrompt: string, dumpGroupName = 'MidScene / Web', dumpCaseName = 'AI Action') {
+  async aiAction(taskPrompt: string, dumpCaseName = 'AI Action', dumpGroupName = 'MidScene / Web') {
     const actionAgent = new PageTaskExecutor(this.page, { taskName: dumpCaseName });
     let error: Error | undefined;
     try {
@@ -48,7 +48,7 @@ export class PageAgent {
     }
   }
 
-  async aiQuery(demand: any, dumpGroupName = 'MidScene / Web', dumpCaseName = 'AI Query') {
+  async aiQuery(demand: any, dumpCaseName = 'AI Query', dumpGroupName = 'MidScene / Web') {
     const actionAgent = new PageTaskExecutor(this.page, { taskName: dumpCaseName });
     let error: Error | undefined;
     let result: any;
@@ -69,11 +69,11 @@ export class PageAgent {
     return result;
   }
 
-  async ai(taskPrompt: string, type = 'action', dumpGroupName = 'MidScene / Web', dumpCaseName = 'AI') {
+  async ai(taskPrompt: string, type = 'action', dumpCaseName = 'AI', dumpGroupName = 'MidScene / Web') {
     if (type === 'action') {
-      return this.aiAction(taskPrompt, dumpGroupName, dumpCaseName);
+      return this.aiAction(taskPrompt, dumpCaseName, dumpGroupName);
     } else if (type === 'query') {
-      return this.aiQuery(taskPrompt, dumpGroupName, dumpCaseName);
+      return this.aiQuery(taskPrompt, dumpCaseName, dumpGroupName);
     }
     throw new Error(`Unknown or Unsupported task type: ${type}, only support 'action' or 'query'`);
   }

packages/web-integration/src/playwright/index.ts

@@ -42,22 +42,22 @@ export const PlaywrightAiFixture = () => {
       await use(async (taskPrompt: string, type = 'action') => {
         const { groupName, caseName } = groupAndCaseForTest(testInfo);
         const agent = agentForPage(page);
-        return agent.ai(taskPrompt, type, groupName, caseName);
+        return agent.ai(taskPrompt, type, caseName, groupName);
       });
     },
     aiAction: async ({ page }: any, use: any, testInfo: TestInfo) => {
       await use(async (taskPrompt: string) => {
         const agent = agentForPage(page);
 
         const { groupName, caseName } = groupAndCaseForTest(testInfo);
-        await agent.aiAction(taskPrompt, groupName, caseName);
+        await agent.aiAction(taskPrompt, caseName, groupName);
       });
     },
     aiQuery: async ({ page }: any, use: any, testInfo: TestInfo) => {
       await use(async function (demand: any) {
         const agent = agentForPage(page);
         const { groupName, caseName } = groupAndCaseForTest(testInfo);
-        return agent.aiQuery(demand, groupName, caseName);
+        return agent.aiQuery(demand, caseName, groupName);
       });
     },
   };

packages/web-integration/src/puppeteer/index.ts

@@ -1,5 +1 @@
-import { PageAgent } from '@/common/agent';
-
-const PuppeteerAgent = PageAgent;
-
-export { PuppeteerAgent };
+export { PageAgent as PuppeteerAgent } from '@/common/agent';

packages/web-integration/tests/puppeteer/utils.ts

@@ -7,7 +7,9 @@ export async function launchPage(
     viewport?: Viewport;
   },
 ) {
-  const browser = await puppeteer.launch();
+  const browser = await puppeteer.launch({
+    headless: false,
+  });
 
   const page = (await browser.pages())[0];
   const viewportConfig = {

packages/web-integration/vitest.config.ts

@@ -8,6 +8,6 @@ export default defineConfig({
     },
   },
   test: {
-    include: ['./tests/puppeteer/*.spec.ts'],
+    include: ['./tests/puppeteer/**/*.spec.ts'],
   },
 });