diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index a20117ae6..583f70fe2 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -1,4 +1,6 @@ -import 'dotenv/config'; +import { existsSync } from 'node:fs'; +import { join } from 'node:path'; +import dotenv from 'dotenv'; import { matchYamlFiles, parseProcessArgs } from './cli-utils'; import { playYamlFiles } from './yaml-runner'; @@ -9,6 +11,12 @@ Promise.resolve( const welcome = '\nWelcome to @midscene/cli\n'; console.log(welcome); + const dotEnvConfigFile = join(process.cwd(), '.env'); + if (existsSync(dotEnvConfigFile)) { + console.log(`loading .env file from ${dotEnvConfigFile}`); + dotenv.config({ path: dotEnvConfigFile }); + } + if (options.url) { console.error( 'the cli mode is no longer supported, please use yaml file instead. See https://midscenejs.com/automate-with-scripts-in-yaml for more information. Sorry for the inconvenience.', diff --git a/packages/web-integration/src/chrome-extension/page.ts b/packages/web-integration/src/chrome-extension/page.ts index 0f0f16cbd..8651d355d 100644 --- a/packages/web-integration/src/chrome-extension/page.ts +++ b/packages/web-integration/src/chrome-extension/page.ts @@ -372,17 +372,14 @@ export default class ChromeExtensionProxyPage implements AbstractPage { commands: ['selectAll'], }); - await this.sendCommandToDebugger('Input.dispatchKeyEvent', { - type: 'keyDown', - key: 'Backspace', - code: 'Backspace', - }); - await this.sendCommandToDebugger('Input.dispatchKeyEvent', { type: 'keyUp', - key: 'Backspace', - code: 'Backspace', + commands: ['selectAll'], }); + + await sleep(100); + + await this.keyboard.press('Backspace'); } mouse = { diff --git a/packages/web-integration/src/extractor/util.ts b/packages/web-integration/src/extractor/util.ts index 218f96958..dfc97fe01 100644 --- a/packages/web-integration/src/extractor/util.ts +++ b/packages/web-integration/src/extractor/util.ts @@ -295,6 +295,7 @@ export function visibleRect( return false; } + // check if the element is hidden by an ancestor let parent: HTMLElement | Node | null = el; while (parent && parent !== document.body) { if (!(parent instanceof HTMLElement)) { @@ -319,6 +320,10 @@ export function visibleRect( return false; } } + // if the parent is a fixed element, stop the search + if (parentStyle.position === 'fixed') { + break; + } parent = parent.parentElement; } diff --git a/packages/web-integration/src/puppeteer/agent-launcher.ts b/packages/web-integration/src/puppeteer/agent-launcher.ts index 35d096ae2..fa90a1df4 100644 --- a/packages/web-integration/src/puppeteer/agent-launcher.ts +++ b/packages/web-integration/src/puppeteer/agent-launcher.ts @@ -80,6 +80,8 @@ export async function puppeteerAgentForTarget( ...(isWindows ? [] : ['--no-sandbox', '--disable-setuid-sandbox']), '--disable-features=PasswordLeakDetection', '--disable-save-password-bubble', + '--start-maximized', + `--window-size=${width},${height}`, ], }); freeFn.push({ diff --git a/packages/web-integration/src/puppeteer/base-page.ts b/packages/web-integration/src/puppeteer/base-page.ts index 27b7b9b53..056ef78c2 100644 --- a/packages/web-integration/src/puppeteer/base-page.ts +++ b/packages/web-integration/src/puppeteer/base-page.ts @@ -1,5 +1,5 @@ import type { Point, Size } from '@midscene/core'; -import { getTmpFile } from '@midscene/core/utils'; +import { getTmpFile, sleep } from '@midscene/core/utils'; import { base64Encoded } from '@midscene/shared/img'; import type { Page as PlaywrightPage } from 'playwright'; import type { Page as PuppeteerPage } from 'puppeteer'; @@ -79,9 +79,14 @@ export class Page< get mouse() { return { - click: async (x: number, y: number, options?: { button: MouseButton }) => + click: async ( + x: number, + y: number, + options?: { button?: MouseButton; count?: number }, + ) => this.underlyingPage.mouse.click(x, y, { button: options?.button || 'left', + count: options?.count || 1, }), wheel: async (deltaX: number, deltaY: number) => { if (this.pageType === 'puppeteer') { @@ -118,18 +123,26 @@ export class Page< return; } - await this.mouse.click(element.center[0], element.center[1]); - const isMac = process.platform === 'darwin'; if (isMac) { - await this.underlyingPage.keyboard.down('Meta'); - await this.underlyingPage.keyboard.press('a'); - await this.underlyingPage.keyboard.up('Meta'); + if (this.pageType === 'puppeteer') { + // https://github.com/segment-boneyard/nightmare/issues/810#issuecomment-452669866 + await this.mouse.click(element.center[0], element.center[1], { + count: 3, + }); + } else { + await this.mouse.click(element.center[0], element.center[1]); + await this.underlyingPage.keyboard.down('Meta'); + await this.underlyingPage.keyboard.press('a'); + await this.underlyingPage.keyboard.up('Meta'); + } } else { + await this.mouse.click(element.center[0], element.center[1]); await this.underlyingPage.keyboard.down('Control'); await this.underlyingPage.keyboard.press('a'); await this.underlyingPage.keyboard.up('Control'); } + await sleep(100); await this.keyboard.press('Backspace'); } diff --git a/packages/web-integration/tests/ai/web/playwright/ai-auto-todo.spec.ts b/packages/web-integration/tests/ai/web/playwright/ai-auto-todo.spec.ts index e228c1df1..f113a4542 100644 --- a/packages/web-integration/tests/ai/web/playwright/ai-auto-todo.spec.ts +++ b/packages/web-integration/tests/ai/web/playwright/ai-auto-todo.spec.ts @@ -12,7 +12,8 @@ test('ai todo', async ({ ai, aiQuery }) => { test.setTimeout(1000 * 50); } - await ai('Enter "Learn" in the task box, don\'t press enter'); + await ai('Enter "Happy Birthday" in the task box'); + await ai('Enter "Learn" in the task box'); await ai( 'Add "JS today" to base on the existing content(important) of the task box, then press enter', diff --git a/packages/web-integration/tests/ai/web/puppeteer/showcase.test.ts b/packages/web-integration/tests/ai/web/puppeteer/showcase.test.ts index d30e00927..50f359385 100644 --- a/packages/web-integration/tests/ai/web/puppeteer/showcase.test.ts +++ b/packages/web-integration/tests/ai/web/puppeteer/showcase.test.ts @@ -85,13 +85,14 @@ describe( }); it('search engine', async () => { - const { originPage, reset } = await launchPage('https://www.bing.com/'); + const { originPage, reset } = await launchPage('https://www.baidu.com/'); const mid = new PuppeteerAgent(originPage); + await mid.aiAction('type "AI 101" in search box'); await mid.aiAction( - 'type "AI 101" in search box, hit Enter, wait 2s, click the second result, wait 4s', + 'type "Hello world" in search box, hit Enter, wait 2s, click the second result, wait 4s', ); - await mid.aiWaitFor('there are some search results'); + await mid.aiWaitFor('there are some search results about "Hello world"'); await reset(); }); diff --git a/packages/web-integration/tests/unit-test/__snapshots__/web-extractor.test.ts.snap b/packages/web-integration/tests/unit-test/__snapshots__/web-extractor.test.ts.snap index d373445fc..650df4766 100644 --- a/packages/web-integration/tests/unit-test/__snapshots__/web-extractor.test.ts.snap +++ b/packages/web-integration/tests/unit-test/__snapshots__/web-extractor.test.ts.snap @@ -559,6 +559,27 @@ exports[`extractor > basic 1`] = ` }, "content": "hidden label", }, + { + "attributes": { + "htmlTagName": "
", + "nodeType": "TEXT Node", + }, + "content": "i am fixed child content", + }, + { + "attributes": { + "htmlTagName": "", + "nodeType": "TEXT Node", + }, + "content": "abcd efg", + }, + { + "attributes": { + "htmlTagName": "
", + "nodeType": "TEXT Node", + }, + "content": "content editable div content. We should collect the parent.", + }, { "attributes": { "htmlTagName": "
", diff --git a/packages/web-integration/tests/unit-test/fixtures/web-extractor/index.html b/packages/web-integration/tests/unit-test/fixtures/web-extractor/index.html index 4e6fafe45..0b09a2f8b 100644 --- a/packages/web-integration/tests/unit-test/fixtures/web-extractor/index.html +++ b/packages/web-integration/tests/unit-test/fixtures/web-extractor/index.html @@ -293,6 +293,23 @@

Form

hidden label + +
+
+
i am fixed child content
+
+
+ + +
+ + abcd efg +
+ content editable div content. We should collect the parent. +
+
+ +
absolute child content
diff --git a/packages/web-integration/tests/unit-test/fixtures/web-extractor/input.png b/packages/web-integration/tests/unit-test/fixtures/web-extractor/input.png index 9c89affc3..fd1662a45 100644 Binary files a/packages/web-integration/tests/unit-test/fixtures/web-extractor/input.png and b/packages/web-integration/tests/unit-test/fixtures/web-extractor/input.png differ diff --git a/packages/web-integration/tests/unit-test/fixtures/web-extractor/output.png b/packages/web-integration/tests/unit-test/fixtures/web-extractor/output.png index 985aad6b2..232c736e1 100644 Binary files a/packages/web-integration/tests/unit-test/fixtures/web-extractor/output.png and b/packages/web-integration/tests/unit-test/fixtures/web-extractor/output.png differ