From 3db0c9bf11dcba40782241f3886cd79f1912f00f Mon Sep 17 00:00:00 2001 From: Mitchell Lee Date: Wed, 7 Feb 2024 14:52:46 -0600 Subject: [PATCH] feat: adding docker setup for scene-composer ui tests - Moves all snapshots to use git lfs (replaces with pointers, you will need git lfs to checkout from this point on) - Updates snapshots for scene-composer ui tests, now just chromium-linux for now (you can technically run darwin if you like, but our master snapshots are linux based from now on). - Add new Github workflow anyone can add to a pull request to trigger a reliability check. Adding the "test:reliability" label to a pull request will now force all UI tests to run 5 times, useful for checking for flaky tests when new tests are being added in a PR --- .github/workflows/codeql-analysis.yml | 1 - .github/workflows/publish-storybook.yml | 1 - .github/workflows/ui-test-for-commit.yml | 4 +- .github/workflows/ui-test-full-suite.yml | 4 +- .github/workflows/ui-test-reliability.yml | 56 +++++++++++++++++++ commitlint.config.js | 2 +- docs/development.md | 5 ++ package.json | 4 +- packages/scene-composer/README.md | 20 +++++++ packages/scene-composer/docker-compose.yml | 13 +++++ .../scene-composer--local-scene.spec.ts | 13 +++-- .../local-scene-canvas-chromium-darwin.png | 3 - .../local-scene-canvas-chromium-linux.png | 4 +- ...otion-indicator-canvas-chromium-darwin.png | 3 - ...motion-indicator-canvas-chromium-linux.png | 4 +- packages/scene-composer/package.json | 19 +++++-- packages/scene-composer/playwright.config.ts | 6 +- 17 files changed, 132 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/ui-test-reliability.yml create mode 100644 packages/scene-composer/docker-compose.yml delete mode 100644 packages/scene-composer/e2e/tests/local-scene/scene-composer--local-scene.spec.ts-snapshots/local-scene-canvas-chromium-darwin.png delete mode 100644 packages/scene-composer/e2e/tests/motion-indicator/scene-composer--motion-indicator.spec.ts-snapshots/motion-indicator-canvas-chromium-darwin.png diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 530dcea15..04221d85e 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -10,7 +10,6 @@ # supported CodeQL languages. # name: "CodeQL" - on: push: branches: [ main, rc ] diff --git a/.github/workflows/publish-storybook.yml b/.github/workflows/publish-storybook.yml index e11943160..327626ff3 100644 --- a/.github/workflows/publish-storybook.yml +++ b/.github/workflows/publish-storybook.yml @@ -3,7 +3,6 @@ on: pull_request: paths: - 'packages/scene-composer/**' - jobs: publish-storybook: runs-on: ubuntu-latest diff --git a/.github/workflows/ui-test-for-commit.yml b/.github/workflows/ui-test-for-commit.yml index 4b936a17f..497ca5acc 100644 --- a/.github/workflows/ui-test-for-commit.yml +++ b/.github/workflows/ui-test-for-commit.yml @@ -1,5 +1,7 @@ name: UI Tests - Associated to last commit on: [pull_request] +env: + PW_TEST_HTML_REPORT_OPEN: 'never' jobs: e2e-tests-playwright: runs-on: ubuntu-latest @@ -30,7 +32,7 @@ jobs: - name: test UI run: - npm run test:ui -w @iot-app-kit/dashboard --filter=[HEAD~1] && npm run test:ui -w @iot-app-kit/react-components --filter=[HEAD~1] && npm run test:ui -w @iot-app-kit/scene-composer --filter=[HEAD~1] + npm run test:ui -w @iot-app-kit/dashboard --filter=[HEAD~1] && npm run test:ui -w @iot-app-kit/react-components --filter=[HEAD~1] && npm run test:ui -w @iot-app-kit/scene-composer --filter=[HEAD~1] --production - uses: actions/upload-artifact@v4 if: failure() with: diff --git a/.github/workflows/ui-test-full-suite.yml b/.github/workflows/ui-test-full-suite.yml index 590769615..273e55e41 100644 --- a/.github/workflows/ui-test-full-suite.yml +++ b/.github/workflows/ui-test-full-suite.yml @@ -1,5 +1,7 @@ name: UI Tests - Full run on: [pull_request] +env: + PW_TEST_HTML_REPORT_OPEN: 'never' jobs: e2e-tests-playwright-full-suite: runs-on: ubuntu-latest @@ -30,7 +32,7 @@ jobs: - name: test UI run: - npm run test:ui -w @iot-app-kit/dashboard && npm run test:ui -w @iot-app-kit/react-components && npm run test:ui:reliability -w @iot-app-kit/scene-composer + npm run test:ui -w @iot-app-kit/dashboard && npm run test:ui -w @iot-app-kit/react-components && npm run test:ui -w @iot-app-kit/scene-composer --production - uses: actions/upload-artifact@v4 if: failure() with: diff --git a/.github/workflows/ui-test-reliability.yml b/.github/workflows/ui-test-reliability.yml new file mode 100644 index 000000000..96bbf0986 --- /dev/null +++ b/.github/workflows/ui-test-reliability.yml @@ -0,0 +1,56 @@ +name: "UI Test Reliability Check" + +# We only run this on PRs that have the label "test:reliability", otherwise it's excluded. +# Add this label anytime there are new UI tests added, or you want to verify a change still +# reliabily passes tests +on: + pull_request: + types: [ labeled ] + +env: + PW_TEST_HTML_REPORT_OPEN: 'never' + +jobs: + test-reliability: + if: ${{ github.event.label.name == 'test:reliability' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + lfs: 'true' + - uses: actions/setup-node@v4 + with: + node-version: 16 + cache: 'npm' + cache-dependency-path: '**/package-lock.json' + + - name: Cache playwright binaries + uses: actions/cache@v4 + id: playwright-cache + with: + path: | + ~/.cache/ms-playwright + key: ${{ runner.os }}-playwright-${{ env.PLAYWRIGHT_VERSION }} + - name: Install + run: + npm ci --prefer-offline --no-audit --no-fund && npm run bootstrap + + - name: Install Playwright's dependencies + run: npx playwright install --with-deps + + - name: test reliability + run: + npm run test:reliability --production + - uses: actions/upload-artifact@v4 + if: failure() + with: + name: Playwright failed test-results + path: packages/**/test-results + retention-days: 30 + - uses: actions/upload-artifact@v4 + if: always() + with: + name: playwright-report + path: packages/**/playwright-report + retention-days: 30 diff --git a/commitlint.config.js b/commitlint.config.js index d26f7b149..d486aa64c 100644 --- a/commitlint.config.js +++ b/commitlint.config.js @@ -1,6 +1,6 @@ module.exports = { extends: ['@commitlint/config-conventional'], rules: { - 'body-max-line-length': [2, 'always', 250], + 'body-max-line-length': [0], }, }; diff --git a/docs/development.md b/docs/development.md index b35913300..cac8270ea 100644 --- a/docs/development.md +++ b/docs/development.md @@ -8,6 +8,8 @@ Ensure you have `node` version 16 and `npm` > 8.0. - Node: any `v16` or higher - Npm: `v8.0.0` or higher +- git lfs: https://git-lfs.com/ - We use git lfs to keep binary files and screenshots from ballooning our repo size +- docker: https://www.docker.com/products/docker-desktop/ - Docker is needed by our UI Tests to ensure reliability/consistency across dev local and ci If you need to setup node, consult https://nodejs.org/en/download/package-manager @@ -51,6 +53,7 @@ Learn more at the [npm workspaces documentation](https://docs.npmjs.com/cli/v7/u Every package in IoT App Kit follows a similar structure: - every package with tests, contains a `test` command that runs jest based tests +- every package with UI Tests, contains a `test:ui` command that runs playwright tests, and a `test:ui:reliability` command that stress tests these tests to ensure they aren't flaky - every package contains a `lint` command that executes eslint - every package contains a `fix` command that fixes eslint errors - every package contains a `build` command that builds the package @@ -78,6 +81,8 @@ the correct [semver](https://semver.org/) version for publishing. Due to the changelog and versioning being determined by the commit messaging, commit messages are considered part of the review process, so "squash merging" and other techniques that allow changing the commit message after approval right before merging, are not allowed. +You should add UI Tests for any major functionality, and if you do, label your pull request with the `test:reliability` label on first checkin, which will stress test your UI tests. + **git good tip**: `git rebase -i HEAD~${NUM_COMMITS}` is a useful way to squash together the last `NUM_COMMITS` before making your pull-request. Learn more [here](https://git-scm.com/book/en/v2/Git-Tools-Rewriting-History). ### 6. Additional Coding guidelines and requirements diff --git a/package.json b/package.json index d5fe955d6..4a17fb423 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "fix:stylelint": "stylelint '**/*.css' --fix", "test": "turbo run test", "test:ui": "turbo run test:ui", + "test:reliability": "turbo run test:ui:reliability", "test:stylelint": "stylelint '**/*.css' --max-warnings 0", "test:git": "git diff --exit-code", "release": "npm run build", @@ -34,7 +35,8 @@ "versionup:patch": "turbo run version --no-git-tag-version patch", "versionup:minor": "turbo run version --no-git-tag-version minor", "versionup:major": "turbo run version --no-git-tag-version major", - "prepare": "husky install" + "prepare": "husky install", + "test:ci": "if test \"$NODE_ENV\" = \"production\"; then echo \"Yes!\"; fi" }, "dependencies": { "@cloudscape-design/collection-hooks": "1.0.22", diff --git a/packages/scene-composer/README.md b/packages/scene-composer/README.md index d371df996..e7751f191 100644 --- a/packages/scene-composer/README.md +++ b/packages/scene-composer/README.md @@ -13,6 +13,26 @@ The library will be built and copied to the `dist` folder. npm run build ``` +**UI Tests** + +Pre-requisites: You need to have [Docker](https://docs.docker.com/get-docker/) installed locally to run UI Tests, we rely on it to provide the necessary browsers to generate consistent screenshots. + +To setup the docker image, there's a post install script, so you may need to trigger it with: + +```bash +npm install + +# Or if you don't want to re-install everything +docker compose up +``` + +Commands: + +```bash +# Run all tests once + +``` + **Analyze command** You can use the following tool to analyze the releasing bundle content: diff --git a/packages/scene-composer/docker-compose.yml b/packages/scene-composer/docker-compose.yml new file mode 100644 index 000000000..d641a5568 --- /dev/null +++ b/packages/scene-composer/docker-compose.yml @@ -0,0 +1,13 @@ +version: '3.0' + +services: + playwright: + image: mcr.microsoft.com/playwright:v1.39.0-focal + build: . + volumes: + - ../../:/iot-app-kit + ports: + - 9323:9323/tcp + - 7006:7006/tcp + working_dir: '/iot-app-kit/packages/scene-composer' + command: sh -c "npx playwright install chromium" diff --git a/packages/scene-composer/e2e/tests/local-scene/scene-composer--local-scene.spec.ts b/packages/scene-composer/e2e/tests/local-scene/scene-composer--local-scene.spec.ts index 7003d0e34..fa7b871dd 100644 --- a/packages/scene-composer/e2e/tests/local-scene/scene-composer--local-scene.spec.ts +++ b/packages/scene-composer/e2e/tests/local-scene/scene-composer--local-scene.spec.ts @@ -6,13 +6,16 @@ const localScene = '/iframe.html?args=&id=developer-scene-composer--local-scene' const canvas = '#tm-scene-unselectable-canvas'; test.describe('scene-composer--local-scene', () => { - test('visual regression', async ({ page }) => { + test('visual regression', async ({ page }, testInfo) => { await page.goto(localScene); const frame = page.locator('#root'); - expect(await frame.locator(canvas).screenshot()).toMatchSnapshot({ - name: 'local-scene-canvas.png', - threshold: 1, - }); + + // Example on how to make sure the screenshot shows up in the test report + // TODO: Automate this so every screenshot is attached automatically to it's test. + const screenshot = await frame.locator(canvas).screenshot(); + await testInfo.attach('local-scene-canvas.png', { body: screenshot, contentType: 'image/png' }); + + expect(screenshot).toMatchSnapshot('local-scene-canvas.png'); }); test('get object by name', async ({ page }) => { diff --git a/packages/scene-composer/e2e/tests/local-scene/scene-composer--local-scene.spec.ts-snapshots/local-scene-canvas-chromium-darwin.png b/packages/scene-composer/e2e/tests/local-scene/scene-composer--local-scene.spec.ts-snapshots/local-scene-canvas-chromium-darwin.png deleted file mode 100644 index 18062e4c3..000000000 --- a/packages/scene-composer/e2e/tests/local-scene/scene-composer--local-scene.spec.ts-snapshots/local-scene-canvas-chromium-darwin.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ae35e9ee935222b1ba3cb13ff038d13896ee581e5dea31a140bfa466148afe88 -size 47817 diff --git a/packages/scene-composer/e2e/tests/local-scene/scene-composer--local-scene.spec.ts-snapshots/local-scene-canvas-chromium-linux.png b/packages/scene-composer/e2e/tests/local-scene/scene-composer--local-scene.spec.ts-snapshots/local-scene-canvas-chromium-linux.png index 4cd699f8d..8cf5794ff 100644 --- a/packages/scene-composer/e2e/tests/local-scene/scene-composer--local-scene.spec.ts-snapshots/local-scene-canvas-chromium-linux.png +++ b/packages/scene-composer/e2e/tests/local-scene/scene-composer--local-scene.spec.ts-snapshots/local-scene-canvas-chromium-linux.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:96c419571e1206746185b90515e1099a6f8748eba77be317a3859e855224cad4 -size 33790 +oid sha256:8fc21efe4f78ca8bfc5060a9126e9f60a693ce00413032134aeee4b829bdc2f5 +size 34065 diff --git a/packages/scene-composer/e2e/tests/motion-indicator/scene-composer--motion-indicator.spec.ts-snapshots/motion-indicator-canvas-chromium-darwin.png b/packages/scene-composer/e2e/tests/motion-indicator/scene-composer--motion-indicator.spec.ts-snapshots/motion-indicator-canvas-chromium-darwin.png deleted file mode 100644 index 9d0b756c3..000000000 --- a/packages/scene-composer/e2e/tests/motion-indicator/scene-composer--motion-indicator.spec.ts-snapshots/motion-indicator-canvas-chromium-darwin.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6b31a8e309febd0a6be8580ad76711425eb7ba61d390f96003c5602f5e157054 -size 15301 diff --git a/packages/scene-composer/e2e/tests/motion-indicator/scene-composer--motion-indicator.spec.ts-snapshots/motion-indicator-canvas-chromium-linux.png b/packages/scene-composer/e2e/tests/motion-indicator/scene-composer--motion-indicator.spec.ts-snapshots/motion-indicator-canvas-chromium-linux.png index 7cea248ed..75a9e9018 100644 --- a/packages/scene-composer/e2e/tests/motion-indicator/scene-composer--motion-indicator.spec.ts-snapshots/motion-indicator-canvas-chromium-linux.png +++ b/packages/scene-composer/e2e/tests/motion-indicator/scene-composer--motion-indicator.spec.ts-snapshots/motion-indicator-canvas-chromium-linux.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:56cedd5b794b99bc66dfad7c4e348f5bade2ba95261eb6bb383addabc6d7567d -size 47047 +oid sha256:14cfb93bdf97a95c95b9ee9fa745456c611716aadbab3822378c74c9797385b8 +size 45949 diff --git a/packages/scene-composer/package.json b/packages/scene-composer/package.json index e03b181a5..91775defb 100644 --- a/packages/scene-composer/package.json +++ b/packages/scene-composer/package.json @@ -36,6 +36,7 @@ } }, "scripts": { + "postinstall": "docker compose up", "build": "run-s compile", "build:watch": "node ./tools/watch-build.js", "dev": "npm-watch build", @@ -49,13 +50,23 @@ "lint": "eslint . --max-warnings=476", "fix": "eslint --fix .", "test": "jest --config jest.config.ts --coverage --silent", + "test:reliability": "npm run test:ui:reliability", "test:dev": "jest --config jest.config.ts --coverage", "test:update-threashold": "jest-coverage-ratchet", "test:coverage": "npm test -- --updateSnapshot --coverage", "test:update": "jest --config jest.config.ts --updateSnapshot", "test:unit": "jest --config jest.config.ts", "test:watch": "jest --config jest.config.ts --watch", - "storybook": "npm run copy-assets && start-storybook -h 0.0.0.0 -p 6006", + "playwright": "docker compose run playwright", + "test:ui": "if test \"$NODE_ENV\" = \"production\"; then npm run _test:ui:ci ; else npm run _test:ui:local ; fi", + "test:ui:reliability": "if test \"$NODE_ENV\" = \"production\"; then npm run _test:ui:reliability:ci ; else npm run _test:ui:reliability:local ; fi", + "_test:ui:ci": "npx playwright test", + "_test:ui:local": "npm run playwright -- npx playwright test", + "_test:ui:reliability:ci": "npx playwright test --repeat-each 5 --workers=1", + "_test:ui:reliability:local": "npm run playwright -- npx playwright test --repeat-each 5 --workers=1", + "test:ui:update": "npm run playwright -- npx playwright test --update-snapshots", + "test:ui:dev": "npx playwright test --ui", + "storybook": "npm run copy-assets && start-storybook -h 0.0.0.0 -p 7006", "start": "npm run storybook", "build-storybook": "build-storybook", "extract-msgs": "formatjs extract 'src/**/*.ts*' --ignore='**/*.d.ts' --out-file translations/IotAppKitSceneComposer.en_US.json --id-interpolation-pattern '[sha512:contenthash:base64:6]' --format tools/totoro-formatter.js", @@ -64,11 +75,7 @@ "copy:notice": "cp ../../NOTICE NOTICE", "prepack": "npm run copy:license && npm run copy:notice", "pack": "npm pack", - "svglint": "svglint src/assets/**/*.svg", - "test:ui": "npx playwright test --update-snapshots", - "test:ui:reliability": "npx playwright test --repeat-each 5 --workers=1", - "test:ui:dev": "npx playwright test --ui", - "test:ui-update": "npx playwright test --update-snapshots" + "svglint": "svglint src/assets/**/*.svg" }, "peerDependencies": { "react": "^18", diff --git a/packages/scene-composer/playwright.config.ts b/packages/scene-composer/playwright.config.ts index 850dfd162..e155ac5bd 100644 --- a/packages/scene-composer/playwright.config.ts +++ b/packages/scene-composer/playwright.config.ts @@ -24,7 +24,7 @@ export default defineConfig({ /* Opt out of parallel tests on CI. */ workers: process.env.CI ? 1 : undefined, /* Reporter to use. See https://playwright.dev/docs/test-reporters */ - reporter: 'html', + reporter: [['html', { host: '0.0.0.0' }]], /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { viewport: { height: 1500, width: 1028 }, @@ -35,7 +35,7 @@ export default defineConfig({ /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ trace: 'on-first-retry', - baseURL: 'http://localhost:6006/', + baseURL: 'http://localhost:7006/', }, /* Configure projects for major browsers */ @@ -53,7 +53,7 @@ export default defineConfig({ webServer: { command: 'npm run start', reuseExistingServer: true, - url: 'http://localhost:6006', + url: 'http://0.0.0.0:7006', timeout: 300 * 1000, // 5 minutes stdout: 'pipe', stderr: 'pipe',