feat: adding docker setup for scene-composer ui tests

- Moves all snapshots to use git lfs (replaces with pointers, you will need git lfs to checkout from this point on)
- Updates snapshots for scene-composer ui tests, now just chromium-linux for now (you can technically run darwin if you like, but our master snapshots are linux based from now on).
- Add new Github workflow anyone can add to a pull request to trigger a reliability check. Adding the "test:reliability" label to a pull request will now force all UI tests to run 5 times, useful for checking for flaky tests when new tests are being added in a PR

.github/workflows/codeql-analysis.yml

@@ -10,7 +10,6 @@
 # supported CodeQL languages.
 #
 name: "CodeQL"
-
 on:
   push:
     branches: [ main, rc ]

.github/workflows/publish-storybook.yml

@@ -3,7 +3,6 @@ on:
   pull_request:
     paths:
     - 'packages/scene-composer/**'
-
 jobs:
   publish-storybook:
     runs-on: ubuntu-latest

.github/workflows/ui-test-for-commit.yml

@@ -1,5 +1,7 @@
 name: UI Tests - Associated to last commit
 on: [pull_request]
+env:
+  PW_TEST_HTML_REPORT_OPEN: 'never'
 jobs:
   e2e-tests-playwright:
     runs-on: ubuntu-latest
@@ -30,7 +32,7 @@ jobs:
 
       - name: test UI
         run:
-          npm run test:ui -w @iot-app-kit/dashboard --filter=[HEAD~1] && npm run test:ui -w @iot-app-kit/react-components --filter=[HEAD~1] && npm run test:ui -w @iot-app-kit/scene-composer --filter=[HEAD~1]
+          npm run test:ui -w @iot-app-kit/dashboard --filter=[HEAD~1] && npm run test:ui -w @iot-app-kit/react-components --filter=[HEAD~1] && npm run test:ui -w @iot-app-kit/scene-composer --filter=[HEAD~1] --production
       - uses: actions/upload-artifact@v4
         if: failure()
         with:

.github/workflows/ui-test-full-suite.yml

@@ -1,5 +1,7 @@
 name: UI Tests - Full run
 on: [pull_request]
+env:
+  PW_TEST_HTML_REPORT_OPEN: 'never'
 jobs:
   e2e-tests-playwright-full-suite:
     runs-on: ubuntu-latest
@@ -30,7 +32,7 @@ jobs:
 
       - name: test UI
         run:
-          npm run test:ui -w @iot-app-kit/dashboard && npm run test:ui -w @iot-app-kit/react-components && npm run test:ui:reliability -w @iot-app-kit/scene-composer
+          npm run test:ui -w @iot-app-kit/dashboard && npm run test:ui -w @iot-app-kit/react-components && npm run test:ui -w @iot-app-kit/scene-composer --production
       - uses: actions/upload-artifact@v4
         if: failure()
         with:

.github/workflows/ui-test-reliability.yml

@@ -0,0 +1,56 @@
+name: "UI Test Reliability Check"
+
+# We only run this on PRs that have the label "test:reliability", otherwise it's excluded.
+# Add this label anytime there are new UI tests added, or you want to verify a change still
+# reliabily passes tests
+on:
+  pull_request:
+    types: [ labeled ]
+
+env:
+  PW_TEST_HTML_REPORT_OPEN: 'never'
+
+jobs:
+  test-reliability:
+    if: ${{ github.event.label.name == 'test:reliability' }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+          lfs: 'true'
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 16
+          cache: 'npm'
+          cache-dependency-path: '**/package-lock.json'
+
+      - name: Cache playwright binaries
+        uses: actions/cache@v4
+        id: playwright-cache
+        with:
+          path: |
+            ~/.cache/ms-playwright
+          key: ${{ runner.os }}-playwright-${{ env.PLAYWRIGHT_VERSION }}
+      - name: Install
+        run:
+          npm ci --prefer-offline --no-audit --no-fund && npm run bootstrap
+
+      - name: Install Playwright's dependencies
+        run: npx playwright install --with-deps
+
+      - name: test reliability
+        run:
+          npm run test:reliability --production
+      - uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: Playwright failed test-results
+          path: packages/**/test-results
+          retention-days: 30
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: playwright-report
+          path: packages/**/playwright-report
+          retention-days: 30

commitlint.config.js

@@ -1,6 +1,6 @@
 module.exports = {
   extends: ['@commitlint/config-conventional'],
   rules: {
-    'body-max-line-length': [2, 'always', 250],
+    'body-max-line-length': [0],
   },
 };

docs/development.md

@@ -8,6 +8,8 @@ Ensure you have `node` version 16 and `npm` > 8.0.
 
 - Node: any `v16` or higher
 - Npm: `v8.0.0` or higher
+- git lfs: https://git-lfs.com/ - We use git lfs to keep binary files and screenshots from ballooning our repo size
+- docker: https://www.docker.com/products/docker-desktop/ - Docker is needed by our UI Tests to ensure reliability/consistency across dev local and ci
 
 If you need to setup node, consult https://nodejs.org/en/download/package-manager
 
@@ -51,6 +53,7 @@ Learn more at the [npm workspaces documentation](https://docs.npmjs.com/cli/v7/u
 Every package in IoT App Kit follows a similar structure:
 
 - every package with tests, contains a `test` command that runs jest based tests
+- every package with UI Tests, contains a `test:ui` command that runs playwright tests, and a `test:ui:reliability` command that stress tests these tests to ensure they aren't flaky
 - every package contains a `lint` command that executes eslint
 - every package contains a `fix` command that fixes eslint errors
 - every package contains a `build` command that builds the package
@@ -78,6 +81,8 @@ the correct [semver](https://semver.org/) version for publishing.
 Due to the changelog and versioning being determined by the commit messaging, commit messages are considered part of the review process,
 so "squash merging" and other techniques that allow changing the commit message after approval right before merging, are not allowed.
 
+You should add UI Tests for any major functionality, and if you do, label your pull request with the `test:reliability` label on first checkin, which will stress test your UI tests.
+
 **git good tip**: `git rebase -i HEAD~${NUM_COMMITS}` is a useful way to squash together the last `NUM_COMMITS` before making your pull-request. Learn more [here](https://git-scm.com/book/en/v2/Git-Tools-Rewriting-History). 
 
 ### 6. Additional Coding guidelines and requirements

package.json

@@ -27,14 +27,16 @@
     "fix:stylelint": "stylelint '**/*.css' --fix",
     "test": "turbo run test",
     "test:ui": "turbo run test:ui",
+    "test:reliability": "turbo run test:ui:reliability",
     "test:stylelint": "stylelint '**/*.css' --max-warnings 0",
     "test:git": "git diff --exit-code",
     "release": "npm run build",
     "pack": "turbo run pack",
     "versionup:patch": "turbo run version --no-git-tag-version patch",
     "versionup:minor": "turbo run version --no-git-tag-version minor",
     "versionup:major": "turbo run version --no-git-tag-version major",
-    "prepare": "husky install"
+    "prepare": "husky install",
+    "test:ci": "if test \"$NODE_ENV\" = \"production\"; then echo \"Yes!\"; fi"
   },
   "dependencies": {
     "@cloudscape-design/collection-hooks": "1.0.22",

packages/scene-composer/README.md

@@ -13,6 +13,26 @@ The library will be built and copied to the `dist` folder.
 npm run build
 ```
 
+**UI Tests**
+
+Pre-requisites: You need to have [Docker](https://docs.docker.com/get-docker/) installed locally to run UI Tests, we rely on it to provide the necessary browsers to generate consistent screenshots.
+
+To setup the docker image, there's a post install script, so you may need to trigger it with:
+
+```bash
+npm install
+
+# Or if you don't want to re-install everything
+docker compose up
+```
+
+Commands:
+
+```bash
+# Run all tests once
+
+```
+
 **Analyze command**
 
 You can use the following tool to analyze the releasing bundle content:

packages/scene-composer/docker-compose.yml

@@ -0,0 +1,13 @@
+version: '3.0'
+
+services:
+  playwright:
+    image: mcr.microsoft.com/playwright:v1.39.0-focal
+    build: .
+    volumes:
+      - ../../:/iot-app-kit
+    ports:
+      - 9323:9323/tcp
+      - 7006:7006/tcp
+    working_dir: '/iot-app-kit/packages/scene-composer'
+    command: sh -c "npx playwright install chromium"

packages/scene-composer/e2e/tests/local-scene/scene-composer--local-scene.spec.ts

@@ -6,13 +6,16 @@ const localScene = '/iframe.html?args=&id=developer-scene-composer--local-scene'
 const canvas = '#tm-scene-unselectable-canvas';
 
 test.describe('scene-composer--local-scene', () => {
-  test('visual regression', async ({ page }) => {
+  test('visual regression', async ({ page }, testInfo) => {
     await page.goto(localScene);
     const frame = page.locator('#root');
-    expect(await frame.locator(canvas).screenshot()).toMatchSnapshot({
-      name: 'local-scene-canvas.png',
-      threshold: 1,
-    });
+
+    // Example on how to make sure the screenshot shows up in the test report
+    // TODO: Automate this so every screenshot is attached automatically to it's test.
+    const screenshot = await frame.locator(canvas).screenshot();
+    await testInfo.attach('local-scene-canvas.png', { body: screenshot, contentType: 'image/png' });
+
+    expect(screenshot).toMatchSnapshot('local-scene-canvas.png');
   });
 
   test('get object by name', async ({ page }) => {

packages/scene-composer/package.json

@@ -36,6 +36,7 @@
     }
   },
   "scripts": {
+    "postinstall": "docker compose up",
     "build": "run-s compile",
     "build:watch": "node ./tools/watch-build.js",
     "dev": "npm-watch build",
@@ -49,13 +50,23 @@
     "lint": "eslint . --max-warnings=476",
     "fix": "eslint --fix .",
     "test": "jest --config jest.config.ts --coverage --silent",
+    "test:reliability": "npm run test:ui:reliability",
     "test:dev": "jest --config jest.config.ts --coverage",
     "test:update-threashold": "jest-coverage-ratchet",
     "test:coverage": "npm test -- --updateSnapshot --coverage",
     "test:update": "jest --config jest.config.ts --updateSnapshot",
     "test:unit": "jest --config jest.config.ts",
     "test:watch": "jest --config jest.config.ts --watch",
-    "storybook": "npm run copy-assets && start-storybook -h 0.0.0.0 -p 6006",
+    "playwright": "docker compose run playwright",
+    "test:ui": "if test \"$NODE_ENV\" = \"production\"; then npm run _test:ui:ci ; else npm run _test:ui:local ; fi",
+    "test:ui:reliability": "if test \"$NODE_ENV\" = \"production\"; then npm run _test:ui:reliability:ci ; else npm run _test:ui:reliability:local ; fi",
+    "_test:ui:ci": "npx playwright test",
+    "_test:ui:local": "npm run playwright -- npx playwright test",
+    "_test:ui:reliability:ci": "npx playwright test --repeat-each 5 --workers=1",
+    "_test:ui:reliability:local": "npm run playwright -- npx playwright test --repeat-each 5 --workers=1",
+    "test:ui:update": "npm run playwright -- npx playwright test --update-snapshots",
+    "test:ui:dev": "npx playwright test --ui",
+    "storybook": "npm run copy-assets && start-storybook -h 0.0.0.0 -p 7006",
     "start": "npm run storybook",
     "build-storybook": "build-storybook",
     "extract-msgs": "formatjs extract 'src/**/*.ts*' --ignore='**/*.d.ts' --out-file translations/IotAppKitSceneComposer.en_US.json --id-interpolation-pattern '[sha512:contenthash:base64:6]' --format tools/totoro-formatter.js",
@@ -64,11 +75,7 @@
     "copy:notice": "cp ../../NOTICE NOTICE",
     "prepack": "npm run copy:license && npm run copy:notice",
     "pack": "npm pack",
-    "svglint": "svglint src/assets/**/*.svg",
-    "test:ui": "npx playwright test --update-snapshots",
-    "test:ui:reliability": "npx playwright test --repeat-each 5 --workers=1",
-    "test:ui:dev": "npx playwright test --ui",
-    "test:ui-update": "npx playwright test --update-snapshots"
+    "svglint": "svglint src/assets/**/*.svg"
   },
   "peerDependencies": {
     "react": "^18",

packages/scene-composer/playwright.config.ts

@@ -24,7 +24,7 @@ export default defineConfig({
   /* Opt out of parallel tests on CI. */
   workers: process.env.CI ? 1 : undefined,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
-  reporter: 'html',
+  reporter: [['html', { host: '0.0.0.0' }]],
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
   use: {
     viewport: { height: 1500, width: 1028 },
@@ -35,7 +35,7 @@ export default defineConfig({
 
     /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
     trace: 'on-first-retry',
-    baseURL: 'http://localhost:6006/',
+    baseURL: 'http://localhost:7006/',
   },
 
   /* Configure projects for major browsers */
@@ -53,7 +53,7 @@ export default defineConfig({
   webServer: {
     command: 'npm run start',
     reuseExistingServer: true,
-    url: 'http://localhost:6006',
+    url: 'http://0.0.0.0:7006',
     timeout: 300 * 1000, // 5 minutes
     stdout: 'pipe',
     stderr: 'pipe',