Port UI tests to vscode-extension-tester (#1873)

This installs the redhat extension tester and removes the wdio test runner and its dependencies from the project. This allows us to test treeviews, which is what the HCP integration uses, locally and in CI in a reliable manner.

This is large port of existing UI tests to a new API and it required a new way to run the tests. So a script that runs the tests has been created that will run the tests in a headless mode when running in CI and in a windowed mode when running locally. This is to ensure that the tests can be run in a reliable manner in both environments. It also mocks the HCP API so that the tests can be run without needing to have a connection to HCP infrastructure.

A bug in the login workflow was found during this effort. After successfully authenticing, the organization picker was left open and could not be closed. This was fixed by disposing the organization picker after successfully authenticating.

Since the primary focus was on the HCP treeview tests (most likely to break when changing this area), the Terraform treeview tests (which have much simpler functionality) were not updated and left for a future PR.

This also fixes the issue with the `test.yml` file where the `VSCODE_VERSION` was not being set correctly. This was causing the tests to not actually run with different versions of VSCode.

A minor addition to the eslint ruleset to allow for the use of `unsafe` access or `any` in the codebase where it is needed, specifically in places where we interact with libraries that are not typed or where we need to access properties that are not typed.

.changes/unreleased/INTERNAL-20241105-111825.yaml

@@ -0,0 +1,6 @@
+kind: INTERNAL
+body: Port UI tests to vscode-extension-tester and remove wdio
+time: 2024-11-05T11:18:25.988501-05:00
+custom:
+    Issue: "1873"
+    Repository: vscode-terraform

.github/workflows/test.yml

@@ -50,6 +50,8 @@ jobs:
           - ubuntu-latest
     runs-on: ${{ matrix.os }}
     timeout-minutes: 10
+    env:
+      VSCODE_VERSION: ${{ matrix.vscode }}
     steps:
       - name: Checkout Repo
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -72,3 +74,9 @@ jobs:
       - name: Run Tests
         run: npm test
         if: runner.os != 'Linux'
+      - name: Run UI Tests
+        run: xvfb-run --auto-servernum --server-args='-screen 0 1920x1080x24' npm run test:ui
+        if: runner.os == 'Linux'
+      - name: Run UI Tests
+        run: npm run test:ui
+        if: runner.os != 'Linux'

.gitignore

@@ -2,7 +2,8 @@
 .terraform
 .vscode-test
 .vscode-test-web
-.wdio-vscode-service
+.test-storage
+.test-extensions
 *.vsix
 bin
 node_modules

.prettierignore

@@ -2,6 +2,8 @@
 .vscode-test
 .vscode-test-web
 .wdio-vscode-service
+.test-storage
+.test-extensions
 language-configuration.json
 node_modules
 dist

.vscode-uitest.mjs

@@ -0,0 +1,45 @@
+/**
+ * Copyright (c) HashiCorp, Inc.
+ * SPDX-License-Identifier: MPL-2.0
+ */
+
+import { ExTester, ReleaseQuality } from 'vscode-extension-tester';
+
+async function runTests() {
+  let code_version;
+  let code_type;
+
+  if (process.env.VSCODE_VERSION === 'insiders') {
+    code_type = ReleaseQuality.Insider;
+  } else if (process.env.VSCODE_VERSION === 'stable') {
+    code_type = ReleaseQuality.Stable;
+  } else {
+    code_version = process.env.VSCODE_VERSION;
+  }
+
+  if (code_type) {
+    console.log(`Running tests for ${code_version} ${code_type} version of VS Code`);
+  } else {
+    console.log(`Running tests for ${code_version} version of VS Code`);
+  }
+
+  process.env.HASHI_CODE_TEST = 'true';
+
+  const tester = new ExTester('.test-storage', code_type, '.test-extensions', false);
+  await tester.setupAndRunTests(
+    'out/test/e2e/specs/**/*.e2e.js',
+    code_version,
+    {
+      installDependencies: false,
+    },
+    {
+      settings: 'src/test/e2e/settings.json',
+      cleanup: true,
+      config: 'src/test/e2e/.mocharc.js',
+      logLevel: 'info',
+      resources: [],
+    },
+  );
+}
+
+runTests().catch(console.error);

.vscodeignore

@@ -1,3 +1,4 @@
+.mocharc.js
 .changes
 .changie.yaml
 .copywrite.hcl
@@ -17,6 +18,8 @@ META.d
 .vscode-test.mjs
 .vscodeignore
 .wdio-vscode-service
+.test-storage
+.test-extensions
 **/__mocks__
 build/
 DEVELOPMENT.md
@@ -28,3 +31,6 @@ out/
 esbuild.mjs
 eslint.config.mjs
 tsconfig.json
+esbuild.mjs
+eslint.config.mjs
+uitest.mjs

eslint.config.mjs

@@ -37,9 +37,10 @@ export default [
       semi: 'warn',
       '@typescript-eslint/no-explicit-any': ['warn', { ignoreRestArgs: true }],
       '@typescript-eslint/naming-convention': 'off',
-      '@typescript-eslint/no-unsafe-assignment': 'warn',
-      '@typescript-eslint/no-unsafe-argument': 'warn',
-      '@typescript-eslint/no-unsafe-member-access': 'warn',
+      '@typescript-eslint/no-unsafe-assignment': 'off',
+      '@typescript-eslint/no-unsafe-argument': 'off',
+      '@typescript-eslint/no-unsafe-member-access': 'off',
+      '@typescript-eslint/no-unsafe-call': 'off',
       '@typescript-eslint/restrict-template-expressions': [
         'error',
         {
@@ -64,6 +65,8 @@ export default [
     ignores: [
       '.vscode-test',
       '.wdio-vscode-service',
+      '.test-storage',
+      '.test-extensions',
       'dist',
       'out',
       'src/test',