From 2ac7d6609bad785da93300f2a5f86b842c7bd3ad Mon Sep 17 00:00:00 2001 From: Fabricio-ESP Date: Mon, 13 Jan 2025 15:39:23 +0000 Subject: [PATCH 01/19] Update IDF version to 5.4 --- tests/runs/CNmirrors.test.js | 2 +- tests/runs/defaultInstall.test.js | 4 ++-- tests/runs/nonInteractiveInstall.test.js | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/runs/CNmirrors.test.js b/tests/runs/CNmirrors.test.js index e1171d1..ac7d9ce 100644 --- a/tests/runs/CNmirrors.test.js +++ b/tests/runs/CNmirrors.test.js @@ -25,7 +25,7 @@ if (process.env.EIM_FILE_PATH) { } const targetList = ["esp32c6"]; // targets used for IDF installation -const idfVersionList = ["v5.3.2"]; // IDF versions to be installed +const idfVersionList = ["v5.4"]; // IDF versions to be installed const installFolder = ".espressif4"; const projectFolder = "project"; diff --git a/tests/runs/defaultInstall.test.js b/tests/runs/defaultInstall.test.js index e5d7fa7..33a5120 100644 --- a/tests/runs/defaultInstall.test.js +++ b/tests/runs/defaultInstall.test.js @@ -35,8 +35,8 @@ if (process.env.EIM_VERSION) { const pathToIDFScript = os.platform() !== "win32" - ? path.join(os.homedir(), ".espressif/activate_idf_v5.3.2.sh") - : "C:\\esp\\v5.3.2\\Microsoft.PowerShell_profile.ps1"; + ? path.join(os.homedir(), ".espressif/activate_idf_v5.4.sh") + : "C:\\esp\\v5.4\\Microsoft.PowerShell_profile.ps1"; const pathToProjectFolder = os.platform() !== "win32" diff --git a/tests/runs/nonInteractiveInstall.test.js b/tests/runs/nonInteractiveInstall.test.js index a96c5b3..ca35f1d 100644 --- a/tests/runs/nonInteractiveInstall.test.js +++ b/tests/runs/nonInteractiveInstall.test.js @@ -25,7 +25,7 @@ if (process.env.EIM_FILE_PATH) { } const targetList = ["esp32c6"]; // targets used for IDF installation -const idfVersionList = ["v5.3.1"]; // IDF versions to be installed +const idfVersionList = ["v5.3.2"]; // IDF versions to be installed const installFolder = ".espressif3"; const projectFolder = "project"; From 3a8d3c40db6cf9d03a4898585e69776854d015cd Mon Sep 17 00:00:00 2001 From: Fabricio-ESP Date: Mon, 13 Jan 2025 16:43:53 +0000 Subject: [PATCH 02/19] Add default IDF read on workflow, change report --- .github/workflows/test.yml | 66 ++++++++++++++++++++++--------- tests/package.json | 14 +++---- tests/run_test.ps1 | 4 ++ tests/run_test.sh | 1 + tests/runs/CNmirrors.test.js | 10 ++++- tests/runs/defaultInstall.test.js | 15 ++++++- 6 files changed, 82 insertions(+), 28 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fb7bc41..52748c6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -57,6 +57,13 @@ jobs: STRIPPED_TAG=${LATEST_TAG#v} echo "CLI_TAG=$STRIPPED_TAG" >> $GITHUB_ENV + - name: Fetch latest IDF release + run: | + curl -s https://api.github.com/repos/espressif/esp-idf/releases/latest > IDF_latest_release.json + IDF_TAG=$(jq -r .tag_name IDF_latest_release.json) + echo "IDF_TAG=$IDF_TAG" >> $GITHUB_ENV + echo "Latest IDF for default install: ${{IDF_TAG}}" + - name: Extract artifact (non-Windows) if: runner.os != 'Windows' run: | @@ -91,14 +98,14 @@ jobs: run: | export LOG_TO_FILE="true" chmod +x ./tests/run_test.sh - . ./tests/run_test.sh "../test-bin/eim" "eim ${{ env.CLI_TAG }}" + . ./tests/run_test.sh "../test-bin/eim" "eim ${{ env.CLI_TAG }}" "${{ env.IDF_TAG }}" - name: Run IDF installation from alternative mirrors in mainland China if: matrix.run_on == 'CNRunner' run: | export LOG_TO_FILE="true" chmod +x ./tests/run_cnrunner.sh - . ./tests/run_cnrunner.sh "../test-bin/eim" "eim ${{ env.CLI_TAG }}" + . ./tests/run_cnrunner.sh "../test-bin/eim" "eim ${{ env.CLI_TAG }}" "${{ env.IDF_TAG }}" # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -110,6 +117,13 @@ jobs: $STRIPPED_TAG = $LATEST_TAG -replace '^v', '' echo "CLI_TAG=$STRIPPED_TAG" | Out-File -FilePath $env:GITHUB_ENV -Append + - name: Fetch latest IDF release + run: | + curl -s https://api.github.com/repos/espressif/esp-idf/releases/latest -o IDF_latest_release.json + $IDF_TAG = (Get-Content IDF_latest_release.json | ConvertFrom-Json).tag_name + echo "IDF_TAG=$IDF_TAG" | Out-File -FilePath $env:GITHUB_ENV -Append + echo "Latest IDF for default install: ${{IDF_TAG}}" + - name: Extract artifact (Windows) if: runner.os == 'Windows' run: | @@ -138,7 +152,7 @@ jobs: if: runner.os == 'Windows' run: | $env:LOG_TO_FILE="true" - .\tests\run_test.ps1 "..\test-bin\eim.exe" "eim ${{ env.CLI_TAG }}" + .\tests\run_test.ps1 "..\test-bin\eim.exe" "eim ${{ env.CLI_TAG }}" "${{ env.IDF_TAG }}" # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -148,13 +162,13 @@ jobs: with: name: test-results-${{ matrix.package_name }}-${{matrix.run_on}}.zip path: | - ./tests/results-pre-test.xml - ./tests/results-pre-install-test.xml - ./tests/results-default-test.xml - ./tests/results-variation1-test.xml - ./tests/results-non-interactive-test.xml - ./tests/results-CNMirror-test.xml - ./tests/results-CNMirror2-test.xml + ./tests/results-pre-test.json + ./tests/results-pre-install-test.json + ./tests/results-default-test.json + ./tests/results-variation1-test.json + ./tests/results-non-interactive-test.json + ./tests/results-CNMirror-test.json + ./tests/results-CNMirror2-test.json ./tests/test.log publish-test-results: @@ -164,13 +178,29 @@ jobs: if: always() steps: - - name: Download Artifacts - uses: actions/download-artifact@v4 - with: - path: ./artifacts - - name: Publish Test Results - uses: EnricoMi/publish-unit-test-result-action@v2 + uses: dorny/test-reporter@v1 with: - action_fail: true - files: "./artifacts/**/*.xml" + artifact: /test-results-(.*)/ + name: "Auto Tests $1" + path: "*.json" + reporter: mocha-json + fail-on-empty: "true" + + # publish-test-results: + # name: Automated Test Results + # needs: test + # runs-on: ubuntu-latest + # if: always() + + # steps: + # - name: Download Artifacts + # uses: actions/download-artifact@v4 + # with: + # path: ./artifacts + + # - name: Publish Test Results + # uses: EnricoMi/publish-unit-test-result-action@v2 + # with: + # action_fail: true + # files: "./artifacts/**/*.xml" diff --git a/tests/package.json b/tests/package.json index 4402fc6..c38abe5 100644 --- a/tests/package.json +++ b/tests/package.json @@ -9,12 +9,12 @@ "winston": "^3.15.0" }, "scripts": { - "pre-test": "mocha --exit --bail --reporter mocha-junit-reporter --reporter-options mochaFile=./results-pre-test.xml script/prerequisites.test.js", - "pre-install": "mocha --exit --bail --reporter mocha-junit-reporter --reporter-options mochaFile=./results-pre-install-test.xml script/prerequisitesInstallRunner.test.js", - "default-test": "mocha --exit --bail --reporter mocha-junit-reporter --reporter-options mochaFile=./results-default-test.xml runs/defaultInstall.test.js", - "variation1-test": "mocha --exit --bail --reporter mocha-junit-reporter --reporter-options mochaFile=./results-variation1-test.xml runs/variation1.test.js", - "non-interactive-test": "mocha --exit --bail --reporter mocha-junit-reporter --reporter-options mochaFile=./results-non-interactive-test.xml runs/nonInteractiveInstall.test.js", - "cnrunner1-test": "mocha --exit --bail --reporter mocha-junit-reporter --reporter-options mochaFile=./results-CNMirror-test.xml runs/CNmirrors.test.js", - "cnrunner2-test": "mocha --exit --bail --reporter mocha-junit-reporter --reporter-options mochaFile=./results-CNmirrors2-test.xml runs/CNmirrors2.test.js" + "pre-test": "mocha --exit --bail --reporter json --reporter-options output=./results-pre-test.json script/prerequisites.test.js", + "pre-install": "mocha --exit --bail --reporter json --reporter-options output=./results-pre-install-test.json script/prerequisitesInstallRunner.test.js", + "default-test": "mocha --exit --bail --reporter json --reporter-options output=./results-default-test.json runs/defaultInstall.test.js", + "variation1-test": "mocha --exit --bail --reporter json --reporter-options output=./results-variation1-test.json runs/variation1.test.js", + "non-interactive-test": "mocha --exit --bail --reporter json --reporter-options output=./results-non-interactive-test.json runs/nonInteractiveInstall.test.js", + "cnrunner1-test": "mocha --exit --bail --reporter json --reporter-options output=./results-CNMirror-test.json runs/CNmirrors.test.js", + "cnrunner2-test": "mocha --exit --bail --reporter json --reporter-options output=./results-CNmirrors2-test.json runs/CNmirrors2.test.js" } } diff --git a/tests/run_test.ps1 b/tests/run_test.ps1 index f09e362..4eace4a 100644 --- a/tests/run_test.ps1 +++ b/tests/run_test.ps1 @@ -6,11 +6,15 @@ param ( [Parameter(Mandatory=$true)] [string]$Version + + [Parameter(Mandatory=$true)] + [string]$IDFVersion ) # Save the arguments as environment variables $env:EIM_FILE_PATH = $Path_to_eim $env:EIM_VERSION = $Version +$env:IDF_VERSION = $IDFVersion Set-Location -Path "./tests" diff --git a/tests/run_test.sh b/tests/run_test.sh index 82f0977..13a8d5e 100644 --- a/tests/run_test.sh +++ b/tests/run_test.sh @@ -3,6 +3,7 @@ # Save the arguments as environment variables export EIM_FILE_PATH="$1" export EIM_VERSION="$2" +export IDF_VERSION="$3" cd tests diff --git a/tests/runs/CNmirrors.test.js b/tests/runs/CNmirrors.test.js index ac7d9ce..8f2aa5a 100644 --- a/tests/runs/CNmirrors.test.js +++ b/tests/runs/CNmirrors.test.js @@ -9,6 +9,7 @@ import path from "path"; * Setup the following environmental variables to execute this test: * * EIM_FILE_PATH to point to the eim application. + * IDF_VERSION to specify the default version of the IDF to be installed. * * use: * Windows: $env:="" @@ -17,6 +18,7 @@ import path from "path"; */ let pathToEim; +let IDFLatestVersion; if (process.env.EIM_FILE_PATH) { pathToEim = process.env.EIM_FILE_PATH; @@ -24,8 +26,14 @@ if (process.env.EIM_FILE_PATH) { pathToEim = path.join(os.homedir(), "eim-cli/eim"); } +if (process.env.IDF_VERSION) { + IDFLatestVersion = process.env.IDF_VERSION; +} else { + IDFLatestVersion = "v5.4"; +} + const targetList = ["esp32c6"]; // targets used for IDF installation -const idfVersionList = ["v5.4"]; // IDF versions to be installed +const idfVersionList = [IDFLatestVersion]; // IDF versions to be installed const installFolder = ".espressif4"; const projectFolder = "project"; diff --git a/tests/runs/defaultInstall.test.js b/tests/runs/defaultInstall.test.js index 33a5120..6a2cafa 100644 --- a/tests/runs/defaultInstall.test.js +++ b/tests/runs/defaultInstall.test.js @@ -11,6 +11,7 @@ import path from "path"; * * EIM_FILE_PATH to point to the eim application. * EIM_VERSION to specify expected version to be printed by the application. + * IDF_VERSION to specify the default version of the IDF to be installed. * * use: * Windows: $env:="" @@ -20,6 +21,7 @@ import path from "path"; let pathToEim; let eimVersion; +let IDFLatestVersion; if (process.env.EIM_FILE_PATH) { pathToEim = process.env.EIM_FILE_PATH; @@ -33,10 +35,19 @@ if (process.env.EIM_VERSION) { eimVersion = "idf-im-cli 0.1.5"; } +if (process.env.IDF_VERSION) { + IDFLatestVersion = process.env.IDF_VERSION; +} else { + IDFLatestVersion = "v5.4"; +} + const pathToIDFScript = os.platform() !== "win32" - ? path.join(os.homedir(), ".espressif/activate_idf_v5.4.sh") - : "C:\\esp\\v5.4\\Microsoft.PowerShell_profile.ps1"; + ? path.join( + os.homedir(), + `.espressif/activate_idf_${IDFLatestVersion}.sh` + ) + : `C:\\esp\\${IDFLatestVersion}\\Microsoft.PowerShell_profile.ps1`; const pathToProjectFolder = os.platform() !== "win32" From 6a365bacf9afa7a95a0c24ddee652f1e417cafe6 Mon Sep 17 00:00:00 2001 From: Fabricio-ESP Date: Mon, 13 Jan 2025 16:53:21 +0000 Subject: [PATCH 03/19] Fix workflow file --- .github/workflows/test.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 52748c6..5626eb9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -59,10 +59,10 @@ jobs: - name: Fetch latest IDF release run: | - curl -s https://api.github.com/repos/espressif/esp-idf/releases/latest > IDF_latest_release.json - IDF_TAG=$(jq -r .tag_name IDF_latest_release.json) - echo "IDF_TAG=$IDF_TAG" >> $GITHUB_ENV - echo "Latest IDF for default install: ${{IDF_TAG}}" + curl -s https://api.github.com/repos/espressif/esp-idf/releases/latest > IDF_latest_release.json + IDF_TAG=$(jq -r .tag_name IDF_latest_release.json) + echo "IDF_TAG=$IDF_TAG" >> $GITHUB_ENV + echo "Latest IDF for default install: $IDF_TAG" - name: Extract artifact (non-Windows) if: runner.os != 'Windows' @@ -119,10 +119,10 @@ jobs: - name: Fetch latest IDF release run: | - curl -s https://api.github.com/repos/espressif/esp-idf/releases/latest -o IDF_latest_release.json - $IDF_TAG = (Get-Content IDF_latest_release.json | ConvertFrom-Json).tag_name - echo "IDF_TAG=$IDF_TAG" | Out-File -FilePath $env:GITHUB_ENV -Append - echo "Latest IDF for default install: ${{IDF_TAG}}" + curl -s https://api.github.com/repos/espressif/esp-idf/releases/latest -o IDF_latest_release.json + $IDF_TAG = (Get-Content IDF_latest_release.json | ConvertFrom-Json).tag_name + echo "IDF_TAG=$IDF_TAG" | Out-File -FilePath $env:GITHUB_ENV -Append + echo "Latest IDF for default install: $IDF_TAG" - name: Extract artifact (Windows) if: runner.os == 'Windows' From 149b182c3305ec4dd23100e47617ffa90dcd7509 Mon Sep 17 00:00:00 2001 From: Fabricio-ESP Date: Mon, 13 Jan 2025 17:27:15 +0000 Subject: [PATCH 04/19] Fix workflow get IDF version action --- .github/workflows/test.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5626eb9..db83ac4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -58,7 +58,13 @@ jobs: echo "CLI_TAG=$STRIPPED_TAG" >> $GITHUB_ENV - name: Fetch latest IDF release + if: runner.os != 'Windows' run: | + if [[ "$RUNNER_OS" == "macOS" ]]; then + brew install jq + else + sudo apt-get install -y jq + fi curl -s https://api.github.com/repos/espressif/esp-idf/releases/latest > IDF_latest_release.json IDF_TAG=$(jq -r .tag_name IDF_latest_release.json) echo "IDF_TAG=$IDF_TAG" >> $GITHUB_ENV @@ -118,6 +124,7 @@ jobs: echo "CLI_TAG=$STRIPPED_TAG" | Out-File -FilePath $env:GITHUB_ENV -Append - name: Fetch latest IDF release + if: runner.os == 'Windows' run: | curl -s https://api.github.com/repos/espressif/esp-idf/releases/latest -o IDF_latest_release.json $IDF_TAG = (Get-Content IDF_latest_release.json | ConvertFrom-Json).tag_name From dc751f4de998f8ad13d7f6f7451a91f6502c8162 Mon Sep 17 00:00:00 2001 From: Fabricio-ESP Date: Mon, 13 Jan 2025 19:04:28 +0000 Subject: [PATCH 05/19] Typo on script --- tests/run_test.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/run_test.ps1 b/tests/run_test.ps1 index 4eace4a..af48413 100644 --- a/tests/run_test.ps1 +++ b/tests/run_test.ps1 @@ -5,7 +5,7 @@ param ( [string]$Path_to_eim, [Parameter(Mandatory=$true)] - [string]$Version + [string]$Version, [Parameter(Mandatory=$true)] [string]$IDFVersion From 9281964d8253b9be477f30039eeadbfc9caf911c Mon Sep 17 00:00:00 2001 From: Fabricio-ESP Date: Tue, 14 Jan 2025 14:20:15 +0000 Subject: [PATCH 06/19] Fix typo on report name, add verbose to curl --- .github/workflows/test.yml | 16 +++------------- tests/package.json | 2 +- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index db83ac4..cf6adb6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,12 +60,8 @@ jobs: - name: Fetch latest IDF release if: runner.os != 'Windows' run: | - if [[ "$RUNNER_OS" == "macOS" ]]; then - brew install jq - else - sudo apt-get install -y jq - fi - curl -s https://api.github.com/repos/espressif/esp-idf/releases/latest > IDF_latest_release.json + curl -v -s https://api.github.com/repos/espressif/esp-idf/releases/latest > IDF_latest_release.json + cat IDF_latest_release.json IDF_TAG=$(jq -r .tag_name IDF_latest_release.json) echo "IDF_TAG=$IDF_TAG" >> $GITHUB_ENV echo "Latest IDF for default install: $IDF_TAG" @@ -169,13 +165,7 @@ jobs: with: name: test-results-${{ matrix.package_name }}-${{matrix.run_on}}.zip path: | - ./tests/results-pre-test.json - ./tests/results-pre-install-test.json - ./tests/results-default-test.json - ./tests/results-variation1-test.json - ./tests/results-non-interactive-test.json - ./tests/results-CNMirror-test.json - ./tests/results-CNMirror2-test.json + ./tests/results-*.json ./tests/test.log publish-test-results: diff --git a/tests/package.json b/tests/package.json index c38abe5..ccd0b7d 100644 --- a/tests/package.json +++ b/tests/package.json @@ -15,6 +15,6 @@ "variation1-test": "mocha --exit --bail --reporter json --reporter-options output=./results-variation1-test.json runs/variation1.test.js", "non-interactive-test": "mocha --exit --bail --reporter json --reporter-options output=./results-non-interactive-test.json runs/nonInteractiveInstall.test.js", "cnrunner1-test": "mocha --exit --bail --reporter json --reporter-options output=./results-CNMirror-test.json runs/CNmirrors.test.js", - "cnrunner2-test": "mocha --exit --bail --reporter json --reporter-options output=./results-CNmirrors2-test.json runs/CNmirrors2.test.js" + "cnrunner2-test": "mocha --exit --bail --reporter json --reporter-options output=./results-CNMirrors2-test.json runs/CNmirrors2.test.js" } } From 663d7cc480b81aa59f16f255bc074322b1a7a254 Mon Sep 17 00:00:00 2001 From: Fabricio-ESP Date: Thu, 16 Jan 2025 18:25:14 +0000 Subject: [PATCH 07/19] Refactor of the test infrastructure --- tests/run_cnrunner.sh | 16 --- tests/run_pre_test.ps1 | 26 ---- tests/run_pre_test.sh | 14 -- tests/run_test.ps1 | 32 ----- tests/run_test.sh | 18 --- tests/runs/CNmirrors.test.js | 77 ---------- tests/runs/CNmirrors2.test.js | 69 --------- tests/runs/basicRun.test.js | 23 +++ tests/runs/defaultInstall.test.js | 69 --------- tests/runs/extendedRun.test.js | 26 ++++ tests/runs/mirrorsRun.test.js | 23 +++ tests/runs/nonInteractiveInstall.test.js | 67 --------- tests/runs/suites/basic_test.json | 30 ++++ tests/runs/suites/extended_test.json | 54 +++++++ tests/runs/suites/mirrors_test.json | 44 ++++++ tests/runs/testRun.test.js | 133 ++++++++++++++++++ tests/runs/variation1.test.js | 67 --------- tests/script/commandLineArguments.test.js | 2 +- tests/script/installCustom.test.js | 155 ++++++++++++--------- tests/script/installNonInteractive.test.js | 81 ----------- tests/script/installWizard.test.js | 20 ++- tests/script/postInstall.test.js | 25 +++- 22 files changed, 458 insertions(+), 613 deletions(-) delete mode 100644 tests/run_cnrunner.sh delete mode 100644 tests/run_pre_test.ps1 delete mode 100644 tests/run_pre_test.sh delete mode 100644 tests/run_test.ps1 delete mode 100644 tests/run_test.sh delete mode 100644 tests/runs/CNmirrors.test.js delete mode 100644 tests/runs/CNmirrors2.test.js create mode 100644 tests/runs/basicRun.test.js delete mode 100644 tests/runs/defaultInstall.test.js create mode 100644 tests/runs/extendedRun.test.js create mode 100644 tests/runs/mirrorsRun.test.js delete mode 100644 tests/runs/nonInteractiveInstall.test.js create mode 100644 tests/runs/suites/basic_test.json create mode 100644 tests/runs/suites/extended_test.json create mode 100644 tests/runs/suites/mirrors_test.json create mode 100644 tests/runs/testRun.test.js delete mode 100644 tests/runs/variation1.test.js delete mode 100644 tests/script/installNonInteractive.test.js diff --git a/tests/run_cnrunner.sh b/tests/run_cnrunner.sh deleted file mode 100644 index 265ed81..0000000 --- a/tests/run_cnrunner.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -# Save the arguments as environment variables -export EIM_FILE_PATH="$1" -export EIM_VERSION="$2" - -cd tests - -# install node modules -# The zip file is currently being expanded in the pre-test, if it was not executed before please run this line locally -npm ci - -# run tests -set +e -npm run cnrunner1-test -npm run cnrunner2-test diff --git a/tests/run_pre_test.ps1 b/tests/run_pre_test.ps1 deleted file mode 100644 index 50b0c3a..0000000 --- a/tests/run_pre_test.ps1 +++ /dev/null @@ -1,26 +0,0 @@ -# Load arguments as environmental variables - -param ( - [Parameter(Mandatory=$true)] - [string]$Path_to_eim, - - [Parameter(Mandatory=$true)] - [string]$Version -) - -# Save the arguments as environment variables -$env:EIM_FILE_PATH = $Path_to_eim -$env:EIM_VERSION = $Version - -Set-Location -Path "./tests" - -# Expand Node modules folder -Expand-Archive node_modules.zip - -# Install node modules using npm ci -# This can be used if the node modules folder is not packed with the repo -# npm ci - -# Run tests using npm run AllTest -npm run pre-test -# npm run pre-install \ No newline at end of file diff --git a/tests/run_pre_test.sh b/tests/run_pre_test.sh deleted file mode 100644 index cedd3fe..0000000 --- a/tests/run_pre_test.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -# Save the arguments as environment variables -export EIM_FILE_PATH="$1" -export EIM_VERSION="$2" - -cd tests - -# install node modules -npm ci - -# run tests -set +e -npm run pre-test \ No newline at end of file diff --git a/tests/run_test.ps1 b/tests/run_test.ps1 deleted file mode 100644 index af48413..0000000 --- a/tests/run_test.ps1 +++ /dev/null @@ -1,32 +0,0 @@ -# Load arguments as environmental variables - -param ( - [Parameter(Mandatory=$true)] - [string]$Path_to_eim, - - [Parameter(Mandatory=$true)] - [string]$Version, - - [Parameter(Mandatory=$true)] - [string]$IDFVersion -) - -# Save the arguments as environment variables -$env:EIM_FILE_PATH = $Path_to_eim -$env:EIM_VERSION = $Version -$env:IDF_VERSION = $IDFVersion - -Set-Location -Path "./tests" - -# Expand Node modules folder -# The zip file is currently being expanded in the pre-test, if it was not executed before please run this line locally -# Expand-Archive node_modules.zip - -# Install node modules using npm ci -# This can be used if the node modules folder is not packed with the repo -# npm ci - -# Run tests using npm run AllTest -npm run default-test -# npm run variation1-test -npm run non-interactive-test \ No newline at end of file diff --git a/tests/run_test.sh b/tests/run_test.sh deleted file mode 100644 index 13a8d5e..0000000 --- a/tests/run_test.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -# Save the arguments as environment variables -export EIM_FILE_PATH="$1" -export EIM_VERSION="$2" -export IDF_VERSION="$3" - -cd tests - -# install node modules -# The zip file is currently being expanded in the pre-test, if it was not executed before please run this line locally -# npm ci - -# run tests -set +e -npm run default-test -# npm run variation1-test -npm run non-interactive-test \ No newline at end of file diff --git a/tests/runs/CNmirrors.test.js b/tests/runs/CNmirrors.test.js deleted file mode 100644 index 8f2aa5a..0000000 --- a/tests/runs/CNmirrors.test.js +++ /dev/null @@ -1,77 +0,0 @@ -import { describe, it, before, after } from "mocha"; -import { runPostInstallTest } from "../script/postInstall.test.js"; -import { runInstallNonInteractive } from "../script/installNonInteractive.test.js"; -import logger from "../classes/logger.class.js"; -import os from "os"; -import path from "path"; - -/** - * Setup the following environmental variables to execute this test: - * - * EIM_FILE_PATH to point to the eim application. - * IDF_VERSION to specify the default version of the IDF to be installed. - * - * use: - * Windows: $env:="" - * Linux/mac: export ="" - * - */ - -let pathToEim; -let IDFLatestVersion; - -if (process.env.EIM_FILE_PATH) { - pathToEim = process.env.EIM_FILE_PATH; -} else { - pathToEim = path.join(os.homedir(), "eim-cli/eim"); -} - -if (process.env.IDF_VERSION) { - IDFLatestVersion = process.env.IDF_VERSION; -} else { - IDFLatestVersion = "v5.4"; -} - -const targetList = ["esp32c6"]; // targets used for IDF installation -const idfVersionList = [IDFLatestVersion]; // IDF versions to be installed -const installFolder = ".espressif4"; -const projectFolder = "project"; - -let installArgs = []; -installArgs.push(` -p ${path.join(os.homedir(), installFolder)}`); // Install Path -// installArgs.push(` -t ${targetList.join(",")}`); // Targets // removed to install for all targets -// installArgs.push(` -i ${idfVersionList.join(",")}`); // IDF versions Removed to install latest version -installArgs.push(` -m https://dl.espressif.com/github_assets`); // IDF tools mirror -installArgs.push(` --idf-mirror https://jihulab.com/esp-mirror`); // ESP-IDF mirror -installArgs.push(` -r false`); // recursive submodules init - -const pathToIDFScript = - os.platform() !== "win32" - ? path.join( - os.homedir(), - installFolder, - `activate_idf_${idfVersionList[0]}.sh` - ) - : path.join( - os.homedir(), - installFolder, - idfVersionList[0], - `Microsoft.PowerShell_profile.ps1` - ); - -logger.info( - `Starting installation using mirror jihulab and dl.espressif.com and EIM on ${pathToEim}` -); - -describe("Installation using mirror jihulab and dl.espressif.com", function () { - this.timeout(2400000); - - runInstallNonInteractive(pathToEim, installArgs); - - runPostInstallTest( - pathToIDFScript, - path.join(os.homedir(), installFolder, projectFolder), - targetList[0], - "esp32c6" - ); -}); diff --git a/tests/runs/CNmirrors2.test.js b/tests/runs/CNmirrors2.test.js deleted file mode 100644 index 20b6439..0000000 --- a/tests/runs/CNmirrors2.test.js +++ /dev/null @@ -1,69 +0,0 @@ -import { describe, it, before, after } from "mocha"; -import { runPostInstallTest } from "../script/postInstall.test.js"; -import { runInstallNonInteractive } from "../script/installNonInteractive.test.js"; -import logger from "../classes/logger.class.js"; -import os from "os"; -import path from "path"; - -/** - * Setup the following environmental variables to execute this test: - * - * EIM_FILE_PATH to point to the eim application. - * - * use: - * Windows: $env:="" - * Linux/mac: export ="" - * - */ - -let pathToEim; - -if (process.env.EIM_FILE_PATH) { - pathToEim = process.env.EIM_FILE_PATH; -} else { - pathToEim = path.join(os.homedir(), "eim-cli/eim"); -} - -const targetList = ["esp32"]; // targets used for IDF installation -const idfVersionList = ["v5.0.7"]; // IDF versions to be installed -const installFolder = ".espressif5"; -const projectFolder = "project"; - -let installArgs = []; -installArgs.push(` -p ${path.join(os.homedir(), installFolder)}`); // Install Path -installArgs.push(` -t ${targetList.join(",")}`); // Targets -installArgs.push(` -i ${idfVersionList.join(",")}`); // IDF versions -installArgs.push(` -m https://dl.espressif.cn/github_assets`); // IDF tools mirror -installArgs.push(` --idf-mirror https://jihulab.com/esp-mirror`); // ESP-IDF mirror -installArgs.push(` -r false`); // recursive submodules init - -const pathToIDFScript = - os.platform() !== "win32" - ? path.join( - os.homedir(), - installFolder, - `activate_idf_${idfVersionList[0]}.sh` - ) - : path.join( - os.homedir(), - installFolder, - idfVersionList[0], - `Microsoft.PowerShell_profile.ps1` - ); - -logger.info( - `Starting installation using mirror jihulab and dl.espressif.cn and EIM on ${pathToEim}` -); - -describe("using mirror jihulab and dl.espressif.cn", function () { - this.timeout(2400000); - - runInstallNonInteractive(pathToEim, installArgs); - - runPostInstallTest( - pathToIDFScript, - path.join(os.homedir(), installFolder, projectFolder), - targetList[0], - "esp32c6" - ); -}); diff --git a/tests/runs/basicRun.test.js b/tests/runs/basicRun.test.js new file mode 100644 index 0000000..51a92f8 --- /dev/null +++ b/tests/runs/basicRun.test.js @@ -0,0 +1,23 @@ +import { testRun } from "./testRun.test.js"; +import path from "path"; +import fs from "fs"; + +/** + * Setup the following environmental variables to execute this test: + * + * EIM_FILE_PATH to point to the eim application. + * EIM_VERSION to specify expected version to be printed by the application. + * IDF_VERSION to specify the default version of the IDF to be installed. + * + * use: + * Windows: $env:="" + * Linux/mac: export ="" + * + * This script relies on test suite data stores on: + * suites/basic_test.json + */ + +const jsonFilePath = path.join(import.meta.dirname, "suites/basic_test.json"); +const basicTest = JSON.parse(fs.readFileSync(jsonFilePath, "utf-8")); + +testRun(basicTest); diff --git a/tests/runs/defaultInstall.test.js b/tests/runs/defaultInstall.test.js deleted file mode 100644 index 6a2cafa..0000000 --- a/tests/runs/defaultInstall.test.js +++ /dev/null @@ -1,69 +0,0 @@ -import { describe, it, before, after } from "mocha"; -import { runArgumentsTests } from "../script/commandLineArguments.test.js"; -import { runInstallWizardTests } from "../script/installWizard.test.js"; -import { runPostInstallTest } from "../script/postInstall.test.js"; -import logger from "../classes/logger.class.js"; -import os from "os"; -import path from "path"; - -/** - * Setup the following environmental variables to execute this test: - * - * EIM_FILE_PATH to point to the eim application. - * EIM_VERSION to specify expected version to be printed by the application. - * IDF_VERSION to specify the default version of the IDF to be installed. - * - * use: - * Windows: $env:="" - * Linux/mac: export ="" - * - */ - -let pathToEim; -let eimVersion; -let IDFLatestVersion; - -if (process.env.EIM_FILE_PATH) { - pathToEim = process.env.EIM_FILE_PATH; -} else { - pathToEim = path.join(os.homedir(), "eim-cli/eim"); -} - -if (process.env.EIM_VERSION) { - eimVersion = process.env.EIM_VERSION; -} else { - eimVersion = "idf-im-cli 0.1.5"; -} - -if (process.env.IDF_VERSION) { - IDFLatestVersion = process.env.IDF_VERSION; -} else { - IDFLatestVersion = "v5.4"; -} - -const pathToIDFScript = - os.platform() !== "win32" - ? path.join( - os.homedir(), - `.espressif/activate_idf_${IDFLatestVersion}.sh` - ) - : `C:\\esp\\${IDFLatestVersion}\\Microsoft.PowerShell_profile.ps1`; - -const pathToProjectFolder = - os.platform() !== "win32" - ? path.join(os.homedir(), ".espressif/project") - : "C:\\esp\\project"; - -logger.info( - `Starting installation using wizard and default settings using EIM on ${pathToEim}` -); - -describe("Installation using default settings", function () { - this.timeout(2400000); - - runArgumentsTests(pathToEim, eimVersion); - - runInstallWizardTests(pathToEim); - - runPostInstallTest(pathToIDFScript, pathToProjectFolder); -}); diff --git a/tests/runs/extendedRun.test.js b/tests/runs/extendedRun.test.js new file mode 100644 index 0000000..d1c73f5 --- /dev/null +++ b/tests/runs/extendedRun.test.js @@ -0,0 +1,26 @@ +import { testRun } from "./testRun.test.js"; +import path from "path"; +import fs from "fs"; + +/** + * Setup the following environmental variables to execute this test: + * + * EIM_FILE_PATH to point to the eim application. + * EIM_VERSION to specify expected version to be printed by the application. + * IDF_VERSION to specify the default version of the IDF to be installed. + * + * use: + * Windows: $env:="" + * Linux/mac: export ="" + * + * This script relies on test suite data stores on: + * suites/basic_test.json + */ + +const jsonFilePath = path.join( + import.meta.dirname, + "suites/extended_test.json" +); +const extendedTest = JSON.parse(fs.readFileSync(jsonFilePath, "utf-8")); + +testRun(extendedTest); diff --git a/tests/runs/mirrorsRun.test.js b/tests/runs/mirrorsRun.test.js new file mode 100644 index 0000000..5b64d59 --- /dev/null +++ b/tests/runs/mirrorsRun.test.js @@ -0,0 +1,23 @@ +import { testRun } from "./testRun.test.js"; +import path from "path"; +import fs from "fs"; + +/** + * Setup the following environmental variables to execute this test: + * + * EIM_FILE_PATH to point to the eim application. + * EIM_VERSION to specify expected version to be printed by the application. + * IDF_VERSION to specify the default version of the IDF to be installed. + * + * use: + * Windows: $env:="" + * Linux/mac: export ="" + * + * This script relies on test suite data stores on: + * suites/basic_test.json + */ + +const jsonFilePath = path.join(import.meta.dirname, "suites/mirrors_test.json"); +const mirrorsTest = JSON.parse(fs.readFileSync(jsonFilePath, "utf-8")); + +testRun(mirrorsTest); diff --git a/tests/runs/nonInteractiveInstall.test.js b/tests/runs/nonInteractiveInstall.test.js deleted file mode 100644 index ca35f1d..0000000 --- a/tests/runs/nonInteractiveInstall.test.js +++ /dev/null @@ -1,67 +0,0 @@ -import { describe, it, before, after } from "mocha"; -import { runPostInstallTest } from "../script/postInstall.test.js"; -import { runInstallNonInteractive } from "../script/installNonInteractive.test.js"; -import logger from "../classes/logger.class.js"; -import os from "os"; -import path from "path"; - -/** - * Setup the following environmental variables to execute this test: - * - * EIM_FILE_PATH to point to the eim application. - * - * use: - * Windows: $env:="" - * Linux/mac: export ="" - * - */ - -let pathToEim; - -if (process.env.EIM_FILE_PATH) { - pathToEim = process.env.EIM_FILE_PATH; -} else { - pathToEim = path.join(os.homedir(), "eim-cli/eim"); -} - -const targetList = ["esp32c6"]; // targets used for IDF installation -const idfVersionList = ["v5.3.2"]; // IDF versions to be installed -const installFolder = ".espressif3"; -const projectFolder = "project"; - -let installArgs = []; -installArgs.push(` -p ${path.join(os.homedir(), installFolder)}`); // Install Path -installArgs.push(` -t ${targetList.join(",")}`); // Targets -installArgs.push(` -i ${idfVersionList.join(",")}`); // IDF versions -installArgs.push(` -m https://github.com`); // IDF tools mirror -installArgs.push(` --idf-mirror https://github.com`); // ESP-IDF mirror -installArgs.push(` -r true`); // recursive submodules init - -const pathToIDFScript = - os.platform() !== "win32" - ? path.join( - os.homedir(), - installFolder, - `activate_idf_${idfVersionList[0]}.sh` - ) - : path.join( - os.homedir(), - installFolder, - idfVersionList[0], - `Microsoft.PowerShell_profile.ps1` - ); - -logger.debug(`Starting non-interactive installation using EIM on ${pathToEim}`); - -describe("Installation using non-interactive settings", function () { - this.timeout(2400000); - - runInstallNonInteractive(pathToEim, installArgs); - - runPostInstallTest( - pathToIDFScript, - path.join(os.homedir(), installFolder, projectFolder), - targetList[0], - "esp32c6" - ); -}); diff --git a/tests/runs/suites/basic_test.json b/tests/runs/suites/basic_test.json new file mode 100644 index 0000000..c81575d --- /dev/null +++ b/tests/runs/suites/basic_test.json @@ -0,0 +1,30 @@ +[ + { + "type": "arguments", + "name": "command-line-arguments" + }, + { + "type": "default", + "name": "default-installation-wizard" + }, + { + "type": "custom", + "name": "non-interactive-basic", + "data": { + "nonInteractive": "true" + } + }, + { + "type": "custom", + "name": "non-interactive-full-parameters", + "data": { + "targetList": "esp32s2", + "idfList": "v5.3.2", + "installFolder": ".espcustom", + "idfMirror": "github", + "toolsMirror": "github", + "recursive": "false", + "nonInteractive": "false" + } + } +] diff --git a/tests/runs/suites/extended_test.json b/tests/runs/suites/extended_test.json new file mode 100644 index 0000000..60f6d6e --- /dev/null +++ b/tests/runs/suites/extended_test.json @@ -0,0 +1,54 @@ +[ + { + "type": "custom", + "name": "Extended 1", + "data": { + "targetList": "esp32s2|esp32h2", + "idfList": "v5.3.2|v5.4", + "installFolder": ".espext1", + "idfMirror": "github", + "toolsMirror": "github", + "recursive": "true", + "nonInteractive": "true" + } + }, + { + "type": "custom", + "name": "Extended 2", + "data": { + "targetList": "esp32", + "idfList": "v5.2.3", + "installFolder": ".espext2", + "idfMirror": "github", + "toolsMirror": "github", + "recursive": "true", + "nonInteractive": "true" + } + }, + { + "type": "custom", + "name": "Extended 2", + "data": { + "targetList": "esp32", + "idfList": "v5.1.5", + "installFolder": ".esprext3", + "idfMirror": "github", + "toolsMirror": "github", + "recursive": "true", + "nonInteractive": "true" + } + }, + { + "type": "custom", + "name": "Extended 2", + "data": { + "targetList": "esp32", + "idfList": "v5.0.7", + "installFolder": ".esprext4", + "idfMirror": "github", + "toolsMirror": "github", + "recursive": "true", + "nonInteractive": "true" + } + } +] diff --git a/tests/runs/suites/mirrors_test.json b/tests/runs/suites/mirrors_test.json new file mode 100644 index 0000000..46859fc --- /dev/null +++ b/tests/runs/suites/mirrors_test.json @@ -0,0 +1,44 @@ +[ + { + "type": "custom", + "name": "Mirrors jihulab and dl_com", + "data": { + "idfMirror": "jihulab", + "toolsMirror": "dl_com", + "recursive": "true", + "nonInteractive": "true" + } + }, + { + "type": "custom", + "name": "Mirrors jihulab and dl_cn", + "data": { + "idfMirror": "jihulab", + "toolsMirror": "dl_cn", + "recursive": "true", + "nonInteractive": "true" + } + }, + { + "type": "custom", + "name": "Mirrors jihulab and dl_com", + "data": { + "idfList": "v5.2.3", + "idfMirror": "jihulab", + "toolsMirror": "dl_com", + "recursive": "true", + "nonInteractive": "true" + } + }, + { + "type": "custom", + "name": "Mirrors jihulab and dl_cn", + "data": { + "idfList": "v5.0.7", + "idfMirror": "jihulab", + "toolsMirror": "dl_cn", + "recursive": "true", + "nonInteractive": "true" + } + } +] diff --git a/tests/runs/testRun.test.js b/tests/runs/testRun.test.js new file mode 100644 index 0000000..576c57f --- /dev/null +++ b/tests/runs/testRun.test.js @@ -0,0 +1,133 @@ +import { describe } from "mocha"; +import { runArgumentsTests } from "../script/commandLineArguments.test.js"; +import { runInstallWizardTests } from "../script/installWizard.test.js"; +import { runInstallCustom } from "../script/installCustom.test.js"; +import { runPostInstallTest } from "../script/postInstall.test.js"; +import logger from "../classes/logger.class.js"; +import os from "os"; +import path from "path"; + +export function testRun(jsonScript) { + const IDFMIRRORS = { + github: "https://github.com", + jihulab: "https://jihulab.com/esp-mirror", + }; + const TOOLSMIRRORS = { + github: "https://github.com", + dl_com: "https://dl.espressif.com/github_assets", + dl_cn: "https://dl.espressif.cn/github_assets", + }; + + const PATHTOEIM = + process.env.EIM_FILE_PATH || path.join(os.homedir(), "eim-cli/eim"); + + const EIMVERSION = process.env.EIM_VERSION || "eim 0.1.5"; + + const IDFDEFAULTVERSION = process.env.IDF_VERSION || "v5.4"; + + jsonScript.forEach((test) => { + if (test.type === "arguments") { + //routine for arguments tests + describe("EIM command line arguments", function () { + this.timeout(20000); + + runArgumentsTests(PATHTOEIM, EIMVERSION); + }); + } else if (test.type === "default") { + //routine for default installation tests + const installFolder = + os.platform() !== "win32" + ? path.join(os.homedir(), `.espressif`) + : `C:\\esp`; + + const pathToIDFScript = + os.platform() !== "win32" + ? path.join( + installFolder, + `activate_idf_${IDFDEFAULTVERSION}.sh` + ) + : path.join( + installFolder, + IDFDEFAULTVERSION, + "Microsoft.PowerShell_profile.ps1" + ); + + describe("Installation manager default installation", function () { + this.timeout(2400000); + + runInstallWizardTests(PATHTOEIM); + + runPostInstallTest(pathToIDFScript, installFolder); + }); + } else if (test.type === "custom") { + //routine for custom installation tests + logger.info(`Starting custom installation ${test.data.name}`); + let installFolder; + if (test.data.installFolder) { + installFolder = path.join( + os.homedir(), + test.data.installFolder + ); + } else { + installFolder = + os.platform() !== "win32" + ? path.join(os.homedir(), `.espressif`) + : `C:\\esp`; + } + + const targetList = test.data.targetList || "esp32"; + const idfVersionList = test.data.idfList || IDFDEFAULTVERSION; + + let installArgs = []; + + test.data.installFolder && installArgs.push(`-p ${installFolder}`); + + test.data.targetList && + installArgs.push( + `-t ${test.data.targetList.split("|").join(",")}` + ); + + test.data.idfList && + installArgs.push( + `-i ${test.data.idfList.split("|").join(",")}` + ); + + test.data.toolsMirror && + installArgs.push(`-m ${TOOLSMIRRORS[test.data.toolsMirror]}`); + + test.data.idfMirror && + installArgs.push( + `--idf-mirror ${IDFMIRRORS[test.data.idfMirror]}` + ); + + test.data.recursive && + installArgs.push(`-r ${test.data.recursive}`); + + test.data.nonInteractive && + installArgs.push(`-n ${test.data.nonInteractive}`); + + const pathToIDFScript = + os.platform() !== "win32" + ? path.join( + installFolder, + `activate_idf_${idfVersionList.split("|")[0]}.sh` + ) + : path.join( + installFolder, + idfVersionList.split("|")[0], + `Microsoft.PowerShell_profile.ps1` + ); + describe("Installation using non-interactive settings", function () { + this.timeout(2400000); + + runInstallCustom(PATHTOEIM, installArgs); + + runPostInstallTest( + pathToIDFScript, + installFolder, + targetList.split("|")[0] + ); + }); + } + }); +} diff --git a/tests/runs/variation1.test.js b/tests/runs/variation1.test.js deleted file mode 100644 index fe806be..0000000 --- a/tests/runs/variation1.test.js +++ /dev/null @@ -1,67 +0,0 @@ -import { describe, it, before, after } from "mocha"; -import { runInstallCustom } from "../script/installCustom.test.js"; -import { runPostInstallTest } from "../script/postInstall.test.js"; -import logger from "../classes/logger.class.js"; -import os from "os"; -import path from "path"; - -/** - * Setup the following environmental variables to execute this test: - * - * EIM_FILE_PATH to point to the eim application. - * - * use: - * Windows: $env:="" - * Linux/mac: export ="" - * - */ - -let pathToEim; - -if (process.env.EIM_FILE_PATH) { - pathToEim = process.env.EIM_FILE_PATH; -} else { - pathToEim = path.join(os.homedir(), "eim-cli/eim"); -} - -const targetList = ["esp32s2"]; // targets used for IDF installation -const idfVersionList = ["v5.2.3"]; // IDF versions to be installed -const installFolder = ".espressif2"; -const projectFolder = "project"; - -let installArgs = []; -installArgs.push(` -p ${path.join(os.homedir(), installFolder)}`); // Install Path -installArgs.push(` -t ${targetList.join(",")}`); // Targets -installArgs.push(` -i ${idfVersionList.join(",")}`); // IDF versions -installArgs.push(` -m https://github.com`); // IDF tools mirror -installArgs.push(` --idf-mirror https://github.com`); // ESP-IDF mirror -installArgs.push(` -r true`); // recursive submodules init - -const pathToIDFScript = - os.platform() !== "win32" - ? path.join( - os.homedir(), - installFolder, - `activate_idf_${idfVersionList[0]}.sh` - ) - : path.join( - os.homedir(), - installFolder, - idfVersionList[0], - `Microsoft.PowerShell_profile.ps1` - ); - -logger.debug(`Starting custom installation using EIM on ${pathToEim}`); - -describe("Installation using custom settings", function () { - this.timeout(2400000); - - runInstallCustom(pathToEim, installArgs); - - runPostInstallTest( - pathToIDFScript, - path.join(os.homedir(), installFolder, projectFolder), - targetList[0], - "esp32c6" - ); -}); diff --git a/tests/script/commandLineArguments.test.js b/tests/script/commandLineArguments.test.js index 95c098b..bc1e117 100644 --- a/tests/script/commandLineArguments.test.js +++ b/tests/script/commandLineArguments.test.js @@ -4,7 +4,7 @@ import { InteractiveCLITestRunner } from "../classes/CLITestRunner.class.js"; import logger from "../classes/logger.class.js"; export function runArgumentsTests(pathToEim, eimVersion) { - describe("CLI Arguments Tests", function () { + describe("Basic Arguments Tests", function () { let testRunner = null; beforeEach(function () { diff --git a/tests/script/installCustom.test.js b/tests/script/installCustom.test.js index c61fa04..3f94e3a 100644 --- a/tests/script/installCustom.test.js +++ b/tests/script/installCustom.test.js @@ -4,81 +4,100 @@ import { InteractiveCLITestRunner } from "../classes/CLITestRunner.class.js"; import logger from "../classes/logger.class.js"; export function runInstallCustom(pathToEim, args = []) { - describe("Run custom installation using given parameters", function () { - let testRunner = null; + describe("Run custom installation using given parameters", function () { + let testRunner = null; - before(async function () { - logger.debug( - `Installing custom IDF version with parameters ${args.join(" ")}` - ); - this.timeout(5000); - testRunner = new InteractiveCLITestRunner(); - try { - await testRunner.start(); - } catch { - logger.info("Error to start terminal"); - } - }); + before(async function () { + logger.debug( + `Installing custom IDF version with parameters ${args.join( + " " + )}` + ); + this.timeout(5000); + testRunner = new InteractiveCLITestRunner(); + try { + await testRunner.start(); + } catch { + logger.info("Error to start terminal"); + } + }); - afterEach(function () { - if (this.currentTest.state === "failed") { - logger.info(`Terminal output on failure: >>\r ${testRunner.output}`); - } - }); + afterEach(function () { + if (this.currentTest.state === "failed") { + logger.info( + `Terminal output on failure: >>\r ${testRunner.output}` + ); + } + }); - after(async function () { - logger.info("Custom installation routine completed"); - this.timeout(20000); - try { - await testRunner.stop(); - } catch (error) { - logger.info("Error to clean up terminal after test"); - logger.info(` Error: ${error}`); - } - }); + after(async function () { + logger.info("Custom installation routine completed"); + this.timeout(20000); + try { + await testRunner.stop(); + } catch (error) { + logger.info("Error to clean up terminal after test"); + logger.info(` Error: ${error}`); + } + }); + + /** Run installation with full parameters, no need to ask questions + * + * It is expected to have all requirements installed + * + */ + + it("Should install IDF using specified parameters", async function () { + logger.info(`Starting test - IDF custom installation`); + testRunner.sendInput(`${pathToEim} ${args.join(" ")}\r`); + if (!"-n true" in args) { + const installationCompleted = await testRunner.waitForOutput( + "Do you want to save the installer configuration", + 1200000 + ); + expect( + installationCompleted, + "Failed to ask to save installation configuration - failure to install using full arguments on run time" + ).to.be.true; + expect( + testRunner.output, + "Error message during installation" + ).to.not.include("error"); + expect( + testRunner.output, + "Failed to download submodules, missing 'Finished fetching submodules'" + ).to.include("Finished fetching submodules"); - /** Run installation with full parameters, no need to ask questions - * - * It is expected to have all requirements installed - * - */ + logger.info("Installation completed"); + testRunner.output = ""; + testRunner.sendInput("n"); + } + const installationSuccessful = await testRunner.waitForOutput( + "Successfully installed IDF", + 1200000 + ); - it("Should install IDF using specified parameters", async function () { - logger.info(`Starting test - IDF custom installation`); - testRunner.sendInput(`${pathToEim} ${args.join(" ")}\r`); - const installationCompleted = await testRunner.waitForOutput( - "Do you want to save the installer configuration", - 1200000 - ); - expect( - installationCompleted, - "Failed to ask to save installation configuration - failure to install using full arguments on run time" - ).to.be.true; - expect( - testRunner.output, - "Error message during installation" - ).to.not.include("error"); - expect( - testRunner.output, - "Failed to download submodules, missing 'Finished fetching submodules'" - ).to.include("Finished fetching submodules"); + expect( + installationSuccessful, + "Failed to complete installation, missing 'Successfully Installed IDF'" + ).to.be.true; - logger.info("Installation completed"); - testRunner.output = ""; - testRunner.sendInput("n"); + // expect( + // testRunner.output, + // "Error message during installation" + // ).to.not.include("error"); - const installationSuccessful = await testRunner.waitForOutput( - "Successfully installed IDF" - ); + expect( + testRunner.output, + "Failed to complete installation, missing 'Now you can start using IDF tools'" + ).to.include("Now you can start using IDF tools"); - expect( - installationSuccessful, - "Failed to complete installation, missing 'Successfully Installed IDF'" - ).to.be.true; - expect( - testRunner.output, - "Failed to complete installation, missing 'Now you can start using IDF tools'" - ).to.include("Now you can start using IDF tools"); + if ("-r true" in args || "--recursive-submodules" in args) { + expect( + testRunner.output, + "Failed to download submodules, missing 'Finished fetching submodules'" + ).to.include("Finished fetching submodules"); + } + }); }); - }); } diff --git a/tests/script/installNonInteractive.test.js b/tests/script/installNonInteractive.test.js deleted file mode 100644 index ef49e71..0000000 --- a/tests/script/installNonInteractive.test.js +++ /dev/null @@ -1,81 +0,0 @@ -import { expect } from "chai"; -import { describe, it, before, after, beforeEach, afterEach } from "mocha"; -import { InteractiveCLITestRunner } from "../classes/CLITestRunner.class.js"; -import logger from "../classes/logger.class.js"; - -export function runInstallNonInteractive(pathToEim, args = []) { - describe("run custom installation using non interactive terminal", function () { - let testRunner = null; - - before(async function () { - logger.debug(`Starting non interactive installation`); - logger.debug(`Using parameters ${args.join(" ")}`); - this.timeout(5000); - testRunner = new InteractiveCLITestRunner(); - try { - await testRunner.start(); - } catch { - logger.info("Error to start terminal"); - } - }); - - afterEach(function () { - if (this.currentTest.state === "failed") { - logger.info( - `Terminal output on failure: >>\r ${testRunner.output}` - ); - testRunner.sendInput("\x03"); - } - }); - - after(async function () { - logger.info("Custom installation routine completed"); - this.timeout(20000); - try { - await testRunner.stop(); - } catch (error) { - logger.info("Error to clean up terminal after test"); - logger.info(` Error: ${error}`); - } - }); - - /** Run installation with non-interactive shell. - * - * It is expected for all setting to fall back to default in case it is not passed as argument - * Config file would also overwrite default setting. - * - */ - - it("Should install IDF using specified parameters", async function () { - logger.info(`Starting test - IDF non-interactive installation`); - testRunner.sendInput(`${pathToEim} ${args.join(" ")} -n true\r`); - - const installationSuccessful = await testRunner.waitForOutput( - "Successfully installed IDF", - 1200000 - ); - - expect( - installationSuccessful, - "Failed to complete installation, missing 'Successfully Installed IDF'" - ).to.be.true; - - // expect( - // testRunner.output, - // "Error message during installation" - // ).to.not.include("error"); - - expect( - testRunner.output, - "Failed to complete installation, missing 'Now you can start using IDF tools'" - ).to.include("Now you can start using IDF tools"); - - if ("-r true" in args || "--recursive-submodules" in args) { - expect( - testRunner.output, - "Failed to download submodules, missing 'Finished fetching submodules'" - ).to.include("Finished fetching submodules"); - } - }); - }); -} diff --git a/tests/script/installWizard.test.js b/tests/script/installWizard.test.js index 75e25c3..eb80601 100644 --- a/tests/script/installWizard.test.js +++ b/tests/script/installWizard.test.js @@ -1,11 +1,14 @@ import { expect } from "chai"; -import { describe, it, before, after, beforeEach, afterEach } from "mocha"; +import { describe, it, before, after, afterEach } from "mocha"; import { InteractiveCLITestRunner } from "../classes/CLITestRunner.class.js"; import logger from "../classes/logger.class.js"; +import os from "os"; export function runInstallWizardTests(pathToEim) { describe("Check IDF Install Wizard steps", function () { + this.timeout(800000); let testRunner = null; + let installationFailed = false; before(async function () { logger.debug(`Starting installation wizard with default options`); @@ -21,19 +24,20 @@ export function runInstallWizardTests(pathToEim) { afterEach(function () { if (this.currentTest.state === "failed") { logger.info( - `Terminal output on failure: \r >>${testRunner.output}<<` + `Installation using wizard failed -> output: \r >>${testRunner.output}<<` ); + installationFailed = true; } }); after(async function () { - logger.info("Install Wizard routine completed"); + logger.info("Installation wizard test cleanup"); this.timeout(20000); try { await testRunner.stop(); } catch (error) { logger.info("Error to clean up terminal after test"); - logger.info(` Error: ${error}`); + logger.info(`${error}`); } }); @@ -46,6 +50,7 @@ export function runInstallWizardTests(pathToEim) { it("Should install IDF using wizard and default values", async function () { logger.info(`Starting test - IDF installation wizard`); + this.timeout(1500000); testRunner.sendInput(`${pathToEim}\r`); const selectTargetQuestion = await testRunner.waitForOutput( "Please select all of the target platforms", @@ -111,10 +116,15 @@ export function runInstallWizardTests(pathToEim) { ); expect(selectInstallPath, "Failed to ask for installation path").to .be.true; + + const defaultPath = + os.platform() === "win32" + ? "(C:\\esp)" + : `(${os.homedir()}/.espressif)`; expect( testRunner.output, "Failed to provide default installation path" - ).to.include("esp"); + ).to.include(defaultPath); logger.info("Select install path passed"); testRunner.output = ""; diff --git a/tests/script/postInstall.test.js b/tests/script/postInstall.test.js index e989ede..2037b00 100644 --- a/tests/script/postInstall.test.js +++ b/tests/script/postInstall.test.js @@ -3,18 +3,26 @@ import { describe, it, before, after, beforeEach, afterEach } from "mocha"; import { InteractiveCLITestRunner } from "../classes/CLITestRunner.class.js"; import logger from "../classes/logger.class.js"; import os from "os"; +import path from "path"; +import fs from "fs"; export function runPostInstallTest( pathToIDFScript, - pathToProjectFolder, + installFolder, validTarget = "esp32", invalidTarget = "" ) { describe("create and build sample project", function () { this.timeout(600000); let testRunner = null; + let pathToProjectFolder = path.join(installFolder, "project"); + let postInstallStepFailed = false; beforeEach(async function () { + if (postInstallStepFailed) { + logger.info("Test failed, skipping next tests"); + this.skip(); + } this.timeout(10000); logger.debug( `Starting IDF terminal using activation script ${pathToIDFScript}, sample project copied at ${pathToProjectFolder}` @@ -34,8 +42,9 @@ export function runPostInstallTest( this.timeout(20000); if (this.currentTest.state === "failed") { logger.info( - `Terminal output on failure: >>\r ${testRunner.output}` + `Post install test step failed -> output: >>\r ${testRunner.output}` ); + postInstallStepFailed = true; } try { await testRunner.stop(); @@ -45,6 +54,16 @@ export function runPostInstallTest( } }); + after(function () { + logger.info("Post install test completed, starting cleanup"); + try { + fs.rmSync(installFolder, { recursive: true, force: true }); + logger.info(`Successfully deleted ${installFolder}`); + } catch (err) { + logger.info(`Error deleting ${installFolder}`); + } + }); + it("Should create a new project based on a template", async function () { /** * This test should attempt to create a copy of the Hello World Project into the ~/esp folder @@ -58,7 +77,7 @@ export function runPostInstallTest( testRunner.sendInput( os.platform() !== "win32" ? `cp -r $IDF_PATH/examples/get-started/hello_world .\r` - : `xcopy /e /i $env:IDF_PATH\\examples\\get-started\\hello_world hello_world\r` + : `xcopy /E /I $env:IDF_PATH\\examples\\get-started\\hello_world hello_world\r` ); if (os.platform() === "win32") { const confirmFilesCopied = await testRunner.waitForOutput( From 7411fab63fdf43e702ca52b3626870aeda598d9c Mon Sep 17 00:00:00 2001 From: Fabricio-ESP Date: Thu, 16 Jan 2025 18:25:47 +0000 Subject: [PATCH 08/19] Update CI workflow --- .github/workflows/test.yml | 67 ++++++++++++++++++++++++++++---------- tests/package.json | 10 +++--- 2 files changed, 54 insertions(+), 23 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cf6adb6..1d3436a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -82,8 +82,11 @@ jobs: if: runner.os != 'Windows' && matrix.run_on != 'CNRunner' run: | export LOG_TO_FILE="true" - chmod +x ./tests/run_pre_test.sh - . ./tests/run_pre_test.sh "../test-bin/eim" "eim ${{ env.CLI_TAG }}" + export EIM_FILE_PATH="../test-bin/eim" + export EIM_VERSION="eim ${{ env.CLI_TAG }}" + cd tests + npm ci + npm run pre-test - name: Install dependencies (Ubuntu) if: runner.os == 'Linux' @@ -95,19 +98,36 @@ jobs: run: | brew install cmake ninja dfu-util - - name: Run IDF installation and post install test script (non-Windows) + - name: Run IDF basic install test script (non-Windows) if: runner.os != 'Windows' && matrix.run_on != 'CNRunner' run: | export LOG_TO_FILE="true" - chmod +x ./tests/run_test.sh - . ./tests/run_test.sh "../test-bin/eim" "eim ${{ env.CLI_TAG }}" "${{ env.IDF_TAG }}" + export EIM_FILE_PATH="../test-bin/eim" + export EIM_VERSION="eim ${{ env.CLI_TAG }}" + export IDF_VERSION="${{ env.IDF_TAG }}" + cd tests + npm run basic-test - - name: Run IDF installation from alternative mirrors in mainland China + - name: Run IDF extended install test script (non-Windows) + if: runner.os != 'Windows' && matrix.run_on != 'CNRunner' + run: | + export LOG_TO_FILE="true" + export EIM_FILE_PATH="../test-bin/eim" + export EIM_VERSION="eim ${{ env.CLI_TAG }}" + export IDF_VERSION="${{ env.IDF_TAG }}" + cd tests + npm run extended-test + + - name: Run IDF installation from alternative mirrors if: matrix.run_on == 'CNRunner' run: | export LOG_TO_FILE="true" - chmod +x ./tests/run_cnrunner.sh - . ./tests/run_cnrunner.sh "../test-bin/eim" "eim ${{ env.CLI_TAG }}" "${{ env.IDF_TAG }}" + export EIM_FILE_PATH="../test-bin/eim" + export EIM_VERSION="eim ${{ env.CLI_TAG }}" + export IDF_VERSION="${{ env.IDF_TAG }}" + cd tests + npm ci + npm run mirrors-test # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -134,28 +154,41 @@ jobs: 7z x ./artifacts/eim-${{ inputs.run_id }}-${{ matrix.package_name }}/eim.zip -otest-bin # 7z x ./test-bin/eim.zip -otest-bin - - name: Print powershell and windows version (Windows) - if: runner.os == 'Windows' - run: | - $PSVersionTable - [System.Environment]::OSVersion.Version - - name: Run prerequisites test script (Windows) if: runner.os == 'Windows' run: | $env:LOG_TO_FILE="true" - .\tests\run_pre_test.ps1 "..\test-bin\eim.exe" "eim ${{ env.CLI_TAG }}" + $env:EIM_FILE_PATH = "..\test-bin\eim.exe" + $env:EIM_VERSION = "eim ${{ env.CLI_TAG }}" + Set-Location -Path "./tests" + Expand-Archive node_modules.zip + npm run pre-test + npm run pre-install - name: Install dependencies (Windows) if: runner.os == 'windows' run: | choco install ninja -y - - name: Run IDF installation and post install test script (Windows) + - name: Run IDF basic install test script (Windows) + if: runner.os == 'Windows' + run: | + $env:LOG_TO_FILE="true" + $env:EIM_FILE_PATH = "..\test-bin\eim.exe" + $env:EIM_VERSION = "eim ${{ env.CLI_TAG }}" + $env:IDF_VERSION = "${{ env.IDF_TAG }}" + Set-Location -Path "./tests" + npm run basic-test + + - name: Run IDF extended install test script (Windows) if: runner.os == 'Windows' run: | $env:LOG_TO_FILE="true" - .\tests\run_test.ps1 "..\test-bin\eim.exe" "eim ${{ env.CLI_TAG }}" "${{ env.IDF_TAG }}" + $env:EIM_FILE_PATH = "..\test-bin\eim.exe" + $env:EIM_VERSION = "eim ${{ env.CLI_TAG }}" + $env:IDF_VERSION = "${{ env.IDF_TAG }}" + Set-Location -Path "./tests" + npm run extended-test # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ diff --git a/tests/package.json b/tests/package.json index ccd0b7d..b42c173 100644 --- a/tests/package.json +++ b/tests/package.json @@ -9,12 +9,10 @@ "winston": "^3.15.0" }, "scripts": { - "pre-test": "mocha --exit --bail --reporter json --reporter-options output=./results-pre-test.json script/prerequisites.test.js", + "pre-test": "mocha --exit --reporter json --reporter-options output=./results-pre-test.json script/prerequisites.test.js", "pre-install": "mocha --exit --bail --reporter json --reporter-options output=./results-pre-install-test.json script/prerequisitesInstallRunner.test.js", - "default-test": "mocha --exit --bail --reporter json --reporter-options output=./results-default-test.json runs/defaultInstall.test.js", - "variation1-test": "mocha --exit --bail --reporter json --reporter-options output=./results-variation1-test.json runs/variation1.test.js", - "non-interactive-test": "mocha --exit --bail --reporter json --reporter-options output=./results-non-interactive-test.json runs/nonInteractiveInstall.test.js", - "cnrunner1-test": "mocha --exit --bail --reporter json --reporter-options output=./results-CNMirror-test.json runs/CNmirrors.test.js", - "cnrunner2-test": "mocha --exit --bail --reporter json --reporter-options output=./results-CNMirrors2-test.json runs/CNmirrors2.test.js" + "basic-test": "mocha --exit --reporter json --reporter-options output=./results-basic-test.json runs/basicRun.test.js", + "extended-test": "mocha --exit --reporter json --reporter-options output=./results-extended-test.json runs/extendedRun.test.js", + "mirrors-test": "mocha --exit --reporter json --reporter-options output=./results-mirrors-test.json runs/mirrorsRun.test.js" } } From 68c681a04c60f787697926d7e3793202b0280bf0 Mon Sep 17 00:00:00 2001 From: Fabricio Ribeiro Date: Thu, 16 Jan 2025 19:37:34 +0000 Subject: [PATCH 09/19] Fix conditionals in CI workflow --- .github/workflows/test.yml | 46 ++++++++++++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1d3436a..c6ac99e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -74,11 +74,13 @@ jobs: # unzip ./test-bin/eim.zip -d test-bin - name: Set executable permissions (non-Windows) + id: set-executable-permissions-non-windows if: runner.os != 'Windows' run: | chmod +x ./test-bin/eim - name: Run prerequisites test script (non-Windows), skip for CNRunner + id: pre-test-non-windows if: runner.os != 'Windows' && matrix.run_on != 'CNRunner' run: | export LOG_TO_FILE="true" @@ -89,17 +91,30 @@ jobs: npm run pre-test - name: Install dependencies (Ubuntu) - if: runner.os == 'Linux' + id: install-deps-ubuntu + if: | + runner.os == 'Linux' && + (success() || steps.pre-test-non-windows.outcome == 'failure') run: | sudo apt-get install -y git cmake ninja-build wget flex bison gperf ccache libffi-dev libssl-dev dfu-util libusb-1.0-0-dev python3 python3-venv python3-pip - name: Install dependencies (MacOS) - if: runner.os == 'macOS' + id: install-deps-macos + if: | + runner.os == 'macOS' && + (success() || steps.pre-test-non-windows.outcome == 'failure') run: | brew install cmake ninja dfu-util - name: Run IDF basic install test script (non-Windows) - if: runner.os != 'Windows' && matrix.run_on != 'CNRunner' + id: basic-test-non-windows + if: | + runner.os != 'Windows' && + matrix.run_on != 'CNRunner' && + (success() || steps.pre-test-non-windows.outcome == 'failure' && + steps.install-deps-ubuntu == 'success' || + steps.pre-test-non-windows.outcome == 'failure' && + steps.install-deps-macos == 'success') run: | export LOG_TO_FILE="true" export EIM_FILE_PATH="../test-bin/eim" @@ -109,7 +124,13 @@ jobs: npm run basic-test - name: Run IDF extended install test script (non-Windows) - if: runner.os != 'Windows' && matrix.run_on != 'CNRunner' + if: | + runner.os != 'Windows' && + matrix.run_on != 'CNRunner' && + (success() || steps.pre-test-non-windows.outcome == 'failure' && + steps.install-deps-ubuntu == 'success' || + steps.pre-test-non-windows.outcome == 'failure' && + steps.install-deps-macos == 'success') run: | export LOG_TO_FILE="true" export EIM_FILE_PATH="../test-bin/eim" @@ -155,6 +176,7 @@ jobs: # 7z x ./test-bin/eim.zip -otest-bin - name: Run prerequisites test script (Windows) + id: pre-test-windows if: runner.os == 'Windows' run: | $env:LOG_TO_FILE="true" @@ -166,12 +188,19 @@ jobs: npm run pre-install - name: Install dependencies (Windows) - if: runner.os == 'windows' + id: install-deps-windows + if: | + runner.os == 'windows' && + (success() || steps.pre-test-windows.outcome == 'failure') run: | choco install ninja -y - name: Run IDF basic install test script (Windows) - if: runner.os == 'Windows' + id: basic-test-windows + if: | + runner.os == 'Windows' && + (success() || steps.pre-test-windows.outcome == 'failure' && + steps.install-deps-windows == 'success') run: | $env:LOG_TO_FILE="true" $env:EIM_FILE_PATH = "..\test-bin\eim.exe" @@ -181,7 +210,10 @@ jobs: npm run basic-test - name: Run IDF extended install test script (Windows) - if: runner.os == 'Windows' + if: | + runner.os == 'Windows' && + (success() || steps.pre-test-windows.outcome == 'failure' && + steps.install-deps-windows == 'success') run: | $env:LOG_TO_FILE="true" $env:EIM_FILE_PATH = "..\test-bin\eim.exe" From 930db8a24965f1cb8b09f3799b69f8c056b38e95 Mon Sep 17 00:00:00 2001 From: Fabricio Ribeiro Date: Fri, 17 Jan 2025 09:24:26 +0000 Subject: [PATCH 10/19] Fix worflow error and incorrect log veriable --- .github/workflows/test.yml | 12 ++++++------ tests/runs/testRun.test.js | 5 ++++- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c6ac99e..ac52913 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -112,9 +112,9 @@ jobs: runner.os != 'Windows' && matrix.run_on != 'CNRunner' && (success() || steps.pre-test-non-windows.outcome == 'failure' && - steps.install-deps-ubuntu == 'success' || + steps.install-deps-ubuntu.outcome == 'success' || steps.pre-test-non-windows.outcome == 'failure' && - steps.install-deps-macos == 'success') + steps.install-deps-macos.outcome == 'success') run: | export LOG_TO_FILE="true" export EIM_FILE_PATH="../test-bin/eim" @@ -128,9 +128,9 @@ jobs: runner.os != 'Windows' && matrix.run_on != 'CNRunner' && (success() || steps.pre-test-non-windows.outcome == 'failure' && - steps.install-deps-ubuntu == 'success' || + steps.install-deps-ubuntu.outcome == 'success' || steps.pre-test-non-windows.outcome == 'failure' && - steps.install-deps-macos == 'success') + steps.install-deps-macos.outcome == 'success') run: | export LOG_TO_FILE="true" export EIM_FILE_PATH="../test-bin/eim" @@ -200,7 +200,7 @@ jobs: if: | runner.os == 'Windows' && (success() || steps.pre-test-windows.outcome == 'failure' && - steps.install-deps-windows == 'success') + steps.install-deps-windows.outcome == 'success') run: | $env:LOG_TO_FILE="true" $env:EIM_FILE_PATH = "..\test-bin\eim.exe" @@ -213,7 +213,7 @@ jobs: if: | runner.os == 'Windows' && (success() || steps.pre-test-windows.outcome == 'failure' && - steps.install-deps-windows == 'success') + steps.install-deps-windows.outcome == 'success') run: | $env:LOG_TO_FILE="true" $env:EIM_FILE_PATH = "..\test-bin\eim.exe" diff --git a/tests/runs/testRun.test.js b/tests/runs/testRun.test.js index 576c57f..34d064d 100644 --- a/tests/runs/testRun.test.js +++ b/tests/runs/testRun.test.js @@ -61,7 +61,10 @@ export function testRun(jsonScript) { }); } else if (test.type === "custom") { //routine for custom installation tests - logger.info(`Starting custom installation ${test.data.name}`); + logger.info( + `################################################################################` + ); + logger.info(`Starting custom installation ${test.name}`); let installFolder; if (test.data.installFolder) { installFolder = path.join( From 0b807701f5cf9a32135412566fe384ff4f2494db Mon Sep 17 00:00:00 2001 From: Fabricio Ribeiro Date: Fri, 17 Jan 2025 11:32:51 +0000 Subject: [PATCH 11/19] Change log order and fix CI step triggers. --- .github/workflows/test.yml | 24 +++++++++++------------- tests/runs/suites/extended_test.json | 4 ++-- tests/runs/testRun.test.js | 20 +++++++++++++++----- 3 files changed, 28 insertions(+), 20 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ac52913..35ec40e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -94,7 +94,8 @@ jobs: id: install-deps-ubuntu if: | runner.os == 'Linux' && - (success() || steps.pre-test-non-windows.outcome == 'failure') + (success() || steps.pre-test-non-windows.outcome == 'failure' && + steps.set-executable-permissions-non-windows.outcome == 'success') run: | sudo apt-get install -y git cmake ninja-build wget flex bison gperf ccache libffi-dev libssl-dev dfu-util libusb-1.0-0-dev python3 python3-venv python3-pip @@ -102,7 +103,8 @@ jobs: id: install-deps-macos if: | runner.os == 'macOS' && - (success() || steps.pre-test-non-windows.outcome == 'failure') + (success() || steps.pre-test-non-windows.outcome == 'failure' && + steps.set-executable-permissions-non-windows.outcome == 'success') run: | brew install cmake ninja dfu-util @@ -111,9 +113,7 @@ jobs: if: | runner.os != 'Windows' && matrix.run_on != 'CNRunner' && - (success() || steps.pre-test-non-windows.outcome == 'failure' && - steps.install-deps-ubuntu.outcome == 'success' || - steps.pre-test-non-windows.outcome == 'failure' && + (steps.install-deps-ubuntu.outcome == 'success' || steps.install-deps-macos.outcome == 'success') run: | export LOG_TO_FILE="true" @@ -127,9 +127,7 @@ jobs: if: | runner.os != 'Windows' && matrix.run_on != 'CNRunner' && - (success() || steps.pre-test-non-windows.outcome == 'failure' && - steps.install-deps-ubuntu.outcome == 'success' || - steps.pre-test-non-windows.outcome == 'failure' && + (steps.install-deps-ubuntu.outcome == 'success' || steps.install-deps-macos.outcome == 'success') run: | export LOG_TO_FILE="true" @@ -169,6 +167,7 @@ jobs: echo "Latest IDF for default install: $IDF_TAG" - name: Extract artifact (Windows) + id: extract-artifact-windows if: runner.os == 'Windows' run: | mkdir -p test-bin @@ -191,7 +190,8 @@ jobs: id: install-deps-windows if: | runner.os == 'windows' && - (success() || steps.pre-test-windows.outcome == 'failure') + (success() || steps.pre-test-windows.outcome == 'failure' && + steps.extract-artifact-windows.outcome == 'success') run: | choco install ninja -y @@ -199,8 +199,7 @@ jobs: id: basic-test-windows if: | runner.os == 'Windows' && - (success() || steps.pre-test-windows.outcome == 'failure' && - steps.install-deps-windows.outcome == 'success') + steps.install-deps-windows.outcome == 'success' run: | $env:LOG_TO_FILE="true" $env:EIM_FILE_PATH = "..\test-bin\eim.exe" @@ -212,8 +211,7 @@ jobs: - name: Run IDF extended install test script (Windows) if: | runner.os == 'Windows' && - (success() || steps.pre-test-windows.outcome == 'failure' && - steps.install-deps-windows.outcome == 'success') + steps.install-deps-windows.outcome == 'success' run: | $env:LOG_TO_FILE="true" $env:EIM_FILE_PATH = "..\test-bin\eim.exe" diff --git a/tests/runs/suites/extended_test.json b/tests/runs/suites/extended_test.json index 60f6d6e..16f8d1c 100644 --- a/tests/runs/suites/extended_test.json +++ b/tests/runs/suites/extended_test.json @@ -27,7 +27,7 @@ }, { "type": "custom", - "name": "Extended 2", + "name": "Extended 3", "data": { "targetList": "esp32", "idfList": "v5.1.5", @@ -40,7 +40,7 @@ }, { "type": "custom", - "name": "Extended 2", + "name": "Extended 4", "data": { "targetList": "esp32", "idfList": "v5.0.7", diff --git a/tests/runs/testRun.test.js b/tests/runs/testRun.test.js index 34d064d..dc8001f 100644 --- a/tests/runs/testRun.test.js +++ b/tests/runs/testRun.test.js @@ -28,13 +28,19 @@ export function testRun(jsonScript) { jsonScript.forEach((test) => { if (test.type === "arguments") { //routine for arguments tests + describe("EIM command line arguments", function () { + logger.info( + `################################################################################` + ); + logger.info(`Starting CLI arguments test ${test.name}`); this.timeout(20000); runArgumentsTests(PATHTOEIM, EIMVERSION); }); } else if (test.type === "default") { //routine for default installation tests + const installFolder = os.platform() !== "win32" ? path.join(os.homedir(), `.espressif`) @@ -53,6 +59,10 @@ export function testRun(jsonScript) { ); describe("Installation manager default installation", function () { + logger.info( + `################################################################################` + ); + logger.info(`Starting default installation ${test.name}`); this.timeout(2400000); runInstallWizardTests(PATHTOEIM); @@ -61,10 +71,6 @@ export function testRun(jsonScript) { }); } else if (test.type === "custom") { //routine for custom installation tests - logger.info( - `################################################################################` - ); - logger.info(`Starting custom installation ${test.name}`); let installFolder; if (test.data.installFolder) { installFolder = path.join( @@ -120,7 +126,11 @@ export function testRun(jsonScript) { idfVersionList.split("|")[0], `Microsoft.PowerShell_profile.ps1` ); - describe("Installation using non-interactive settings", function () { + describe(`Installation using custom settings -> ${test.name}`, function () { + logger.info( + `################################################################################` + ); + logger.info(`Starting custom installation ${test.name}`); this.timeout(2400000); runInstallCustom(PATHTOEIM, installArgs); From 9db68f0c6d65105a8660b603b55e25720bf530cc Mon Sep 17 00:00:00 2001 From: Fabricio Ribeiro Date: Fri, 17 Jan 2025 12:50:41 +0000 Subject: [PATCH 12/19] Change conditional for workflow steps --- .github/workflows/test.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 35ec40e..4c17c78 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -94,8 +94,7 @@ jobs: id: install-deps-ubuntu if: | runner.os == 'Linux' && - (success() || steps.pre-test-non-windows.outcome == 'failure' && - steps.set-executable-permissions-non-windows.outcome == 'success') + steps.set-executable-permissions-non-windows.outcome == 'success' run: | sudo apt-get install -y git cmake ninja-build wget flex bison gperf ccache libffi-dev libssl-dev dfu-util libusb-1.0-0-dev python3 python3-venv python3-pip @@ -103,8 +102,7 @@ jobs: id: install-deps-macos if: | runner.os == 'macOS' && - (success() || steps.pre-test-non-windows.outcome == 'failure' && - steps.set-executable-permissions-non-windows.outcome == 'success') + steps.set-executable-permissions-non-windows.outcome == 'success' run: | brew install cmake ninja dfu-util @@ -190,8 +188,7 @@ jobs: id: install-deps-windows if: | runner.os == 'windows' && - (success() || steps.pre-test-windows.outcome == 'failure' && - steps.extract-artifact-windows.outcome == 'success') + steps.extract-artifact-windows.outcome == 'success' run: | choco install ninja -y From 04ec13c42e551a52931dd7386d0a450a84b2bb06 Mon Sep 17 00:00:00 2001 From: Fabricio Ribeiro Date: Fri, 17 Jan 2025 14:03:14 +0000 Subject: [PATCH 13/19] Change workflow steps logic control --- .github/workflows/test.yml | 39 ++++++++++++++------------------------ 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4c17c78..ffce407 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -74,13 +74,12 @@ jobs: # unzip ./test-bin/eim.zip -d test-bin - name: Set executable permissions (non-Windows) - id: set-executable-permissions-non-windows if: runner.os != 'Windows' run: | chmod +x ./test-bin/eim - name: Run prerequisites test script (non-Windows), skip for CNRunner - id: pre-test-non-windows + continue-on-error: true if: runner.os != 'Windows' && matrix.run_on != 'CNRunner' run: | export LOG_TO_FILE="true" @@ -91,28 +90,22 @@ jobs: npm run pre-test - name: Install dependencies (Ubuntu) - id: install-deps-ubuntu if: | - runner.os == 'Linux' && - steps.set-executable-permissions-non-windows.outcome == 'success' + runner.os == 'Linux' run: | sudo apt-get install -y git cmake ninja-build wget flex bison gperf ccache libffi-dev libssl-dev dfu-util libusb-1.0-0-dev python3 python3-venv python3-pip - name: Install dependencies (MacOS) - id: install-deps-macos if: | - runner.os == 'macOS' && - steps.set-executable-permissions-non-windows.outcome == 'success' + runner.os == 'macOS' run: | brew install cmake ninja dfu-util - name: Run IDF basic install test script (non-Windows) - id: basic-test-non-windows + continue-on-error: true if: | runner.os != 'Windows' && - matrix.run_on != 'CNRunner' && - (steps.install-deps-ubuntu.outcome == 'success' || - steps.install-deps-macos.outcome == 'success') + matrix.run_on != 'CNRunner' run: | export LOG_TO_FILE="true" export EIM_FILE_PATH="../test-bin/eim" @@ -122,11 +115,11 @@ jobs: npm run basic-test - name: Run IDF extended install test script (non-Windows) + continue-on-error: true if: | runner.os != 'Windows' && - matrix.run_on != 'CNRunner' && - (steps.install-deps-ubuntu.outcome == 'success' || - steps.install-deps-macos.outcome == 'success') + matrix.run_on != 'CNRunner' + run: | export LOG_TO_FILE="true" export EIM_FILE_PATH="../test-bin/eim" @@ -165,7 +158,6 @@ jobs: echo "Latest IDF for default install: $IDF_TAG" - name: Extract artifact (Windows) - id: extract-artifact-windows if: runner.os == 'Windows' run: | mkdir -p test-bin @@ -173,7 +165,7 @@ jobs: # 7z x ./test-bin/eim.zip -otest-bin - name: Run prerequisites test script (Windows) - id: pre-test-windows + continue-on-error: true if: runner.os == 'Windows' run: | $env:LOG_TO_FILE="true" @@ -185,18 +177,15 @@ jobs: npm run pre-install - name: Install dependencies (Windows) - id: install-deps-windows if: | - runner.os == 'windows' && - steps.extract-artifact-windows.outcome == 'success' + runner.os == 'windows' run: | choco install ninja -y - name: Run IDF basic install test script (Windows) - id: basic-test-windows + continue-on-error: true if: | - runner.os == 'Windows' && - steps.install-deps-windows.outcome == 'success' + runner.os == 'Windows' run: | $env:LOG_TO_FILE="true" $env:EIM_FILE_PATH = "..\test-bin\eim.exe" @@ -206,9 +195,9 @@ jobs: npm run basic-test - name: Run IDF extended install test script (Windows) + continue-on-error: true if: | - runner.os == 'Windows' && - steps.install-deps-windows.outcome == 'success' + runner.os == 'Windows' run: | $env:LOG_TO_FILE="true" $env:EIM_FILE_PATH = "..\test-bin\eim.exe" From 01e7851b83dc2c5f2cf9766e1e04cd3023a96de0 Mon Sep 17 00:00:00 2001 From: Fabricio Ribeiro Date: Mon, 20 Jan 2025 11:28:03 +0000 Subject: [PATCH 14/19] Update test descriptions, add check for IDF ver. --- tests/runs/suites/basic_test.json | 4 ++++ tests/runs/suites/extended_test.json | 4 ++++ tests/runs/suites/mirrors_test.json | 4 ++++ tests/runs/testRun.test.js | 24 +++++++------------ tests/script/commandLineArguments.test.js | 2 +- tests/script/installCustom.test.js | 2 +- tests/script/installWizard.test.js | 2 +- tests/script/postInstall.test.js | 2 +- tests/script/prerequisites.test.js | 2 +- tests/script/prerequisitesInstall.test.js | 2 +- .../script/prerequisitesInstallRunner.test.js | 2 +- 11 files changed, 27 insertions(+), 23 deletions(-) diff --git a/tests/runs/suites/basic_test.json b/tests/runs/suites/basic_test.json index c81575d..71e8fc4 100644 --- a/tests/runs/suites/basic_test.json +++ b/tests/runs/suites/basic_test.json @@ -1,13 +1,16 @@ [ { + "id": 1, "type": "arguments", "name": "command-line-arguments" }, { + "id": 2, "type": "default", "name": "default-installation-wizard" }, { + "id": 3, "type": "custom", "name": "non-interactive-basic", "data": { @@ -15,6 +18,7 @@ } }, { + "id": 4, "type": "custom", "name": "non-interactive-full-parameters", "data": { diff --git a/tests/runs/suites/extended_test.json b/tests/runs/suites/extended_test.json index 16f8d1c..2a59e17 100644 --- a/tests/runs/suites/extended_test.json +++ b/tests/runs/suites/extended_test.json @@ -1,5 +1,6 @@ [ { + "id": 1, "type": "custom", "name": "Extended 1", "data": { @@ -13,6 +14,7 @@ } }, { + "id": 2, "type": "custom", "name": "Extended 2", "data": { @@ -26,6 +28,7 @@ } }, { + "id": 3, "type": "custom", "name": "Extended 3", "data": { @@ -39,6 +42,7 @@ } }, { + "id": 4, "type": "custom", "name": "Extended 4", "data": { diff --git a/tests/runs/suites/mirrors_test.json b/tests/runs/suites/mirrors_test.json index 46859fc..11bba65 100644 --- a/tests/runs/suites/mirrors_test.json +++ b/tests/runs/suites/mirrors_test.json @@ -1,5 +1,6 @@ [ { + "id": 1, "type": "custom", "name": "Mirrors jihulab and dl_com", "data": { @@ -10,6 +11,7 @@ } }, { + "id": 2, "type": "custom", "name": "Mirrors jihulab and dl_cn", "data": { @@ -20,6 +22,7 @@ } }, { + "id": 3, "type": "custom", "name": "Mirrors jihulab and dl_com", "data": { @@ -31,6 +34,7 @@ } }, { + "id": 4, "type": "custom", "name": "Mirrors jihulab and dl_cn", "data": { diff --git a/tests/runs/testRun.test.js b/tests/runs/testRun.test.js index dc8001f..521058c 100644 --- a/tests/runs/testRun.test.js +++ b/tests/runs/testRun.test.js @@ -23,17 +23,17 @@ export function testRun(jsonScript) { const EIMVERSION = process.env.EIM_VERSION || "eim 0.1.5"; - const IDFDEFAULTVERSION = process.env.IDF_VERSION || "v5.4"; + const IDFDEFAULTVERSION = + process.env.IDF_VERSION & (process.env.IDF_VERSION !== "null") + ? process.env.IDF_VERSION + : "v5.4"; + // Test Runs jsonScript.forEach((test) => { if (test.type === "arguments") { //routine for arguments tests - describe("EIM command line arguments", function () { - logger.info( - `################################################################################` - ); - logger.info(`Starting CLI arguments test ${test.name}`); + describe(`${test.id} - EIM command line arguments ->`, function () { this.timeout(20000); runArgumentsTests(PATHTOEIM, EIMVERSION); @@ -58,11 +58,7 @@ export function testRun(jsonScript) { "Microsoft.PowerShell_profile.ps1" ); - describe("Installation manager default installation", function () { - logger.info( - `################################################################################` - ); - logger.info(`Starting default installation ${test.name}`); + describe(`${test.id} - Installation manager default installation ->`, function () { this.timeout(2400000); runInstallWizardTests(PATHTOEIM); @@ -126,11 +122,7 @@ export function testRun(jsonScript) { idfVersionList.split("|")[0], `Microsoft.PowerShell_profile.ps1` ); - describe(`Installation using custom settings -> ${test.name}`, function () { - logger.info( - `################################################################################` - ); - logger.info(`Starting custom installation ${test.name}`); + describe(`${test.id} - Installation using custom settings -> ${test.name} ->`, function () { this.timeout(2400000); runInstallCustom(PATHTOEIM, installArgs); diff --git a/tests/script/commandLineArguments.test.js b/tests/script/commandLineArguments.test.js index bc1e117..7304506 100644 --- a/tests/script/commandLineArguments.test.js +++ b/tests/script/commandLineArguments.test.js @@ -4,7 +4,7 @@ import { InteractiveCLITestRunner } from "../classes/CLITestRunner.class.js"; import logger from "../classes/logger.class.js"; export function runArgumentsTests(pathToEim, eimVersion) { - describe("Basic Arguments Tests", function () { + describe("Basic Arguments Tests ->", function () { let testRunner = null; beforeEach(function () { diff --git a/tests/script/installCustom.test.js b/tests/script/installCustom.test.js index 3f94e3a..719d98c 100644 --- a/tests/script/installCustom.test.js +++ b/tests/script/installCustom.test.js @@ -4,7 +4,7 @@ import { InteractiveCLITestRunner } from "../classes/CLITestRunner.class.js"; import logger from "../classes/logger.class.js"; export function runInstallCustom(pathToEim, args = []) { - describe("Run custom installation using given parameters", function () { + describe("Run custom installation using given parameters ->", function () { let testRunner = null; before(async function () { diff --git a/tests/script/installWizard.test.js b/tests/script/installWizard.test.js index eb80601..da03d32 100644 --- a/tests/script/installWizard.test.js +++ b/tests/script/installWizard.test.js @@ -5,7 +5,7 @@ import logger from "../classes/logger.class.js"; import os from "os"; export function runInstallWizardTests(pathToEim) { - describe("Check IDF Install Wizard steps", function () { + describe("Run Install Wizard steps ->", function () { this.timeout(800000); let testRunner = null; let installationFailed = false; diff --git a/tests/script/postInstall.test.js b/tests/script/postInstall.test.js index 2037b00..e98e821 100644 --- a/tests/script/postInstall.test.js +++ b/tests/script/postInstall.test.js @@ -12,7 +12,7 @@ export function runPostInstallTest( validTarget = "esp32", invalidTarget = "" ) { - describe("create and build sample project", function () { + describe("Post installation test (create and build project) ->", function () { this.timeout(600000); let testRunner = null; let pathToProjectFolder = path.join(installFolder, "project"); diff --git a/tests/script/prerequisites.test.js b/tests/script/prerequisites.test.js index 10057c7..8f40173 100644 --- a/tests/script/prerequisites.test.js +++ b/tests/script/prerequisites.test.js @@ -24,7 +24,7 @@ if (process.env.EIM_FILE_PATH) { pathToEim = path.join(os.homedir(), "eim-cli/eim"); } -describe("Check if prerequisites are installed", function () { +describe("Check if prerequisites are installed ->", function () { this.timeout(600000); let testRunner; diff --git a/tests/script/prerequisitesInstall.test.js b/tests/script/prerequisitesInstall.test.js index ebb7f24..17ed6c0 100644 --- a/tests/script/prerequisitesInstall.test.js +++ b/tests/script/prerequisitesInstall.test.js @@ -23,7 +23,7 @@ if (process.env.EIM_FILE_PATH) { pathToEim = path.join(os.homedir(), "eim-cli/eim"); } -describe("Check Pre-requisites installation on Windows", function () { +describe("Check Pre-requisites installation on Windows ->", function () { this.timeout(600000); let testRunner; diff --git a/tests/script/prerequisitesInstallRunner.test.js b/tests/script/prerequisitesInstallRunner.test.js index 777c032..1b93bd1 100644 --- a/tests/script/prerequisitesInstallRunner.test.js +++ b/tests/script/prerequisitesInstallRunner.test.js @@ -23,7 +23,7 @@ if (process.env.EIM_FILE_PATH) { pathToEim = path.join(os.homedir(), "eim-cli/eim"); } -describe("Check Pre-requisites installation on Windows", function () { +describe("Check Pre-requisites installation on Windows ->", function () { this.timeout(600000); let testRunner; From 15069c1f9782bdddddb0197b9e6aebe69f377349 Mon Sep 17 00:00:00 2001 From: Fabricio Ribeiro Date: Mon, 20 Jan 2025 15:34:21 +0000 Subject: [PATCH 15/19] Source for IDF version, update test descriptions --- .github/workflows/test.yml | 10 +++++----- tests/script/installCustom.test.js | 2 +- tests/script/installWizard.test.js | 2 +- tests/script/postInstall.test.js | 8 ++++---- tests/script/prerequisitesInstallRunner.test.js | 6 +++--- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ffce407..0566ee8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,9 +60,8 @@ jobs: - name: Fetch latest IDF release if: runner.os != 'Windows' run: | - curl -v -s https://api.github.com/repos/espressif/esp-idf/releases/latest > IDF_latest_release.json - cat IDF_latest_release.json - IDF_TAG=$(jq -r .tag_name IDF_latest_release.json) + curl -v -s https://dl.espressif.com/dl/esp-idf/idf_versions.json > IDF_latest_release.json + IDF_TAG=$(jq -r .VERSIONS[1].name IDF_latest_release.json) echo "IDF_TAG=$IDF_TAG" >> $GITHUB_ENV echo "Latest IDF for default install: $IDF_TAG" @@ -129,6 +128,7 @@ jobs: npm run extended-test - name: Run IDF installation from alternative mirrors + continue-on-error: true if: matrix.run_on == 'CNRunner' run: | export LOG_TO_FILE="true" @@ -152,8 +152,8 @@ jobs: - name: Fetch latest IDF release if: runner.os == 'Windows' run: | - curl -s https://api.github.com/repos/espressif/esp-idf/releases/latest -o IDF_latest_release.json - $IDF_TAG = (Get-Content IDF_latest_release.json | ConvertFrom-Json).tag_name + $json = Invoke-RestMethod -Uri "https://dl.espressif.com/dl/esp-idf/idf_versions.json" + $IDF_TAG = $json.VERSIONS[1].name echo "IDF_TAG=$IDF_TAG" | Out-File -FilePath $env:GITHUB_ENV -Append echo "Latest IDF for default install: $IDF_TAG" diff --git a/tests/script/installCustom.test.js b/tests/script/installCustom.test.js index 719d98c..ff9165d 100644 --- a/tests/script/installCustom.test.js +++ b/tests/script/installCustom.test.js @@ -4,7 +4,7 @@ import { InteractiveCLITestRunner } from "../classes/CLITestRunner.class.js"; import logger from "../classes/logger.class.js"; export function runInstallCustom(pathToEim, args = []) { - describe("Run custom installation using given parameters ->", function () { + describe("1 - Run custom installation using given parameters ->", function () { let testRunner = null; before(async function () { diff --git a/tests/script/installWizard.test.js b/tests/script/installWizard.test.js index da03d32..3d11bcf 100644 --- a/tests/script/installWizard.test.js +++ b/tests/script/installWizard.test.js @@ -5,7 +5,7 @@ import logger from "../classes/logger.class.js"; import os from "os"; export function runInstallWizardTests(pathToEim) { - describe("Run Install Wizard steps ->", function () { + describe("1 - Run Install Wizard steps ->", function () { this.timeout(800000); let testRunner = null; let installationFailed = false; diff --git a/tests/script/postInstall.test.js b/tests/script/postInstall.test.js index e98e821..5a3efce 100644 --- a/tests/script/postInstall.test.js +++ b/tests/script/postInstall.test.js @@ -12,7 +12,7 @@ export function runPostInstallTest( validTarget = "esp32", invalidTarget = "" ) { - describe("Post installation test (create and build project) ->", function () { + describe("2 - Post installation test (create and build project) ->", function () { this.timeout(600000); let testRunner = null; let pathToProjectFolder = path.join(installFolder, "project"); @@ -64,7 +64,7 @@ export function runPostInstallTest( } }); - it("Should create a new project based on a template", async function () { + it("1 - Should create a new project based on a template", async function () { /** * This test should attempt to create a copy of the Hello World Project into the ~/esp folder * The commands might differ for each operating system. @@ -109,7 +109,7 @@ export function runPostInstallTest( logger.info("sample project creation Passed"); }); - it("Should set the target", async function () { + it("2 - Should set the target", async function () { /** * This test attempts to set a target MCU for the project created in the previous test. */ @@ -140,7 +140,7 @@ export function runPostInstallTest( logger.info("Set Target Passed"); }); - it("Should build project for the selected target", async function () { + it("3 - Should build project for the selected target", async function () { /** * This test attempts to build artifacts for the project and targets selected above. * The test is successful if the success message is printed in the terminal. diff --git a/tests/script/prerequisitesInstallRunner.test.js b/tests/script/prerequisitesInstallRunner.test.js index 1b93bd1..e4f9555 100644 --- a/tests/script/prerequisitesInstallRunner.test.js +++ b/tests/script/prerequisitesInstallRunner.test.js @@ -55,7 +55,7 @@ describe("Check Pre-requisites installation on Windows ->", function () { testRunner = null; }); - it("should install prerequisites and offer to install python and exit upon negative answer", async function () { + it("1 - should install prerequisites and offer to install python and exit upon negative answer", async function () { logger.info(`Starting test - check python requirement`); this.timeout(240000); const promptRequisites = await testRunner.waitForOutput( @@ -81,7 +81,7 @@ describe("Check Pre-requisites installation on Windows ->", function () { ).to.be.true; }); - it("should detect all prerequisites are installed", async function () { + it("2 - should detect all prerequisites are installed", async function () { logger.info(`Starting test - all requirements installed`); this.timeout(22000); const selectTargetQuestion2 = await testRunner.waitForOutput( @@ -90,7 +90,7 @@ describe("Check Pre-requisites installation on Windows ->", function () { ); expect( selectTargetQuestion2, - "EIM did not ask to select target, error detecting prerequisites" + "EIM did not ask to select target, error installing prerequisites" ).to.be.true; }); }); From 814a1177ea2aa749710fea07baa2d8c54f7d8b5d Mon Sep 17 00:00:00 2001 From: Fabricio Ribeiro Date: Tue, 21 Jan 2025 09:44:53 +0000 Subject: [PATCH 16/19] Update test README --- tests/README.md | 52 ++++++++++++-------------------------- tests/runs/testRun.test.js | 2 +- 2 files changed, 17 insertions(+), 37 deletions(-) diff --git a/tests/README.md b/tests/README.md index b3f1a7d..16a92a6 100644 --- a/tests/README.md +++ b/tests/README.md @@ -104,49 +104,29 @@ Install ESP-IDF pre-requisites Navigate to the idf-im-cli folder, where the repository was cloned. -The scripts should be executed passing as arguments the path to the `eim` application and the version of the file being tested. +The test runs are split into multiple files, each of them with an associated JSON test script and a NPM run script for each access. +The tests relies on environmental variables for the information below, the test scripts also have default values associated to this variables in case +environmental variables are not found. -#### Windows +EIM_FILE_PATH -> Specify the path to the EIM application -> default value Windows: `$USERPROFILE\eim-cli\` Linux/MacOS: `$HOME/eim-cli/` +EIM_VERSION -> Version of the EIM application being tested -> default value "eim 0.1.6" +IDF_VERSION -> The latest released version of ESP-IDF, used on express installation by EIM -> default "v5.4" -Open Powershell, and enable script execution: -`Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass` +Option variables -Prerequisites test can be executed by running: -`.\tests\run_pre_test.ps1 "" ""` -Default arguments are: -`.\tests\run_pre_test.ps1 "$env:USERPROFILE\eim-cli\eim.exe" "idf-im-cli 0.1.5"` +LOG_TO_FILE="true" -> Enable logs saved to text file on the execution folder -> default false +DEBUG="true" -> Enable debug level messages generated by the test scripts -> default false -To execute tests on windows, use the script -`.\tests\run_test.ps1 "" ""` -Default arguments are: -`.\tests\run_test.ps1 "$env:USERPROFILE\eim-cli\eim.exe" "idf-im-cli 0.1.5"` +To modify the test parameters, modify the json files located at `/src/tests/runs/suites` +then execute the tests by running the associated npm script (one script for each json file) -#### Linux +`npm run basic-test` +`npm run extended-test` +`npm run mirrors-test` -(if needed) Give execution permission to the test script -`chmod +x ./tests/run_test.sh` +The test for prerequisites test can be executed to check the detection of missing prerequisites (before they are installed in the system) by running: -Prerequisites test can be executed by running: -`. ./tests/run_pre_test.sh "" ""` -Default arguments are: -`. ./tests/run_pre_test.sh "$HOME/eim-cli/eim" "idf-im-cli 0.1.5"` - -To execute tests on linux, use the script: -`. ./tests/run_test.sh "" ""` -Default arguments are: -`. ./tests/run_test.sh "$HOME/eim-cli/eim" "idf-im-cli 0.1.5"` - -#### MacOS - -Prerequisites test can be executed by running: -`. ./tests/run_pre_test.sh "" ""` -Default arguments are: -`. ./tests/run_pre_test.sh "$HOME/eim-cli/eim" "idf-im-cli 0.1.5"` - -To execute tests on linux, use the script: -`. ./tests/run_test.sh "" ""` -Default arguments are: -`. ./tests/run_test.sh "$HOME/eim-cli/eim" "idf-im-cli 0.1.5"` +`npm run pre-test` # Installation Manager Usage diff --git a/tests/runs/testRun.test.js b/tests/runs/testRun.test.js index 521058c..8c2a125 100644 --- a/tests/runs/testRun.test.js +++ b/tests/runs/testRun.test.js @@ -21,7 +21,7 @@ export function testRun(jsonScript) { const PATHTOEIM = process.env.EIM_FILE_PATH || path.join(os.homedir(), "eim-cli/eim"); - const EIMVERSION = process.env.EIM_VERSION || "eim 0.1.5"; + const EIMVERSION = process.env.EIM_VERSION || "eim 0.1.6"; const IDFDEFAULTVERSION = process.env.IDF_VERSION & (process.env.IDF_VERSION !== "null") From d51ede51c14d479e9be89ddcf7ec9ca5432e7e83 Mon Sep 17 00:00:00 2001 From: Fabricio Ribeiro Date: Tue, 21 Jan 2025 19:04:57 +0000 Subject: [PATCH 17/19] Single test script controlled by env variable. --- .github/workflows/test.yml | 33 +--- tests/README.md | 8 +- tests/eim_config.toml | 16 ++ tests/package.json | 7 +- tests/results-basic_test.json | 306 +++++++++++++++++++++++++++++ tests/runs/basicRun.test.js | 23 --- tests/runs/extendedRun.test.js | 26 --- tests/runs/mirrorsRun.test.js | 23 --- tests/runs/testRun.test.js | 13 +- tests/script/installCustom.test.js | 5 - 10 files changed, 351 insertions(+), 109 deletions(-) create mode 100644 tests/eim_config.toml create mode 100644 tests/results-basic_test.json delete mode 100644 tests/runs/basicRun.test.js delete mode 100644 tests/runs/extendedRun.test.js delete mode 100644 tests/runs/mirrorsRun.test.js diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0566ee8..71d38de 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -110,8 +110,9 @@ jobs: export EIM_FILE_PATH="../test-bin/eim" export EIM_VERSION="eim ${{ env.CLI_TAG }}" export IDF_VERSION="${{ env.IDF_TAG }}" + export JSON_FILENAME="basic_test" cd tests - npm run basic-test + npm run test - name: Run IDF extended install test script (non-Windows) continue-on-error: true @@ -124,8 +125,9 @@ jobs: export EIM_FILE_PATH="../test-bin/eim" export EIM_VERSION="eim ${{ env.CLI_TAG }}" export IDF_VERSION="${{ env.IDF_TAG }}" + export JSON_FILENAME="extended_test" cd tests - npm run extended-test + npm run test - name: Run IDF installation from alternative mirrors continue-on-error: true @@ -135,9 +137,10 @@ jobs: export EIM_FILE_PATH="../test-bin/eim" export EIM_VERSION="eim ${{ env.CLI_TAG }}" export IDF_VERSION="${{ env.IDF_TAG }}" + export JSON_FILENAME="mirrors_test" cd tests npm ci - npm run mirrors-test + npm run test # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -191,8 +194,9 @@ jobs: $env:EIM_FILE_PATH = "..\test-bin\eim.exe" $env:EIM_VERSION = "eim ${{ env.CLI_TAG }}" $env:IDF_VERSION = "${{ env.IDF_TAG }}" + $env:JSON_FILENAME = "basic_test" Set-Location -Path "./tests" - npm run basic-test + npm run test-win - name: Run IDF extended install test script (Windows) continue-on-error: true @@ -203,8 +207,9 @@ jobs: $env:EIM_FILE_PATH = "..\test-bin\eim.exe" $env:EIM_VERSION = "eim ${{ env.CLI_TAG }}" $env:IDF_VERSION = "${{ env.IDF_TAG }}" + $env:JSON_FILENAME = "extended_test" Set-Location -Path "./tests" - npm run extended-test + npm run test-win # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -232,21 +237,3 @@ jobs: path: "*.json" reporter: mocha-json fail-on-empty: "true" - - # publish-test-results: - # name: Automated Test Results - # needs: test - # runs-on: ubuntu-latest - # if: always() - - # steps: - # - name: Download Artifacts - # uses: actions/download-artifact@v4 - # with: - # path: ./artifacts - - # - name: Publish Test Results - # uses: EnricoMi/publish-unit-test-result-action@v2 - # with: - # action_fail: true - # files: "./artifacts/**/*.xml" diff --git a/tests/README.md b/tests/README.md index 16a92a6..f7a5462 100644 --- a/tests/README.md +++ b/tests/README.md @@ -118,11 +118,11 @@ LOG_TO_FILE="true" -> Enable logs saved to text file on the execution folder -> DEBUG="true" -> Enable debug level messages generated by the test scripts -> default false To modify the test parameters, modify the json files located at `/src/tests/runs/suites` -then execute the tests by running the associated npm script (one script for each json file) +then execute the tests by running the test npm script passing the test script file name as argument: -`npm run basic-test` -`npm run extended-test` -`npm run mirrors-test` +`npm run test --file=basic_test` +`npm run test --file=extended_test` +`npm run test --file=mirrors-test` The test for prerequisites test can be executed to check the detection of missing prerequisites (before they are installed in the system) by running: diff --git a/tests/eim_config.toml b/tests/eim_config.toml new file mode 100644 index 0000000..78ad08a --- /dev/null +++ b/tests/eim_config.toml @@ -0,0 +1,16 @@ +path = "/home/fabricio/.espressif" +idf_path = "/home/fabricio/.espressif/v5.4/esp-idf" +esp_idf_json_path = "/home/fabricio/.espressif/tools" +tool_download_folder_name = "dist" +tool_install_folder_name = "tools" +target = ["all"] +idf_versions = ["v5.4"] +tools_json_file = "tools/tools.json" +idf_tools_path = "tools/idf_tools.py" +config_file_save_path = "eim_config.toml" +non_interactive = true +wizard_all_questions = false +mirror = "https://github.com" +idf_mirror = "https://github.com" +recurse_submodules = false +install_all_prerequisites = false diff --git a/tests/package.json b/tests/package.json index b42c173..0baa814 100644 --- a/tests/package.json +++ b/tests/package.json @@ -9,10 +9,9 @@ "winston": "^3.15.0" }, "scripts": { - "pre-test": "mocha --exit --reporter json --reporter-options output=./results-pre-test.json script/prerequisites.test.js", + "pre-test": "mocha --exit --reporter json --reporter-options output=./results-pre-requisites-test.json script/prerequisites.test.js", "pre-install": "mocha --exit --bail --reporter json --reporter-options output=./results-pre-install-test.json script/prerequisitesInstallRunner.test.js", - "basic-test": "mocha --exit --reporter json --reporter-options output=./results-basic-test.json runs/basicRun.test.js", - "extended-test": "mocha --exit --reporter json --reporter-options output=./results-extended-test.json runs/extendedRun.test.js", - "mirrors-test": "mocha --exit --reporter json --reporter-options output=./results-mirrors-test.json runs/mirrorsRun.test.js" + "test": "mocha --exit --reporter json --reporter-options output=./results-$JSON_FILENAME.json runs/testRun.test.js", + "test-win": "mocha --exit --reporter json --reporter-options output=./results-$env:JSON_FILENAME.json runs/testRun.test.js" } } diff --git a/tests/results-basic_test.json b/tests/results-basic_test.json new file mode 100644 index 0000000..03fde97 --- /dev/null +++ b/tests/results-basic_test.json @@ -0,0 +1,306 @@ +{ + "stats": { + "suites": 11, + "tests": 15, + "passes": 11, + "pending": 2, + "failures": 2, + "start": "2025-01-21T18:30:45.482Z", + "end": "2025-01-21T19:02:35.885Z", + "duration": 1910403 + }, + "tests": [ + { + "title": "should show correct version number", + "fullTitle": "1 - EIM command line arguments -> Basic Arguments Tests -> should show correct version number", + "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", + "duration": 2107, + "currentRetry": 0, + "speed": "slow", + "err": {} + }, + { + "title": "should show help with --help argument", + "fullTitle": "1 - EIM command line arguments -> Basic Arguments Tests -> should show help with --help argument", + "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", + "duration": 2107, + "currentRetry": 0, + "speed": "slow", + "err": {} + }, + { + "title": "should handle invalid arguments", + "fullTitle": "1 - EIM command line arguments -> Basic Arguments Tests -> should handle invalid arguments", + "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", + "duration": 2105, + "currentRetry": 0, + "speed": "slow", + "err": {} + }, + { + "title": "Should install IDF using wizard and default values", + "fullTitle": "2 - Installation manager default installation -> 1 - Run Install Wizard steps -> Should install IDF using wizard and default values", + "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", + "duration": 222008, + "currentRetry": 0, + "speed": "slow", + "err": {} + }, + { + "title": "1 - Should create a new project based on a template", + "fullTitle": "2 - Installation manager default installation -> 2 - Post installation test (create and build project) -> 1 - Should create a new project based on a template", + "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", + "duration": 102, + "currentRetry": 0, + "speed": "slow", + "err": {} + }, + { + "title": "2 - Should set the target", + "fullTitle": "2 - Installation manager default installation -> 2 - Post installation test (create and build project) -> 2 - Should set the target", + "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", + "duration": 97875, + "currentRetry": 0, + "speed": "slow", + "err": {} + }, + { + "title": "3 - Should build project for the selected target", + "fullTitle": "2 - Installation manager default installation -> 2 - Post installation test (create and build project) -> 3 - Should build project for the selected target", + "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", + "duration": 11035, + "currentRetry": 0, + "speed": "slow", + "err": {} + }, + { + "title": "Should install IDF using specified parameters", + "fullTitle": "3 - Installation using custom settings -> non-interactive-basic -> 1 - Run custom installation using given parameters -> Should install IDF using specified parameters", + "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", + "duration": 221105, + "currentRetry": 0, + "speed": "slow", + "err": {} + }, + { + "title": "1 - Should create a new project based on a template", + "fullTitle": "3 - Installation using custom settings -> non-interactive-basic -> 2 - Post installation test (create and build project) -> 1 - Should create a new project based on a template", + "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", + "duration": 100, + "currentRetry": 0, + "speed": "slow", + "err": {} + }, + { + "title": "2 - Should set the target", + "fullTitle": "3 - Installation using custom settings -> non-interactive-basic -> 2 - Post installation test (create and build project) -> 2 - Should set the target", + "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", + "duration": 97430, + "currentRetry": 0, + "speed": "slow", + "err": {} + }, + { + "title": "3 - Should build project for the selected target", + "fullTitle": "3 - Installation using custom settings -> non-interactive-basic -> 2 - Post installation test (create and build project) -> 3 - Should build project for the selected target", + "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", + "duration": 10835, + "currentRetry": 0, + "speed": "slow", + "err": {} + }, + { + "title": "Should install IDF using specified parameters", + "fullTitle": "4 - Installation using custom settings -> non-interactive-full-parameters -> 1 - Run custom installation using given parameters -> Should install IDF using specified parameters", + "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", + "duration": 1200007, + "currentRetry": 0, + "err": { + "stack": "AssertionError: Failed to complete installation, missing 'Successfully Installed IDF': expected false to be true\n at Context. (file:///home/fabricio/idf-im-cli/tests/script/installCustom.test.js:83:20)", + "message": "Failed to complete installation, missing 'Successfully Installed IDF': expected false to be true", + "actual": "false", + "expected": "true", + "showDiff": true, + "operator": "strictEqual" + } + }, + { + "title": "1 - Should create a new project based on a template", + "fullTitle": "4 - Installation using custom settings -> non-interactive-full-parameters -> 2 - Post installation test (create and build project) -> 1 - Should create a new project based on a template", + "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", + "duration": 10048, + "currentRetry": 0, + "err": { + "stack": "AssertionError: sdkconfig.ci file not shown after a ls command, file copy failed: expected false to be true\n at Context. (file:///home/fabricio/idf-im-cli/tests/script/postInstall.test.js:99:20)", + "message": "sdkconfig.ci file not shown after a ls command, file copy failed: expected false to be true", + "actual": "false", + "expected": "true", + "showDiff": true, + "operator": "strictEqual" + } + }, + { + "title": "2 - Should set the target", + "fullTitle": "4 - Installation using custom settings -> non-interactive-full-parameters -> 2 - Post installation test (create and build project) -> 2 - Should set the target", + "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", + "currentRetry": 0, + "err": {} + }, + { + "title": "3 - Should build project for the selected target", + "fullTitle": "4 - Installation using custom settings -> non-interactive-full-parameters -> 2 - Post installation test (create and build project) -> 3 - Should build project for the selected target", + "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", + "currentRetry": 0, + "err": {} + } + ], + "pending": [ + { + "title": "2 - Should set the target", + "fullTitle": "4 - Installation using custom settings -> non-interactive-full-parameters -> 2 - Post installation test (create and build project) -> 2 - Should set the target", + "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", + "currentRetry": 0, + "err": {} + }, + { + "title": "3 - Should build project for the selected target", + "fullTitle": "4 - Installation using custom settings -> non-interactive-full-parameters -> 2 - Post installation test (create and build project) -> 3 - Should build project for the selected target", + "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", + "currentRetry": 0, + "err": {} + } + ], + "failures": [ + { + "title": "Should install IDF using specified parameters", + "fullTitle": "4 - Installation using custom settings -> non-interactive-full-parameters -> 1 - Run custom installation using given parameters -> Should install IDF using specified parameters", + "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", + "duration": 1200007, + "currentRetry": 0, + "err": { + "stack": "AssertionError: Failed to complete installation, missing 'Successfully Installed IDF': expected false to be true\n at Context. (file:///home/fabricio/idf-im-cli/tests/script/installCustom.test.js:83:20)", + "message": "Failed to complete installation, missing 'Successfully Installed IDF': expected false to be true", + "actual": "false", + "expected": "true", + "showDiff": true, + "operator": "strictEqual" + } + }, + { + "title": "1 - Should create a new project based on a template", + "fullTitle": "4 - Installation using custom settings -> non-interactive-full-parameters -> 2 - Post installation test (create and build project) -> 1 - Should create a new project based on a template", + "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", + "duration": 10048, + "currentRetry": 0, + "err": { + "stack": "AssertionError: sdkconfig.ci file not shown after a ls command, file copy failed: expected false to be true\n at Context. (file:///home/fabricio/idf-im-cli/tests/script/postInstall.test.js:99:20)", + "message": "sdkconfig.ci file not shown after a ls command, file copy failed: expected false to be true", + "actual": "false", + "expected": "true", + "showDiff": true, + "operator": "strictEqual" + } + } + ], + "passes": [ + { + "title": "should show correct version number", + "fullTitle": "1 - EIM command line arguments -> Basic Arguments Tests -> should show correct version number", + "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", + "duration": 2107, + "currentRetry": 0, + "speed": "slow", + "err": {} + }, + { + "title": "should show help with --help argument", + "fullTitle": "1 - EIM command line arguments -> Basic Arguments Tests -> should show help with --help argument", + "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", + "duration": 2107, + "currentRetry": 0, + "speed": "slow", + "err": {} + }, + { + "title": "should handle invalid arguments", + "fullTitle": "1 - EIM command line arguments -> Basic Arguments Tests -> should handle invalid arguments", + "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", + "duration": 2105, + "currentRetry": 0, + "speed": "slow", + "err": {} + }, + { + "title": "Should install IDF using wizard and default values", + "fullTitle": "2 - Installation manager default installation -> 1 - Run Install Wizard steps -> Should install IDF using wizard and default values", + "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", + "duration": 222008, + "currentRetry": 0, + "speed": "slow", + "err": {} + }, + { + "title": "1 - Should create a new project based on a template", + "fullTitle": "2 - Installation manager default installation -> 2 - Post installation test (create and build project) -> 1 - Should create a new project based on a template", + "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", + "duration": 102, + "currentRetry": 0, + "speed": "slow", + "err": {} + }, + { + "title": "2 - Should set the target", + "fullTitle": "2 - Installation manager default installation -> 2 - Post installation test (create and build project) -> 2 - Should set the target", + "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", + "duration": 97875, + "currentRetry": 0, + "speed": "slow", + "err": {} + }, + { + "title": "3 - Should build project for the selected target", + "fullTitle": "2 - Installation manager default installation -> 2 - Post installation test (create and build project) -> 3 - Should build project for the selected target", + "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", + "duration": 11035, + "currentRetry": 0, + "speed": "slow", + "err": {} + }, + { + "title": "Should install IDF using specified parameters", + "fullTitle": "3 - Installation using custom settings -> non-interactive-basic -> 1 - Run custom installation using given parameters -> Should install IDF using specified parameters", + "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", + "duration": 221105, + "currentRetry": 0, + "speed": "slow", + "err": {} + }, + { + "title": "1 - Should create a new project based on a template", + "fullTitle": "3 - Installation using custom settings -> non-interactive-basic -> 2 - Post installation test (create and build project) -> 1 - Should create a new project based on a template", + "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", + "duration": 100, + "currentRetry": 0, + "speed": "slow", + "err": {} + }, + { + "title": "2 - Should set the target", + "fullTitle": "3 - Installation using custom settings -> non-interactive-basic -> 2 - Post installation test (create and build project) -> 2 - Should set the target", + "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", + "duration": 97430, + "currentRetry": 0, + "speed": "slow", + "err": {} + }, + { + "title": "3 - Should build project for the selected target", + "fullTitle": "3 - Installation using custom settings -> non-interactive-basic -> 2 - Post installation test (create and build project) -> 3 - Should build project for the selected target", + "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", + "duration": 10835, + "currentRetry": 0, + "speed": "slow", + "err": {} + } + ] +} \ No newline at end of file diff --git a/tests/runs/basicRun.test.js b/tests/runs/basicRun.test.js deleted file mode 100644 index 51a92f8..0000000 --- a/tests/runs/basicRun.test.js +++ /dev/null @@ -1,23 +0,0 @@ -import { testRun } from "./testRun.test.js"; -import path from "path"; -import fs from "fs"; - -/** - * Setup the following environmental variables to execute this test: - * - * EIM_FILE_PATH to point to the eim application. - * EIM_VERSION to specify expected version to be printed by the application. - * IDF_VERSION to specify the default version of the IDF to be installed. - * - * use: - * Windows: $env:="" - * Linux/mac: export ="" - * - * This script relies on test suite data stores on: - * suites/basic_test.json - */ - -const jsonFilePath = path.join(import.meta.dirname, "suites/basic_test.json"); -const basicTest = JSON.parse(fs.readFileSync(jsonFilePath, "utf-8")); - -testRun(basicTest); diff --git a/tests/runs/extendedRun.test.js b/tests/runs/extendedRun.test.js deleted file mode 100644 index d1c73f5..0000000 --- a/tests/runs/extendedRun.test.js +++ /dev/null @@ -1,26 +0,0 @@ -import { testRun } from "./testRun.test.js"; -import path from "path"; -import fs from "fs"; - -/** - * Setup the following environmental variables to execute this test: - * - * EIM_FILE_PATH to point to the eim application. - * EIM_VERSION to specify expected version to be printed by the application. - * IDF_VERSION to specify the default version of the IDF to be installed. - * - * use: - * Windows: $env:="" - * Linux/mac: export ="" - * - * This script relies on test suite data stores on: - * suites/basic_test.json - */ - -const jsonFilePath = path.join( - import.meta.dirname, - "suites/extended_test.json" -); -const extendedTest = JSON.parse(fs.readFileSync(jsonFilePath, "utf-8")); - -testRun(extendedTest); diff --git a/tests/runs/mirrorsRun.test.js b/tests/runs/mirrorsRun.test.js deleted file mode 100644 index 5b64d59..0000000 --- a/tests/runs/mirrorsRun.test.js +++ /dev/null @@ -1,23 +0,0 @@ -import { testRun } from "./testRun.test.js"; -import path from "path"; -import fs from "fs"; - -/** - * Setup the following environmental variables to execute this test: - * - * EIM_FILE_PATH to point to the eim application. - * EIM_VERSION to specify expected version to be printed by the application. - * IDF_VERSION to specify the default version of the IDF to be installed. - * - * use: - * Windows: $env:="" - * Linux/mac: export ="" - * - * This script relies on test suite data stores on: - * suites/basic_test.json - */ - -const jsonFilePath = path.join(import.meta.dirname, "suites/mirrors_test.json"); -const mirrorsTest = JSON.parse(fs.readFileSync(jsonFilePath, "utf-8")); - -testRun(mirrorsTest); diff --git a/tests/runs/testRun.test.js b/tests/runs/testRun.test.js index 8c2a125..32218d1 100644 --- a/tests/runs/testRun.test.js +++ b/tests/runs/testRun.test.js @@ -6,8 +6,19 @@ import { runPostInstallTest } from "../script/postInstall.test.js"; import logger from "../classes/logger.class.js"; import os from "os"; import path from "path"; +import fs from "fs"; -export function testRun(jsonScript) { +const jsonFilePath = path.join( + import.meta.dirname, + "suites", + `${process.env.JSON_FILENAME}.json` +); +const testScript = JSON.parse(fs.readFileSync(jsonFilePath, "utf-8")); +logger.info(`Running test script: ${jsonFilePath}`); + +testRun(testScript); + +function testRun(jsonScript) { const IDFMIRRORS = { github: "https://github.com", jihulab: "https://jihulab.com/esp-mirror", diff --git a/tests/script/installCustom.test.js b/tests/script/installCustom.test.js index ff9165d..d6e05a4 100644 --- a/tests/script/installCustom.test.js +++ b/tests/script/installCustom.test.js @@ -82,11 +82,6 @@ export function runInstallCustom(pathToEim, args = []) { "Failed to complete installation, missing 'Successfully Installed IDF'" ).to.be.true; - // expect( - // testRunner.output, - // "Error message during installation" - // ).to.not.include("error"); - expect( testRunner.output, "Failed to complete installation, missing 'Now you can start using IDF tools'" From 5e8eb0b5a52ea19965c6414970145c6cb90f334e Mon Sep 17 00:00:00 2001 From: Fabricio Ribeiro Date: Wed, 22 Jan 2025 10:55:51 +0000 Subject: [PATCH 18/19] Simply test command, update report upload --- .github/workflows/test.yml | 29 ++-- tests/package.json | 4 +- tests/results-basic_test.json | 306 ---------------------------------- 3 files changed, 14 insertions(+), 325 deletions(-) delete mode 100644 tests/results-basic_test.json diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 71d38de..2a89792 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -110,9 +110,8 @@ jobs: export EIM_FILE_PATH="../test-bin/eim" export EIM_VERSION="eim ${{ env.CLI_TAG }}" export IDF_VERSION="${{ env.IDF_TAG }}" - export JSON_FILENAME="basic_test" cd tests - npm run test + npm run test --file=basic-test - name: Run IDF extended install test script (non-Windows) continue-on-error: true @@ -124,10 +123,9 @@ jobs: export LOG_TO_FILE="true" export EIM_FILE_PATH="../test-bin/eim" export EIM_VERSION="eim ${{ env.CLI_TAG }}" - export IDF_VERSION="${{ env.IDF_TAG }}" - export JSON_FILENAME="extended_test" + export IDF_VERSION="${{ env.IDF_TAG }}" cd tests - npm run test + npm run test --file=extended-test - name: Run IDF installation from alternative mirrors continue-on-error: true @@ -136,11 +134,10 @@ jobs: export LOG_TO_FILE="true" export EIM_FILE_PATH="../test-bin/eim" export EIM_VERSION="eim ${{ env.CLI_TAG }}" - export IDF_VERSION="${{ env.IDF_TAG }}" - export JSON_FILENAME="mirrors_test" + export IDF_VERSION="${{ env.IDF_TAG }}" cd tests npm ci - npm run test + npm run test --file=mirrors-test # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -193,10 +190,9 @@ jobs: $env:LOG_TO_FILE="true" $env:EIM_FILE_PATH = "..\test-bin\eim.exe" $env:EIM_VERSION = "eim ${{ env.CLI_TAG }}" - $env:IDF_VERSION = "${{ env.IDF_TAG }}" - $env:JSON_FILENAME = "basic_test" + $env:IDF_VERSION = "${{ env.IDF_TAG }}" Set-Location -Path "./tests" - npm run test-win + npm run test-win --file=basic-test - name: Run IDF extended install test script (Windows) continue-on-error: true @@ -206,10 +202,9 @@ jobs: $env:LOG_TO_FILE="true" $env:EIM_FILE_PATH = "..\test-bin\eim.exe" $env:EIM_VERSION = "eim ${{ env.CLI_TAG }}" - $env:IDF_VERSION = "${{ env.IDF_TAG }}" - $env:JSON_FILENAME = "extended_test" + $env:IDF_VERSION = "${{ env.IDF_TAG }}" Set-Location -Path "./tests" - npm run test-win + npm run test-win --file=extended-test # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -230,10 +225,10 @@ jobs: steps: - name: Publish Test Results - uses: dorny/test-reporter@v1 + uses: phoenix-actions/test-reporting@v8 with: artifact: /test-results-(.*)/ - name: "Auto Tests $1" + name: "Tests $1" path: "*.json" reporter: mocha-json - fail-on-empty: "true" + fail-on-error: "true" diff --git a/tests/package.json b/tests/package.json index 0baa814..2ba2bc9 100644 --- a/tests/package.json +++ b/tests/package.json @@ -11,7 +11,7 @@ "scripts": { "pre-test": "mocha --exit --reporter json --reporter-options output=./results-pre-requisites-test.json script/prerequisites.test.js", "pre-install": "mocha --exit --bail --reporter json --reporter-options output=./results-pre-install-test.json script/prerequisitesInstallRunner.test.js", - "test": "mocha --exit --reporter json --reporter-options output=./results-$JSON_FILENAME.json runs/testRun.test.js", - "test-win": "mocha --exit --reporter json --reporter-options output=./results-$env:JSON_FILENAME.json runs/testRun.test.js" + "test": "export JSON_FILENAME=$npm_config_file && mocha --exit --reporter json --reporter-options output=./results-$JSON_FILENAME.json runs/testRun.test.js", + "test-win": "powershell -Command \"$env:JSON_FILENAME=$env:npm_config_file; mocha --exit --reporter json --reporter-options output=./results-$env:JSON_FILENAME.json runs/testRun.test.js\"" } } diff --git a/tests/results-basic_test.json b/tests/results-basic_test.json deleted file mode 100644 index 03fde97..0000000 --- a/tests/results-basic_test.json +++ /dev/null @@ -1,306 +0,0 @@ -{ - "stats": { - "suites": 11, - "tests": 15, - "passes": 11, - "pending": 2, - "failures": 2, - "start": "2025-01-21T18:30:45.482Z", - "end": "2025-01-21T19:02:35.885Z", - "duration": 1910403 - }, - "tests": [ - { - "title": "should show correct version number", - "fullTitle": "1 - EIM command line arguments -> Basic Arguments Tests -> should show correct version number", - "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", - "duration": 2107, - "currentRetry": 0, - "speed": "slow", - "err": {} - }, - { - "title": "should show help with --help argument", - "fullTitle": "1 - EIM command line arguments -> Basic Arguments Tests -> should show help with --help argument", - "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", - "duration": 2107, - "currentRetry": 0, - "speed": "slow", - "err": {} - }, - { - "title": "should handle invalid arguments", - "fullTitle": "1 - EIM command line arguments -> Basic Arguments Tests -> should handle invalid arguments", - "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", - "duration": 2105, - "currentRetry": 0, - "speed": "slow", - "err": {} - }, - { - "title": "Should install IDF using wizard and default values", - "fullTitle": "2 - Installation manager default installation -> 1 - Run Install Wizard steps -> Should install IDF using wizard and default values", - "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", - "duration": 222008, - "currentRetry": 0, - "speed": "slow", - "err": {} - }, - { - "title": "1 - Should create a new project based on a template", - "fullTitle": "2 - Installation manager default installation -> 2 - Post installation test (create and build project) -> 1 - Should create a new project based on a template", - "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", - "duration": 102, - "currentRetry": 0, - "speed": "slow", - "err": {} - }, - { - "title": "2 - Should set the target", - "fullTitle": "2 - Installation manager default installation -> 2 - Post installation test (create and build project) -> 2 - Should set the target", - "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", - "duration": 97875, - "currentRetry": 0, - "speed": "slow", - "err": {} - }, - { - "title": "3 - Should build project for the selected target", - "fullTitle": "2 - Installation manager default installation -> 2 - Post installation test (create and build project) -> 3 - Should build project for the selected target", - "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", - "duration": 11035, - "currentRetry": 0, - "speed": "slow", - "err": {} - }, - { - "title": "Should install IDF using specified parameters", - "fullTitle": "3 - Installation using custom settings -> non-interactive-basic -> 1 - Run custom installation using given parameters -> Should install IDF using specified parameters", - "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", - "duration": 221105, - "currentRetry": 0, - "speed": "slow", - "err": {} - }, - { - "title": "1 - Should create a new project based on a template", - "fullTitle": "3 - Installation using custom settings -> non-interactive-basic -> 2 - Post installation test (create and build project) -> 1 - Should create a new project based on a template", - "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", - "duration": 100, - "currentRetry": 0, - "speed": "slow", - "err": {} - }, - { - "title": "2 - Should set the target", - "fullTitle": "3 - Installation using custom settings -> non-interactive-basic -> 2 - Post installation test (create and build project) -> 2 - Should set the target", - "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", - "duration": 97430, - "currentRetry": 0, - "speed": "slow", - "err": {} - }, - { - "title": "3 - Should build project for the selected target", - "fullTitle": "3 - Installation using custom settings -> non-interactive-basic -> 2 - Post installation test (create and build project) -> 3 - Should build project for the selected target", - "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", - "duration": 10835, - "currentRetry": 0, - "speed": "slow", - "err": {} - }, - { - "title": "Should install IDF using specified parameters", - "fullTitle": "4 - Installation using custom settings -> non-interactive-full-parameters -> 1 - Run custom installation using given parameters -> Should install IDF using specified parameters", - "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", - "duration": 1200007, - "currentRetry": 0, - "err": { - "stack": "AssertionError: Failed to complete installation, missing 'Successfully Installed IDF': expected false to be true\n at Context. (file:///home/fabricio/idf-im-cli/tests/script/installCustom.test.js:83:20)", - "message": "Failed to complete installation, missing 'Successfully Installed IDF': expected false to be true", - "actual": "false", - "expected": "true", - "showDiff": true, - "operator": "strictEqual" - } - }, - { - "title": "1 - Should create a new project based on a template", - "fullTitle": "4 - Installation using custom settings -> non-interactive-full-parameters -> 2 - Post installation test (create and build project) -> 1 - Should create a new project based on a template", - "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", - "duration": 10048, - "currentRetry": 0, - "err": { - "stack": "AssertionError: sdkconfig.ci file not shown after a ls command, file copy failed: expected false to be true\n at Context. (file:///home/fabricio/idf-im-cli/tests/script/postInstall.test.js:99:20)", - "message": "sdkconfig.ci file not shown after a ls command, file copy failed: expected false to be true", - "actual": "false", - "expected": "true", - "showDiff": true, - "operator": "strictEqual" - } - }, - { - "title": "2 - Should set the target", - "fullTitle": "4 - Installation using custom settings -> non-interactive-full-parameters -> 2 - Post installation test (create and build project) -> 2 - Should set the target", - "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", - "currentRetry": 0, - "err": {} - }, - { - "title": "3 - Should build project for the selected target", - "fullTitle": "4 - Installation using custom settings -> non-interactive-full-parameters -> 2 - Post installation test (create and build project) -> 3 - Should build project for the selected target", - "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", - "currentRetry": 0, - "err": {} - } - ], - "pending": [ - { - "title": "2 - Should set the target", - "fullTitle": "4 - Installation using custom settings -> non-interactive-full-parameters -> 2 - Post installation test (create and build project) -> 2 - Should set the target", - "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", - "currentRetry": 0, - "err": {} - }, - { - "title": "3 - Should build project for the selected target", - "fullTitle": "4 - Installation using custom settings -> non-interactive-full-parameters -> 2 - Post installation test (create and build project) -> 3 - Should build project for the selected target", - "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", - "currentRetry": 0, - "err": {} - } - ], - "failures": [ - { - "title": "Should install IDF using specified parameters", - "fullTitle": "4 - Installation using custom settings -> non-interactive-full-parameters -> 1 - Run custom installation using given parameters -> Should install IDF using specified parameters", - "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", - "duration": 1200007, - "currentRetry": 0, - "err": { - "stack": "AssertionError: Failed to complete installation, missing 'Successfully Installed IDF': expected false to be true\n at Context. (file:///home/fabricio/idf-im-cli/tests/script/installCustom.test.js:83:20)", - "message": "Failed to complete installation, missing 'Successfully Installed IDF': expected false to be true", - "actual": "false", - "expected": "true", - "showDiff": true, - "operator": "strictEqual" - } - }, - { - "title": "1 - Should create a new project based on a template", - "fullTitle": "4 - Installation using custom settings -> non-interactive-full-parameters -> 2 - Post installation test (create and build project) -> 1 - Should create a new project based on a template", - "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", - "duration": 10048, - "currentRetry": 0, - "err": { - "stack": "AssertionError: sdkconfig.ci file not shown after a ls command, file copy failed: expected false to be true\n at Context. (file:///home/fabricio/idf-im-cli/tests/script/postInstall.test.js:99:20)", - "message": "sdkconfig.ci file not shown after a ls command, file copy failed: expected false to be true", - "actual": "false", - "expected": "true", - "showDiff": true, - "operator": "strictEqual" - } - } - ], - "passes": [ - { - "title": "should show correct version number", - "fullTitle": "1 - EIM command line arguments -> Basic Arguments Tests -> should show correct version number", - "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", - "duration": 2107, - "currentRetry": 0, - "speed": "slow", - "err": {} - }, - { - "title": "should show help with --help argument", - "fullTitle": "1 - EIM command line arguments -> Basic Arguments Tests -> should show help with --help argument", - "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", - "duration": 2107, - "currentRetry": 0, - "speed": "slow", - "err": {} - }, - { - "title": "should handle invalid arguments", - "fullTitle": "1 - EIM command line arguments -> Basic Arguments Tests -> should handle invalid arguments", - "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", - "duration": 2105, - "currentRetry": 0, - "speed": "slow", - "err": {} - }, - { - "title": "Should install IDF using wizard and default values", - "fullTitle": "2 - Installation manager default installation -> 1 - Run Install Wizard steps -> Should install IDF using wizard and default values", - "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", - "duration": 222008, - "currentRetry": 0, - "speed": "slow", - "err": {} - }, - { - "title": "1 - Should create a new project based on a template", - "fullTitle": "2 - Installation manager default installation -> 2 - Post installation test (create and build project) -> 1 - Should create a new project based on a template", - "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", - "duration": 102, - "currentRetry": 0, - "speed": "slow", - "err": {} - }, - { - "title": "2 - Should set the target", - "fullTitle": "2 - Installation manager default installation -> 2 - Post installation test (create and build project) -> 2 - Should set the target", - "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", - "duration": 97875, - "currentRetry": 0, - "speed": "slow", - "err": {} - }, - { - "title": "3 - Should build project for the selected target", - "fullTitle": "2 - Installation manager default installation -> 2 - Post installation test (create and build project) -> 3 - Should build project for the selected target", - "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", - "duration": 11035, - "currentRetry": 0, - "speed": "slow", - "err": {} - }, - { - "title": "Should install IDF using specified parameters", - "fullTitle": "3 - Installation using custom settings -> non-interactive-basic -> 1 - Run custom installation using given parameters -> Should install IDF using specified parameters", - "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", - "duration": 221105, - "currentRetry": 0, - "speed": "slow", - "err": {} - }, - { - "title": "1 - Should create a new project based on a template", - "fullTitle": "3 - Installation using custom settings -> non-interactive-basic -> 2 - Post installation test (create and build project) -> 1 - Should create a new project based on a template", - "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", - "duration": 100, - "currentRetry": 0, - "speed": "slow", - "err": {} - }, - { - "title": "2 - Should set the target", - "fullTitle": "3 - Installation using custom settings -> non-interactive-basic -> 2 - Post installation test (create and build project) -> 2 - Should set the target", - "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", - "duration": 97430, - "currentRetry": 0, - "speed": "slow", - "err": {} - }, - { - "title": "3 - Should build project for the selected target", - "fullTitle": "3 - Installation using custom settings -> non-interactive-basic -> 2 - Post installation test (create and build project) -> 3 - Should build project for the selected target", - "file": "/home/fabricio/idf-im-cli/tests/runs/testRun.test.js", - "duration": 10835, - "currentRetry": 0, - "speed": "slow", - "err": {} - } - ] -} \ No newline at end of file From f32cedd7e398c46fb927fdfd17fbaf2d8729e3a1 Mon Sep 17 00:00:00 2001 From: Fabricio Ribeiro Date: Wed, 22 Jan 2025 11:43:05 +0000 Subject: [PATCH 19/19] Rename json files --- .github/workflows/test.yml | 4 ++-- tests/README.md | 2 ++ tests/runs/suites/{basic_test.json => basic-test.json} | 0 tests/runs/suites/{extended_test.json => extended-test.json} | 0 tests/runs/suites/{mirrors_test.json => mirrors-test.json} | 0 5 files changed, 4 insertions(+), 2 deletions(-) rename tests/runs/suites/{basic_test.json => basic-test.json} (100%) rename tests/runs/suites/{extended_test.json => extended-test.json} (100%) rename tests/runs/suites/{mirrors_test.json => mirrors-test.json} (100%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2a89792..52c6ebf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -225,10 +225,10 @@ jobs: steps: - name: Publish Test Results - uses: phoenix-actions/test-reporting@v8 + uses: dorny/test-reporter@v1 with: artifact: /test-results-(.*)/ name: "Tests $1" path: "*.json" reporter: mocha-json - fail-on-error: "true" + fail-on-empty: "true" diff --git a/tests/README.md b/tests/README.md index f7a5462..d057fe2 100644 --- a/tests/README.md +++ b/tests/README.md @@ -124,6 +124,8 @@ then execute the tests by running the test npm script passing the test script fi `npm run test --file=extended_test` `npm run test --file=mirrors-test` +> For Windows use `test-win` + The test for prerequisites test can be executed to check the detection of missing prerequisites (before they are installed in the system) by running: `npm run pre-test` diff --git a/tests/runs/suites/basic_test.json b/tests/runs/suites/basic-test.json similarity index 100% rename from tests/runs/suites/basic_test.json rename to tests/runs/suites/basic-test.json diff --git a/tests/runs/suites/extended_test.json b/tests/runs/suites/extended-test.json similarity index 100% rename from tests/runs/suites/extended_test.json rename to tests/runs/suites/extended-test.json diff --git a/tests/runs/suites/mirrors_test.json b/tests/runs/suites/mirrors-test.json similarity index 100% rename from tests/runs/suites/mirrors_test.json rename to tests/runs/suites/mirrors-test.json