Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

random shuffling of test implemented for mocha and jasmine runner #58

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

random shuffling of test implemented for mocha and jasmine runner #58

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3f7ca2d
random shuffling of test implemented for mocha and jasmine runner
HRanjan-11 Jul 31, 2022
6c4bd9a
removed white spaces
HRanjan-11 Jul 31, 2022
611adbc
fixed lint issues
HRanjan-11 Jul 31, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions packages/core/src/util.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,17 @@ export class Util {
return locator;
}

static shuffle(array: Array<any>)
{
let currentIndex = array.length, randomIndex;
while (currentIndex != 0) {
randomIndex = Math.floor(Math.random() * currentIndex);
currentIndex--;
[array[currentIndex], array[randomIndex]] = [
array[randomIndex], array[currentIndex]];
}
return array;
}
/**
* Loads blocked tests by reading a file at path specified by env var BLOCK_TESTS_FILE.
* This works on the assumption that the format of JSON stored in file is in the following format:
Expand Down
3 changes: 3 additions & 0 deletions packages/core/src/validations.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ export class Validations {
if (process.env.BUILD_ID === undefined) {
throw new ValidationException("'BUILD_ID' is not set in environment variables.")
}
if (process.env.SHUFFLE_TEST === undefined) {
throw new ValidationException("'SHUFFLE_TEST' is not set in environment variables.")
}
}
}

Expand Down
23 changes: 15 additions & 8 deletions packages/jasmine-runner/src/jasmine-runner.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,8 @@ class JasmineRunner implements TestRunner {
}
}

async execute(testFilesGlob: string | string[], config: string, locators: string[] = []): Promise<ExecutionResult> {
async execute(testFilesGlob: string | string[], shuffleTest: boolean, config: string,
locators: string[] = []): Promise<ExecutionResult> {
const testLocators = new Set<string>(locators)
const blockTestLocators = new Set<string>()
const entityIdFilenameMap = new Map<number, string>();
Expand Down Expand Up @@ -89,7 +90,7 @@ class JasmineRunner implements TestRunner {
const rootSuite = jasmineObj.env.topSuite();
const specIdsToRun: number[] = [];

this.fetchSpecIdsToRun(rootSuite, specIdsToRun, entityIdFilenameMap,
this.fetchSpecIdsToRunAndDetermineOrder(rootSuite, specIdsToRun, shuffleTest, entityIdFilenameMap,
testLocators, blockTestLocators);

if (specIdsToRun.length == 0) {
Expand Down Expand Up @@ -127,6 +128,7 @@ class JasmineRunner implements TestRunner {
const orgID = process.env.ORG_ID as ID;
const repoID = process.env.REPO_ID as ID;
const commitID = process.env.COMMIT_ID as ID;
const shuffleTest = JSON.parse(process.env.SHUFFLE_TEST as string) as boolean;
Validations.validateExecutionEnv(argv);
const testFilesGlob = argv.pattern as string | string[];
const locatorFile = argv.locatorFile as string;
Expand All @@ -140,11 +142,11 @@ class JasmineRunner implements TestRunner {
);
if (locatorFile) {
const locators = Util.getLocatorsFromFile(locatorFile)
const result = await this.execute(testFilesGlob, argv.config, locators)
const result = await this.execute(testFilesGlob, shuffleTest, argv.config, locators)
executionResults.push(result)
} else {
// run all tests if locator file is not present
const result = await this.execute(testFilesGlob, argv.config)
const result = await this.execute(testFilesGlob, shuffleTest, argv.config)
executionResults.push(result)
}
return executionResults;
Expand Down Expand Up @@ -265,21 +267,26 @@ class JasmineRunner implements TestRunner {
}
}
}

private fetchSpecIdsToRun(
private fetchSpecIdsToRunAndDetermineOrder(
currentSuite: jasmine.Suite,
specIdsToRun: number[],
shuffleTest: boolean,
entityIdFilenameMap: Map<number, string>,
testLocators: Set<string>,
blockTestLocators: Set<string>,
ancestorTitles: string[] = [],
) {
for (const child of currentSuite.children) {
let currentSuiteChildren = currentSuite.children;
if (shuffleTest){
currentSuiteChildren = Util.shuffle(currentSuite.children);
}
for (const child of currentSuiteChildren) {
if ((child as jasmine.Suite).children !== undefined) {
// child is a TestSuite
const childSuite = child as jasmine.Suite;
ancestorTitles.push(child.description);
this.fetchSpecIdsToRun(childSuite, specIdsToRun, entityIdFilenameMap,
this.fetchSpecIdsToRunAndDetermineOrder(childSuite, specIdsToRun, shuffleTest, entityIdFilenameMap,
testLocators, blockTestLocators, ancestorTitles);
ancestorTitles.pop();
} else {
Expand Down
26 changes: 17 additions & 9 deletions packages/mocha-runner/src/mocha-runner.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,13 +91,14 @@ class MochaRunner implements TestRunner {
return result;
}

async execute(testFilesGlob: string | string[], locators: string[] = []): Promise<ExecutionResult> {
async execute(testFilesGlob: string | string[], shuffleTest: boolean,
locators: string [] = []): Promise<ExecutionResult> {
const mocha = this.createMochaInstance()
const testRunTask = new Task<void>();

this._testlocator = new Set<string>(locators);

this.extendNativeRunner();
this.extendNativeRunner(shuffleTest);
mocha.reporter(require.resolve("./mocha-reporter"));

let testFilesToProcess: Set<string> = new Set();
Expand Down Expand Up @@ -140,6 +141,7 @@ class MochaRunner implements TestRunner {
const orgID = process.env.ORG_ID as ID;
const repoID = process.env.REPO_ID as ID;
const commitID = process.env.COMMIT_ID as ID;
const shuffleTest = JSON.parse(process.env.SHUFFLE_TEST as string) as boolean;
Validations.validateExecutionEnv(argv);
const testFilesGlob = argv.pattern as string | string[];
const locatorFile = argv.locatorFile as string;
Expand All @@ -153,11 +155,11 @@ class MochaRunner implements TestRunner {

if (locatorFile) {
const locators = Util.getLocatorsFromFile(locatorFile);
const result = await this.execute(testFilesGlob, locators);
const result = await this.execute(testFilesGlob, shuffleTest, locators);
executionResults.push(result);
} else {
// run all tests if locator file is not present
const result = await this.execute(testFilesGlob)
const result = await this.execute(testFilesGlob, shuffleTest)
executionResults.push(result)
}
return executionResults;
Expand Down Expand Up @@ -219,19 +221,22 @@ class MochaRunner implements TestRunner {
}
}

private extendNativeRunner() {
private extendNativeRunner(shuffleTest: boolean) {
// eslint-disable-next-line @typescript-eslint/no-this-alias
const _self = this;
// This is the hook point where we can randomizee, specify order
// or do any sort of stuffs with suites and tests
Mocha.Runner.prototype.run = function (fn: ((failures: number) => void)) {
_self.filterSpecs(this.suite);
_self.filterSpecsAndDetermineOrder(this.suite, shuffleTest);
return originalRun.call(this, fn);
};
}

private filterSpecs(suite: Mocha.Suite) {
private filterSpecsAndDetermineOrder(suite: Mocha.Suite, shuffleTest:boolean) {
if (suite.tests) {
if(shuffleTest){
suite.tests = Util.shuffle(suite.tests) as Mocha.Test[];
}
const filteredTests: Mocha.Test[] = [];
for (const test of suite.tests) {
const filename = test.file ?? "";
Expand Down Expand Up @@ -271,9 +276,12 @@ class MochaRunner implements TestRunner {
}
}
}
suite.tests = filteredTests;
suite.tests = filteredTests;
}
if (suite.suites) {
if (shuffleTest){
suite.suites = Util.shuffle(suite.suites) as Mocha.Suite[];
}
for (const childSuite of suite.suites) {
const filename = childSuite.file ?? "";
const parentSuites: string[] = [];
Expand All @@ -289,7 +297,7 @@ class MochaRunner implements TestRunner {
);
this._blockedSuites.push(suiteResult);
}
this.filterSpecs(childSuite);
this.filterSpecsAndDetermineOrder(childSuite, shuffleTest);
}
}
}
Expand Down