From 095f57f93f3c34df58ab6565aa70aa8d254fcbde Mon Sep 17 00:00:00 2001 From: Tarik Gul <47201679+TarikGul@users.noreply.github.com> Date: Tue, 20 Dec 2022 11:38:23 -0500 Subject: [PATCH] test(e2e-tests): add e2e-tests for latest runtimes, and blocks (#1155) * test(e2e-tests): outline errors for the process * add correct error code when sidecar fails to build * reorg the tsconfig in e2e tests and where the entry file exists * add endpoints * change yarn script names to historical for init * change request format to return statusCode as well as data * add working script * some cleanup * organize naming * cleanup * fix tiny grumble * add license to file * add script to startup the e2e tests * fix small blunders, and add scripts to package.json * set runE2eTests to runHistoricalE2eTests * add --local to historical e2e tests * get local working, and fix resolving errors * fix is local to fail when a chain is not specified * DRY up code * update docs in scripts * docs in e2e-tests * cleanup * add all statemint end points * make `--local` take in an input * update release notes * add latest to readme * new lint --- README.md | 3 +- RELEASE.md | 4 +- e2e-tests/README.md | 38 ++-- e2e-tests/helpers/request.ts | 14 +- e2e-tests/historical-e2e-tests.spec.ts | 8 +- .../{index.ts => historical/historical.ts} | 5 +- e2e-tests/latest/endpoints/index.ts | 18 ++ e2e-tests/latest/endpoints/polkadot.ts | 182 ++++++++++++++++++ e2e-tests/latest/endpoints/statemint.ts | 35 ++++ e2e-tests/latest/index.ts | 115 +++++++++++ e2e-tests/latest/types/endpoints.ts | 24 +++ e2e-tests/tsconfig.json | 20 +- package.json | 16 +- scripts/README.md | 46 ++++- scripts/config.ts | 47 +++-- scripts/e2eHelpers.ts | 90 +++++++++ ...unE2eTests.ts => runHistoricalE2eTests.ts} | 90 ++------- scripts/runLatestE2eTests.ts | 97 ++++++++++ scripts/sidecarScriptApi.ts | 13 +- scripts/types.ts | 6 +- 20 files changed, 747 insertions(+), 124 deletions(-) rename e2e-tests/{index.ts => historical/historical.ts} (92%) create mode 100644 e2e-tests/latest/endpoints/index.ts create mode 100644 e2e-tests/latest/endpoints/polkadot.ts create mode 100644 e2e-tests/latest/endpoints/statemint.ts create mode 100644 e2e-tests/latest/index.ts create mode 100644 e2e-tests/latest/types/endpoints.ts create mode 100644 scripts/e2eHelpers.ts rename scripts/{runE2eTests.ts => runHistoricalE2eTests.ts} (52%) create mode 100644 scripts/runLatestE2eTests.ts diff --git a/README.md b/README.md index c46db26c2..d37d0f1f1 100644 --- a/README.md +++ b/README.md @@ -311,7 +311,8 @@ and read the release notes for any breaking changes or high priority updates. In yarn build yarn lint yarn test - yarn test:init-e2e-tests + yarn test:historical-e2e-tests + yarn test:latest-e2e-tests ``` 1. Commit the dependency updates with a name like `fix(deps): update pjs api` (title depending on what got updated, see commit history for other examples of this), and wait to get it merged. diff --git a/RELEASE.md b/RELEASE.md index 47b13a92c..e40f5abc7 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -8,9 +8,9 @@ 1. Checkout a branch with the format `name-v5-0-1`. When deciding what version will be released it is important to look over 1) PRs since the last release and 2) release notes for any updated polkadot-js dependencies as they may affect type definitions. -1. The next step is making sure the release will work against all relevant runtimes for Polkadot, Kusama, and Westend. This can be handled by running `yarn test:init-e2e-tests`. If you would like to test on an individual chain, you may run the same command followed by its chain, ex: `yarn test:init-e2e-tests:polkadot`. Before moving forward ensure all tests pass, and if it warns of any missing types feel free to make an issue [here](https://github.com/paritytech/substrate-api-sidecar/issues). +1. The next step is to run the e2e tests. There are two types of e2e tests: `yarn test:historical-e2e-tests`, and `yarn test:latest-e2e-tests`. If you would like to run either tests against a single chain you may use the flag `--chain` to specify the chain. If you would also like to test against a local node you may use the `--local` flag in conjunction with `--chain`. Before moving forward ensure all tests pass, and if it warns of any missing types feel free to make an issue [here](https://github.com/paritytech/substrate-api-sidecar/issues). - Note: that the e2e tests will connect to running nodes in order to test sidecar against real data, and they may fail owing to those connections taking too long to establish. If you run into any failures, try running tests just for the chain that failed with something like `yarn test:init-e2e-tests:polkadot`. + Note: that the e2e tests will connect to running nodes in order to test sidecar against real data, and they may fail owing to those connections taking too long to establish. If you run into any failures, try running tests the tests again. 1. Update the version in the package.json (this is very important for releasing on NPM). diff --git a/e2e-tests/README.md b/e2e-tests/README.md index 398ffb016..60186d7f5 100644 --- a/e2e-tests/README.md +++ b/e2e-tests/README.md @@ -1,22 +1,36 @@ ## Summary -This is a helper library for Sidecar to run e2e tests against specific chains, at certain blocks. +This is a helper library to run e2e tests. There are two types of tests to run: -## Testing +1. Historical: Tests that focus on the integrity of the api service with older runtimes. -The below instructions are specific to running the e2e-tests against one chain. -If you are looking to run the e2e-tests against all chains (Polkadot, Kusama, Westend, Statemine) then run `yarn test:init-e2e-tests` in -the root directory of sidecar. +2. Latest: Tests that focus on the integrity of the api service with the current runtime, and most recent block. -### Polkadot +### Historical -To run the tests against a single chain, you may use the following below. For more examples, reference the `/package.json` +Historical tests use jest to run the api service at specific blocks where the result is known. Jest compares the known result to what we receive, and operates like any other test. It requires having the api service running against a live chain (Archive nodes are preferred). -`yarn test:init-e2e-tests:polkadot` +To run the tests locally: -That's it! -All the tests should come back with green checkmarks. If you find a bug file an issue [here](https://github.com/paritytech/substrate-api-sidecar/issues). +```bash +$ yarn build:e2e-tests +$ node ./e2e-tests/build/historical/historical.js --config=./e2e-tests/jest.config.js +``` -### Config +`--chain`: The chain you would like to test against. It defaults to `polkadot`. +`--config`: The path to the jest config. -If you are looking to update the e2e-tests config, the file to do so exists in `/scripts/config.ts`. +### Latest + +Latest tests run a set of known endpoints with the latest block against the chain. The tests will pass as long as all the responses are succesfull. If any test is 400 or above it will fail, and log each failed query along with its error. + +To run the tests locally: + +```bash +$ yarn build:e2e-tests +$ node ./e2e-tests/build/latest/index.js +``` + +#### Flags + +`--chain`: The chain you would like to test against. It defaults to `polkadot`. diff --git a/e2e-tests/helpers/request.ts b/e2e-tests/helpers/request.ts index ec6f006f8..7b1c32359 100644 --- a/e2e-tests/helpers/request.ts +++ b/e2e-tests/helpers/request.ts @@ -16,16 +16,26 @@ import http from 'http'; +export interface IRequest { + data: string; + path: string; + statusCode?: number; +} + export const request = ( path: string, hostname: string, port: number -): Promise => { +): Promise => { return new Promise((resolve) => { http.get({ path, hostname, port }, (response) => { let data = ''; response.on('data', (_data) => (data += _data)); - response.on('end', () => resolve(data)); + response.on('end', () => resolve({ + data, + path, + statusCode: response.statusCode, + })); }); }); }; diff --git a/e2e-tests/historical-e2e-tests.spec.ts b/e2e-tests/historical-e2e-tests.spec.ts index 40ac5dd2b..bc1958cc5 100644 --- a/e2e-tests/historical-e2e-tests.spec.ts +++ b/e2e-tests/historical-e2e-tests.spec.ts @@ -45,7 +45,7 @@ describe('Runtime Tests for blocks', () => { 'Given path %p, it should return the correct JSON response', async (blockPath, blockResponse) => { const res = await request(blockPath, HOST, PORT); - const responseJson = JSON.parse(res) as IBlockResponse; + const responseJson = JSON.parse(res.data) as IBlockResponse; expect(responseJson).toStrictEqual(JSON.parse(blockResponse)); } @@ -64,7 +64,7 @@ describe('Runtime Tests for accounts', () => { 'Given path %p, it should return the correct JSON response', async (accountsPath, accountsResponse) => { const res = await request(accountsPath, HOST, PORT); - const responseJson = JSON.parse(res) as AccountsResponse; + const responseJson = JSON.parse(res.data) as AccountsResponse; expect(responseJson).toStrictEqual(JSON.parse(accountsResponse)); } @@ -80,7 +80,7 @@ describe('Runtime Tests for `/runtime/*`', () => { 'Given path %p, it should return the correct JSON response', async (runtimePath, runtimeResponse) => { const res = await request(runtimePath, HOST, PORT); - const responseJson = JSON.parse(res) as RuntimeResponse; + const responseJson = JSON.parse(res.data) as RuntimeResponse; expect(responseJson).toStrictEqual(JSON.parse(runtimeResponse)); } @@ -97,7 +97,7 @@ describe('Runtime Tests for `/paras/*`', () => { async (runtimePath, runtimeResponse) => { const res = await request(runtimePath, HOST, PORT); // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - const responseJson = JSON.parse(res); + const responseJson = JSON.parse(res.data); expect(responseJson).toStrictEqual(JSON.parse(runtimeResponse)); } diff --git a/e2e-tests/index.ts b/e2e-tests/historical/historical.ts similarity index 92% rename from e2e-tests/index.ts rename to e2e-tests/historical/historical.ts index 4289054f7..0ec274ce7 100644 --- a/e2e-tests/index.ts +++ b/e2e-tests/historical/historical.ts @@ -16,7 +16,7 @@ import { ArgumentParser } from 'argparse'; -import { IParser } from './historical/types'; +import { IParser } from './types'; const config = { chain: 'polkadot', @@ -25,7 +25,8 @@ const config = { const argv = process.argv.slice(0, 2); /** - * The arg parser takes in two commands. + * The arg parser takes in two commands. This file also directly relates to the historical e2e-tests. + * * @arg --chain The chain to be passed into the jest test * @arg --config The path to the correct jest config. This is important as the * jest config inside of /runtime-tests ignores all the other tests. diff --git a/e2e-tests/latest/endpoints/index.ts b/e2e-tests/latest/endpoints/index.ts new file mode 100644 index 000000000..eb90052e1 --- /dev/null +++ b/e2e-tests/latest/endpoints/index.ts @@ -0,0 +1,18 @@ +// Copyright 2017-2022 Parity Technologies (UK) Ltd. +// This file is part of Substrate API Sidecar. +// +// Substrate API Sidecar is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +export * from './polkadot'; +export * from './statemint'; diff --git a/e2e-tests/latest/endpoints/polkadot.ts b/e2e-tests/latest/endpoints/polkadot.ts new file mode 100644 index 000000000..56fe877fb --- /dev/null +++ b/e2e-tests/latest/endpoints/polkadot.ts @@ -0,0 +1,182 @@ +// Copyright 2017-2022 Parity Technologies (UK) Ltd. +// This file is part of Substrate API Sidecar. +// +// Substrate API Sidecar is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +import { IConfig } from '../types/endpoints'; + +export const polkadot: IConfig = { + '/accounts/{accountId}/balance-info': { + path: '/accounts/12xtAYsRUrmbniiWQqJtECiBQrMn8AypQcXhnQAc6RB6XkLW/balance-info', + queryParams: [ + 'denominated=true', + 'at={blockId}' + ], + }, + '/accounts/{accountId}/convert': { + path: '/accounts/0xde1894014026720b9918b1b21b488af8a0d4f15953621233830946ec0b4d7b75/convert', + queryParams: [], + }, + '/accounts/{accountId}/vesting-info': { + path: '/accounts/15aKvwRqGVAwuBMaogtQXhuz9EQqUWsZJSAzomyb5xYwgBXA/vesting-info', + queryParams: [ + 'at={blockId}', + ], + }, + '/accounts/{accountId}/staking-info': { + path: '/accounts/12BnVhXxGBZXoq9QAkSv9UtVcdBs1k38yNx6sHUJWasTgYrm/staking-info', + queryParams: [ + 'at={blockId}' + ] + }, + '/accounts/{accountId}/staking-payouts': { + path: '/accounts/12BnVhXxGBZXoq9QAkSv9UtVcdBs1k38yNx6sHUJWasTgYrm/staking-payouts', + queryParams: [ + 'at={blockId}', + 'unclaimedOnly=false', + ], + }, + '/accounts/{accountId}/validate': { + path: '/accounts/DXgXPAT5zWtPHo6FhVvrDdiaDPgCNGxhJAeVBYLtiwW9hAc/validate', + queryParams: [], + }, + '/blocks': { + path: '/blocks?range=1-5', + queryParams: [], + }, + '/blocks/{blockId}': { + path: '/blocks/{blockId}', + queryParams: [ + 'eventDocs=true', + 'extrinsicDocs=true', + ], + }, + '/blocks/{blockId}/header': { + path: '/blocks/{blockId}/header', + queryParams: [], + }, + '/blocks/{blockId}/extrinsics/{extrinsicIndex}': { + path: `/blocks/{blockId}/extrinsics/0`, + queryParams: [ + 'eventDocs=true', + 'extrinsicDocs=true', + ], + }, + '/blocks/head': { + path: `/blocks/head`, + queryParams: [ + 'eventDocs=true', + 'extrinsicDocs=true', + ], + }, + '/blocks/head/header': { + path: '/blocks/head', + queryParams: [], + }, + '/node/network': { + path: '/node/network', + queryParams: [], + }, + '/node/transaction-pool': { + path: '/node/transaction-pool', + queryParams: [ + 'includeFee=true', + ], + }, + '/node/version': { + path: '/node/version', + queryParams: [], + }, + '/pallets/staking/progress': { + path: '/pallets/staking/progress', + queryParams: [ + 'at={blockId}', + ], + }, + '/pallets/{palletId}/storage': { + path: '/pallets/System/storage', + queryParams: [ + 'onlyIds=true', + 'at={blockId}', + ], + }, + '/pallets/{palletId}/storage/{storageItemId}': { + path: '/pallets/System/storage/BlockWeight', + queryParams: [ + 'metadata=true', + 'at={blockId}', + ], + }, + '/runtime/metadata': { + path: '/runtime/metadata', + queryParams: [ + 'at={blockId}', + ], + }, + '/runtime/code': { + path: '/runtime/code', + queryParams: [ + 'at={blockId}', + ], + }, + '/runtime/spec': { + path: '/runtime/spec', + queryParams: [ + 'at={blockId}', + ], + }, + '/transaction/material': { + path: '/transaction/material', + queryParams: [ + 'noMeta=true', + ], + }, + '/paras': { + path: '/paras', + queryParams: [ + 'at={blockId}', + ], + }, + '/paras/leases/current': { + path: '/paras/leases/current', + queryParams: [ + 'at={blockId}', + 'currentLeaseHolders=false' + ], + }, + '/paras/auctions/current': { + path: '/paras/auctions/current', + queryParams: [ + 'at={blockId}', + ], + }, + '/paras/crowdloans': { + path: '/paras/crowdloans', + queryParams: [ + 'at={blockId}', + ], + }, + '/paras/{paraId}/crowdloan-info': { + path: '/paras/2021/crowdloan-info', + queryParams: [ + 'at={blockId}', + ], + }, + '/paras/{paraId}/lease-info': { + path: '/paras/2021/lease-info', + queryParams: [ + 'at={blockId}', + ], + } +} diff --git a/e2e-tests/latest/endpoints/statemint.ts b/e2e-tests/latest/endpoints/statemint.ts new file mode 100644 index 000000000..503c13607 --- /dev/null +++ b/e2e-tests/latest/endpoints/statemint.ts @@ -0,0 +1,35 @@ +// Copyright 2017-2022 Parity Technologies (UK) Ltd. +// This file is part of Substrate API Sidecar. +// +// Substrate API Sidecar is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +export const statemint = { + '/accounts/{accountId}/asset-balances': { + path: '/accounts/1ULZhwpUPLLg5VRYiq6rBHY8XaShAmBW7kqGBfvHBqrgBcN/asset-balances', + queryParams: [ + 'at={blockId}', + 'assets[]=100&assets[]=123' + ] + }, + '/accounts/{accountId}/asset-approvals': { + path: '/accounts/13zCwRqhAj4D33czsm1G82EgHBNq58CCcWRsbwABaby64p1A/asset-approvals?at={blockId}&assetId=1984&delegate=12jU3Wn96uJgfiAe7Zk9s1vKWDz8SBNnqQ8t7s8kj1hDxMMc', + queryParams: [], + }, + '/pallets/assets/{assetId}/asset-info': { + path: '/pallets/assets/123/asset-info', + queryParams: [ + 'at={blockId}' + ], + }, +}; diff --git a/e2e-tests/latest/index.ts b/e2e-tests/latest/index.ts new file mode 100644 index 000000000..12f8bedd3 --- /dev/null +++ b/e2e-tests/latest/index.ts @@ -0,0 +1,115 @@ +// Copyright 2017-2022 Parity Technologies (UK) Ltd. +// This file is part of Substrate API Sidecar. +// +// Substrate API Sidecar is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +import { ArgumentParser } from 'argparse'; +import { request, IRequest } from '../helpers/request'; +import { polkadot, statemint } from './endpoints'; +import { IConfig } from './types/endpoints'; +import { HOST, PORT } from '../helpers/consts'; + +enum StatusCode { + Success = 0, + Failed = 1, +} + +interface ILatestE2eParser { + chain: string +} + +const main = async (args: ILatestE2eParser): Promise => { + const { Success, Failed } = StatusCode; + + let config: IConfig; + switch(args.chain) { + case 'polkadot': + config = polkadot + break; + case 'statemint': + config = statemint + break; + default: + config = polkadot; + break; + } + + let blockId: string; + try { + const res = await request('/blocks/head', HOST, PORT); + blockId = JSON.parse(res.data).number as string; + } catch (err) { + throw (`Error fetching the latest block: ${err}`); + } + + /** + * Urls will contains all the urls including the base path of an endpoint, + * and seperate urls with the query params added. Anything that contains `{blockId}` + * will be replaced by the current blockId we are testing + */ + const urls: string[] = []; + for (const endpoint in config) { + const { path, queryParams } = config[endpoint]; + const formattedPath = path.replace('{blockId}', blockId) + // Base endpoint with no query params attached + urls.push(formattedPath); + // Attach each of the query params to their own base path + if (queryParams.length) { + const tempPath = formattedPath.concat('?') + const urlsWithParams = queryParams.map((param) => { + return tempPath.concat(param).replace('{blockId}', blockId); + + }); + urls.push(...urlsWithParams); + } + } + + const responses = await Promise.all(urls.map((u) => request(u, HOST, PORT))); + const errors: IRequest[] = []; + responses.forEach((res) => { + if (res.statusCode && res.statusCode >= 400) { + errors.push(res) + } + }) + logErrors(errors); + + if (errors.length > 0) { + console.log(`Finished with a status code of ${Failed}`) + return Failed + } else { + console.log(`Finished with a status code of ${Success}`) + return Success + } +} + +const logErrors = (errors: IRequest[]) => { + console.log('Received the following errors:'); + errors.forEach((err) => { + console.log('----------------------------------------------'); + console.log(`Queried Endpoint: ${err.path}`); + console.log(`Status Code: ${err.statusCode}`); + console.log(`Received logging: ${err.data}`); + }); +} + +const parser = new ArgumentParser(); + +parser.add_argument('--chain', { + choices: ['polkadot', 'statemint'], + default: 'polkadot' +}); + +const args = parser.parse_args() as ILatestE2eParser; + +main(args).finally(() => process.exit()); diff --git a/e2e-tests/latest/types/endpoints.ts b/e2e-tests/latest/types/endpoints.ts new file mode 100644 index 000000000..8db2b932c --- /dev/null +++ b/e2e-tests/latest/types/endpoints.ts @@ -0,0 +1,24 @@ +// Copyright 2017-2022 Parity Technologies (UK) Ltd. +// This file is part of Substrate API Sidecar. +// +// Substrate API Sidecar is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +export type IConfig = { + [key: string]: IEndpoints +} + +interface IEndpoints { + path: string; + queryParams: string[] +} diff --git a/e2e-tests/tsconfig.json b/e2e-tests/tsconfig.json index b3d794f0a..b18c537de 100644 --- a/e2e-tests/tsconfig.json +++ b/e2e-tests/tsconfig.json @@ -1,9 +1,19 @@ { "extends": "@substrate/dev/config/tsconfig.json", "compilerOptions": { - "baseUrl": ".", - "outDir": "build", - "suppressImplicitAnyIndexErrors": true, - "resolveJsonModule": true, + "baseUrl": ".", + "outDir": "build", + "rootDirs": ["./src"], + "suppressImplicitAnyIndexErrors": true, + "resolveJsonModule": true, }, -} + "typeRoots": [ + "../node_modules/@types" + ], + "include": [ + "./historical", "./helpers", "./latest", + ], + "exclude": [ + "./historical-e2e-tests.spec.ts" + ] + } diff --git a/package.json b/package.json index 18c90e91f..dad241243 100644 --- a/package.json +++ b/package.json @@ -38,15 +38,13 @@ "test:watch": "NODE_ENV=test substrate-exec-jest --watch", "test:ci": "NODE_ENV=test substrate-exec-jest --runInBand", "lint:e2e-tests": "cd e2e-tests && substrate-dev-run-lint", - "build:e2e-tests": "substrate-exec-tsc --project e2e-tests/tsconfig.json", - "test:e2e-tests": "yarn build:e2e-tests && node ./e2e-tests/build/index.js --config=./e2e-tests/jest.config.js", - "test:init-e2e-tests": "yarn start:e2e-scripts --log-level=http", - "test:init-e2e-tests:polkadot": "yarn test:init-e2e-tests --chain polkadot", - "test:init-e2e-tests:kusama": "yarn test:init-e2e-tests --chain kusama", - "test:init-e2e-tests:westend": "yarn test:init-e2e-tests --chain westend", - "test:init-e2e-tests:statemine": "yarn test:init-e2e-tests --chain statemine", - "test:init-e2e-tests:statemint": "yarn test:init-e2e-tests --chain statemint", - "start:e2e-scripts": "yarn build:scripts && node scripts/build/runE2eTests.js", + "build:e2e-tests": "substrate-exec-rimraf e2e-tests/build/ && substrate-exec-tsc --project e2e-tests/tsconfig.json", + "test:latest-e2e-tests": "yarn state:latest-e2e-scripts", + "test:historical-e2e-tests": "yarn start:historical-e2e-scripts", + "start:historical-e2e-tests": "yarn build:e2e-tests && node ./e2e-tests/build/historical/historical.js --config=./e2e-tests/jest.config.js", + "start:latest-e2e-tests": "yarn build:e2e-tests && node ./e2e-tests/build/latest/index.js", + "start:historical-e2e-scripts": "yarn build:scripts && node scripts/build/runHistoricalE2eTests.js", + "state:latest-e2e-scripts": "yarn build:scripts && node scripts/build/runLatestE2eTests.js", "build:scripts": "substrate-exec-rimraf scripts/build/ && substrate-exec-tsc --project scripts/tsconfig.json", "lint:scripts": "cd scripts && substrate-dev-run-lint", "start:test-release": "yarn build:scripts && node scripts/build/runYarnPack.js", diff --git a/scripts/README.md b/scripts/README.md index 2bcc072cc..a0d70e9a4 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -6,21 +6,49 @@

+## Script `runHistoricalE2eTests.ts` -## Script `runChainTests.ts` +### Summary + +This script calls the local historical e2e tests helper library in order to test the current branch or development environment against a collection of different blocks, across different runtimes. + +### Flags + +`--chain`: This sets the specific chain to run the script against. Acceptable values are `['polkadot', 'westend', 'kusama', 'statemine', 'statemint']`. If no chain is selected it will default to run the e2e-tests against all chains. + +`--local`: This sets the websocket url for the e2e test. Its to be used along with `--chain`. If `--chain` is not present it will throw an error. + +`--log-level`: Acceptable values are `['error', 'warn', 'info', 'http', 'verbose', 'debug', 'silly']`. This will enable logging in Sidecar. + +### Run the script + +In order to run the program: + +```bash +$ yarn test:historical-e2e-tests +``` + +## Script `runLatestE2eTests.ts` ### Summary -This script calls the local e2e-tests helper library in order to test the current branch or development environment against -a collection of different blocks, across different runtimes. It does this for Polkadot, Kusama, and Westend. +This script calls the local latest e2e tests helper library in order to test the api service against the most recent runtime. It works through all the available endpoints, and calls them each against the most recent block. ### Flags -`--chain`: This sets the specific chain to run the script against. Acceptable values are `['polkadot', 'westend', 'kusama']`. If no chain is selected it will default to run the e2e-tests against all chains. +`--chain`: This sets the specific chain to run the script against. Acceptable values are `['polkadot', 'statemint']`. If no chain is selected it will default to run the e2e-tests against all chains. + +`--local`: This sets the websocket url for the e2e test. Its to be used along with `--chain`. If `--chain` is not present it will throw an error. `--log-level`: Acceptable values are `['error', 'warn', 'info', 'http', 'verbose', 'debug', 'silly']`. This will enable logging in Sidecar. +### Run the script +In order to run the program: + +```bash +$ yarn test:latest-e2e-tests +``` ## Script `runYarnPack.ts` @@ -41,3 +69,13 @@ If the cleanup has any issues, you may run from the root directory of this repos $ yarn remove @substrate/api-sidecar $ rm -rf ./package.tgz ``` + +## Errors + +`0`: The process has finished succesfully and all tests have passed. + +`1`: The process has finished succesfully but the tests did not come back all successful. Either one or many have failed. + +`2`: The process has exited with an error, and building sidecar has failed. + +`3`: CLI args are used incorrectly internally. diff --git a/scripts/config.ts b/scripts/config.ts index 443d12621..bd7e75e23 100644 --- a/scripts/config.ts +++ b/scripts/config.ts @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -import { IChainConfig } from './types'; +import { IChainConfigE2E } from './types'; const defaultJestOpts = { proc: 'jest', @@ -41,45 +41,68 @@ export const defaultSasPackOpts = { args: ['pack'], }; -export const config: Record = { +export const historicalE2eConfig: Record = { polkadot: { wsUrl: 'wss://rpc.polkadot.io', - JestProcOpts: { + e2eStartOpts: { ...defaultJestOpts, - args: ['test:e2e-tests', '--chain', 'polkadot'], + args: ['start:historical-e2e-tests', '--chain', 'polkadot'], }, SasStartOpts: defaultSasStartOpts, }, kusama: { wsUrl: 'wss://kusama.api.onfinality.io/public-ws', - JestProcOpts: { + e2eStartOpts: { ...defaultJestOpts, - args: ['test:e2e-tests', '--chain', 'kusama'], + args: ['start:historical-e2e-tests', '--chain', 'kusama'], }, SasStartOpts: defaultSasStartOpts, }, westend: { wsUrl: 'wss://westend.api.onfinality.io/public-ws', - JestProcOpts: { + e2eStartOpts: { ...defaultJestOpts, - args: ['test:e2e-tests', '--chain', 'westend'], + args: ['start:historical-e2e-tests', '--chain', 'westend'], }, SasStartOpts: defaultSasStartOpts, }, statemine: { wsUrl: 'wss://statemine.api.onfinality.io/public-ws', - JestProcOpts: { + e2eStartOpts: { ...defaultJestOpts, - args: ['test:e2e-tests', '--chain', 'statemine'], + args: ['start:historical-e2e-tests', '--chain', 'statemine'], }, SasStartOpts: defaultSasStartOpts, }, statemint: { wsUrl: 'wss://statemint-rpc.polkadot.io', - JestProcOpts: { + e2eStartOpts: { ...defaultJestOpts, - args: ['test:e2e-tests', '--chain', 'statemint'], + args: ['start:historical-e2e-tests', '--chain', 'statemint'], }, SasStartOpts: defaultSasStartOpts } }; + +export const latestE2eConfig: Record = { + polkadot: { + wsUrl: 'wss://rpc.polkadot.io', + SasStartOpts: defaultSasStartOpts, + e2eStartOpts: { + proc: 'latest-e2e', + resolver: 'Finished with a status code of 0', + resolverFailed: 'Finished with a status code of 1', + args: ['start:latest-e2e-tests', '--chain', 'polkadot'] + } + }, + statemint: { + wsUrl: 'wss://statemint-rpc.polkadot.io', + SasStartOpts: defaultSasStartOpts, + e2eStartOpts: { + proc: 'latest-e2e', + resolver: 'Finished with a status code of 0', + resolverFailed: 'Finished with a status code of 1', + args: ['start:latest-e2e-tests', '--chain', 'statemint'] + } + } +} diff --git a/scripts/e2eHelpers.ts b/scripts/e2eHelpers.ts new file mode 100644 index 000000000..04ee90e42 --- /dev/null +++ b/scripts/e2eHelpers.ts @@ -0,0 +1,90 @@ +// Copyright 2017-2022 Parity Technologies (UK) Ltd. +// This file is part of Substrate API Sidecar. +// +// Substrate API Sidecar is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . +import { IChainConfigE2E, ProcsType, StatusCode } from './types'; +import { + killAll, + launchProcess, + setWsUrl, +} from './sidecarScriptApi'; + +/** + * Check each chain test returned by `launchChainTest`, and exit the program + * with the correct process. + * + * @param args The results of each test. + */ +export const checkTests = (...args: boolean[]): void => { + const testStatus = args.every((test) => test); + + if (testStatus) { + console.log('[PASSED] All Tests Passed!'); + process.exit(0); + } else { + console.log('[FAILED] Some Tests Failed!'); + process.exit(1); + } +}; + +/** + * Launch a e2e test for a chain. + * + * @param chain The chain to test against. + * @param config The config specific to a chain. + * @param isLocal Boolean declaring if this chain is local. + * @param procs Object containing all the processes. + */ +export const launchChainTest = async ( + chain: string, + config: Record, + procs: ProcsType, + localUrl?: string, +): Promise => { + const { wsUrl, SasStartOpts, e2eStartOpts } = config[chain]; + const { Success } = StatusCode; + + // Set the ws url env var + localUrl ? setWsUrl(localUrl) : setWsUrl(wsUrl); + + console.log('Launching Sidecar...'); + const sidecarStart = await launchProcess('yarn', procs, SasStartOpts); + + if (sidecarStart === Success) { + // Sidecar successfully launched, and jest will now get called + console.log('Launching jest...'); + const jest = await launchProcess('yarn', procs, e2eStartOpts); + + if (jest === Success) { + killAll(procs); + return true; + } else { + killAll(procs); + return false; + } + } else { + console.error('Error launching sidecar... exiting...'); + killAll(procs); + process.exit(2); + } +} + +export const checkWsType = (arg: string) => { + const res = /^(wss?:\/\/)([0-9]{1,3}(?:\.[0-9]{1,3}){3}|[a-zA-Z]+):([0-9]{1,5})$/.test(arg); + if (!res) { + console.error('Invalid Ws Url format'); + process.exit(3); + } + return arg +} diff --git a/scripts/runE2eTests.ts b/scripts/runHistoricalE2eTests.ts similarity index 52% rename from scripts/runE2eTests.ts rename to scripts/runHistoricalE2eTests.ts index 38e4f31ee..65160f936 100644 --- a/scripts/runE2eTests.ts +++ b/scripts/runHistoricalE2eTests.ts @@ -16,104 +16,50 @@ import { ArgumentParser, Namespace } from 'argparse'; -import { config, defaultSasBuildOpts } from './config'; +import { historicalE2eConfig, defaultSasBuildOpts } from './config'; import { killAll, launchProcess, setLogLevel, - setWsUrl, } from './sidecarScriptApi'; import { ProcsType, StatusCode } from './types'; +import { checkTests, launchChainTest, checkWsType } from './e2eHelpers'; // Stores all the processes const procs: ProcsType = {}; -/** - * Launches Sidecar, and if successful it will launch the jest runner. This operation - * handles killing all the processes after the jest runner is done. - * - * @param chain The chain in which to target the e2e tests too. - */ -const launchChainTest = async (chain: string): Promise => { - const { wsUrl, SasStartOpts, JestProcOpts } = config[chain]; - const { Success } = StatusCode; - - // Set the ws url env var - setWsUrl(wsUrl); - - console.log('Launching Sidecar...'); - const sidecarStart = await launchProcess('yarn', procs, SasStartOpts); - - if (sidecarStart === Success) { - // Sidecar successfully launched, and jest will now get called - console.log('Launching jest...'); - const jest = await launchProcess('yarn', procs, JestProcOpts); - - if (jest === Success) { - killAll(procs); - return true; - } else { - killAll(procs); - return false; - } - } else { - console.error('Error launching sidecar... exiting...'); - killAll(procs); - process.exit(1); - } -}; - -const checkTests = (...args: boolean[]) => { - const testStatus = args.every((test) => test); - - if (testStatus) { - console.log('[PASSED] All Tests Passed!'); - process.exit(0); - } else { - console.log('[FAILED] Some Tests Failed!'); - process.exit(1); - } -}; - const main = async (args: Namespace): Promise => { const { Failed } = StatusCode; + const localUrl: string | undefined = args.local ? args.local : undefined; + + if (localUrl && !args.chain) { + console.error('error: `--local` must be used in conjunction with `--chain`'); + process.exit(3); + } if (args.log_level) { setLogLevel(args.log_level); } - // Build sidecar console.log('Building Sidecar...'); const sidecarBuild = await launchProcess('yarn', procs, defaultSasBuildOpts); - // When sidecar fails to build, we kill all process's and exit if (sidecarBuild === Failed) { console.error('Sidecar failed to build, exiting...'); - // Kill all processes killAll(procs); - // Exit program - process.exit(); + process.exit(2); } if (args.chain) { - const selectedChain = await launchChainTest(args.chain); + const selectedChain = await launchChainTest(args.chain, historicalE2eConfig, procs, localUrl); checkTests(selectedChain); } else { - // Test the e2e tests against polkadot - const polkadotTest = await launchChainTest('polkadot'); - - // Test the e2e tests against kusama - const kusamaTest = await launchChainTest('kusama'); - - // Test the e2e tests against westend - const westendTest = await launchChainTest('westend'); - - // Test the e2e tests against statemine - const statemineTest = await launchChainTest('statemine'); - - // Test the e2e tests against statemint - const statemintTest = await launchChainTest('statemint'); + const polkadotTest = await launchChainTest('polkadot', historicalE2eConfig, procs); + const kusamaTest = await launchChainTest('kusama', historicalE2eConfig, procs); + const westendTest = await launchChainTest('westend', historicalE2eConfig, procs); + const statemineTest = await launchChainTest('statemine', historicalE2eConfig, procs); + const statemintTest = await launchChainTest('statemint', historicalE2eConfig, procs); checkTests(polkadotTest, kusamaTest, westendTest, statemineTest, statemintTest); } @@ -124,11 +70,17 @@ const main = async (args: Namespace): Promise => { */ const parser = new ArgumentParser(); +parser.add_argument('--local', { + required: false, + nargs: '?', + type: checkWsType +}) parser.add_argument('--chain', { choices: ['polkadot', 'kusama', 'westend', 'statemine', 'statemint'], }); parser.add_argument('--log-level', { choices: ['error', 'warn', 'info', 'http', 'verbose', 'debug', 'silly'], + default: 'http', }); const args = parser.parse_args() as Namespace; diff --git a/scripts/runLatestE2eTests.ts b/scripts/runLatestE2eTests.ts new file mode 100644 index 000000000..fe1be28e8 --- /dev/null +++ b/scripts/runLatestE2eTests.ts @@ -0,0 +1,97 @@ +// Copyright 2017-2022 Parity Technologies (UK) Ltd. +// This file is part of Substrate API Sidecar. +// +// Substrate API Sidecar is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +import { ArgumentParser, Namespace } from 'argparse'; +import { launchProcess, setLogLevel, killAll } from './sidecarScriptApi'; +import { latestE2eConfig, defaultSasBuildOpts } from './config'; +import { checkTests, launchChainTest, checkWsType } from './e2eHelpers'; + +import { ProcsType, StatusCode } from './types'; + +// Stores all the processes +const procs: ProcsType = {}; + +const main = async (args: Namespace) => { + const { Failed } = StatusCode; + const localUrl: string | undefined = args.local ? args.local : undefined; + + if (localUrl && !args.chain) { + console.error('error: `--local` must be used in conjunction with `--chain`'); + process.exit(3); + } + + if (args.log_level) { + setLogLevel(args.log_level); + } + + console.log('Building Sidecar...'); + const sidecarBuild = await launchProcess('yarn', procs, defaultSasBuildOpts); + + if (sidecarBuild === Failed) { + console.log('Sidecar failed to build, exiting...'); + killAll(procs); + process.exit(2); + } + + // CheckTests will either return a success exit code of 0, or a failed exit code of 1. + if (args.chain) { + const selectedChain = await launchChainTest(args.chain, latestE2eConfig, procs, localUrl); + + checkTests(selectedChain); + } else { + const polkadotTest = await launchChainTest('polkadot', latestE2eConfig, procs); + const statemintTest = await launchChainTest('statemint', latestE2eConfig, procs); + + checkTests(polkadotTest, statemintTest); + } +} + +const parser = new ArgumentParser(); + +parser.add_argument('--local', { + required: false, + nargs: '?', + type: checkWsType +}); +parser.add_argument('--chain', { + choices: ['polkadot', 'statemint'], +}); +parser.add_argument('--log-level', { + choices: ['error', 'warn', 'info', 'http', 'verbose', 'debug', 'silly'], + default: 'http', +}); + +const args = parser.parse_args() as Namespace; + +/** + * Signal interrupt + */ + process.on('SIGINT', function () { + console.log('Caught interrupt signal'); + killAll(procs); + process.exit(); +}); + +/** + * Signal hangup terminal + */ +process.on('SIGHUP', function () { + console.log('Caught terminal termination'); + killAll(procs); + process.exit(); +}); + +main(args).finally(() => process.exit()); diff --git a/scripts/sidecarScriptApi.ts b/scripts/sidecarScriptApi.ts index 7f77ef0a0..43969a514 100644 --- a/scripts/sidecarScriptApi.ts +++ b/scripts/sidecarScriptApi.ts @@ -83,7 +83,14 @@ export const killAll = (procs: ProcsType): void => { export const launchProcess = ( cmd: string, procs: ProcsType, - { proc, resolver, resolverJestErr, resolverStartupErr, args }: IProcOpts + { + proc, + resolver, + resolverJestErr, + resolverStartupErr, + resolverFailed, + args + }: IProcOpts ): Promise => { return new Promise((resolve, reject) => { const { Success, Failed } = StatusCode; @@ -98,6 +105,10 @@ export const launchProcess = ( if (data.toString().includes(resolver)) { resolve(Success); } + + if (resolverFailed && data.toString().includes(resolverFailed)) { + resolve(Failed); + } }); procs[proc].stderr.on('data', (data: Buffer) => { diff --git a/scripts/types.ts b/scripts/types.ts index a49dabf61..7719a1c58 100644 --- a/scripts/types.ts +++ b/scripts/types.ts @@ -25,15 +25,19 @@ export enum StatusCode { export interface IChainConfig { wsUrl: string; - JestProcOpts: IProcOpts; SasStartOpts: IProcOpts; } +export interface IChainConfigE2E extends IChainConfig { + e2eStartOpts: IProcOpts; +} + // Process options export interface IProcOpts { proc: string; resolver: string; resolverStartupErr?: string; resolverJestErr?: string; + resolverFailed?: string; args: string[]; }