From 7381ad7941a259d12264a6e454a4b2febe3a5526 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Thu, 23 Mar 2023 14:50:09 +0100 Subject: [PATCH] e2e: add prometheus flag to Generator CLI (#563) (#569) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add prometheus flag to CLI * Fix CLI help message * Enable prometheus in nightly tests (cherry picked from commit 01c0188de85c931cf65f59b3be0c4f790d256276) # Conflicts: # .github/workflows/e2e-nightly-37x.yml Co-authored-by: Hernán Vanzetto <15466498+hvanz@users.noreply.github.com> --- .github/workflows/e2e-manual-multiversion.yml | 2 +- .github/workflows/e2e-manual.yml | 2 +- .github/workflows/e2e-nightly-34x.yml | 2 +- .github/workflows/e2e-nightly-37x.yml | 77 +++++++++++++++++++ .github/workflows/e2e-nightly-main.yml | 2 +- test/e2e/generator/generate.go | 6 +- test/e2e/generator/main.go | 12 ++- 7 files changed, 94 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/e2e-nightly-37x.yml diff --git a/.github/workflows/e2e-manual-multiversion.yml b/.github/workflows/e2e-manual-multiversion.yml index 9332ef91d..164ee3b9e 100644 --- a/.github/workflows/e2e-manual-multiversion.yml +++ b/.github/workflows/e2e-manual-multiversion.yml @@ -31,7 +31,7 @@ jobs: # When changing -g, also change the matrix groups above # Generate multi-version tests with double the quantity of E2E nodes # based on the current branch as compared to the latest version. - run: ./build/generator -g 5 -m "latest:1,local:2" -d networks/nightly/ + run: ./build/generator -g 5 -m "latest:1,local:2" -d networks/nightly/ -p - name: Run ${{ matrix.p2p }} p2p testnets working-directory: test/e2e diff --git a/.github/workflows/e2e-manual.yml b/.github/workflows/e2e-manual.yml index 302e83bb9..d69cdf892 100644 --- a/.github/workflows/e2e-manual.yml +++ b/.github/workflows/e2e-manual.yml @@ -29,7 +29,7 @@ jobs: - name: Generate testnets working-directory: test/e2e # When changing -g, also change the matrix groups above - run: ./build/generator -g 5 -d networks/nightly/ + run: ./build/generator -g 5 -d networks/nightly/ -p - name: Run ${{ matrix.p2p }} p2p testnets working-directory: test/e2e diff --git a/.github/workflows/e2e-nightly-34x.yml b/.github/workflows/e2e-nightly-34x.yml index 5bae7b4e1..a727e88a5 100644 --- a/.github/workflows/e2e-nightly-34x.yml +++ b/.github/workflows/e2e-nightly-34x.yml @@ -41,7 +41,7 @@ jobs: - name: Generate testnets working-directory: test/e2e # When changing -g, also change the matrix groups above - run: ./build/generator -g 2 -d networks/nightly + run: ./build/generator -g 2 -d networks/nightly -p - name: Run testnets in group ${{ matrix.group }} working-directory: test/e2e diff --git a/.github/workflows/e2e-nightly-37x.yml b/.github/workflows/e2e-nightly-37x.yml new file mode 100644 index 000000000..26614fe4b --- /dev/null +++ b/.github/workflows/e2e-nightly-37x.yml @@ -0,0 +1,77 @@ +# Runs randomly generated E2E testnets nightly on the v0.37.x branch. + +# !! This file should be kept in sync with the e2e-nightly-main.yml file, +# modulo changes to the version labels. + +name: e2e-nightly-37x +on: + schedule: + - cron: '0 2 * * *' + +jobs: + e2e-nightly-test: + # Run parallel jobs for the listed testnet groups (must match the + # ./build/generator -g flag) + strategy: + fail-fast: false + matrix: + group: ['00', '01', '02', '03', "04"] + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/setup-go@v4 + with: + go-version: '1.20' + + - uses: actions/checkout@v3 + with: + ref: 'v0.37.x' + + - name: Capture git repo info + id: git-info + run: | + echo "branch=`git branch --show-current`" >> $GITHUB_OUTPUT + + - name: Build + working-directory: test/e2e + # Run make jobs in parallel, since we can't run steps in parallel. + run: make -j2 docker generator runner tests + + - name: Generate testnets + working-directory: test/e2e + # When changing -g, also change the matrix groups above + run: ./build/generator -g 5 -d networks/nightly/ -p + + - name: Run ${{ matrix.p2p }} p2p testnets + working-directory: test/e2e + run: ./run-multiple.sh networks/nightly/*-group${{ matrix.group }}-*.toml + + outputs: + git-branch: ${{ steps.git-info.outputs.branch }} + + e2e-nightly-fail: + needs: e2e-nightly-test + if: ${{ failure() }} + runs-on: ubuntu-latest + steps: + - name: Notify Slack on failure + uses: slackapi/slack-github-action@v1.23.0 + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + BRANCH: ${{ needs.e2e-nightly-test.outputs.git-branch }} + RUN_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + COMMITS_URL: "${{ github.server_url }}/${{ github.repository }}/commits/${{ needs.e2e-nightly-test.outputs.git-branch }}" + with: + payload: | + { + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": ":skull: Nightly E2E tests for `${{ env.BRANCH }}` failed. See the <${{ env.RUN_URL }}|run details> and the <${{ env.COMMITS_URL }}|latest commits> possibly related to the failure." + } + } + ] + } diff --git a/.github/workflows/e2e-nightly-main.yml b/.github/workflows/e2e-nightly-main.yml index 8418616cd..f4f066e22 100644 --- a/.github/workflows/e2e-nightly-main.yml +++ b/.github/workflows/e2e-nightly-main.yml @@ -34,7 +34,7 @@ jobs: - name: Generate testnets working-directory: test/e2e # When changing -g, also change the matrix groups above - run: ./build/generator -g 5 -d networks/nightly/ + run: ./build/generator -g 5 -d networks/nightly/ -p - name: Run ${{ matrix.p2p }} p2p testnets working-directory: test/e2e diff --git a/test/e2e/generator/generate.go b/test/e2e/generator/generate.go index d3e30a3c5..bd61032d3 100644 --- a/test/e2e/generator/generate.go +++ b/test/e2e/generator/generate.go @@ -66,6 +66,7 @@ type generateConfig struct { randSource *rand.Rand outputDir string multiVersion string + prometheus bool } // Generate generates random testnets using the given RNG. @@ -107,7 +108,7 @@ func Generate(cfg *generateConfig) ([]e2e.Manifest, error) { } manifests := []e2e.Manifest{} for _, opt := range combinations(testnetCombinations) { - manifest, err := generateTestnet(cfg.randSource, opt, upgradeVersion) + manifest, err := generateTestnet(cfg.randSource, opt, upgradeVersion, cfg.prometheus) if err != nil { return nil, err } @@ -117,7 +118,7 @@ func Generate(cfg *generateConfig) ([]e2e.Manifest, error) { } // generateTestnet generates a single testnet with the given options. -func generateTestnet(r *rand.Rand, opt map[string]interface{}, upgradeVersion string) (e2e.Manifest, error) { +func generateTestnet(r *rand.Rand, opt map[string]interface{}, upgradeVersion string, prometheus bool) (e2e.Manifest, error) { manifest := e2e.Manifest{ IPv6: ipv6.Choose(r).(bool), ABCIProtocol: nodeABCIProtocols.Choose(r).(string), @@ -128,6 +129,7 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}, upgradeVersion st Evidence: evidence.Choose(r).(int), Nodes: map[string]*e2e.ManifestNode{}, UpgradeVersion: upgradeVersion, + Prometheus: prometheus, } switch abciDelays.Choose(r).(string) { diff --git a/test/e2e/generator/main.go b/test/e2e/generator/main.go index c0cc6ebb5..f4f8140c0 100644 --- a/test/e2e/generator/main.go +++ b/test/e2e/generator/main.go @@ -31,7 +31,7 @@ type CLI struct { func NewCLI() *CLI { cli := &CLI{} cli.root = &cobra.Command{ - Use: "generator -d dir [-g int] [-m version_weight_csv]", + Use: "generator -d dir [-g int] [-m version_weight_csv] [-p]", Short: "End-to-end testnet generator", SilenceUsage: true, SilenceErrors: true, // we'll output them ourselves in Run() @@ -48,7 +48,11 @@ func NewCLI() *CLI { if err != nil { return err } - return cli.generate(dir, groups, multiVersion) + prometheus, err := cmd.Flags().GetBool("prometheus") + if err != nil { + return err + } + return cli.generate(dir, groups, multiVersion, prometheus) }, } @@ -57,12 +61,13 @@ func NewCLI() *CLI { cli.root.PersistentFlags().StringP("multi-version", "m", "", "Comma-separated list of versions of CometBFT to test in the generated testnets, "+ "or empty to only use this branch's version") cli.root.PersistentFlags().IntP("groups", "g", 0, "Number of groups") + cli.root.PersistentFlags().BoolP("prometheus", "p", false, "Enable generation of Prometheus metrics on all manifests") return cli } // generate generates manifests in a directory. -func (cli *CLI) generate(dir string, groups int, multiVersion string) error { +func (cli *CLI) generate(dir string, groups int, multiVersion string, prometheus bool) error { err := os.MkdirAll(dir, 0o755) if err != nil { return err @@ -71,6 +76,7 @@ func (cli *CLI) generate(dir string, groups int, multiVersion string) error { cfg := &generateConfig{ randSource: rand.New(rand.NewSource(randomSeed)), //nolint:gosec multiVersion: multiVersion, + prometheus: prometheus, } manifests, err := Generate(cfg) if err != nil {