e2e: add prometheus flag to Generator CLI (#563) (#569)

* Add prometheus flag to CLI * Fix CLI help message * Enable prometheus in nightly tests (cherry picked from commit 01c0188de85c931cf65f59b3be0c4f790d256276) # Conflicts: # .github/workflows/e2e-nightly-37x.yml Co-authored-by: Hernán Vanzetto <[email protected]>
skip-mev · Mar 23, 2023 · 7381ad7 · 7381ad7
1 parent 9a9334b
commit 7381ad7
Show file tree

Hide file tree

Showing 7 changed files with 94 additions and 9 deletions.
diff --git a/.github/workflows/e2e-manual-multiversion.yml b/.github/workflows/e2e-manual-multiversion.yml
@@ -31,7 +31,7 @@ jobs:
         # When changing -g, also change the matrix groups above
         # Generate multi-version tests with double the quantity of E2E nodes
         # based on the current branch as compared to the latest version.
-        run: ./build/generator -g 5 -m "latest:1,local:2" -d networks/nightly/
+        run: ./build/generator -g 5 -m "latest:1,local:2" -d networks/nightly/ -p
 
       - name: Run ${{ matrix.p2p }} p2p testnets
         working-directory: test/e2e

diff --git a/.github/workflows/e2e-manual.yml b/.github/workflows/e2e-manual.yml
@@ -29,7 +29,7 @@ jobs:
       - name: Generate testnets
         working-directory: test/e2e
         # When changing -g, also change the matrix groups above
-        run: ./build/generator -g 5 -d networks/nightly/
+        run: ./build/generator -g 5 -d networks/nightly/ -p
 
       - name: Run ${{ matrix.p2p }} p2p testnets
         working-directory: test/e2e

diff --git a/.github/workflows/e2e-nightly-34x.yml b/.github/workflows/e2e-nightly-34x.yml
@@ -41,7 +41,7 @@ jobs:
       - name: Generate testnets
         working-directory: test/e2e
         # When changing -g, also change the matrix groups above
-        run: ./build/generator -g 2 -d networks/nightly
+        run: ./build/generator -g 2 -d networks/nightly -p
 
       - name: Run testnets in group ${{ matrix.group }}
         working-directory: test/e2e

diff --git a/.github/workflows/e2e-nightly-37x.yml b/.github/workflows/e2e-nightly-37x.yml
@@ -0,0 +1,77 @@
+# Runs randomly generated E2E testnets nightly on the v0.37.x branch.
+
+# !! This file should be kept in sync with the e2e-nightly-main.yml file,
+# modulo changes to the version labels.
+
+name: e2e-nightly-37x
+on:
+  schedule:
+    - cron: '0 2 * * *'
+
+jobs:
+  e2e-nightly-test:
+    # Run parallel jobs for the listed testnet groups (must match the
+    # ./build/generator -g flag)
+    strategy:
+      fail-fast: false
+      matrix:
+        group: ['00', '01', '02', '03', "04"]
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.20'
+
+      - uses: actions/checkout@v3
+        with:
+          ref: 'v0.37.x'
+
+      - name: Capture git repo info
+        id: git-info
+        run: |
+          echo "branch=`git branch --show-current`" >> $GITHUB_OUTPUT
+
+      - name: Build
+        working-directory: test/e2e
+        # Run make jobs in parallel, since we can't run steps in parallel.
+        run: make -j2 docker generator runner tests
+
+      - name: Generate testnets
+        working-directory: test/e2e
+        # When changing -g, also change the matrix groups above
+        run: ./build/generator -g 5 -d networks/nightly/ -p
+
+      - name: Run ${{ matrix.p2p }} p2p testnets
+        working-directory: test/e2e
+        run: ./run-multiple.sh networks/nightly/*-group${{ matrix.group }}-*.toml
+
+    outputs:
+      git-branch: ${{ steps.git-info.outputs.branch }}
+
+  e2e-nightly-fail:
+    needs: e2e-nightly-test
+    if: ${{ failure() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Notify Slack on failure
+        uses: slackapi/[email protected]
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
+          BRANCH: ${{ needs.e2e-nightly-test.outputs.git-branch }}
+          RUN_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          COMMITS_URL: "${{ github.server_url }}/${{ github.repository }}/commits/${{ needs.e2e-nightly-test.outputs.git-branch }}"
+        with:
+          payload: |
+            {
+              "blocks": [
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": ":skull: Nightly E2E tests for `${{ env.BRANCH }}` failed. See the <${{ env.RUN_URL }}|run details> and the <${{ env.COMMITS_URL }}|latest commits> possibly related to the failure."
+                  }
+                }
+              ]
+            }
diff --git a/.github/workflows/e2e-nightly-main.yml b/.github/workflows/e2e-nightly-main.yml
@@ -34,7 +34,7 @@ jobs:
       - name: Generate testnets
         working-directory: test/e2e
         # When changing -g, also change the matrix groups above
-        run: ./build/generator -g 5 -d networks/nightly/
+        run: ./build/generator -g 5 -d networks/nightly/ -p
 
       - name: Run ${{ matrix.p2p }} p2p testnets
         working-directory: test/e2e

diff --git a/test/e2e/generator/generate.go b/test/e2e/generator/generate.go
@@ -66,6 +66,7 @@ type generateConfig struct {
 	randSource   *rand.Rand
 	outputDir    string
 	multiVersion string
+	prometheus   bool
 }
 
 // Generate generates random testnets using the given RNG.
@@ -107,7 +108,7 @@ func Generate(cfg *generateConfig) ([]e2e.Manifest, error) {
 	}
 	manifests := []e2e.Manifest{}
 	for _, opt := range combinations(testnetCombinations) {
-		manifest, err := generateTestnet(cfg.randSource, opt, upgradeVersion)
+		manifest, err := generateTestnet(cfg.randSource, opt, upgradeVersion, cfg.prometheus)
 		if err != nil {
 			return nil, err
 		}
@@ -117,7 +118,7 @@ func Generate(cfg *generateConfig) ([]e2e.Manifest, error) {
 }
 
 // generateTestnet generates a single testnet with the given options.
-func generateTestnet(r *rand.Rand, opt map[string]interface{}, upgradeVersion string) (e2e.Manifest, error) {
+func generateTestnet(r *rand.Rand, opt map[string]interface{}, upgradeVersion string, prometheus bool) (e2e.Manifest, error) {
 	manifest := e2e.Manifest{
 		IPv6:             ipv6.Choose(r).(bool),
 		ABCIProtocol:     nodeABCIProtocols.Choose(r).(string),
@@ -128,6 +129,7 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}, upgradeVersion st
 		Evidence:         evidence.Choose(r).(int),
 		Nodes:            map[string]*e2e.ManifestNode{},
 		UpgradeVersion:   upgradeVersion,
+		Prometheus:       prometheus,
 	}
 
 	switch abciDelays.Choose(r).(string) {

diff --git a/test/e2e/generator/main.go b/test/e2e/generator/main.go
@@ -31,7 +31,7 @@ type CLI struct {
 func NewCLI() *CLI {
 	cli := &CLI{}
 	cli.root = &cobra.Command{
-		Use:           "generator -d dir [-g int] [-m version_weight_csv]",
+		Use:           "generator -d dir [-g int] [-m version_weight_csv] [-p]",
 		Short:         "End-to-end testnet generator",
 		SilenceUsage:  true,
 		SilenceErrors: true, // we'll output them ourselves in Run()
@@ -48,7 +48,11 @@ func NewCLI() *CLI {
 			if err != nil {
 				return err
 			}
-			return cli.generate(dir, groups, multiVersion)
+			prometheus, err := cmd.Flags().GetBool("prometheus")
+			if err != nil {
+				return err
+			}
+			return cli.generate(dir, groups, multiVersion, prometheus)
 		},
 	}
 
@@ -57,12 +61,13 @@ func NewCLI() *CLI {
 	cli.root.PersistentFlags().StringP("multi-version", "m", "", "Comma-separated list of versions of CometBFT to test in the generated testnets, "+
 		"or empty to only use this branch's version")
 	cli.root.PersistentFlags().IntP("groups", "g", 0, "Number of groups")
+	cli.root.PersistentFlags().BoolP("prometheus", "p", false, "Enable generation of Prometheus metrics on all manifests")
 
 	return cli
 }
 
 // generate generates manifests in a directory.
-func (cli *CLI) generate(dir string, groups int, multiVersion string) error {
+func (cli *CLI) generate(dir string, groups int, multiVersion string, prometheus bool) error {
 	err := os.MkdirAll(dir, 0o755)
 	if err != nil {
 		return err
@@ -71,6 +76,7 @@ func (cli *CLI) generate(dir string, groups int, multiVersion string) error {
 	cfg := &generateConfig{
 		randSource:   rand.New(rand.NewSource(randomSeed)), //nolint:gosec
 		multiVersion: multiVersion,
+		prometheus:   prometheus,
 	}
 	manifests, err := Generate(cfg)
 	if err != nil {