test(e2e): Add log level option in e2e generator and manual debug wor…

…kflow (backport #3881) (#3887) close: #3880 #### PR checklist - [ ] Tests written/updated - [X] Changelog entry added in `.changelog` (we use [unclog](https://github.com/informalsystems/unclog) to manage our changelog) - [X] Updated relevant documentation (`docs/` or `spec/`) and code comments <hr>This is an automatic backport of pull request #3881 done by [Mergify](https://mergify.com). Co-authored-by: Andy Nogueira <me@andynogueira.dev>
cometbft · Aug 28, 2024 · 50564cf · 50564cf
1 parent 49a80fd
commit 50564cf
Show file tree

Hide file tree

Showing 6 changed files with 74 additions and 5 deletions.
diff --git a/.changelog/unreleased/improvements/3880-e2e-generator-debug.md b/.changelog/unreleased/improvements/3880-e2e-generator-debug.md
@@ -0,0 +1,2 @@
+- `[e2e]` Add log level option in e2e generator
+  ([\#3880](https://github.com/cometbft/cometbft/issues/3880))
diff --git a/.github/workflows/e2e-manual-debug.yml b/.github/workflows/e2e-manual-debug.yml
@@ -0,0 +1,45 @@
+# Runs randomly generated E2E testnets nightly on main with debug logs (but p2p module only info log since it's too verbose)
+# manually run e2e tests with debug logs
+name: e2e-manual-debug
+on:
+  workflow_dispatch:
+
+jobs:
+  e2e-nightly-test:
+    # Run parallel jobs for the listed testnet groups (must match the
+    # ./build/generator -g flag)
+    strategy:
+      fail-fast: false
+      matrix:
+        group: ['00', '01', '02', '03', '04', '05']
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+
+      - run: echo "GO_VERSION=$(cat .github/workflows/go-version.env | grep GO_VERSION | cut -d '=' -f2)" >> $GITHUB_ENV
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - name: Build
+        working-directory: test/e2e
+        # Run make jobs in parallel, since we can't run steps in parallel.
+        run: make -j2 docker generator runner tests
+
+      - name: Generate testnets
+        if: matrix.group != 5
+        working-directory: test/e2e
+        # When changing -g, also change the matrix groups above
+        run: ./build/generator -g 5 -d networks/nightly/ -p -l "debug,p2p:info"
+
+      - name: Run p2p testnets (${{ matrix.group }})
+        if: matrix.group != 5
+        working-directory: test/e2e
+        run: ./run-multiple.sh networks/nightly/*-group${{ matrix.group }}-*.toml
+
+      - name: Run p2p testnets (regression)
+        if: matrix.group == 5
+        working-directory: test/e2e
+        run: ./run-multiple.sh networks_regressions/*.toml
diff --git a/test/e2e/README.md b/test/e2e/README.md
@@ -107,6 +107,19 @@ The generator generates this type of perturbation both on full nodes and on ligh
 Perturbations of type `upgrade` are a noop if the node's version matches the
 one in `upgrade_version`.
 
+If you need to generate manifests with a specific `log_level` that will configure the log level parameter in the
+CometBFT's config file for each node, you can specify the level using the flags `-l` or `--log-level`.
+
+```
+./build/generator -g 2 -d networks/nightly/ -l debug
+```
+
+This will add the specified log level on each generated manifest (TOML file):
+
+```toml
+log_level = "debug"
+```
+
 ## Test Stages
 
 The test runner has the following stages, which can also be executed explicitly by running `./build/runner -f <manifest> <stage>`:

diff --git a/test/e2e/app/app.go b/test/e2e/app/app.go
@@ -279,6 +279,7 @@ func (app *Application) CheckTx(_ context.Context, req *abci.CheckTxRequest) (*a
 			panic(fmt.Sprintf("txHeight is less than current height; txHeight %v, height %v", txHeight, stHeight))
 		}
 		if stHeight > txHeight.(uint64)+txTTL {
+			app.logger.Debug("Application CheckTx", "msg", "transaction expired", "tx_hash", cmttypes.Tx.Hash(req.Tx), "seen_height", txHeight, "current_height", stHeight)
 			app.seenTxs.Delete(txKey)
 			return &abci.CheckTxResponse{
 				Code: kvstore.CodeTypeExpired,

diff --git a/test/e2e/generator/generate.go b/test/e2e/generator/generate.go
@@ -75,6 +75,7 @@ type generateConfig struct {
 	outputDir    string
 	multiVersion string
 	prometheus   bool
+	logLevel     string
 }
 
 // Generate generates random testnets using the given RNG.
@@ -116,7 +117,7 @@ func Generate(cfg *generateConfig) ([]e2e.Manifest, error) {
 	}
 	manifests := []e2e.Manifest{}
 	for _, opt := range combinations(testnetCombinations) {
-		manifest, err := generateTestnet(cfg.randSource, opt, upgradeVersion, cfg.prometheus)
+		manifest, err := generateTestnet(cfg.randSource, opt, upgradeVersion, cfg.prometheus, cfg.logLevel)
 		if err != nil {
 			return nil, err
 		}
@@ -126,7 +127,7 @@ func Generate(cfg *generateConfig) ([]e2e.Manifest, error) {
 }
 
 // generateTestnet generates a single testnet with the given options.
-func generateTestnet(r *rand.Rand, opt map[string]any, upgradeVersion string, prometheus bool) (e2e.Manifest, error) {
+func generateTestnet(r *rand.Rand, opt map[string]any, upgradeVersion string, prometheus bool, logLevel string) (e2e.Manifest, error) {
 	manifest := e2e.Manifest{
 		IPv6:             ipv6.Choose(r).(bool),
 		ABCIProtocol:     nodeABCIProtocols.Choose(r).(string),
@@ -138,6 +139,7 @@ func generateTestnet(r *rand.Rand, opt map[string]any, upgradeVersion string, pr
 		Nodes:            map[string]*e2e.ManifestNode{},
 		UpgradeVersion:   upgradeVersion,
 		Prometheus:       prometheus,
+		LogLevel:         logLevel,
 	}
 
 	switch abciDelays.Choose(r).(string) {

diff --git a/test/e2e/generator/main.go b/test/e2e/generator/main.go
@@ -31,7 +31,7 @@ type CLI struct {
 func NewCLI() *CLI {
 	cli := &CLI{}
 	cli.root = &cobra.Command{
-		Use:           "generator -d dir [-g int] [-m version_weight_csv] [-p]",
+		Use:           "generator -d dir [-g int] [-m version_weight_csv] [-p] [-l log_level]",
 		Short:         "End-to-end testnet generator",
 		SilenceUsage:  true,
 		SilenceErrors: true, // we'll output them ourselves in Run()
@@ -52,7 +52,11 @@ func NewCLI() *CLI {
 			if err != nil {
 				return err
 			}
-			return cli.generate(dir, groups, multiVersion, prometheus)
+			logLevel, err := cmd.Flags().GetString("log-level")
+			if err != nil {
+				return err
+			}
+			return cli.generate(dir, groups, multiVersion, prometheus, logLevel)
 		},
 	}
 
@@ -62,12 +66,13 @@ func NewCLI() *CLI {
 		"or empty to only use this branch's version")
 	cli.root.PersistentFlags().IntP("groups", "g", 0, "Number of groups")
 	cli.root.PersistentFlags().BoolP("prometheus", "p", false, "Enable generation of Prometheus metrics on all manifests")
+	cli.root.PersistentFlags().StringP("log-level", "l", "", "Log level to use in the CometBFT config file, e.g. 'debug'")
 
 	return cli
 }
 
 // generate generates manifests in a directory.
-func (*CLI) generate(dir string, groups int, multiVersion string, prometheus bool) error {
+func (*CLI) generate(dir string, groups int, multiVersion string, prometheus bool, logLevel string) error {
 	err := os.MkdirAll(dir, 0o755)
 	if err != nil {
 		return err
@@ -77,6 +82,7 @@ func (*CLI) generate(dir string, groups int, multiVersion string, prometheus boo
 		randSource:   rand.New(rand.NewSource(randomSeed)), //nolint:gosec
 		multiVersion: multiVersion,
 		prometheus:   prometheus,
+		logLevel:     logLevel,
 	}
 	manifests, err := Generate(cfg)
 	if err != nil {
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		- `[e2e]` Add log level option in e2e generator
		([\#3880](https://github.com/cometbft/cometbft/issues/3880))