test(localnet): Add monitoring tools for localnet (#2108)

Closes #2107  This PR introduces a new folder `test/monitoring` which contains a Docker Compose file that creates a Grafana and a Prometheus server locally for the developer. It connects up with a localnet installation. The localnet Docker compose file (`compose.yml`) was extended to allow Prometheus traffic from the container and documentation was updated to reflect the changes. Short way to execute: ``` make build-linux localnet-start monitoring-start ``` --- The JSON files are pre-populated Grafana dashboards. Two of them are from the qa-infra repository, one is part of host monitoring with "node-explorer" and one is a new one created for storage experiments that are currently ongoing. #### PR checklist - [ ] Tests written/updated - no Go code change - [X] Changelog entry added in `.changelog` (we use [unclog](https://github.com/informalsystems/unclog) to manage our changelog) - [X] Updated relevant documentation (`docs/` or `spec/`) and code comments - test docs were updated - [X] Title follows the [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) spec --------- Co-authored-by: Anton Kaliaev <anton.kalyaev@gmail.com>
cometbft · Jan 24, 2024 · e8b4efa · e8b4efa
1 parent 1d99e05
commit e8b4efa
Show file tree

Hide file tree

Showing 19 changed files with 28,368 additions and 4 deletions.
diff --git a/.changelog/unreleased/features/2107-localnet-monitoring.md b/.changelog/unreleased/features/2107-localnet-monitoring.md
@@ -0,0 +1 @@
+- `[test]` Added monitoring tools and dashboards for local testing with `localnet`. ([\#2107](https://github.com/cometbft/cometbft/issues/2107))
diff --git a/Makefile b/Makefile
@@ -327,14 +327,22 @@ build_c-amazonlinux:
 #? localnet-start: Run a 4-node testnet locally
 localnet-start: localnet-stop build-docker-localnode
 	@if ! [ -f build/node0/config/genesis.json ]; then docker run --rm -v $(CURDIR)/build:/cometbft:Z cometbft/localnode testnet --config /etc/cometbft/config-template.toml --o . --starting-ip-address 192.167.10.2; fi
-	docker-compose up
+	docker-compose up -d
 .PHONY: localnet-start
 
 #? localnet-stop: Stop testnet
 localnet-stop:
 	docker-compose down
 .PHONY: localnet-stop
 
+#? monitoring-start: Start Prometheus and Grafana servers for localnet monitoring
+monitoring-start:
+	cd test/monitoring && docker-compose up -d
+
+#? monitoring-stop: Stop the Prometheus and Grafana servers
+monitoring-stop:
+	cd test/monitoring && docker-compose down
+
 #? build-contract-tests-hooks: Build hooks for dredd, to skip or add information on some steps
 build-contract-tests-hooks:
 ifeq ($(OS),Windows_NT)

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -6,6 +6,7 @@ services:
     image: "cometbft/localnode"
     ports:
       - "26656-26657:26656-26657"
+      - "26670:26660"
     environment:
       - ID=0
       - LOG=${LOG:-cometbft.log}
@@ -20,6 +21,7 @@ services:
     image: "cometbft/localnode"
     ports:
       - "26659-26660:26656-26657"
+      - "26671:26660"
     environment:
       - ID=1
       - LOG=${LOG:-cometbft.log}
@@ -37,6 +39,7 @@ services:
       - LOG=${LOG:-cometbft.log}
     ports:
       - "26661-26662:26656-26657"
+      - "26672:26660"
     volumes:
       - ./build:/cometbft:Z
     networks:
@@ -51,6 +54,7 @@ services:
       - LOG=${LOG:-cometbft.log}
     ports:
       - "26663-26664:26656-26657"
+      - "26673:26660"
     volumes:
       - ./build:/cometbft:Z
     networks:

diff --git a/networks/local/localnode/Dockerfile b/networks/local/localnode/Dockerfile
@@ -6,7 +6,7 @@ RUN apk update && \
 
 VOLUME /cometbft
 WORKDIR /cometbft
-EXPOSE 26656 26657
+EXPOSE 26656 26657 26660
 ENTRYPOINT ["/usr/bin/wrapper.sh"]
 CMD ["node", "--proxy_app", "kvstore"]
 STOPSIGNAL SIGTERM

diff --git a/networks/local/localnode/config-template.toml b/networks/local/localnode/config-template.toml
@@ -1,2 +1,4 @@
 [rpc]
 laddr = "tcp://0.0.0.0:26657"
+[instrumentation]
+prometheus = true
diff --git a/test/README.md b/test/README.md
@@ -1,7 +1,11 @@
 # CometBFT Tests
 
-The unit tests (ie. the `go test` s) can be run with `make test`.
-The integration tests can be run with `make test_integrations`.
+## Unit tests
+The unit tests (ie. `go test`) can be run with `make test` from the root directory of the repository.
+
+## Integration tests
+
+The integration tests can be run with `make test_integrations` from the root directory of the repository.
 
 Running the integrations test will build a docker container with local version of CometBFT
 and run the following tests in docker containers:
@@ -13,6 +17,44 @@ and run the following tests in docker containers:
 - persistence tests
     - crash cometbft at each of many predefined points, restart, and ensure it syncs properly with the app
 
+## End-to-end tests
+
+You can run e2e nightly tests locally by running
+```
+make -j2 docker generator runner && ./build/generator -g 5 -d networks/nightly/ -p && ./run-multiple.sh networks/nightly/*-group*-*.toml
+```
+from the root directory of the repository.
+
+Please refer to the [README.MD](e2e/README.md) in the `e2e` folder for more information.
+
+## `localnet` tests
+During development you might need a local network that runs modified code.
+
+1. Build a linux binary with the code modifications
+```
+make build-linux
+```
+
+2. Run a local network with four nodes
+```
+make localnet-start
+```
+This command will build a `localnet` docker image and run it using Docker Compose.
+
+3. You can send arbitrary transactions on your local network by building and running the `loadtime` application.
+```
+make -C test/loadtime build
+test/loadtime/build/load -c 2 --broadcast-tx-method sync -T 1200 --endpoints ws://127.0.0.1:26657/websocket
+```
+Refer to the [README.md](loadtime/README.md) for more information.
+
+4. You can also monitor your network using Prometheus and Grafana.
+```
+make monitoring-start
+```
+Refer to the [README.md](monitoring/README.md) for more information.
+
+
 ## Fuzzing
 
 [Fuzzing](https://en.wikipedia.org/wiki/Fuzzing) of various system inputs.

diff --git a/test/monitoring/.gitignore b/test/monitoring/.gitignore
@@ -0,0 +1,3 @@
+data-grafana
+data-prometheus
+
diff --git a/test/monitoring/README.md b/test/monitoring/README.md
@@ -0,0 +1,26 @@
+# localnet monitoring
+Prometheus and Grafana server for localnet.
+
+# How to run
+```bash
+docker-compose up -d
+```
+This will start both Prometheus and Grafana.
+
+# Details
+This docker compose (`compose.yml`) creates a local Granafa and Prometheus server.
+It is useful for local debugging and monitoring, especially for `localnet` setups.
+
+Prometheus will connect to the host machine's ports for data. By default it will connect to 26670,26671,26672,26673
+which are the prometheus ports defined for `node0`, `node1`, `node2`, `node3` in `make localnet-start`.
+
+Ports and other settings can be changed in the `config-prometheus/prometheus.yml` file before startup.
+
+You can access the Grafana web interface at `http://localhost:3000` and the Prometheus web interface at `http://localhost:9090`.
+
+The default Grafana username and password is `admin`/`admin`. You will only need it if you want to change something. The
+pre-loaded dashboards can be viewed without a password.
+
+Data from Grafana and Prometheus end up in the `data-grafana` and `data-prometheus` folders on your host machine.
+This allows you to stop and and restart the servers without data loss. The folders are excluded from Git.
+
diff --git a/test/monitoring/compose.yml b/test/monitoring/compose.yml
@@ -0,0 +1,49 @@
+services:
+  prometheus:
+    image: prom/prometheus
+    container_name: prometheus
+    ports:
+      - 127.0.0.1:9090:9090
+    volumes:
+      - ./config-prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
+      - ./data-prometheus:/prometheus
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+  node-exporter:
+    image: prom/node-exporter
+    container_name: node-exporter
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+    command:
+      - '--path.procfs=/host/proc'
+      - '--path.rootfs=/rootfs'
+      - '--path.sysfs=/host/sys'
+      - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
+  grafana:
+    image: grafana/grafana-oss
+    container_name: grafana
+    # if you are running as root then set it to 0
+    # else find the right id with the id -u command
+    user: '501'
+    ports:
+      - 127.0.0.1:3000:3000
+    volumes:
+      - ./data-grafana:/var/lib/grafana
+      - ./config-grafana/provisioning:/etc/grafana/provisioning
+    environment:
+      GF_LOG_LEVEL: error
+      GF_ANALYTICS_ENABLED: false
+      GF_ANALYTICS_REPORTING_ENABLED: false
+      GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: false
+      GF_ANALYTICS_CHECK_FOR_UPDATES: false
+      GF_ANALYTICS_FEEDBACK_LINKS_ENABLED: false
+      GF_SECURITY_DISABLE_GRAVATAR: true
+      GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /etc/grafana/provisioning/dashboards-data/main.json
+      GF_USERS_DEFAULT_THEME: system
+      GF_USERS_EDITORS_CAN_ADMIN: true
+      GF_AUTH_ANONYMOUS_ENABLED: true
+      GF_AUTH_ANONYMOUS_ORG_ROLE: Editor
+      GF_AUTH_BASIC_ENABLED: false
+      GF_NEWS_NEWS_FEED_ENABLED: false
diff --git a/test/monitoring/config-grafana/provisioning/alerting/.placeholder b/test/monitoring/config-grafana/provisioning/alerting/.placeholder
@@ -0,0 +1 @@
+Placeholder so the Grafana log does not complain about missing folders.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		- `[test]` Added monitoring tools and dashboards for local testing with `localnet`. ([\#2107](https://github.com/cometbft/cometbft/issues/2107))
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Placeholder so the Grafana log does not complain about missing folders.