diff --git a/Makefile b/Makefile index c9f42798de..0726b627cc 100644 --- a/Makefile +++ b/Makefile @@ -286,6 +286,17 @@ kill-sei-node: kill-rpc-node: docker ps --filter name=sei-rpc-node --filter status=running -aq | xargs docker kill 2> /dev/null || true +CLUSTER_ENV_VARS = DOCKER_PLATFORM=$(DOCKER_PLATFORM) USERID=$(shell id -u) GROUPID=$(shell id -g) \ + GOCACHE=$(shell go env GOCACHE) NUM_ACCOUNTS=10 \ + INVARIANT_CHECK_INTERVAL=$(INVARIANT_CHECK_INTERVAL) \ + UPGRADE_VERSION_LIST=$(UPGRADE_VERSION_LIST) \ + MOCK_BALANCES=$(MOCK_BALANCES) \ + GIGA_EXECUTOR=$(GIGA_EXECUTOR) \ + GIGA_OCC=$(GIGA_OCC) \ + RECEIPT_BACKEND=$(RECEIPT_BACKEND) \ + AUTOBAHN=$(AUTOBAHN) \ + GIGA_STORAGE=$(GIGA_STORAGE) + # Run a 4-node docker containers docker-cluster-start: docker-cluster-stop build-docker-node @rm -rf $(PROJECT_HOME)/build/generated @@ -297,7 +308,7 @@ docker-cluster-start: docker-cluster-stop build-docker-node else \ DETACH_FLAG=""; \ fi; \ - DOCKER_PLATFORM=$(DOCKER_PLATFORM) USERID=$(shell id -u) GROUPID=$(shell id -g) GOCACHE=$(shell go env GOCACHE) NUM_ACCOUNTS=10 INVARIANT_CHECK_INTERVAL=${INVARIANT_CHECK_INTERVAL} UPGRADE_VERSION_LIST=${UPGRADE_VERSION_LIST} MOCK_BALANCES=${MOCK_BALANCES} GIGA_EXECUTOR=${GIGA_EXECUTOR} GIGA_OCC=${GIGA_OCC} RECEIPT_BACKEND=${RECEIPT_BACKEND} AUTOBAHN=${AUTOBAHN} GIGA_STORAGE=${GIGA_STORAGE} docker compose up $$DETACH_FLAG + $(CLUSTER_ENV_VARS) docker compose up $$DETACH_FLAG .PHONY: localnet-start @@ -318,6 +329,25 @@ docker-cluster-stop: @cd docker && DOCKER_PLATFORM=$(DOCKER_PLATFORM) USERID=$(shell id -u) GROUPID=$(shell id -g) GOCACHE=$(shell go env GOCACHE) docker compose down .PHONY: localnet-stop +# Start 4-node cluster with Prometheus and Grafana monitoring +docker-cluster-start-monitoring: docker-cluster-stop-monitoring build-docker-node + @rm -rf $(PROJECT_HOME)/build/generated + @mkdir -p $(shell go env GOPATH)/pkg/mod + @mkdir -p $(shell go env GOCACHE) + @cd docker && \ + if [ "$${DOCKER_DETACH:-}" = "true" ]; then \ + DETACH_FLAG="-d"; \ + else \ + DETACH_FLAG=""; \ + fi; \ + $(CLUSTER_ENV_VARS) docker compose -f docker-compose.yml -f docker-compose.monitoring.yml up --no-attach grafana --no-attach prometheus $$DETACH_FLAG +.PHONY: docker-cluster-start-monitoring + +# Stop monitoring containers (Prometheus and Grafana) and cluster +docker-cluster-stop-monitoring: + @cd docker && DOCKER_PLATFORM=$(DOCKER_PLATFORM) USERID=$(shell id -u) GROUPID=$(shell id -g) GOCACHE=$(shell go env GOCACHE) docker compose -f docker-compose.yml -f docker-compose.monitoring.yml down +.PHONY: docker-cluster-stop-monitoring + # Run GIGA EVM integration tests with a GIGA-enabled cluster # This starts a fresh cluster with GIGA_EXECUTOR and GIGA_OCC enabled, # runs the EVM GIGA tests, then stops the cluster. diff --git a/docker/README.md b/docker/README.md index cc304e9a8a..7e4ed68c5b 100644 --- a/docker/README.md +++ b/docker/README.md @@ -54,14 +54,26 @@ docker exec -it [container_name] /bin/bash ## Prometheus / Grafana (monitornode) -To run local Prometheus and Grafana containers for metrics visualization: +**Cluster and monitoring together:** from the repo root you can run: + +```sh +make docker-cluster-start-monitoring +``` + +This stops any existing compose stack, rebuilds the node image, starts the four-node local cluster, and brings up the Prometheus and Grafana containers via the monitoring compose overlay, so you do not need to run the scripts below for that flow. To tear down the cluster and monitoring containers together: + +```sh +make docker-cluster-stop-monitoring +``` + +**Scripts only:** to start Prometheus and Grafana by themselves (for example when you are not using the make target above): ```sh ./docker/monitornode/scripts/start-prometheus.sh ./docker/monitornode/scripts/start-grafana.sh ``` -Grafana UI: http://localhost:3000 (login: admin / admin). To stop: +Grafana UI: http://localhost:3000 (login: admin / admin). To stop containers started via the scripts: ```sh ./docker/monitornode/scripts/stop-prometheus.sh diff --git a/docker/docker-compose.monitoring.yml b/docker/docker-compose.monitoring.yml new file mode 100644 index 0000000000..84a3e6f06e --- /dev/null +++ b/docker/docker-compose.monitoring.yml @@ -0,0 +1,33 @@ +services: + prometheus: + container_name: sei-prometheus + image: prom/prometheus:latest + ports: + - "9099:9090" + volumes: + - ./docker_compose_monitoring/prometheus.yaml:/etc/prometheus/prometheus.yml:ro + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --web.enable-lifecycle + networks: + - localnet + + grafana: + container_name: sei-grafana + image: grafana/grafana:latest + ports: + - "3000:3000" + volumes: + - ./docker_compose_monitoring/grafana-datasource.yaml:/etc/grafana/provisioning/datasources/grafana-datasource.yaml:ro + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=admin + - GF_USERS_ALLOW_SIGN_UP=false + depends_on: + - prometheus + networks: + - localnet + +networks: + localnet: \ No newline at end of file diff --git a/docker/docker_compose_monitoring/grafana-datasource.yaml b/docker/docker_compose_monitoring/grafana-datasource.yaml new file mode 100644 index 0000000000..774c799693 --- /dev/null +++ b/docker/docker_compose_monitoring/grafana-datasource.yaml @@ -0,0 +1,8 @@ +apiVersion: 1 +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false \ No newline at end of file diff --git a/docker/docker_compose_monitoring/prometheus.yaml b/docker/docker_compose_monitoring/prometheus.yaml new file mode 100644 index 0000000000..5444c30245 --- /dev/null +++ b/docker/docker_compose_monitoring/prometheus.yaml @@ -0,0 +1,14 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: 'sei-localnet' + metrics_path: '/metrics' + static_configs: + - targets: + - 'sei-node-0:26660' + - 'sei-node-1:26660' + - 'sei-node-2:26660' + - 'sei-node-3:26660' + scrape_interval: 5s