add Makefile and run benchmarks as part of PRs (sourcegraph#130)

This: - Adds a Makefile with `make test` and `make lint` and `make bench` - Adds a new workflow for `main` that runs benchmarks and saves them - Adds a new workflow for PRs that runs benchmarks and compares against `main`. When a benchmark regression occurs, it should fail the workflow
uk0 · Jan 21, 2024 · 1124809 · 1124809
1 parent e454401
commit 1124809
Show file tree

Hide file tree

Showing 3 changed files with 156 additions and 0 deletions.
diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
@@ -0,0 +1,80 @@
+name: Benchmark
+
+on:
+  pull_request:
+    branches: [ "main" ]
+
+permissions:
+  contents: read
+
+jobs:
+  go-bench:
+    strategy:
+      matrix:
+        go-version: [ '1.19', 'stable' ]
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # to be able to retrieve the last commit in main
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ matrix.go-version }}
+
+      - name: Run benchmark and store the output to a file
+        run: |
+          set -o pipefail
+          make bench | tee ${{ github.sha }}_bench_output.txt
+
+      - name: Get CPU information
+        uses: kenchan0130/actions-system-info@v1.2.1
+        id: system-info
+
+      - name: Get Main branch SHA
+        id: get-main-branch-sha
+        run: |
+          SHA=$(git rev-parse origin/main)
+          echo "sha=$SHA" >> $GITHUB_OUTPUT
+
+      - name: Get benchmark JSON from main branch
+        id: cache
+        uses: actions/cache/restore@v3
+        with:
+          path: ./cache/benchmark-data.json
+          key: ${{ steps.get-main-branch-sha.outputs.sha }}-${{ runner.os }}-${{ steps.system-info.outputs.cpu-model }}-go-benchmark
+
+      - name: Compare benchmarks with Main
+        uses: benchmark-action/github-action-benchmark@v1
+        if: steps.cache.outputs.cache-hit == 'true'
+        with:
+          # What benchmark tool the output.txt came from
+          tool: 'go'
+          # Where the output from the benchmark tool is stored
+          output-file-path: ${{ github.sha }}_bench_output.txt
+          # Where the benchmarks in main are (to compare)
+          external-data-json-path: ./cache/benchmark-data.json
+          # Do not save the data
+          save-data-file: false
+          # Workflow will fail when an alert happens
+          fail-on-alert: true
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          # Enable Job Summary for PRs
+          summary-always: true
+
+      - name: Run benchmarks but don't compare to Main branch
+        uses: benchmark-action/github-action-benchmark@v1
+        if: steps.cache.outputs.cache-hit != 'true'
+        with:
+          # What benchmark tool the output.txt came from
+          tool: 'go'
+          # Where the output from the benchmark tool is stored
+          output-file-path: ${{ github.sha }}_bench_output.txt
+          # Write benchmarks to this file, do not publish to GitHub Pages
+          save-data-file: false
+          external-data-json-path: ./cache/benchmark-data.json
+          # Enable Job Summary for PRs
+          summary-always: true
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -0,0 +1,52 @@
+name: Main
+on:
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: read
+
+jobs:
+  go-bench:
+    strategy:
+      matrix:
+        go-version: [ '1.19', 'stable' ]
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ${{ matrix.go-version }}
+
+      - name: Run benchmark and store the output to a file
+        run: |
+          set -o pipefail
+          make bench | tee bench_output.txt
+
+      - name: Get benchmark as JSON
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          # What benchmark tool the output.txt came from
+          tool: 'go'
+          # Where the output from the benchmark tool is stored
+          output-file-path: bench_output.txt
+          # Write benchmarks to this file
+          external-data-json-path: ./cache/benchmark-data.json
+          # Workflow will fail when an alert happens
+          fail-on-alert: true
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Get CPU information
+        uses: kenchan0130/actions-system-info@v1.2.1
+        id: system-info
+
+      - name: Save benchmark JSON to cache
+        uses: actions/cache/save@v3
+        with:
+          path: ./cache/benchmark-data.json
+          # Save with commit hash to avoid "cache already exists"
+          # Save with OS & CPU info to prevent comparing against results from different CPUs
+          key: ${{ github.sha }}-${{ runner.os }}-${{ steps.system-info.outputs.cpu-model }}-go-benchmark
diff --git a/Makefile b/Makefile
@@ -0,0 +1,24 @@
+.DEFAULT_GOAL := help
+
+GO_BIN ?= $(shell go env GOPATH)/bin
+
+.PHONY: help
+help:
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+$(GO_BIN)/golangci-lint:
+	@echo "==> Installing golangci-lint within "${GO_BIN}""
+	@go install -v github.com/golangci/golangci-lint/cmd/golangci-lint@latest
+
+.PHONY: lint
+lint: $(GO_BIN)/golangci-lint ## Run linting on Go files
+	@echo "==> Linting Go source files"
+	@golangci-lint run -v --fix -c .golangci.yml ./...
+
+.PHONY: test
+test: ## Run tests
+	go test -race -v ./... -coverprofile ./coverage.txt
+
+.PHONY: bench
+bench: ## Run benchmarks. See https://pkg.go.dev/cmd/go#hdr-Testing_flags
+	go test ./... -bench . -benchtime 5s -timeout 0 -run=XXX -cpu 1 -benchmem