diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c8562f805..1a707761c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -50,6 +50,7 @@ jobs: matrix: include: - image: swiftlang/swift:nightly-jammy + swift-version: main env: MAX_ALLOCS_ALLOWED_bidi_1k_rpcs_10_requests: 323000 MAX_ALLOCS_ALLOWED_bidi_1k_rpcs_1_request: 161000 @@ -60,6 +61,7 @@ jobs: MAX_ALLOCS_ALLOWED_unary_1k_ping_pong_interceptors_client: 170000 MAX_ALLOCS_ALLOWED_unary_1k_ping_pong_interceptors_server: 170000 - image: swift:5.9-jammy + swift-version: 5.9 env: MAX_ALLOCS_ALLOWED_bidi_1k_rpcs_10_requests: 323000 MAX_ALLOCS_ALLOWED_bidi_1k_rpcs_1_request: 161000 @@ -70,6 +72,7 @@ jobs: MAX_ALLOCS_ALLOWED_unary_1k_ping_pong_interceptors_client: 170000 MAX_ALLOCS_ALLOWED_unary_1k_ping_pong_interceptors_server: 170000 - image: swift:5.8-focal + swift-version: 5.8 env: MAX_ALLOCS_ALLOWED_bidi_1k_rpcs_10_requests: 323000 MAX_ALLOCS_ALLOWED_bidi_1k_rpcs_1_request: 161000 @@ -80,6 +83,7 @@ jobs: MAX_ALLOCS_ALLOWED_unary_1k_ping_pong_interceptors_client: 170000 MAX_ALLOCS_ALLOWED_unary_1k_ping_pong_interceptors_server: 170000 - image: swift:5.7-focal + swift-version: 5.7 env: MAX_ALLOCS_ALLOWED_bidi_1k_rpcs_10_requests: 323000 MAX_ALLOCS_ALLOWED_bidi_1k_rpcs_1_request: 161000 @@ -99,6 +103,13 @@ jobs: run: ./Performance/allocations/test-allocation-counts.sh env: ${{ matrix.env }} timeout-minutes: 20 + - name: Install jemalloc for benchmarking + run: apt update && apt-get install -y libjemalloc-dev + timeout-minutes: 20 + - name: Run Benchmarks + working-directory: ./Performance/Benchmarks + run: swift package benchmark baseline check --check-absolute-path Thresholds/${{ matrix.swift-version }}/ + timeout-minutes: 20 integration-tests: strategy: fail-fast: false diff --git a/.gitignore b/.gitignore index b0c63f998..4c81ad261 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ dev/codegen-tests/**/generated/* /scripts/.swift-format-source/ Package.resolved *.out.* +/Performance/Benchmarks/.benchmarkBaselines/ \ No newline at end of file diff --git a/Performance/Benchmarks/Benchmarks/GRPCSwiftBenchmark/Benchmarks.swift b/Performance/Benchmarks/Benchmarks/GRPCSwiftBenchmark/Benchmarks.swift new file mode 100644 index 000000000..59e30ad49 --- /dev/null +++ b/Performance/Benchmarks/Benchmarks/GRPCSwiftBenchmark/Benchmarks.swift @@ -0,0 +1,37 @@ +/* + * Copyright 2023, gRPC Authors All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import Benchmark +import Foundation + +let benchmarks = { + Benchmark.defaultConfiguration = .init( + metrics: [ + .mallocCountTotal, + .syscalls, + .readSyscalls, + .writeSyscalls, + .memoryLeaked, + .retainCount, + .releaseCount, + ] + ) + + // async code is currently still quite flaky in the number of retain/release it does so we don't measure them today + var configWithoutRetainRelease = Benchmark.defaultConfiguration + configWithoutRetainRelease.metrics.removeAll(where: { $0 == .retainCount || $0 == .releaseCount }) + + // Add Benchmarks here +} diff --git a/Performance/Benchmarks/Package.swift b/Performance/Benchmarks/Package.swift new file mode 100644 index 000000000..74d324150 --- /dev/null +++ b/Performance/Benchmarks/Package.swift @@ -0,0 +1,41 @@ +// swift-tools-version: 5.7 +/* + * Copyright 2023, gRPC Authors All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import PackageDescription + +let package = Package( + name: "benchmarks", + platforms: [ + .macOS(.v13), + ], + dependencies: [ + .package(path: "../../"), + .package(url: "https://github.com/ordo-one/package-benchmark", from: "1.11.2") + ], + targets: [ + .executableTarget( + name: "GRPCSwiftBenchmark", + dependencies: [ + .product(name: "Benchmark", package: "package-benchmark"), + .product(name: "GRPC", package: "grpc-swift") + ], + path: "Benchmarks/GRPCSwiftBenchmark", + plugins: [ + .plugin(name: "BenchmarkPlugin", package: "package-benchmark") + ] + ), + ] +) diff --git a/README.md b/README.md index 47fa297e8..91155b7e8 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,25 @@ The `docs` directory contains documentation, including: - How to configure keepalive in [`docs/keepalive.md`][docs-keepalive] - Support for Apple Platforms and NIO Transport Services in [`docs/apple-platforms.md`][docs-apple] + +## Benchmarks + +Benchmarks for `grpc-swift` are in a separate Swift Package in the `Performance/Benchmarks` subfolder of this repository. +They use the [`package-benchmark`](https://github.com/ordo-one/package-benchmark) plugin. +Benchmarks depends on the [`jemalloc`](https://jemalloc.net) memory allocation library, which is used by `package-benchmark` to capture memory allocation statistics. +An installation guide can be found in the [Getting Started article](https://swiftpackageindex.com/ordo-one/package-benchmark/documentation/benchmark/gettingstarted#Installing-Prerequisites-and-Platform-Support) of `package-benchmark`. +Afterwards you can run the benchmarks from CLI by going to the `Performance/Benchmarks` subfolder (e.g. `cd Performance/Benchmarks`) and invoking: +``` +swift package benchmark +``` + +Profiling benchmarks or building the benchmarks in release mode in Xcode with `jemalloc` is currently not supported and requires disabling `jemalloc`. +Make sure Xcode is closed and then open it from the CLI with the `BENCHMARK_DISABLE_JEMALLOC=true` environment variable set e.g.: +``` +BENCHMARK_DISABLE_JEMALLOC=true xed . +``` + +For more information please refer to `swift package benchmark --help` or the [documentation of `package-benchmark`](https://swiftpackageindex.com/ordo-one/package-benchmark/documentation/benchmark). ## Security