From 358488faf2780db0fc43128c7cf423fa8e9043c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Noah=20H=C3=BCsser?= Date: Wed, 15 Jan 2025 01:05:48 +0100 Subject: [PATCH] test --- .github/workflows/smoketest.yml | 24 +- Cargo.lock | 108 +++++++++ probe-rs-tools/Cargo.toml | 14 ++ .../src/bin/probe-rs/cmd/benchmark.rs | 211 ++++++++++++++---- .../src/bin/probe-rs/util/common_options.rs | 4 + 5 files changed, 313 insertions(+), 48 deletions(-) diff --git a/.github/workflows/smoketest.yml b/.github/workflows/smoketest.yml index 4a29573852..afe0a8cde8 100644 --- a/.github/workflows/smoketest.yml +++ b/.github/workflows/smoketest.yml @@ -46,7 +46,7 @@ jobs: path: target/aarch64-unknown-linux-gnu/release/smoke_tester - name: Build probe-rs - run: cross build --release --target aarch64-unknown-linux-gnu -p probe-rs-tools + run: cross build --release --target aarch64-unknown-linux-gnu -p probe-rs-tools --features ci - uses: actions/upload-artifact@v4 with: @@ -90,3 +90,25 @@ jobs: if [ -n "$EMBEDDED_TEST_RUNNER" ]; then . $EMBEDDED_TEST_RUNNER fi + + benchmark: + runs-on: ["self-hosted", "linux", "ARM64", "${{ matrix.runner }}"] + strategy: + matrix: + runner: ["smoke-tester-2"] + env: + RUN_ID: ${{ github.run_id }} + + needs: [build, test, embedded-test] + + steps: + - uses: actions/download-artifact@v4 + with: + name: probe-rs + + - name: Run benchmark + run: | + chmod +x probe-rs + ./probe-rs benchmark --chip stm32f429zitx --probe "0403:6010:FT929K9H" --address 0x20000000 --word-size 32 --iterations 5 + ./probe-rs benchmark --chip stm32f051r8tx --probe "1366:0101:000778807372" --address 0x20000000 --word-size 32 --iterations 5 + ./probe-rs benchmark --chip stm32l432kcux --probe "0483:374b:0664FF505271754867105124" --address 0x20000000 --word-size 32 --iterations 5 diff --git a/Cargo.lock b/Cargo.lock index 86e2087069..7724033195 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -60,6 +60,21 @@ dependencies = [ "once_cell", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "ansi-parser" version = "0.9.1" @@ -455,8 +470,13 @@ version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", "num-traits", "serde", + "wasm-bindgen", + "windows-targets 0.52.6", ] [[package]] @@ -979,6 +999,15 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "env_filter" version = "0.1.2" @@ -1365,6 +1394,25 @@ dependencies = [ "scroll", ] +[[package]] +name = "h2" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hash32" version = "0.3.1" @@ -1508,6 +1556,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", + "h2", "http", "http-body", "httparse", @@ -1556,6 +1605,29 @@ dependencies = [ "tracing", ] +[[package]] +name = "iana-time-zone" +version = "0.1.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "1.5.0" @@ -2551,6 +2623,7 @@ dependencies = [ "capstone", "cargo-config2", "cargo_metadata", + "chrono", "clap", "clap_complete", "colored", @@ -2577,6 +2650,7 @@ dependencies = [ "probe-rs-mi", "ratatui", "regex", + "reqwest", "rustyline", "sanitize-filename", "schemafy", @@ -2829,8 +2903,11 @@ checksum = "43e734407157c3c2034e0258f5e4473ddb361b1e85f95a66690d67264d7cd1da" dependencies = [ "base64", "bytes", + "encoding_rs", + "futures-channel", "futures-core", "futures-util", + "h2", "http", "http-body", "http-body-util", @@ -2853,6 +2930,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "sync_wrapper", + "system-configuration", "tokio", "tokio-rustls", "tokio-util", @@ -3540,6 +3618,27 @@ dependencies = [ "syn 2.0.89", ] +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags 2.6.0", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tap" version = "1.0.1" @@ -4329,6 +4428,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-registry" version = "0.2.0" diff --git a/probe-rs-tools/Cargo.toml b/probe-rs-tools/Cargo.toml index 16f0cc0436..2dbaeaba40 100644 --- a/probe-rs-tools/Cargo.toml +++ b/probe-rs-tools/Cargo.toml @@ -93,6 +93,20 @@ zip = { version = "2.0.0", default-features = false, features = [ urlencoding = "2" probe-rs-debug = { version = "0.25.0", path = "../probe-rs-debug" } +reqwest = { version = "0.12", optional = true, default-features = false, features = [ + "rustls-tls", + "charset", + "http2", + "macos-system-configuration", + "blocking", + "json", +] } +chrono = { version = "0.4", optional = true, features = ["serde"] } + +[features] +default = ["ci"] +ci = ["dep:reqwest", "dep:chrono"] + [build-dependencies] git-version = "0.3" diff --git a/probe-rs-tools/src/bin/probe-rs/cmd/benchmark.rs b/probe-rs-tools/src/bin/probe-rs/cmd/benchmark.rs index 1a19f3cc58..c29197b7ea 100644 --- a/probe-rs-tools/src/bin/probe-rs/cmd/benchmark.rs +++ b/probe-rs-tools/src/bin/probe-rs/cmd/benchmark.rs @@ -112,11 +112,12 @@ impl Cmd { self.word_size, self.iterations, ); - if let Err(e) = res { + if let Err(error) = res { println!( - "Test failed for speed {} size {} word_size {}bit - {}", - speed, size, self.word_size, e - ) + "Test failed for speed {} size {} word_size {}bit", + speed, size, self.word_size + ); + println!("{:?}", error); } } } @@ -154,55 +155,84 @@ impl Cmd { iterations: usize, ) -> Result<(), anyhow::Error> { let mut probe = common_options.attach_probe(lister)?; + #[cfg(feature = "ci")] + let probe_name = probe.get_name(); + #[cfg(feature = "ci")] + let speed = probe.speed_khz(); let target = common_options.get_target_selector()?; if probe.set_speed(speed).is_ok() { - let mut session = common_options.attach_session(probe, target)?; - let mut test = TestData::new(address, word_size, size); - println!( - "Test: Speed {}, Word size {}bit, Data length {} bytes, Number of iterations {}", + println!("failed to set speed {}", speed); + return Ok(()); + } + + let mut session = common_options.attach_session(probe, target)?; + let mut test = TestData::new(address, word_size, size); + println!( + "Test: Speed {}, Word size {}bit, Data length {} bytes, Number of iterations {}", + speed, + word_size, + test.data_type.size() * size, + iterations + ); + let mut core = session.core(0).context("Failed to attach to core")?; + core.halt(Duration::from_millis(100)) + .context("Halting failed")?; + + let mut read_results = Vec::::with_capacity(iterations); + let mut write_results = Vec::::with_capacity(iterations); + 'inner: for _ in 0..iterations { + let write_throughput = test.block_write(&mut core)?; + let read_throughput = test.block_read(&mut core)?; + let verify_success = test.block_verify(); + if verify_success { + read_results.push(read_throughput); + write_results.push(write_throughput); + } else { + eprintln!("Verification failed."); + break 'inner; + } + } + let read_mean = mean(&read_results).expect("invalid mean"); + let read_std = std_deviation(&read_results).expect("invalid std deviation"); + let write_mean = mean(&write_results).expect("invalid mean"); + let write_std = std_deviation(&write_results).expect("invalid std deviation"); + + println!( + "Results: Read: {read_mean:.2} bytes/s Std Dev {read_std:.2}, Write: {write_mean:.2} bytes/s Std Dev {write_std:.2}", + ); + + #[cfg(feature = "ci")] + { + let selector = common_options.probe(); + let vid = selector.as_ref().map(|s| s.vendor_id).unwrap_or_default(); + let pid = selector.as_ref().map(|s| s.product_id).unwrap_or_default(); + let chip = common_options + .chip() + .unwrap_or_else(|| "".into()); + ci::send_benchmarks( + format!("{} ({vid}, {pid})", probe_name), + chip, speed, word_size, - test.data_type.size() * size, - iterations - ); - let mut core = session.core(0).context("Failed to attach to core")?; - core.halt(Duration::from_millis(100)) - .context("Halting failed")?; - - let mut read_results = Vec::::with_capacity(iterations); - let mut write_results = Vec::::with_capacity(iterations); - 'inner: for _ in 0..iterations { - let write_throughput = test.block_write(&mut core)?; - let read_throughput = test.block_read(&mut core)?; - let verify_success = test.block_verify(); - if verify_success { - read_results.push(read_throughput); - write_results.push(write_throughput); - } else { - eprintln!("Verification failed."); - break 'inner; - } - } + size, + read_mean, + write_mean, + read_std, + write_std, + )? + }; + + if read_results.len() != iterations || write_results.len() != iterations { println!( - "Results: Read: {:.2} bytes/s Std Dev {:.2}, Write: {:.2} bytes/s Std Dev {:.2}", - mean(&read_results).expect("invalid mean"), - std_deviation(&read_results).expect("invalid std deviation"), - mean(&write_results).expect("invalid mean"), - std_deviation(&write_results).expect("invalid std deviation") - ); - if read_results.len() != iterations || write_results.len() != iterations { - println!( - "Warning: {} reads and {} writes successful (out of {} iterations)", - read_results.len(), - write_results.len(), - iterations - ) - } - // Insert another blank line to visually seperate results - println!(); - } else { - println!("failed to set speed {}", speed); + "Warning: {} reads and {} writes successful (out of {} iterations)", + read_results.len(), + write_results.len(), + iterations + ) } + // Insert another blank line to visually seperate results + println!(); + Ok(()) } } @@ -378,3 +408,90 @@ fn std_deviation(data: &[f64]) -> Option { _ => None, } } + +#[cfg(feature = "ci")] +mod ci { + const PERFBOT_URL: &str = "https://perfbot.drop.huesser.dev"; + + pub(crate) fn send_benchmarks( + probe: String, + chip: String, + speed: u32, + word_size: u32, + size: usize, + read_mean: f64, + write_mean: f64, + read_std: f64, + write_std: f64, + ) -> Result<(), anyhow::Error> { + let run_id = std::env::var("RUN_ID") + .ok() + .and_then(|r| r.parse().ok()) + .expect("Please set the RUN_ID env var."); + let api_endpoint = format!("{PERFBOT_URL}/api/measurements/"); + + let client = reqwest::blocking::Client::new(); + client + .post(&api_endpoint) + .json(&CreateMeasurement { + run: run_id, + datetime: chrono::Utc::now().into(), + name: format!("ram-read-{speed}-{word_size}-{size}"), + description: format!("Read {size} bytes of RAM in {word_size} bit chunks."), + unit: "bit/s".into(), + improves: BenchmarkImproves::Up, + probe: probe.clone(), + chip: chip.clone(), + speed_khz: speed as usize, + value: read_mean, + std: read_std, + }) + .send()? + .text()?; + + client + .post(&api_endpoint) + .json(&CreateMeasurement { + run: run_id, + datetime: chrono::Utc::now().into(), + name: format!("ram-write-{speed}-{word_size}-{size}"), + description: format!("Write {size} bytes of RAM in {word_size} bit chunks."), + unit: "bit/s".into(), + improves: BenchmarkImproves::Up, + probe, + chip, + speed_khz: speed as usize, + value: write_mean, + std: write_std, + }) + .send()? + .text()?; + + Ok(()) + } + + #[derive(serde::Serialize, serde::Deserialize, Clone, PartialEq, Debug)] + #[serde(rename_all = "camelCase")] + pub struct CreateMeasurement { + pub run: i64, + pub datetime: chrono::DateTime, + + pub name: String, + pub description: String, + pub unit: String, + pub improves: BenchmarkImproves, + + pub probe: String, + pub chip: String, + pub speed_khz: usize, + + pub value: f64, + pub std: f64, + } + + #[derive(serde::Serialize, serde::Deserialize, Clone, PartialEq, Debug, Copy)] + pub enum BenchmarkImproves { + Up, + Down, + } +} diff --git a/probe-rs-tools/src/bin/probe-rs/util/common_options.rs b/probe-rs-tools/src/bin/probe-rs/util/common_options.rs index 7bd881c4f3..51661c20e1 100644 --- a/probe-rs-tools/src/bin/probe-rs/util/common_options.rs +++ b/probe-rs-tools/src/bin/probe-rs/util/common_options.rs @@ -324,6 +324,10 @@ impl LoadedProbeOptions { self.0.dry_run } + pub(crate) fn probe(&self) -> Option { + self.0.probe.clone() + } + pub(crate) fn chip(&self) -> Option { self.0.chip.clone() }