//! Side-by-side benchmark comparing polyfill-rs against Polymarket's //! `rs-clob-client-v2`. //! //! Run with: //! `cargo run --release --example official_client_side_by_side_benchmark --features official-client-benchmark` //! //! Optional env vars: //! - `POLYMARKET_BENCH_HOST` defaults to `https://clob.polymarket.com` //! - `POLYMARKET_BENCH_ITERATIONS` defaults to `20` //! - `POLYMARKET_BENCH_WARMUPS` defaults to `3` //! - `POLYMARKET_BENCH_PARSE_ITERATIONS` defaults to `200` //! - `POLYMARKET_BENCH_DELAY_MS` defaults to `100` //! - `POLYMARKET_BENCH_KEEPALIVE` defaults to `true` use std::hint::black_box; use std::time::{Duration, Instant}; use polyfill_rs::ClobClient; use polymarket_client_sdk_v2::clob::types::response::{ Page as OfficialPage, SimplifiedMarketResponse as OfficialSimplifiedMarketResponse, }; use polymarket_client_sdk_v2::clob::{Client as OfficialClient, Config as OfficialConfig}; type BenchResult = Result>; const INITIAL_CURSOR: &str = "MA=="; const OFFICIAL_SDK_REV: &str = "8ba5008733c3c03e92041eef8b1cb8495dbed718"; #[derive(Debug, Clone, Copy)] struct Sample { elapsed: Duration, count: usize, } struct RawHttpVariant { name: &'static str, client: reqwest::Client, } #[derive(Debug, Clone, Copy)] struct Stats { mean_ms: f64, std_dev_ms: f64, min_ms: f64, max_ms: f64, p50_ms: f64, p95_ms: f64, p99_ms: f64, } fn env_usize(name: &str, default: usize) -> usize { std::env::var(name) .ok() .and_then(|value| value.parse().ok()) .unwrap_or(default) } fn env_u64(name: &str, default: u64) -> u64 { std::env::var(name) .ok() .and_then(|value| value.parse().ok()) .unwrap_or(default) } fn env_bool(name: &str, default: bool) -> bool { std::env::var(name) .ok() .and_then(|value| value.parse().ok()) .unwrap_or(default) } fn percentile(sorted: &[f64], percentile: f64) -> f64 { let rank = (percentile * sorted.len() as f64).ceil() as usize; let index = rank.saturating_sub(1).min(sorted.len() - 1); sorted[index] } fn calc_stats(times: &[Duration]) -> Option { if times.is_empty() { return None; } let mut values: Vec = times .iter() .map(|duration| duration.as_micros() as f64 / 1000.0) .collect(); values.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); let mean_ms = values.iter().sum::() / values.len() as f64; let variance = values .iter() .map(|value| (value - mean_ms).powi(2)) .sum::() / values.len() as f64; Some(Stats { mean_ms, std_dev_ms: variance.sqrt(), min_ms: values[0], max_ms: values[values.len() - 1], p50_ms: percentile(&values, 0.50), p95_ms: percentile(&values, 0.95), p99_ms: percentile(&values, 0.99), }) } fn print_stats(name: &str, stats: Option, successes: usize, attempts: usize) { println!("{name}:"); match stats { Some(stats) => { println!( " p50/p95/p99: {:.1} / {:.1} / {:.1} ms", stats.p50_ms, stats.p95_ms, stats.p99_ms ); println!( " mean: {:.1} ms +/- {:.1} ms", stats.mean_ms, stats.std_dev_ms ); println!( " range: {:.1} - {:.1} ms", stats.min_ms, stats.max_ms ); println!(" success: {successes}/{attempts}"); }, None => println!(" success: 0/{attempts}"), } } fn print_single(name: &str, sample: &Result) { match sample { Ok(sample) => println!( "{name}: {:.1} ms ({} items)", sample.elapsed.as_micros() as f64 / 1000.0, sample.count ), Err(error) => println!("{name}: ERROR - {error}"), } } async fn time_polyfill_typed(client: &ClobClient) -> Result { let start = Instant::now(); let page = client .get_simplified_markets(Some(INITIAL_CURSOR)) .await .map_err(|error| error.to_string())?; Ok(Sample { elapsed: start.elapsed(), count: page.data.len(), }) } async fn time_official_typed(client: &OfficialClient) -> Result { let start = Instant::now(); let page = client .simplified_markets(Some(INITIAL_CURSOR.to_string())) .await .map_err(|error| error.to_string())?; Ok(Sample { elapsed: start.elapsed(), count: page.data.len(), }) } async fn time_polyfill_cold(host: &str) -> Result { let start = Instant::now(); let client = ClobClient::new(host); let page = client .get_simplified_markets(Some(INITIAL_CURSOR)) .await .map_err(|error| error.to_string())?; Ok(Sample { elapsed: start.elapsed(), count: page.data.len(), }) } async fn time_official_cold(host: &str) -> Result { let start = Instant::now(); let client = OfficialClient::new(host, OfficialConfig::default()).map_err(|error| error.to_string())?; let page = client .simplified_markets(Some(INITIAL_CURSOR.to_string())) .await .map_err(|error| error.to_string())?; Ok(Sample { elapsed: start.elapsed(), count: page.data.len(), }) } async fn time_reqwest_raw(client: &reqwest::Client, url: &str) -> Result { let start = Instant::now(); let bytes = client .get(url) .send() .await .map_err(|error| error.to_string())? .bytes() .await .map_err(|error| error.to_string())?; Ok(Sample { elapsed: start.elapsed(), count: bytes.len(), }) } async fn run_typed_pairs( polyfill_client: &ClobClient, official_client: &OfficialClient, iterations: usize, delay: Duration, ) -> (Vec, Vec) { let mut polyfill_times = Vec::with_capacity(iterations); let mut official_times = Vec::with_capacity(iterations); for i in 1..=iterations { let polyfill_first = i % 2 == 1; let (polyfill_result, official_result) = if polyfill_first { let polyfill_result = time_polyfill_typed(polyfill_client).await; tokio::time::sleep(delay).await; let official_result = time_official_typed(official_client).await; (polyfill_result, official_result) } else { let official_result = time_official_typed(official_client).await; tokio::time::sleep(delay).await; let polyfill_result = time_polyfill_typed(polyfill_client).await; (polyfill_result, official_result) }; if let Ok(sample) = polyfill_result { polyfill_times.push(sample.elapsed); } if let Ok(sample) = official_result { official_times.push(sample.elapsed); } tokio::time::sleep(delay).await; } (polyfill_times, official_times) } async fn run_raw_warmups(variants: &[RawHttpVariant], url: &str, delay: Duration) { for variant in variants { let _ = time_reqwest_raw(&variant.client, url).await; tokio::time::sleep(delay).await; } } async fn run_raw_matrix( variants: &[RawHttpVariant], url: &str, iterations: usize, delay: Duration, ) -> Vec> { let mut samples = vec![Vec::with_capacity(iterations); variants.len()]; for iteration in 0..iterations { for offset in 0..variants.len() { let idx = (iteration + offset) % variants.len(); if let Ok(sample) = time_reqwest_raw(&variants[idx].client, url).await { samples[idx].push(sample.elapsed); } tokio::time::sleep(delay).await; } } samples } async fn run_typed_warmups( polyfill_client: &ClobClient, official_client: &OfficialClient, warmups: usize, delay: Duration, ) { for _ in 0..warmups { let _ = time_polyfill_typed(polyfill_client).await; tokio::time::sleep(delay).await; let _ = time_official_typed(official_client).await; tokio::time::sleep(delay).await; } } fn official_headers() -> reqwest::header::HeaderMap { let mut headers = reqwest::header::HeaderMap::new(); headers.insert( reqwest::header::USER_AGENT, reqwest::header::HeaderValue::from_static("rs_clob_client"), ); headers.insert( reqwest::header::ACCEPT, reqwest::header::HeaderValue::from_static("*/*"), ); headers.insert( reqwest::header::CONNECTION, reqwest::header::HeaderValue::from_static("keep-alive"), ); headers.insert( reqwest::header::CONTENT_TYPE, reqwest::header::HeaderValue::from_static("application/json"), ); headers } fn polyfill_headers() -> reqwest::header::HeaderMap { let mut headers = reqwest::header::HeaderMap::new(); headers.insert( reqwest::header::USER_AGENT, reqwest::header::HeaderValue::from_static(concat!( "polyfill-rs/", env!("CARGO_PKG_VERSION") )), ); headers.insert( reqwest::header::ACCEPT, reqwest::header::HeaderValue::from_static("*/*"), ); headers.insert( reqwest::header::CONTENT_TYPE, reqwest::header::HeaderValue::from_static("application/json"), ); headers } fn polyfill_tuned_builder() -> reqwest::ClientBuilder { reqwest::Client::builder() .no_proxy() .http2_adaptive_window(true) .http2_initial_stream_window_size(512 * 1024) .tcp_nodelay(true) .pool_max_idle_per_host(10) .pool_idle_timeout(Duration::from_secs(90)) } fn polyfill_light_builder() -> reqwest::ClientBuilder { reqwest::Client::builder() .no_proxy() .tcp_nodelay(true) .pool_max_idle_per_host(10) .pool_idle_timeout(Duration::from_secs(90)) } fn build_raw_http_variants( polyfill_client: &ClobClient, ) -> Result, reqwest::Error> { Ok(vec![ RawHttpVariant { name: "polyfill-rs actual HTTP", client: polyfill_client.http_client.clone(), }, RawHttpVariant { name: "reqwest default", client: reqwest::Client::builder().build()?, }, RawHttpVariant { name: "rs-clob-client-v2 headers", client: reqwest::Client::builder() .default_headers(official_headers()) .build()?, }, RawHttpVariant { name: "polyfill tuned + official headers", client: polyfill_tuned_builder() .default_headers(official_headers()) .build()?, }, RawHttpVariant { name: "polyfill tuned + polyfill headers", client: polyfill_tuned_builder() .default_headers(polyfill_headers()) .build()?, }, RawHttpVariant { name: "polyfill light no h2 tuning", client: polyfill_light_builder() .default_headers(polyfill_headers()) .build()?, }, ]) } fn parse_polyfill_once(bytes: &[u8]) -> BenchResult { let start = Instant::now(); let page: polyfill_rs::types::SimplifiedMarketsResponse = polyfill_rs::decode::fast_parse::parse_json_fast_owned(bytes)?; black_box(page); Ok(start.elapsed()) } fn parse_official_direct_once(bytes: &[u8]) -> BenchResult { let start = Instant::now(); let page: OfficialPage = serde_json::from_slice(bytes)?; black_box(page); Ok(start.elapsed()) } fn parse_official_helper_once(bytes: &[u8]) -> BenchResult { let start = Instant::now(); let value: serde_json::Value = serde_json::from_slice(bytes)?; let page: Option> = serde_json::from_value(value)?; black_box(page); Ok(start.elapsed()) } fn run_parse_samples( bytes: &[u8], iterations: usize, mut parse_once: F, ) -> BenchResult> where F: FnMut(&[u8]) -> BenchResult, { let mut times = Vec::with_capacity(iterations); for _ in 0..iterations { times.push(parse_once(bytes)?); } Ok(times) } #[tokio::main] async fn main() -> Result<(), Box> { let host = std::env::var("POLYMARKET_BENCH_HOST") .unwrap_or_else(|_| "https://clob.polymarket.com".to_string()); let url = format!("{host}/simplified-markets?next_cursor={INITIAL_CURSOR}"); let iterations = env_usize("POLYMARKET_BENCH_ITERATIONS", 20); let warmups = env_usize("POLYMARKET_BENCH_WARMUPS", 3); let parse_iterations = env_usize("POLYMARKET_BENCH_PARSE_ITERATIONS", 200); let delay = Duration::from_millis(env_u64("POLYMARKET_BENCH_DELAY_MS", 100)); let keepalive = env_bool("POLYMARKET_BENCH_KEEPALIVE", true); println!("======================================================="); println!(" polyfill-rs vs rs-clob-client-v2 benchmark"); println!("======================================================="); println!("Endpoint: {url}"); println!("Iterations: {iterations}"); println!("Warmups: {warmups}"); println!("Parse iterations: {parse_iterations}"); println!("Delay: {} ms", delay.as_millis()); println!("polyfill-rs background keepalive: {keepalive}"); println!("rs-clob-client-v2 rev: {OFFICIAL_SDK_REV}"); println!(); println!("Cold start: client construction + first typed request"); println!("-------------------------------------------------------"); let polyfill_cold = time_polyfill_cold(&host).await; tokio::time::sleep(delay).await; let official_cold = time_official_cold(&host).await; print_single("polyfill-rs", &polyfill_cold); print_single("rs-clob-client-v2", &official_cold); println!(); let polyfill_client = ClobClient::new(&host); let official_client = OfficialClient::new(&host, OfficialConfig::default())?; let raw_http_variants = build_raw_http_variants(&polyfill_client)?; if keepalive { polyfill_client .start_keepalive(Duration::from_secs(30)) .await; } run_typed_warmups(&polyfill_client, &official_client, warmups, delay).await; println!("Warm connection: first typed request after warmup"); println!("-------------------------------------------------------"); let polyfill_warm = time_polyfill_typed(&polyfill_client).await; tokio::time::sleep(delay).await; let official_warm = time_official_typed(&official_client).await; print_single("polyfill-rs", &polyfill_warm); print_single("rs-clob-client-v2", &official_warm); println!(); println!("Steady state: typed client total time"); println!("-------------------------------------------------------"); let (polyfill_typed_times, official_typed_times) = run_typed_pairs(&polyfill_client, &official_client, iterations, delay).await; print_stats( "polyfill-rs", calc_stats(&polyfill_typed_times), polyfill_typed_times.len(), iterations, ); println!(); print_stats( "rs-clob-client-v2", calc_stats(&official_typed_times), official_typed_times.len(), iterations, ); println!(); println!("Steady state: network-only byte fetch transport matrix"); println!("-------------------------------------------------------"); run_raw_warmups(&raw_http_variants, &url, delay).await; let raw_samples = run_raw_matrix(&raw_http_variants, &url, iterations, delay).await; for (variant, samples) in raw_http_variants.iter().zip(raw_samples.iter()) { print_stats(variant.name, calc_stats(samples), samples.len(), iterations); println!(); } println!(); println!("CPU-only parse from cached payload"); println!("-------------------------------------------------------"); let payload = polyfill_client .http_client .get(&url) .send() .await? .bytes() .await? .to_vec(); println!("payload: {} bytes", payload.len()); let polyfill_parse_times = run_parse_samples(&payload, parse_iterations, parse_polyfill_once)?; let official_direct_parse_times = run_parse_samples(&payload, parse_iterations, parse_official_direct_once)?; let official_helper_parse_times = run_parse_samples(&payload, parse_iterations, parse_official_helper_once)?; print_stats( "polyfill-rs typed parse", calc_stats(&polyfill_parse_times), polyfill_parse_times.len(), parse_iterations, ); println!(); print_stats( "rs-clob-client-v2 direct typed parse", calc_stats(&official_direct_parse_times), official_direct_parse_times.len(), parse_iterations, ); println!(); print_stats( "rs-clob-client-v2 request-helper parse", calc_stats(&official_helper_parse_times), official_helper_parse_times.len(), parse_iterations, ); if keepalive { polyfill_client.stop_keepalive().await; } Ok(()) }