Integrate CLMM accounts and optimize ordered buffers

This commit is contained in:
0xfnzero
2026-05-26 00:03:22 +08:00
parent 983aedf27c
commit 6149ae2987
7 changed files with 357 additions and 49 deletions
+3 -3
View File
@@ -1,6 +1,6 @@
[package]
name = "solana-streamer-sdk"
version = "1.4.13"
version = "1.4.14"
edition = "2021"
authors = ["William <byteblock6@gmail.com>", "sgxiang <sgxiang@gmail.com>", "wei <1415121722@qq.com>"]
repository = "https://github.com/0xfnzero/solana-streamer"
@@ -15,13 +15,13 @@ crate-type = ["cdylib", "rlib"]
[features]
default = ["sdk-parse-borsh"]
sdk-parse-borsh = ["sol-parser-sdk/parse-borsh"]
# If both parser backend features are enabled, sol-parser-sdk 0.4.18+ uses zero-copy.
# If both parser backend features are enabled, sol-parser-sdk 0.4.19+ uses zero-copy.
sdk-parse-zero-copy = ["sol-parser-sdk/parse-zero-copy"]
sdk-perf-stats = ["sol-parser-sdk/perf-stats"]
sdk-ultra-perf = ["sol-parser-sdk/ultra-perf"]
[dependencies]
sol-parser-sdk = { version = "0.4.18", default-features = false }
sol-parser-sdk = { version = "0.4.19", default-features = false }
solana-sdk = "3.0.0"
solana-client = "3.1.12"
solana-transaction-status = "3.1.12"
+9 -5
View File
@@ -123,30 +123,34 @@ Add the dependency to your `Cargo.toml`:
```toml
# Add to your Cargo.toml
solana-streamer-sdk = { path = "./solana-streamer", version = "1.4.13" }
solana-streamer-sdk = { path = "./solana-streamer", version = "1.4.14" }
```
### Use crates.io
```toml
# Add to your Cargo.toml
solana-streamer-sdk = "1.4.13"
solana-streamer-sdk = "1.4.14"
```
Parser backend features:
```toml
# Default: sol-parser-sdk parse-borsh backend
solana-streamer-sdk = "1.4.13"
solana-streamer-sdk = "1.4.14"
# Zero-copy parser backend for latency-sensitive bots
solana-streamer-sdk = { version = "1.4.13", default-features = false, features = ["sdk-parse-zero-copy"] }
solana-streamer-sdk = { version = "1.4.14", default-features = false, features = ["sdk-parse-zero-copy"] }
```
If both `sdk-parse-borsh` and `sdk-parse-zero-copy` are enabled, `sol-parser-sdk 0.4.18+` uses the zero-copy backend.
If both `sdk-parse-borsh` and `sdk-parse-zero-copy` are enabled, `sol-parser-sdk 0.4.19+` uses the zero-copy backend.
## 🔄 Migration Guide
### Upgrading to v1.4.14
Version 1.4.14 uses `sol-parser-sdk 0.4.19`, wires Raydium CLMM account parsing through the SDK bridge, and reduces ordered-buffer allocation/move overhead in low-latency delivery paths.
### Upgrading to v1.4.13
Version 1.4.13 uses `sol-parser-sdk 0.4.18` and updates Raydium CLMM integration to the official upgraded IDL: current log-side event discriminators, official Swap/Liquidity/Create/Collect layouts, limit-order events, dynamic-fee related events, and the reshaped PoolState/TickState account structs.
+9 -5
View File
@@ -122,30 +122,34 @@ git clone https://github.com/0xfnzero/solana-streamer
```toml
# 添加到您的 Cargo.toml
solana-streamer-sdk = { path = "./solana-streamer", version = "1.4.13" }
solana-streamer-sdk = { path = "./solana-streamer", version = "1.4.14" }
```
### 使用 crates.io
```toml
# 添加到您的 Cargo.toml
solana-streamer-sdk = "1.4.13"
solana-streamer-sdk = "1.4.14"
```
解析后端 feature
```toml
# 默认:sol-parser-sdk parse-borsh 后端
solana-streamer-sdk = "1.4.13"
solana-streamer-sdk = "1.4.14"
# 面向低延迟 Bot 的 zero-copy 解析后端
solana-streamer-sdk = { version = "1.4.13", default-features = false, features = ["sdk-parse-zero-copy"] }
solana-streamer-sdk = { version = "1.4.14", default-features = false, features = ["sdk-parse-zero-copy"] }
```
如果同时启用 `sdk-parse-borsh``sdk-parse-zero-copy``sol-parser-sdk 0.4.18+` 会优先使用 zero-copy 后端。
如果同时启用 `sdk-parse-borsh``sdk-parse-zero-copy``sol-parser-sdk 0.4.19+` 会优先使用 zero-copy 后端。
## 🔄 迁移指南
### 升级到 v1.4.14
v1.4.14 使用 `sol-parser-sdk 0.4.19`,打通 Raydium CLMM 账户解析的 SDK bridge,并降低有序缓冲低延迟投递路径中的分配和搬移开销。
### 升级到 v1.4.13
v1.4.13 使用 `sol-parser-sdk 0.4.18`,并按 Raydium CLMM 官方升级后 IDL 更新集成:当前 log-side event discriminator、官方 Swap/Liquidity/Create/Collect 事件布局、限价单事件、dynamic fee 相关事件,以及重塑后的 PoolState/TickState 账户结构。
+93 -28
View File
@@ -38,15 +38,13 @@ impl SlotBuffer {
}
pub fn flush_before(&mut self, current_slot: u64) -> Vec<DexEvent> {
let slots_to_flush: Vec<u64> =
self.slots.keys().filter(|&&s| s < current_slot).copied().collect();
let keep_slots = self.slots.split_off(&current_slot);
let flush_slots = std::mem::replace(&mut self.slots, keep_slots);
let mut result = Vec::with_capacity(slots_to_flush.len() * 4);
for slot in slots_to_flush {
if let Some(mut events) = self.slots.remove(&slot) {
events.sort_unstable_by_key(|(idx, _)| *idx);
result.extend(events.into_iter().map(|(_, event)| event));
}
let mut result = Vec::with_capacity(flush_slots.values().map(Vec::len).sum());
for (_slot, mut events) in flush_slots {
events.sort_unstable_by_key(|(idx, _)| *idx);
result.extend(events.into_iter().map(|(_, event)| event));
}
if !result.is_empty() {
@@ -56,14 +54,12 @@ impl SlotBuffer {
}
pub fn flush_all(&mut self) -> Vec<DexEvent> {
let all_slots: Vec<u64> = self.slots.keys().copied().collect();
let mut result = Vec::with_capacity(all_slots.len() * 4);
let all_slots = std::mem::take(&mut self.slots);
let mut result = Vec::with_capacity(all_slots.values().map(Vec::len).sum());
for slot in all_slots {
if let Some(mut events) = self.slots.remove(&slot) {
events.sort_unstable_by_key(|(idx, _)| *idx);
result.extend(events.into_iter().map(|(_, event)| event));
}
for (_slot, mut events) in all_slots {
events.sort_unstable_by_key(|(idx, _)| *idx);
result.extend(events.into_iter().map(|(_, event)| event));
}
if !result.is_empty() {
@@ -83,12 +79,12 @@ impl SlotBuffer {
let mut result = Vec::new();
if slot > self.current_slot && self.current_slot > 0 {
let old_slots: Vec<u64> = self.slots.keys().filter(|&&s| s < slot).copied().collect();
for old_slot in old_slots {
if let Some(mut events) = self.slots.remove(&old_slot) {
events.sort_unstable_by_key(|(idx, _)| *idx);
result.extend(events.into_iter().map(|(_, event)| event));
}
let keep_slots = self.slots.split_off(&slot);
let flush_slots = std::mem::replace(&mut self.slots, keep_slots);
result.reserve(flush_slots.values().map(Vec::len).sum());
for (old_slot, mut events) in flush_slots {
events.sort_unstable_by_key(|(idx, _)| *idx);
result.extend(events.into_iter().map(|(_, event)| event));
self.streaming_watermarks.remove(&old_slot);
}
}
@@ -103,12 +99,23 @@ impl SlotBuffer {
result.push(event);
let mut watermark = next_expected + 1;
if let Some(buffered) = self.slots.get_mut(&slot) {
let remove_empty_slot = if let Some(buffered) = self.slots.get_mut(&slot) {
buffered.sort_unstable_by_key(|(idx, _)| *idx);
while let Some(pos) = buffered.iter().position(|(idx, _)| *idx == watermark) {
result.push(buffered.remove(pos).1);
let mut ready_count = 0;
while ready_count < buffered.len() && buffered[ready_count].0 == watermark {
watermark += 1;
ready_count += 1;
}
result.reserve(ready_count);
for (_, event) in buffered.drain(..ready_count) {
result.push(event);
}
buffered.is_empty()
} else {
false
};
if remove_empty_slot {
self.slots.remove(&slot);
}
self.streaming_watermarks.insert(slot, watermark);
} else if tx_index > next_expected {
@@ -125,8 +132,9 @@ impl SlotBuffer {
}
pub fn flush_streaming_timeout(&mut self) -> Vec<DexEvent> {
let mut result = Vec::new();
for (slot, mut events) in std::mem::take(&mut self.slots) {
let flush_slots = std::mem::take(&mut self.slots);
let mut result = Vec::with_capacity(flush_slots.values().map(Vec::len).sum());
for (slot, mut events) in flush_slots {
events.sort_unstable_by_key(|(idx, _)| *idx);
result.extend(events.into_iter().map(|(_, event)| event));
self.streaming_watermarks.remove(&slot);
@@ -172,8 +180,8 @@ impl MicroBatchBuffer {
}
self.events.sort_unstable_by_key(|(slot, tx_index, _)| (*slot, *tx_index));
let result =
std::mem::take(&mut self.events).into_iter().map(|(_, _, event)| event).collect();
let mut result = Vec::with_capacity(self.events.len());
result.extend(self.events.drain(..).map(|(_, _, event)| event));
self.window_start_us = 0;
result
}
@@ -194,3 +202,60 @@ impl Default for MicroBatchBuffer {
Self::new()
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::streaming::event_parser::protocols::BlockMetaEvent;
fn event(id: u64) -> DexEvent {
DexEvent::BlockMetaEvent(BlockMetaEvent::new(id, id.to_string(), 0, 0))
}
fn ids(events: Vec<DexEvent>) -> Vec<u64> {
events
.into_iter()
.map(|event| match event {
DexEvent::BlockMetaEvent(event) => event.slot,
_ => unreachable!("test only creates block meta events"),
})
.collect()
}
#[test]
fn flush_before_keeps_newer_slots_and_sorts_flushed_events() {
let mut buffer = SlotBuffer::new();
buffer.push(7, 2, event(72));
buffer.push(5, 1, event(51));
buffer.push(5, 0, event(50));
assert_eq!(ids(buffer.flush_before(6)), vec![50, 51]);
assert_eq!(ids(buffer.flush_all()), vec![72]);
}
#[test]
fn streaming_order_drains_only_contiguous_ready_prefix() {
let mut buffer = SlotBuffer::new();
assert!(buffer.push_streaming(10, 3, event(103)).is_empty());
assert!(buffer.push_streaming(10, 1, event(101)).is_empty());
assert_eq!(ids(buffer.push_streaming(10, 0, event(100))), vec![100, 101]);
assert_eq!(ids(buffer.push_streaming(10, 2, event(102))), vec![102, 103]);
assert!(buffer.is_empty());
}
#[test]
fn micro_batch_flush_sorts_and_reuses_allocation() {
let mut buffer = MicroBatchBuffer::new();
let initial_capacity = buffer.events.capacity();
assert!(!buffer.push(2, 1, event(21), 0, 100));
assert!(!buffer.push(1, 0, event(10), 10, 100));
assert_eq!(ids(buffer.flush()), vec![10, 21]);
assert!(buffer.events.capacity() >= initial_capacity);
assert!(!buffer.push(3, 0, event(30), 200, 100));
assert_eq!(ids(buffer.flush()), vec![30]);
}
}
@@ -115,6 +115,15 @@ fn streamer_account_event_to_sdk_types(t: &EventType) -> Vec<SdkGrpcEventType> {
vec![SdkGrpcEventType::AccountPumpSwapGlobalConfig]
}
EventType::AccountPumpSwapPool => vec![SdkGrpcEventType::AccountPumpSwapPool],
EventType::AccountRaydiumClmmAmmConfig => {
vec![SdkGrpcEventType::AccountRaydiumClmmAmmConfig]
}
EventType::AccountRaydiumClmmPoolState => {
vec![SdkGrpcEventType::AccountRaydiumClmmPoolState]
}
EventType::AccountRaydiumClmmTickArrayState => {
vec![SdkGrpcEventType::AccountRaydiumClmmTickArrayState]
}
_ => Vec::new(),
}
}
@@ -168,6 +177,24 @@ fn normalize_account_event(event: &mut DexEvent, account: &AccountPretty) {
e.owner = account.owner;
e.rent_epoch = account.rent_epoch;
}
DexEvent::RaydiumClmmAmmConfigAccountEvent(e) => {
e.executable = account.executable;
e.lamports = account.lamports;
e.owner = account.owner;
e.rent_epoch = account.rent_epoch;
}
DexEvent::RaydiumClmmPoolStateAccountEvent(e) => {
e.executable = account.executable;
e.lamports = account.lamports;
e.owner = account.owner;
e.rent_epoch = account.rent_epoch;
}
DexEvent::RaydiumClmmTickArrayStateAccountEvent(e) => {
e.executable = account.executable;
e.lamports = account.lamports;
e.owner = account.owner;
e.rent_epoch = account.rent_epoch;
}
_ => {}
}
}
@@ -642,6 +642,45 @@ pub(crate) fn convert_parser_event(
e, meta,
)))
}
PbDexEvent::RaydiumClmmAmmConfigAccount(e) => {
let meta = adapt_pm(
e.metadata.clone(),
bt,
recv_wall_us,
ProtocolType::RaydiumClmm,
EventType::AccountRaydiumClmmAmmConfig,
raydium_clmm_program(),
);
Some(DexEvent::RaydiumClmmAmmConfigAccountEvent(
raydium_clmm_amm_config_account_from_parser(e, meta),
))
}
PbDexEvent::RaydiumClmmPoolStateAccount(e) => {
let meta = adapt_pm(
e.metadata.clone(),
bt,
recv_wall_us,
ProtocolType::RaydiumClmm,
EventType::AccountRaydiumClmmPoolState,
raydium_clmm_program(),
);
Some(DexEvent::RaydiumClmmPoolStateAccountEvent(
raydium_clmm_pool_state_account_from_parser(e, meta),
))
}
PbDexEvent::RaydiumClmmTickArrayStateAccount(e) => {
let meta = adapt_pm(
e.metadata.clone(),
bt,
recv_wall_us,
ProtocolType::RaydiumClmm,
EventType::AccountRaydiumClmmTickArrayState,
raydium_clmm_program(),
);
Some(DexEvent::RaydiumClmmTickArrayStateAccountEvent(
raydium_clmm_tick_array_state_account_from_parser(e, meta),
))
}
PbDexEvent::RaydiumAmmV4Swap(e) => {
let event_type = if e.max_amount_in != 0 || (e.amount_out != 0 && e.amount_in == 0) {
@@ -9,14 +9,19 @@ use crate::streaming::event_parser::protocols::raydium_amm_v4::events::{
RaydiumAmmV4WithdrawEvent, RaydiumAmmV4WithdrawPnlEvent,
};
use crate::streaming::event_parser::protocols::raydium_clmm::events::{
RaydiumClmmClosePositionEvent, RaydiumClmmCollectFeeEvent, RaydiumClmmConfigChangeEvent,
RaydiumClmmCreatePersonalPositionEvent, RaydiumClmmCreatePoolEvent,
RaydiumClmmDecreaseLimitOrderEvent, RaydiumClmmDecreaseLiquidityV2Event,
RaydiumClmmIncreaseLimitOrderEvent, RaydiumClmmIncreaseLiquidityV2Event,
RaydiumClmmLiquidityCalculateEvent, RaydiumClmmLiquidityChangeEvent,
RaydiumClmmOpenLimitOrderEvent, RaydiumClmmOpenPositionV2Event,
RaydiumClmmOpenPositionWithToken22NftEvent, RaydiumClmmSettleLimitOrderEvent,
RaydiumClmmSwapEvent, RaydiumClmmUpdateRewardInfosEvent,
RaydiumClmmAmmConfigAccountEvent, RaydiumClmmClosePositionEvent, RaydiumClmmCollectFeeEvent,
RaydiumClmmConfigChangeEvent, RaydiumClmmCreatePersonalPositionEvent,
RaydiumClmmCreatePoolEvent, RaydiumClmmDecreaseLimitOrderEvent,
RaydiumClmmDecreaseLiquidityV2Event, RaydiumClmmIncreaseLimitOrderEvent,
RaydiumClmmIncreaseLiquidityV2Event, RaydiumClmmLiquidityCalculateEvent,
RaydiumClmmLiquidityChangeEvent, RaydiumClmmOpenLimitOrderEvent,
RaydiumClmmOpenPositionV2Event, RaydiumClmmOpenPositionWithToken22NftEvent,
RaydiumClmmPoolStateAccountEvent, RaydiumClmmSettleLimitOrderEvent, RaydiumClmmSwapEvent,
RaydiumClmmTickArrayStateAccountEvent, RaydiumClmmUpdateRewardInfosEvent,
};
use crate::streaming::event_parser::protocols::raydium_clmm::types::{
AmmConfig as ClmmAmmConfig, DynamicFeeInfo as ClmmDynamicFeeInfo, PoolState as ClmmPoolState,
RewardInfo as ClmmRewardInfo, TickArrayState as ClmmTickArrayState, TickState as ClmmTickState,
};
use crate::streaming::event_parser::protocols::raydium_cpmm::events::{
RaydiumCpmmDepositEvent, RaydiumCpmmInitializeEvent, RaydiumCpmmSwapEvent,
@@ -609,6 +614,170 @@ pub(crate) fn raydium_clmm_update_reward_infos_from_parser(
}
}
pub(crate) fn raydium_clmm_amm_config_account_from_parser(
e: sol_parser_sdk::core::events::RaydiumClmmAmmConfigAccountEvent,
meta: EventMetadata,
) -> RaydiumClmmAmmConfigAccountEvent {
RaydiumClmmAmmConfigAccountEvent {
metadata: meta,
pubkey: e.pubkey,
amm_config: raydium_clmm_amm_config_from_parser(e.amm_config),
..Default::default()
}
}
pub(crate) fn raydium_clmm_pool_state_account_from_parser(
e: sol_parser_sdk::core::events::RaydiumClmmPoolStateAccountEvent,
meta: EventMetadata,
) -> RaydiumClmmPoolStateAccountEvent {
RaydiumClmmPoolStateAccountEvent {
metadata: meta,
pubkey: e.pubkey,
pool_state: raydium_clmm_pool_state_from_parser(e.pool_state),
..Default::default()
}
}
pub(crate) fn raydium_clmm_tick_array_state_account_from_parser(
e: sol_parser_sdk::core::events::RaydiumClmmTickArrayStateAccountEvent,
meta: EventMetadata,
) -> RaydiumClmmTickArrayStateAccountEvent {
RaydiumClmmTickArrayStateAccountEvent {
metadata: meta,
pubkey: e.pubkey,
tick_array_state: raydium_clmm_tick_array_state_from_parser(e.tick_array_state),
..Default::default()
}
}
fn raydium_clmm_amm_config_from_parser(
e: sol_parser_sdk::core::events::RaydiumClmmAmmConfig,
) -> ClmmAmmConfig {
ClmmAmmConfig {
bump: e.bump,
index: e.index,
owner: e.owner,
protocol_fee_rate: e.protocol_fee_rate,
trade_fee_rate: e.trade_fee_rate,
tick_spacing: e.tick_spacing,
fund_fee_rate: e.fund_fee_rate,
padding_u32: e.padding_u32,
fund_owner: e.fund_owner,
padding: e.padding,
}
}
fn raydium_clmm_pool_state_from_parser(
e: sol_parser_sdk::core::events::RaydiumClmmPoolState,
) -> ClmmPoolState {
ClmmPoolState {
bump: e.bump,
amm_config: e.amm_config,
owner: e.owner,
token_mint_0: e.token_mint_0,
token_mint_1: e.token_mint_1,
token_vault_0: e.token_vault_0,
token_vault_1: e.token_vault_1,
observation_key: e.observation_key,
mint_decimals_0: e.mint_decimals_0,
mint_decimals_1: e.mint_decimals_1,
tick_spacing: e.tick_spacing,
liquidity: e.liquidity,
sqrt_price_x64: e.sqrt_price_x64,
tick_current: e.tick_current,
padding3: e.padding3,
padding4: e.padding4,
fee_growth_global0_x64: e.fee_growth_global_0_x64,
fee_growth_global1_x64: e.fee_growth_global_1_x64,
protocol_fees_token0: e.protocol_fees_token_0,
protocol_fees_token1: e.protocol_fees_token_1,
padding5: e.padding5,
status: e.status,
fee_on: e.fee_on,
padding: e.padding,
reward_infos: [
raydium_clmm_reward_info_from_parser(e.reward_infos[0].clone()),
raydium_clmm_reward_info_from_parser(e.reward_infos[1].clone()),
raydium_clmm_reward_info_from_parser(e.reward_infos[2].clone()),
],
tick_array_bitmap: e.tick_array_bitmap,
padding6: e.padding6,
fund_fees_token0: e.fund_fees_token_0,
fund_fees_token1: e.fund_fees_token_1,
open_time: e.open_time,
recent_epoch: e.recent_epoch,
dynamic_fee_info: raydium_clmm_dynamic_fee_info_from_parser(e.dynamic_fee_info),
padding1: e.padding1,
padding2: e.padding2,
}
}
fn raydium_clmm_reward_info_from_parser(
e: sol_parser_sdk::core::events::RaydiumClmmRewardInfo,
) -> ClmmRewardInfo {
ClmmRewardInfo {
reward_state: e.reward_state,
open_time: e.open_time,
end_time: e.end_time,
last_update_time: e.last_update_time,
emissions_per_second_x64: e.emissions_per_second_x64,
reward_total_emitted: e.reward_total_emitted,
reward_claimed: e.reward_claimed,
token_mint: e.token_mint,
token_vault: e.token_vault,
authority: e.authority,
reward_growth_global_x64: e.reward_growth_global_x64,
}
}
fn raydium_clmm_dynamic_fee_info_from_parser(
e: sol_parser_sdk::core::events::RaydiumClmmDynamicFeeInfo,
) -> ClmmDynamicFeeInfo {
ClmmDynamicFeeInfo {
filter_period: e.filter_period,
decay_period: e.decay_period,
reduction_factor: e.reduction_factor,
dynamic_fee_control: e.dynamic_fee_control,
max_volatility_accumulator: e.max_volatility_accumulator,
tick_spacing_index_reference: e.tick_spacing_index_reference,
volatility_reference: e.volatility_reference,
volatility_accumulator: e.volatility_accumulator,
last_update_timestamp: e.last_update_timestamp,
padding: e.padding,
}
}
fn raydium_clmm_tick_array_state_from_parser(
e: sol_parser_sdk::core::events::RaydiumClmmTickArrayState,
) -> ClmmTickArrayState {
ClmmTickArrayState {
pool_id: e.pool_id,
start_tick_index: e.start_tick_index,
ticks: core::array::from_fn(|i| {
e.ticks.get(i).cloned().map(raydium_clmm_tick_state_from_parser).unwrap_or_default()
}),
initialized_tick_count: e.initialized_tick_count,
recent_epoch: e.recent_epoch,
padding: e.padding,
}
}
fn raydium_clmm_tick_state_from_parser(e: sol_parser_sdk::core::events::Tick) -> ClmmTickState {
ClmmTickState {
tick: e.tick,
liquidity_net: e.liquidity_net,
liquidity_gross: e.liquidity_gross,
fee_growth_outside0_x64: e.fee_growth_outside_0_x64,
fee_growth_outside1_x64: e.fee_growth_outside_1_x64,
reward_growths_outside_x64: e.reward_growths_outside_x64,
order_phase: e.order_phase,
orders_amount: e.orders_amount,
part_filled_orders_remaining: e.part_filled_orders_remaining,
unfilled_ratio_x64: e.unfilled_ratio_x64,
padding: e.padding,
}
}
pub(crate) fn meteora_damm_v2_add_liquidity_from_pb(
e: sol_parser_sdk::core::events::MeteoraDammV2AddLiquidityEvent,
meta: EventMetadata,