feat: refactor to multi-protocol event streaming system

Major architectural refactor, upgrading from simple logging system to comprehensive multi-protocol Solana DEX event streaming system:

 New Features:
- Support for 5 DEX protocols: PumpFun, PumpSwap, Bonk, Raydium CPMM, Raydium CLMM
- Implement unified event interface (UnifiedEvent trait) and event factory pattern
- Add dual streaming support: Yellowstone gRPC and ShredStream
- Add Chinese documentation (README_CN.md)

🏗️ Architectural Improvements:
- Refactor event parsing system with modular design
- Implement protocol-specific parsers and event types
- Optimize dependency management, update Cargo.toml
- Remove legacy logging modules, clean up redundant code

📊 Statistics:
- Added 46 files, 4511 lines of code
- Removed 1381 lines of legacy code
- Net addition of 3130 lines of code

Tech Stack:
- Rust async/await for asynchronous processing
- Protocol Buffers support
- Multi-protocol event parsing
- High-performance event stream subscription
This commit is contained in:
ysq
2025-07-19 23:46:42 +08:00
parent a7c9721877
commit 9e34a01874
46 changed files with 4511 additions and 1381 deletions
+322
View File
@@ -0,0 +1,322 @@
use borsh::BorshDeserialize;
use serde::{Deserialize, Serialize};
use solana_sdk::pubkey::Pubkey;
use crate::streaming::event_parser::common::EventMetadata;
use crate::impl_unified_event;
/// 买入事件
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, BorshDeserialize)]
pub struct PumpSwapBuyEvent {
#[borsh(skip)]
pub metadata: EventMetadata,
pub timestamp: i64,
pub base_amount_out: u64,
pub max_quote_amount_in: u64,
pub user_base_token_reserves: u64,
pub user_quote_token_reserves: u64,
pub pool_base_token_reserves: u64,
pub pool_quote_token_reserves: u64,
pub quote_amount_in: u64,
pub lp_fee_basis_points: u64,
pub lp_fee: u64,
pub protocol_fee_basis_points: u64,
pub protocol_fee: u64,
pub quote_amount_in_with_lp_fee: u64,
pub user_quote_amount_in: u64,
pub pool: Pubkey,
pub user: Pubkey,
pub user_base_token_account: Pubkey,
pub user_quote_token_account: Pubkey,
pub protocol_fee_recipient: Pubkey,
pub protocol_fee_recipient_token_account: Pubkey,
pub coin_creator: Pubkey,
pub coin_creator_fee_basis_points: u64,
pub coin_creator_fee: u64,
#[borsh(skip)]
pub base_mint: Pubkey,
#[borsh(skip)]
pub quote_mint: Pubkey,
#[borsh(skip)]
pub pool_base_token_account: Pubkey,
#[borsh(skip)]
pub pool_quote_token_account: Pubkey,
#[borsh(skip)]
pub coin_creator_vault_ata: Pubkey,
#[borsh(skip)]
pub coin_creator_vault_authority: Pubkey,
}
// 使用宏生成UnifiedEvent实现,指定需要合并的字段
impl_unified_event!(
PumpSwapBuyEvent,
timestamp,
base_amount_out,
max_quote_amount_in,
user_base_token_reserves,
user_quote_token_reserves,
pool_base_token_reserves,
pool_quote_token_reserves,
quote_amount_in,
lp_fee_basis_points,
lp_fee,
protocol_fee_basis_points,
protocol_fee,
quote_amount_in_with_lp_fee,
user_quote_amount_in,
pool,
user,
user_base_token_account,
user_quote_token_account,
protocol_fee_recipient,
protocol_fee_recipient_token_account,
coin_creator,
coin_creator_fee_basis_points,
coin_creator_fee
);
/// 卖出事件
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, BorshDeserialize)]
pub struct PumpSwapSellEvent {
#[borsh(skip)]
pub metadata: EventMetadata,
pub timestamp: i64,
pub base_amount_in: u64,
pub min_quote_amount_out: u64,
pub user_base_token_reserves: u64,
pub user_quote_token_reserves: u64,
pub pool_base_token_reserves: u64,
pub pool_quote_token_reserves: u64,
pub quote_amount_out: u64,
pub lp_fee_basis_points: u64,
pub lp_fee: u64,
pub protocol_fee_basis_points: u64,
pub protocol_fee: u64,
pub quote_amount_out_without_lp_fee: u64,
pub user_quote_amount_out: u64,
pub pool: Pubkey,
pub user: Pubkey,
pub user_base_token_account: Pubkey,
pub user_quote_token_account: Pubkey,
pub protocol_fee_recipient: Pubkey,
pub protocol_fee_recipient_token_account: Pubkey,
pub coin_creator: Pubkey,
pub coin_creator_fee_basis_points: u64,
pub coin_creator_fee: u64,
#[borsh(skip)]
pub base_mint: Pubkey,
#[borsh(skip)]
pub quote_mint: Pubkey,
#[borsh(skip)]
pub pool_base_token_account: Pubkey,
#[borsh(skip)]
pub pool_quote_token_account: Pubkey,
#[borsh(skip)]
pub coin_creator_vault_ata: Pubkey,
#[borsh(skip)]
pub coin_creator_vault_authority: Pubkey,
}
// 使用宏生成UnifiedEvent实现,指定需要合并的字段
impl_unified_event!(
PumpSwapSellEvent,
timestamp,
base_amount_in,
min_quote_amount_out,
user_base_token_reserves,
user_quote_token_reserves,
pool_base_token_reserves,
pool_quote_token_reserves,
quote_amount_out,
lp_fee_basis_points,
lp_fee,
protocol_fee_basis_points,
protocol_fee,
quote_amount_out_without_lp_fee,
user_quote_amount_out,
pool,
user,
user_base_token_account,
user_quote_token_account,
protocol_fee_recipient,
protocol_fee_recipient_token_account,
coin_creator,
coin_creator_fee_basis_points,
coin_creator_fee
);
/// 创建池子事件
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, BorshDeserialize)]
pub struct PumpSwapCreatePoolEvent {
#[borsh(skip)]
pub metadata: EventMetadata,
pub timestamp: i64,
pub index: u16,
pub creator: Pubkey,
pub base_mint: Pubkey,
pub quote_mint: Pubkey,
pub base_mint_decimals: u8,
pub quote_mint_decimals: u8,
pub base_amount_in: u64,
pub quote_amount_in: u64,
pub pool_base_amount: u64,
pub pool_quote_amount: u64,
pub minimum_liquidity: u64,
pub initial_liquidity: u64,
pub lp_token_amount_out: u64,
pub pool_bump: u8,
pub pool: Pubkey,
pub lp_mint: Pubkey,
pub user_base_token_account: Pubkey,
pub user_quote_token_account: Pubkey,
pub coin_creator: Pubkey,
#[borsh(skip)]
pub user_pool_token_account: Pubkey,
#[borsh(skip)]
pub pool_base_token_account: Pubkey,
#[borsh(skip)]
pub pool_quote_token_account: Pubkey,
}
impl_unified_event!(
PumpSwapCreatePoolEvent,
timestamp,
index,
creator,
base_mint,
quote_mint,
base_mint_decimals,
quote_mint_decimals,
base_amount_in,
quote_amount_in,
pool_base_amount,
pool_quote_amount,
minimum_liquidity,
initial_liquidity,
lp_token_amount_out,
pool_bump,
pool,
lp_mint,
user_base_token_account,
user_quote_token_account,
coin_creator
);
/// 存款事件
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, BorshDeserialize)]
pub struct PumpSwapDepositEvent {
#[borsh(skip)]
pub metadata: EventMetadata,
pub timestamp: i64,
pub lp_token_amount_out: u64,
pub max_base_amount_in: u64,
pub max_quote_amount_in: u64,
pub user_base_token_reserves: u64,
pub user_quote_token_reserves: u64,
pub pool_base_token_reserves: u64,
pub pool_quote_token_reserves: u64,
pub base_amount_in: u64,
pub quote_amount_in: u64,
pub lp_mint_supply: u64,
pub pool: Pubkey,
pub user: Pubkey,
pub user_base_token_account: Pubkey,
pub user_quote_token_account: Pubkey,
pub user_pool_token_account: Pubkey,
#[borsh(skip)]
pub base_mint: Pubkey,
#[borsh(skip)]
pub quote_mint: Pubkey,
#[borsh(skip)]
pub pool_base_token_account: Pubkey,
#[borsh(skip)]
pub pool_quote_token_account: Pubkey,
}
impl_unified_event!(
PumpSwapDepositEvent,
timestamp,
lp_token_amount_out,
max_base_amount_in,
max_quote_amount_in,
user_base_token_reserves,
user_quote_token_reserves,
pool_base_token_reserves,
pool_quote_token_reserves,
base_amount_in,
quote_amount_in,
lp_mint_supply,
pool,
user,
user_base_token_account,
user_quote_token_account,
user_pool_token_account
);
/// 提款事件
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, BorshDeserialize)]
pub struct PumpSwapWithdrawEvent {
#[borsh(skip)]
pub metadata: EventMetadata,
pub timestamp: i64,
pub lp_token_amount_in: u64,
pub min_base_amount_out: u64,
pub min_quote_amount_out: u64,
pub user_base_token_reserves: u64,
pub user_quote_token_reserves: u64,
pub pool_base_token_reserves: u64,
pub pool_quote_token_reserves: u64,
pub base_amount_out: u64,
pub quote_amount_out: u64,
pub lp_mint_supply: u64,
pub pool: Pubkey,
pub user: Pubkey,
pub user_base_token_account: Pubkey,
pub user_quote_token_account: Pubkey,
pub user_pool_token_account: Pubkey,
#[borsh(skip)]
pub base_mint: Pubkey,
#[borsh(skip)]
pub quote_mint: Pubkey,
#[borsh(skip)]
pub pool_base_token_account: Pubkey,
#[borsh(skip)]
pub pool_quote_token_account: Pubkey,
}
impl_unified_event!(
PumpSwapWithdrawEvent,
timestamp,
lp_token_amount_in,
min_base_amount_out,
min_quote_amount_out,
user_base_token_reserves,
user_quote_token_reserves,
pool_base_token_reserves,
pool_quote_token_reserves,
base_amount_out,
quote_amount_out,
lp_mint_supply,
pool,
user,
user_base_token_account,
user_quote_token_account,
user_pool_token_account
);
/// 事件鉴别器常量
pub mod discriminators {
// 事件鉴别器
pub const BUY_EVENT: &str = "0xe445a52e51cb9a1d67f4521f2cf57777";
pub const SELL_EVENT: &str = "0xe445a52e51cb9a1d3e2f370aa503dc2a";
pub const CREATE_POOL_EVENT: &str = "0xe445a52e51cb9a1db1310cd2a076a774";
pub const DEPOSIT_EVENT: &str = "0xe445a52e51cb9a1d78f83d531f8e6b90";
pub const WITHDRAW_EVENT: &str = "0xe445a52e51cb9a1d1609851aa02c47c0";
// 指令鉴别器
pub const BUY_IX: &[u8] = &[102, 6, 61, 18, 1, 218, 235, 234];
pub const SELL_IX: &[u8] = &[51, 230, 133, 164, 1, 127, 131, 173];
pub const CREATE_POOL_IX: &[u8] = &[233, 146, 209, 142, 207, 104, 64, 188];
pub const DEPOSIT_IX: &[u8] = &[242, 35, 198, 137, 82, 225, 242, 182];
pub const WITHDRAW_IX: &[u8] = &[183, 18, 70, 156, 148, 109, 161, 34];
}
+5
View File
@@ -0,0 +1,5 @@
pub mod events;
pub mod parser;
pub use events::*;
pub use parser::PumpSwapEventParser;
+386
View File
@@ -0,0 +1,386 @@
use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey};
use solana_transaction_status::UiCompiledInstruction;
use crate::streaming::event_parser::{
common::{EventMetadata, EventType, ProtocolType, read_u64_le},
core::traits::{EventParser, GenericEventParseConfig, GenericEventParser, UnifiedEvent},
protocols::pumpswap::{
discriminators, PumpSwapBuyEvent, PumpSwapCreatePoolEvent, PumpSwapDepositEvent,
PumpSwapSellEvent, PumpSwapWithdrawEvent,
},
};
/// PumpSwap程序ID
pub const PUMPSWAP_PROGRAM_ID: Pubkey =
solana_sdk::pubkey!("pAMMBay6oceH9fJKBRHGP5D4bD4sWpmSwMn52FMfXEA");
/// PumpSwap事件解析器
pub struct PumpSwapEventParser {
inner: GenericEventParser,
}
impl PumpSwapEventParser {
pub fn new() -> Self {
// 配置所有事件类型
let configs = vec![
GenericEventParseConfig {
inner_instruction_discriminator: discriminators::BUY_EVENT,
instruction_discriminator: discriminators::BUY_IX,
event_type: EventType::PumpSwapBuy,
inner_instruction_parser: Self::parse_buy_inner_instruction,
instruction_parser: Self::parse_buy_instruction,
},
GenericEventParseConfig {
inner_instruction_discriminator: discriminators::SELL_EVENT,
instruction_discriminator: discriminators::SELL_IX,
event_type: EventType::PumpSwapSell,
inner_instruction_parser: Self::parse_sell_inner_instruction,
instruction_parser: Self::parse_sell_instruction,
},
GenericEventParseConfig {
inner_instruction_discriminator: discriminators::CREATE_POOL_EVENT,
instruction_discriminator: discriminators::CREATE_POOL_IX,
event_type: EventType::PumpSwapCreatePool,
inner_instruction_parser: Self::parse_create_pool_inner_instruction,
instruction_parser: Self::parse_create_pool_instruction,
},
GenericEventParseConfig {
inner_instruction_discriminator: discriminators::DEPOSIT_EVENT,
instruction_discriminator: discriminators::DEPOSIT_IX,
event_type: EventType::PumpSwapDeposit,
inner_instruction_parser: Self::parse_deposit_inner_instruction,
instruction_parser: Self::parse_deposit_instruction,
},
GenericEventParseConfig {
inner_instruction_discriminator: discriminators::WITHDRAW_EVENT,
instruction_discriminator: discriminators::WITHDRAW_IX,
event_type: EventType::PumpSwapWithdraw,
inner_instruction_parser: Self::parse_withdraw_inner_instruction,
instruction_parser: Self::parse_withdraw_instruction,
},
];
let inner = GenericEventParser::new(PUMPSWAP_PROGRAM_ID, ProtocolType::PumpSwap, configs);
Self { inner }
}
/// 解析买入日志事件
fn parse_buy_inner_instruction(data: &[u8], metadata: EventMetadata) -> Option<Box<dyn UnifiedEvent>> {
if let Ok(event) = borsh::from_slice::<PumpSwapBuyEvent>(data) {
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, event.user, event.pool, event.base_amount_out
));
Some(Box::new(PumpSwapBuyEvent {
metadata: metadata,
..event
}))
} else {
None
}
}
/// 解析卖出日志事件
fn parse_sell_inner_instruction(data: &[u8], metadata: EventMetadata) -> Option<Box<dyn UnifiedEvent>> {
if let Ok(event) = borsh::from_slice::<PumpSwapSellEvent>(data) {
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, event.user, event.pool, event.base_amount_in
));
Some(Box::new(PumpSwapSellEvent {
metadata: metadata,
..event
}))
} else {
None
}
}
/// 解析创建池子日志事件
fn parse_create_pool_inner_instruction(
data: &[u8],
metadata: EventMetadata,
) -> Option<Box<dyn UnifiedEvent>> {
if let Ok(event) = borsh::from_slice::<PumpSwapCreatePoolEvent>(data) {
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, event.pool, event.creator, event.base_amount_in
));
Some(Box::new(PumpSwapCreatePoolEvent {
metadata: metadata,
..event
}))
} else {
None
}
}
/// 解析存款日志事件
fn parse_deposit_inner_instruction(data: &[u8], metadata: EventMetadata) -> Option<Box<dyn UnifiedEvent>> {
if let Ok(event) = borsh::from_slice::<PumpSwapDepositEvent>(data) {
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, event.pool, event.user, event.lp_token_amount_out
));
Some(Box::new(PumpSwapDepositEvent {
metadata: metadata,
..event
}))
} else {
None
}
}
/// 解析提款日志事件
fn parse_withdraw_inner_instruction(data: &[u8], metadata: EventMetadata) -> Option<Box<dyn UnifiedEvent>> {
if let Ok(event) = borsh::from_slice::<PumpSwapWithdrawEvent>(data) {
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, event.pool, event.user, event.lp_token_amount_in
));
Some(Box::new(PumpSwapWithdrawEvent {
metadata: metadata,
..event
}))
} else {
None
}
}
/// 解析买入指令事件
fn parse_buy_instruction(
data: &[u8],
accounts: &[Pubkey],
metadata: EventMetadata,
) -> Option<Box<dyn UnifiedEvent>> {
if data.len() < 16 || accounts.len() < 11 {
return None;
}
let base_amount_out = read_u64_le(data, 0)?;
let max_quote_amount_in = read_u64_le(data, 8)?;
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[1], accounts[0], base_amount_out
));
Some(Box::new(PumpSwapBuyEvent {
metadata,
base_amount_out,
max_quote_amount_in,
pool: accounts[0],
user: accounts[1],
base_mint: accounts[3],
quote_mint: accounts[4],
user_base_token_account: accounts[5],
user_quote_token_account: accounts[6],
pool_base_token_account: accounts[7],
pool_quote_token_account: accounts[8],
protocol_fee_recipient: accounts[9],
protocol_fee_recipient_token_account: accounts[10],
coin_creator_vault_ata: accounts.get(17).copied().unwrap_or_default(),
coin_creator_vault_authority: accounts.get(18).copied().unwrap_or_default(),
..Default::default()
}))
}
/// 解析卖出指令事件
fn parse_sell_instruction(
data: &[u8],
accounts: &[Pubkey],
metadata: EventMetadata,
) -> Option<Box<dyn UnifiedEvent>> {
if data.len() < 16 || accounts.len() < 11 {
return None;
}
let base_amount_in = read_u64_le(data, 0)?;
let min_quote_amount_out = read_u64_le(data, 8)?;
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[1], accounts[0], base_amount_in
));
Some(Box::new(PumpSwapSellEvent {
metadata,
base_amount_in,
min_quote_amount_out,
pool: accounts[0],
user: accounts[1],
base_mint: accounts[3],
quote_mint: accounts[4],
user_base_token_account: accounts[5],
user_quote_token_account: accounts[6],
pool_base_token_account: accounts[7],
pool_quote_token_account: accounts[8],
protocol_fee_recipient: accounts[9],
protocol_fee_recipient_token_account: accounts[10],
coin_creator_vault_ata: accounts.get(17).copied().unwrap_or_default(),
coin_creator_vault_authority: accounts.get(18).copied().unwrap_or_default(),
..Default::default()
}))
}
/// 解析创建池子指令事件
fn parse_create_pool_instruction(
data: &[u8],
accounts: &[Pubkey],
metadata: EventMetadata,
) -> Option<Box<dyn UnifiedEvent>> {
if data.len() < 18 || accounts.len() < 11 {
return None;
}
let index = u16::from_le_bytes(data[0..2].try_into().ok()?);
let base_amount_in = u64::from_le_bytes(data[2..10].try_into().ok()?);
let quote_amount_in = u64::from_le_bytes(data[10..18].try_into().ok()?);
let coin_creator = if data.len() >= 50 {
Pubkey::new_from_array(data[18..50].try_into().ok()?)
} else {
Pubkey::default()
};
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[0], accounts[2], base_amount_in
));
Some(Box::new(PumpSwapCreatePoolEvent {
metadata,
index,
base_amount_in,
quote_amount_in,
pool: accounts[0],
creator: accounts[2],
base_mint: accounts[3],
quote_mint: accounts[4],
lp_mint: accounts[5],
user_base_token_account: accounts[6],
user_quote_token_account: accounts[7],
user_pool_token_account: accounts[8],
pool_base_token_account: accounts[9],
pool_quote_token_account: accounts[10],
coin_creator,
..Default::default()
}))
}
/// 解析存款指令事件
fn parse_deposit_instruction(
data: &[u8],
accounts: &[Pubkey],
metadata: EventMetadata,
) -> Option<Box<dyn UnifiedEvent>> {
if data.len() < 24 || accounts.len() < 11 {
return None;
}
let lp_token_amount_out = u64::from_le_bytes(data[0..8].try_into().ok()?);
let max_base_amount_in = u64::from_le_bytes(data[8..16].try_into().ok()?);
let max_quote_amount_in = u64::from_le_bytes(data[16..24].try_into().ok()?);
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[0], accounts[2], lp_token_amount_out
));
Some(Box::new(PumpSwapDepositEvent {
metadata,
lp_token_amount_out,
max_base_amount_in,
max_quote_amount_in,
pool: accounts[0],
user: accounts[2],
base_mint: accounts[3],
quote_mint: accounts[4],
user_base_token_account: accounts[6],
user_quote_token_account: accounts[7],
user_pool_token_account: accounts[8],
pool_base_token_account: accounts[9],
pool_quote_token_account: accounts[10],
..Default::default()
}))
}
/// 解析提款指令事件
fn parse_withdraw_instruction(
data: &[u8],
accounts: &[Pubkey],
metadata: EventMetadata,
) -> Option<Box<dyn UnifiedEvent>> {
if data.len() < 24 || accounts.len() < 11 {
return None;
}
let lp_token_amount_in = u64::from_le_bytes(data[0..8].try_into().ok()?);
let min_base_amount_out = u64::from_le_bytes(data[8..16].try_into().ok()?);
let min_quote_amount_out = u64::from_le_bytes(data[16..24].try_into().ok()?);
let mut metadata = metadata;
metadata.set_id(format!(
"{}-{}-{}-{}",
metadata.signature, accounts[0], accounts[2], lp_token_amount_in
));
Some(Box::new(PumpSwapWithdrawEvent {
metadata,
lp_token_amount_in,
min_base_amount_out,
min_quote_amount_out,
pool: accounts[0],
user: accounts[2],
base_mint: accounts[3],
quote_mint: accounts[4],
user_base_token_account: accounts[6],
user_quote_token_account: accounts[7],
user_pool_token_account: accounts[8],
pool_base_token_account: accounts[9],
pool_quote_token_account: accounts[10],
..Default::default()
}))
}
}
#[async_trait::async_trait]
impl EventParser for PumpSwapEventParser {
fn parse_events_from_inner_instruction(
&self,
inner_instruction: &UiCompiledInstruction,
signature: &str,
slot: u64,
) -> Vec<Box<dyn UnifiedEvent>> {
self.inner
.parse_events_from_inner_instruction(inner_instruction, signature, slot)
}
fn parse_events_from_instruction(
&self,
instruction: &CompiledInstruction,
accounts: &[Pubkey],
signature: &str,
slot: u64,
) -> Vec<Box<dyn UnifiedEvent>> {
self.inner
.parse_events_from_instruction(instruction, accounts, signature, slot)
}
fn should_handle(&self, program_id: &Pubkey) -> bool {
self.inner.should_handle(program_id)
}
fn supported_program_ids(&self) -> Vec<Pubkey> {
self.inner.supported_program_ids()
}
}