gavindiaz/solana-streamer
1
0
Fork 0
mirror of https://github.com/0xfnzero/solana-streamer.git synced 2026-08-17 10:58:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf: Refactor event processing system for better performance

Browse Source
This commit is contained in:
ysq
2025-08-26 17:57:39 +08:00
parent f87f3dd7f3
commit dee683396d
33 changed files with 1158 additions and 1666 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/streaming/event_parser/common/mod.rs
+14 -8
View File
@@ -2,6 +2,8 @@ pub mod types;
pub mod utils;
pub mod filter;
pub const EMPTY_ID: &str = "";
/// 自动生成UnifiedEvent trait实现的宏
#[macro_export]
macro_rules! impl_unified_event {
@@ -12,6 +14,10 @@ macro_rules! impl_unified_event {
&self.metadata.id
}
fn clear_id(&mut self) {
self.metadata.id = $crate::streaming::event_parser::common::EMPTY_ID.to_string();
}
fn event_type(&self) -> $crate::streaming::event_parser::common::types::EventType {
self.metadata.event_type.clone()
}
@@ -24,16 +30,16 @@ macro_rules! impl_unified_event {
self.metadata.slot
}
fn program_received_time_ms(&self) -> i64 {
self.metadata.program_received_time_ms
fn program_received_time_us(&self) -> i64 {
self.metadata.program_received_time_us
}
fn program_handle_time_consuming_ms(&self) -> i64 {
self.metadata.program_handle_time_consuming_ms
fn program_handle_time_consuming_us(&self) -> i64 {
self.metadata.program_handle_time_consuming_us
}
fn set_program_handle_time_consuming_ms(&mut self, program_handle_time_consuming_ms: i64) {
self.metadata.program_handle_time_consuming_ms = program_handle_time_consuming_ms;
fn set_program_handle_time_consuming_us(&mut self, program_handle_time_consuming_us: i64) {
self.metadata.program_handle_time_consuming_us = program_handle_time_consuming_us;
}
fn as_any(&self) -> &dyn std::any::Any {
@@ -56,8 +62,8 @@ macro_rules! impl_unified_event {
}
}
fn set_transfer_datas(&mut self, transfer_datas: Vec<$crate::streaming::event_parser::common::types::TransferData>, swap_data: Option<$crate::streaming::event_parser::common::types::SwapData>) {
self.metadata.set_transfer_datas(transfer_datas, swap_data);
fn set_swap_data(&mut self, swap_data: $crate::streaming::event_parser::common::types::SwapData) {
self.metadata.set_swap_data(swap_data);
}
fn index(&self) -> String {
src/streaming/event_parser/common/types.rs
+229 -288
View File
@@ -1,13 +1,14 @@
use borsh::{BorshDeserialize, BorshSerialize};
use crossbeam_queue::ArrayQueue;
use serde::{Deserialize, Serialize};
use solana_sdk::pubkey::Pubkey;
use solana_transaction_status::UiInstruction;
use solana_transaction_status::{InnerInstruction, UiInstruction};
use std::{
fmt,
hash::{DefaultHasher, Hash, Hasher},
str::FromStr,
sync::Arc,
};
use tokio::sync::Mutex;
use crate::{
match_event,
@@ -30,7 +31,7 @@ const TRANSFER_DATA_POOL_SIZE: usize = 2000;
/// Event metadata object pool
pub struct EventMetadataPool {
pool: Arc<Mutex<Vec<EventMetadata>>>,
pool: Arc<ArrayQueue<EventMetadata>>,
}
impl Default for EventMetadataPool {
@@ -41,25 +42,22 @@ impl Default for EventMetadataPool {
impl EventMetadataPool {
pub fn new() -> Self {
Self { pool: Arc::new(Mutex::new(Vec::with_capacity(EVENT_METADATA_POOL_SIZE))) }
Self { pool: Arc::new(ArrayQueue::new(EVENT_METADATA_POOL_SIZE)) }
}
pub async fn acquire(&self) -> Option<EventMetadata> {
let mut pool = self.pool.lock().await;
pool.pop()
pub fn acquire(&self) -> Option<EventMetadata> {
self.pool.pop()
}
pub async fn release(&self, metadata: EventMetadata) {
let mut pool = self.pool.lock().await;
if pool.len() < EVENT_METADATA_POOL_SIZE {
pool.push(metadata);
}
pub fn release(&self, metadata: EventMetadata) {
// 如果队列已满，push 会失败，但不会阻塞
let _ = self.pool.push(metadata);
}
}
/// Transfer data object pool
pub struct TransferDataPool {
pool: Arc<Mutex<Vec<TransferData>>>,
pool: Arc<ArrayQueue<TransferData>>,
}
impl Default for TransferDataPool {
@@ -70,19 +68,16 @@ impl Default for TransferDataPool {
impl TransferDataPool {
pub fn new() -> Self {
Self { pool: Arc::new(Mutex::new(Vec::with_capacity(TRANSFER_DATA_POOL_SIZE))) }
Self { pool: Arc::new(ArrayQueue::new(TRANSFER_DATA_POOL_SIZE)) }
}
pub async fn acquire(&self) -> Option<TransferData> {
let mut pool = self.pool.lock().await;
pool.pop()
pub fn acquire(&self) -> Option<TransferData> {
self.pool.pop()
}
pub async fn release(&self, transfer_data: TransferData) {
let mut pool = self.pool.lock().await;
if pool.len() < TRANSFER_DATA_POOL_SIZE {
pool.push(transfer_data);
}
pub fn release(&self, transfer_data: TransferData) {
// 如果队列已满，push 会失败，但不会阻塞
let _ = self.pool.push(transfer_data);
}
}
@@ -199,70 +194,69 @@ pub const ACCOUNT_EVENT_TYPES: &[EventType] = &[
];
pub const BLOCK_EVENT_TYPES: &[EventType] = &[EventType::BlockMeta];
impl EventType {
#[allow(clippy::inherent_to_string)]
pub fn to_string(&self) -> String {
impl fmt::Display for EventType {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
EventType::PumpSwapBuy => "PumpSwapBuy".to_string(),
EventType::PumpSwapSell => "PumpSwapSell".to_string(),
EventType::PumpSwapCreatePool => "PumpSwapCreatePool".to_string(),
EventType::PumpSwapDeposit => "PumpSwapDeposit".to_string(),
EventType::PumpSwapWithdraw => "PumpSwapWithdraw".to_string(),
EventType::PumpFunCreateToken => "PumpFunCreateToken".to_string(),
EventType::PumpFunBuy => "PumpFunBuy".to_string(),
EventType::PumpFunSell => "PumpFunSell".to_string(),
EventType::PumpFunMigrate => "PumpFunMigrate".to_string(),
EventType::BonkBuyExactIn => "BonkBuyExactIn".to_string(),
EventType::BonkBuyExactOut => "BonkBuyExactOut".to_string(),
EventType::BonkSellExactIn => "BonkSellExactIn".to_string(),
EventType::BonkSellExactOut => "BonkSellExactOut".to_string(),
EventType::BonkInitialize => "BonkInitialize".to_string(),
EventType::BonkInitializeV2 => "BonkInitializeV2".to_string(),
EventType::BonkMigrateToAmm => "BonkMigrateToAmm".to_string(),
EventType::BonkMigrateToCpswap => "BonkMigrateToCpswap".to_string(),
EventType::AccountPumpFunBondingCurve => "AccountPumpFunBondingCurve".to_string(),
EventType::AccountPumpFunGlobal => "AccountPumpFunGlobal".to_string(),
EventType::AccountPumpSwapGlobalConfig => "AccountPumpSwapGlobalConfig".to_string(),
EventType::AccountPumpSwapPool => "AccountPumpSwapPool".to_string(),
EventType::AccountBonkPoolState => "AccountBonkPoolState".to_string(),
EventType::AccountBonkGlobalConfig => "AccountBonkGlobalConfig".to_string(),
EventType::AccountBonkPlatformConfig => "AccountBonkPlatformConfig".to_string(),
EventType::AccountBonkVestingRecord => "AccountBonkVestingRecord".to_string(),
EventType::RaydiumCpmmSwapBaseInput => "RaydiumCpmmSwapBaseInput".to_string(),
EventType::RaydiumCpmmSwapBaseOutput => "RaydiumCpmmSwapBaseOutput".to_string(),
EventType::RaydiumCpmmDeposit => "RaydiumCpmmDeposit".to_string(),
EventType::RaydiumCpmmInitialize => "RaydiumCpmmInitialize".to_string(),
EventType::RaydiumCpmmWithdraw => "RaydiumCpmmWithdraw".to_string(),
EventType::RaydiumClmmSwap => "RaydiumClmmSwap".to_string(),
EventType::RaydiumClmmSwapV2 => "RaydiumClmmSwapV2".to_string(),
EventType::RaydiumClmmClosePosition => "RaydiumClmmClosePosition".to_string(),
EventType::PumpSwapBuy => write!(f, "PumpSwapBuy"),
EventType::PumpSwapSell => write!(f, "PumpSwapSell"),
EventType::PumpSwapCreatePool => write!(f, "PumpSwapCreatePool"),
EventType::PumpSwapDeposit => write!(f, "PumpSwapDeposit"),
EventType::PumpSwapWithdraw => write!(f, "PumpSwapWithdraw"),
EventType::PumpFunCreateToken => write!(f, "PumpFunCreateToken"),
EventType::PumpFunBuy => write!(f, "PumpFunBuy"),
EventType::PumpFunSell => write!(f, "PumpFunSell"),
EventType::PumpFunMigrate => write!(f, "PumpFunMigrate"),
EventType::BonkBuyExactIn => write!(f, "BonkBuyExactIn"),
EventType::BonkBuyExactOut => write!(f, "BonkBuyExactOut"),
EventType::BonkSellExactIn => write!(f, "BonkSellExactIn"),
EventType::BonkSellExactOut => write!(f, "BonkSellExactOut"),
EventType::BonkInitialize => write!(f, "BonkInitialize"),
EventType::BonkInitializeV2 => write!(f, "BonkInitializeV2"),
EventType::BonkMigrateToAmm => write!(f, "BonkMigrateToAmm"),
EventType::BonkMigrateToCpswap => write!(f, "BonkMigrateToCpswap"),
EventType::RaydiumCpmmSwapBaseInput => write!(f, "RaydiumCpmmSwapBaseInput"),
EventType::RaydiumCpmmSwapBaseOutput => write!(f, "RaydiumCpmmSwapBaseOutput"),
EventType::RaydiumCpmmDeposit => write!(f, "RaydiumCpmmDeposit"),
EventType::RaydiumCpmmInitialize => write!(f, "RaydiumCpmmInitialize"),
EventType::RaydiumCpmmWithdraw => write!(f, "RaydiumCpmmWithdraw"),
EventType::RaydiumClmmSwap => write!(f, "RaydiumClmmSwap"),
EventType::RaydiumClmmSwapV2 => write!(f, "RaydiumClmmSwapV2"),
EventType::RaydiumClmmClosePosition => write!(f, "RaydiumClmmClosePosition"),
EventType::RaydiumClmmDecreaseLiquidityV2 => {
"RaydiumClmmDecreaseLiquidityV2".to_string()
write!(f, "RaydiumClmmDecreaseLiquidityV2")
}
EventType::RaydiumClmmCreatePool => "RaydiumClmmCreatePool".to_string(),
EventType::RaydiumClmmCreatePool => write!(f, "RaydiumClmmCreatePool"),
EventType::RaydiumClmmIncreaseLiquidityV2 => {
"RaydiumClmmIncreaseLiquidityV2".to_string()
write!(f, "RaydiumClmmIncreaseLiquidityV2")
}
EventType::RaydiumClmmOpenPositionWithToken22Nft => {
"RaydiumClmmOpenPositionWithToken22Nft".to_string()
write!(f, "RaydiumClmmOpenPositionWithToken22Nft")
}
EventType::RaydiumClmmOpenPositionV2 => "RaydiumClmmOpenPositionV2".to_string(),
EventType::RaydiumAmmV4SwapBaseIn => "RaydiumAmmV4SwapBaseIn".to_string(),
EventType::RaydiumAmmV4SwapBaseOut => "RaydiumAmmV4SwapBaseOut".to_string(),
EventType::RaydiumAmmV4Deposit => "RaydiumAmmV4Deposit".to_string(),
EventType::RaydiumAmmV4Initialize2 => "RaydiumAmmV4Initialize2".to_string(),
EventType::RaydiumAmmV4Withdraw => "RaydiumAmmV4Withdraw".to_string(),
EventType::RaydiumAmmV4WithdrawPnl => "RaydiumAmmV4WithdrawPnl".to_string(),
EventType::AccountRaydiumAmmV4AmmInfo => "AccountRaydiumAmmV4AmmInfo".to_string(),
EventType::AccountRaydiumClmmAmmConfig => "AccountRaydiumClmmAmmConfig".to_string(),
EventType::AccountRaydiumClmmPoolState => "AccountRaydiumClmmPoolState".to_string(),
EventType::RaydiumClmmOpenPositionV2 => write!(f, "RaydiumClmmOpenPositionV2"),
EventType::RaydiumAmmV4SwapBaseIn => write!(f, "RaydiumAmmV4SwapBaseIn"),
EventType::RaydiumAmmV4SwapBaseOut => write!(f, "RaydiumAmmV4SwapBaseOut"),
EventType::RaydiumAmmV4Deposit => write!(f, "RaydiumAmmV4Deposit"),
EventType::RaydiumAmmV4Initialize2 => write!(f, "RaydiumAmmV4Initialize2"),
EventType::RaydiumAmmV4Withdraw => write!(f, "RaydiumAmmV4Withdraw"),
EventType::RaydiumAmmV4WithdrawPnl => write!(f, "RaydiumAmmV4WithdrawPnl"),
EventType::AccountRaydiumAmmV4AmmInfo => write!(f, "AccountRaydiumAmmV4AmmInfo"),
EventType::AccountPumpSwapGlobalConfig => write!(f, "AccountPumpSwapGlobalConfig"),
EventType::AccountPumpSwapPool => write!(f, "AccountPumpSwapPool"),
EventType::AccountBonkPoolState => write!(f, "AccountBonkPoolState"),
EventType::AccountBonkGlobalConfig => write!(f, "AccountBonkGlobalConfig"),
EventType::AccountBonkPlatformConfig => write!(f, "AccountBonkPlatformConfig"),
EventType::AccountBonkVestingRecord => write!(f, "AccountBonkVestingRecord"),
EventType::AccountPumpFunBondingCurve => write!(f, "AccountPumpFunBondingCurve"),
EventType::AccountPumpFunGlobal => write!(f, "AccountPumpFunGlobal"),
EventType::AccountRaydiumClmmAmmConfig => write!(f, "AccountRaydiumClmmAmmConfig"),
EventType::AccountRaydiumClmmPoolState => write!(f, "AccountRaydiumClmmPoolState"),
EventType::AccountRaydiumClmmTickArrayState => {
"AccountRaydiumClmmTickArrayState".to_string()
write!(f, "AccountRaydiumClmmTickArrayState")
}
EventType::AccountRaydiumCpmmAmmConfig => "AccountRaydiumCpmmAmmConfig".to_string(),
EventType::AccountRaydiumCpmmPoolState => "AccountRaydiumCpmmPoolState".to_string(),
EventType::BlockMeta => "BlockMeta".to_string(),
EventType::Unknown => "Unknown".to_string(),
EventType::AccountRaydiumCpmmAmmConfig => write!(f, "AccountRaydiumCpmmAmmConfig"),
EventType::AccountRaydiumCpmmPoolState => write!(f, "AccountRaydiumCpmmPoolState"),
EventType::BlockMeta => write!(f, "BlockMeta"),
EventType::Unknown => write!(f, "Unknown"),
}
}
}
@@ -345,11 +339,12 @@ pub struct EventMetadata {
pub slot: u64,
pub block_time: i64,
pub block_time_ms: i64,
pub program_received_time_ms: i64,
pub program_handle_time_consuming_ms: i64,
pub program_received_time_us: i64,
pub program_handle_time_consuming_us: i64,
pub protocol: ProtocolType,
pub event_type: EventType,
pub program_id: Pubkey,
#[deprecated(note = "Please use swap_data instead")]
pub transfer_datas: Vec<TransferData>,
pub swap_data: Option<SwapData>,
pub index: String,
@@ -367,7 +362,7 @@ impl EventMetadata {
event_type: EventType,
program_id: Pubkey,
index: String,
program_received_time_ms: i64,
program_received_time_us: i64,
) -> Self {
Self {
id,
@@ -375,250 +370,196 @@ impl EventMetadata {
slot,
block_time,
block_time_ms,
program_received_time_ms,
program_handle_time_consuming_ms: 0,
program_received_time_us,
program_handle_time_consuming_us: 0,
protocol,
event_type,
program_id,
transfer_datas: Vec::with_capacity(4), // Pre-allocate capacity
transfer_datas: vec![],
swap_data: None,
index,
}
}
pub fn set_id(&mut self, id: String) {
let _id = format!("{}-{}-{}", self.signature, self.event_type.to_string(), id);
let mut hasher = DefaultHasher::new();
_id.hash(&mut hasher);
let hash_value = hasher.finish();
self.id = format!("{:x}", hash_value);
self.id = format!("{}-{}-{}", self.signature, self.event_type, id);
}
pub fn set_transfer_datas(
&mut self,
transfer_datas: Vec<TransferData>,
swap_data: Option<SwapData>,
) {
self.transfer_datas = transfer_datas;
self.swap_data = swap_data;
pub fn set_swap_data(&mut self, swap_data: SwapData) {
self.swap_data = Some(swap_data);
}
/// Recycle EventMetadata to object pool
pub async fn recycle(self) {
EVENT_METADATA_POOL.release(self).await;
pub fn recycle(self) {
EVENT_METADATA_POOL.release(self);
}
}
/// Parse token transfer data from next instructions
pub fn parse_transfer_datas_from_next_instructions(
event: Box<dyn UnifiedEvent>,
inner_instruction: &solana_transaction_status::UiInnerInstructions,
current_index: i8,
accounts: &[Pubkey],
) -> (Vec<TransferData>, Option<SwapData>) {
let mut transfer_datas = vec![];
// Get the next two instructions after the current instruction
let next_instructions: Vec<&UiInstruction> =
inner_instruction.instructions.iter().skip((current_index + 1) as usize).collect();
let system_programs = vec![
// Token Program
lazy_static::lazy_static! {
static ref SOL_MINT: Pubkey = Pubkey::from_str("So11111111111111111111111111111111111111111").unwrap();
static ref SYSTEM_PROGRAMS: [Pubkey; 3] = [
Pubkey::from_str("TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA").unwrap(),
// Token 2022 Program
Pubkey::from_str("TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb").unwrap(),
// System Program
Pubkey::from_str("11111111111111111111111111111111").unwrap(),
];
for instruction in next_instructions {
if let UiInstruction::Compiled(compiled) = instruction {
if !system_programs.contains(&accounts[compiled.program_id_index as usize]) {
break;
}
if let Ok(data) = bs58::decode(compiled.data.clone()).into_vec() {
// Token Program: transferChecked
// Token 2022 Program: transferChecked
if data[0] == 12 {
let account_pubkeys: Vec<Pubkey> =
compiled.accounts.iter().map(|a| accounts[*a as usize]).collect();
if account_pubkeys.len() < 4 {
continue;
}
let (source, mint, destination, authority) = (
account_pubkeys[0],
account_pubkeys[1],
account_pubkeys[2],
account_pubkeys[3],
);
let amount = u64::from_le_bytes(data[1..9].try_into().unwrap());
let decimals = data[9];
let token_program = accounts[compiled.program_id_index as usize];
transfer_datas.push(TransferData {
amount,
decimals: Some(decimals),
mint: Some(mint),
source,
destination,
authority: Some(authority),
token_program,
});
}
// Token Program: transfer
else if data[0] == 3 {
let account_pubkeys: Vec<Pubkey> =
compiled.accounts.iter().map(|a| accounts[*a as usize]).collect();
if account_pubkeys.len() < 3 {
continue;
}
let (source, destination, authority) =
(account_pubkeys[0], account_pubkeys[1], account_pubkeys[2]);
let amount = u64::from_le_bytes(data[1..9].try_into().unwrap());
let token_program = accounts[compiled.program_id_index as usize];
transfer_datas.push(TransferData {
amount,
decimals: None,
mint: None,
source,
destination,
authority: Some(authority),
token_program,
});
}
//System Program: transfer
else if data[0] == 2 {
let account_pubkeys: Vec<Pubkey> =
compiled.accounts.iter().map(|a| accounts[*a as usize]).collect();
if account_pubkeys.len() < 2 {
continue;
}
let (source, destination) = (account_pubkeys[0], account_pubkeys[1]);
let amount = u64::from_le_bytes(data[4..12].try_into().unwrap());
let token_program = accounts[compiled.program_id_index as usize];
transfer_datas.push(TransferData {
amount,
decimals: None,
mint: None,
source,
destination,
authority: None,
token_program,
});
}
}
}
}
let mut swap_data: SwapData = SwapData {
}
/// Parse token transfer data from next instructions
pub fn parse_swap_data_from_next_instructions(
event: Box<dyn UnifiedEvent>,
inner_instruction: &solana_transaction_status::InnerInstructions,
current_index: i8,
accounts: &[Pubkey],
) -> Option<SwapData> {
let mut swap_data = SwapData {
from_mint: Pubkey::default(),
to_mint: Pubkey::default(),
from_amount: 0,
to_amount: 0,
description: None,
};
let sol_mint = Pubkey::from_str("So11111111111111111111111111111111111111111").unwrap();
if transfer_datas.len() > 0 {
let mut user: Option<Pubkey> = None;
let mut from_mint: Option<Pubkey> = None;
let mut to_mint: Option<Pubkey> = None;
let mut user_from_token: Option<Pubkey> = None;
let mut user_to_token: Option<Pubkey> = None;
let mut from_vault: Option<Pubkey> = None;
let mut to_vault: Option<Pubkey> = None;
match_event!(event, {
BonkTradeEvent => |e: BonkTradeEvent| {
user = Some(e.payer);
from_mint = Some(e.base_token_mint);
to_mint = Some(e.quote_token_mint);
user_from_token = Some(e.user_base_token);
user_to_token = Some(e.user_quote_token);
from_vault = Some(e.base_vault);
to_vault = Some(e.quote_vault);
},
PumpFunTradeEvent => |e: PumpFunTradeEvent| {
swap_data.from_mint = if e.is_buy {
sol_mint
} else {
e.mint
};
swap_data.to_mint = if e.is_buy {
e.mint
} else {
sol_mint
};
},
PumpSwapBuyEvent => |e: PumpSwapBuyEvent| {
swap_data.from_mint = e.quote_mint;
swap_data.to_mint = e.base_mint;
},
PumpSwapSellEvent => |e: PumpSwapSellEvent| {
swap_data.from_mint = e.base_mint;
swap_data.to_mint = e.quote_mint;
},
RaydiumCpmmSwapEvent => |e: RaydiumCpmmSwapEvent| {
user = Some(e.payer);
from_mint = Some(e.input_token_mint);
to_mint = Some(e.output_token_mint);
user_from_token = Some(e.input_token_account);
user_to_token = Some(e.output_token_account);
from_vault = Some(e.input_vault);
to_vault = Some(e.output_vault);
},
RaydiumClmmSwapEvent => |e: RaydiumClmmSwapEvent| {
user = Some(e.payer);
swap_data.description = Some("Unable to get from_mint and to_mint from RaydiumClmmSwapEvent".to_string());
user_from_token = Some(e.input_token_account);
user_to_token = Some(e.output_token_account);
from_vault = Some(e.input_vault);
to_vault = Some(e.output_vault);
},
RaydiumClmmSwapV2Event => |e: RaydiumClmmSwapV2Event| {
user = Some(e.payer);
from_mint = Some(e.input_vault_mint);
to_mint = Some(e.output_vault_mint);
user_from_token = Some(e.input_token_account);
user_to_token = Some(e.output_token_account);
from_vault = Some(e.input_vault);
to_vault = Some(e.output_vault);
},
RaydiumAmmV4SwapEvent => |e: RaydiumAmmV4SwapEvent| {
user = Some(e.user_source_owner);
swap_data.description = Some("Unable to get from_mint and to_mint from RaydiumAmmV4SwapEvent".to_string());
user_from_token = Some(e.user_source_token_account);
user_to_token = Some(e.user_destination_token_account);
from_vault = Some(e.pool_pc_token_account);
to_vault = Some(e.pool_coin_token_account);
},
});
for transfer_data in transfer_datas.clone() {
if transfer_data.source == user_to_token.unwrap_or_default()
&& transfer_data.destination == to_vault.unwrap_or_default()
{
swap_data.from_mint = to_mint.unwrap_or_default();
swap_data.from_amount = transfer_data.amount;
} else if transfer_data.source == from_vault.unwrap_or_default()
&& transfer_data.destination == user_from_token.unwrap_or_default()
{
swap_data.to_mint = from_mint.unwrap_or_default();
swap_data.to_amount = transfer_data.amount;
} else if transfer_data.source == user_from_token.unwrap_or_default()
&& transfer_data.destination == from_vault.unwrap_or_default()
{
swap_data.from_mint = from_mint.unwrap_or_default();
swap_data.from_amount = transfer_data.amount;
} else if transfer_data.source == to_vault.unwrap_or_default()
&& transfer_data.destination == user_to_token.unwrap_or_default()
{
swap_data.to_mint = to_mint.unwrap_or_default();
swap_data.to_amount = transfer_data.amount;
// 先根据 event 取出关键信息
let mut user: Option<Pubkey> = None;
let mut from_mint: Option<Pubkey> = None;
let mut to_mint: Option<Pubkey> = None;
let mut user_from_token: Option<Pubkey> = None;
let mut user_to_token: Option<Pubkey> = None;
let mut from_vault: Option<Pubkey> = None;
let mut to_vault: Option<Pubkey> = None;
match_event!(event, {
BonkTradeEvent => |e: BonkTradeEvent| {
user = Some(e.payer);
from_mint = Some(e.base_token_mint);
to_mint = Some(e.quote_token_mint);
user_from_token = Some(e.user_base_token);
user_to_token = Some(e.user_quote_token);
from_vault = Some(e.base_vault);
to_vault = Some(e.quote_vault);
},
PumpFunTradeEvent => |e: PumpFunTradeEvent| {
swap_data.from_mint = if e.is_buy { *SOL_MINT } else { e.mint };
swap_data.to_mint = if e.is_buy { e.mint } else { *SOL_MINT };
},
PumpSwapBuyEvent => |e: PumpSwapBuyEvent| {
swap_data.from_mint = e.quote_mint;
swap_data.to_mint = e.base_mint;
},
PumpSwapSellEvent => |e: PumpSwapSellEvent| {
swap_data.from_mint = e.base_mint;
swap_data.to_mint = e.quote_mint;
},
RaydiumCpmmSwapEvent => |e: RaydiumCpmmSwapEvent| {
user = Some(e.payer);
from_mint = Some(e.input_token_mint);
to_mint = Some(e.output_token_mint);
user_from_token = Some(e.input_token_account);
user_to_token = Some(e.output_token_account);
from_vault = Some(e.input_vault);
to_vault = Some(e.output_vault);
},
RaydiumClmmSwapEvent => |e: RaydiumClmmSwapEvent| {
user = Some(e.payer);
swap_data.description = Some("Unable to get from_mint and to_mint from RaydiumClmmSwapEvent".to_string());
user_from_token = Some(e.input_token_account);
user_to_token = Some(e.output_token_account);
from_vault = Some(e.input_vault);
to_vault = Some(e.output_vault);
},
RaydiumClmmSwapV2Event => |e: RaydiumClmmSwapV2Event| {
user = Some(e.payer);
from_mint = Some(e.input_vault_mint);
to_mint = Some(e.output_vault_mint);
user_from_token = Some(e.input_token_account);
user_to_token = Some(e.output_token_account);
from_vault = Some(e.input_vault);
to_vault = Some(e.output_vault);
},
RaydiumAmmV4SwapEvent => |e: RaydiumAmmV4SwapEvent| {
user = Some(e.user_source_owner);
swap_data.description = Some("Unable to get from_mint and to_mint from RaydiumAmmV4SwapEvent".to_string());
user_from_token = Some(e.user_source_token_account);
user_to_token = Some(e.user_destination_token_account);
from_vault = Some(e.pool_pc_token_account);
to_vault = Some(e.pool_coin_token_account);
},
});
let user_to_token = user_to_token.unwrap_or_default();
let user_from_token = user_from_token.unwrap_or_default();
let to_vault = to_vault.unwrap_or_default();
let from_vault = from_vault.unwrap_or_default();
let to_mint = to_mint.unwrap_or_default();
let from_mint = from_mint.unwrap_or_default();
// 单次循环完成提取和判断
for instruction in inner_instruction.instructions.iter().skip((current_index + 1) as usize) {
let compiled = &instruction.instruction;
let program_id = accounts[compiled.program_id_index as usize];
if !SYSTEM_PROGRAMS.contains(&program_id) {
break;
}
let data = &compiled.data;
let get_pubkey = |i: usize| accounts[compiled.accounts[i] as usize];
let (source, destination, amount) = match data[0] {
12 if compiled.accounts.len() >= 4 => {
let amt = u64::from_le_bytes(data[1..9].try_into().unwrap());
(get_pubkey(0), get_pubkey(2), amt)
}
3 if compiled.accounts.len() >= 3 => {
let amt = u64::from_le_bytes(data[1..9].try_into().unwrap());
(get_pubkey(0), get_pubkey(1), amt)
}
2 if compiled.accounts.len() >= 2 => {
let amt = u64::from_le_bytes(data[4..12].try_into().unwrap());
(get_pubkey(0), get_pubkey(1), amt)
}
_ => continue,
};
match (source, destination) {
(s, d) if s == user_to_token && d == to_vault => {
swap_data.from_mint = to_mint;
swap_data.from_amount = amount;
}
(s, d) if s == from_vault && d == user_from_token => {
swap_data.to_mint = from_mint;
swap_data.to_amount = amount;
}
(s, d) if s == user_from_token && d == from_vault => {
swap_data.from_mint = from_mint;
swap_data.from_amount = amount;
}
(s, d) if s == to_vault && d == user_to_token => {
swap_data.to_mint = to_mint;
swap_data.to_amount = amount;
}
(s, d) if s == user_from_token && d == to_vault => {
swap_data.from_mint = from_mint;
swap_data.from_amount = amount;
}
(s, d) if s == from_vault && d == user_to_token => {
swap_data.to_mint = to_mint;
swap_data.to_amount = amount;
}
_ => {}
}
if swap_data.from_mint != Pubkey::default() && swap_data.to_mint != Pubkey::default() {
break;
}
if swap_data.from_amount != 0 && swap_data.to_amount != 0 {
break;
}
}
if swap_data.from_mint != Pubkey::default()
|| swap_data.to_mint != Pubkey::default()
|| swap_data.from_amount != 0
|| swap_data.to_amount != 0
{
(transfer_datas, Some(swap_data))
Some(swap_data)
} else {
(transfer_datas, None)
None
}
}
src/streaming/event_parser/common/utils.rs
-12
View File
@@ -1,5 +1,3 @@
use base64::engine::general_purpose;
use base64::Engine;
use std::time::{SystemTime, UNIX_EPOCH};
/// 获取当前时间戳
@@ -7,16 +5,6 @@ pub fn current_timestamp() -> i64 {
SystemTime::now().duration_since(UNIX_EPOCH).expect("Time went backwards").as_secs() as i64
}
/// 从base64字符串解码数据
pub fn decode_base64(data: &str) -> Result<Vec<u8>, base64::DecodeError> {
general_purpose::STANDARD.decode(data)
}
/// 将数据编码为base64字符串
pub fn encode_base64(data: &[u8]) -> String {
general_purpose::STANDARD.encode(data)
}
/// 从字节数组中提取鉴别器和剩余数据
pub fn extract_discriminator(length: usize, data: &[u8]) -> Option<(&[u8], &[u8])> {
if data.len() < length {
src/streaming/event_parser/core/account_event_parser.rs
+5 -6
View File
@@ -158,12 +158,11 @@ impl AccountEventParser {
pub fn parse_account_event(
protocols: Vec<Protocol>,
account: AccountPretty,
program_received_time_ms: i64,
event_type_filter: Option<EventTypeFilter>,
) -> Option<Box<dyn UnifiedEvent>> {
let configs = Self::configs(protocols, event_type_filter);
for config in configs {
if account.owner == config.program_id.to_string()
if account.owner == config.program_id
&& account.data[..config.account_discriminator.len()]
== *config.account_discriminator
{
@@ -171,17 +170,17 @@ impl AccountEventParser {
&account,
EventMetadata {
slot: account.slot,
signature: account.signature.clone(),
signature: account.signature.to_string(),
protocol: config.protocol_type,
event_type: config.event_type,
program_id: config.program_id,
program_received_time_ms,
program_received_time_us: account.program_received_time_us,
..Default::default()
},
);
if let Some(mut event) = event {
event.set_program_handle_time_consuming_ms(
chrono::Utc::now().timestamp_millis() - program_received_time_ms,
event.set_program_handle_time_consuming_us(
chrono::Utc::now().timestamp_micros() - account.program_received_time_us,
);
return Some(event);
}
src/streaming/event_parser/core/common_event_parser.rs
+10 -1
View File
@@ -8,8 +8,17 @@ impl CommonEventParser {
slot: u64,
block_hash: &str,
block_time_ms: i64,
program_received_time_us: i64,
) -> Box<dyn UnifiedEvent> {
let block_meta_event = BlockMetaEvent::new(slot, block_hash.to_string(), block_time_ms);
let mut block_meta_event = BlockMetaEvent::new(
slot,
block_hash.to_string(),
block_time_ms,
program_received_time_us,
);
block_meta_event.set_program_handle_time_consuming_us(
chrono::Utc::now().timestamp_micros() - program_received_time_us,
);
Box::new(block_meta_event)
}
}
src/streaming/event_parser/core/traits.rs
+136 -193
View File
@@ -1,17 +1,14 @@
use anyhow::Result;
use prost_types::Timestamp;
use solana_sdk::signature::Signature;
use solana_sdk::{
instruction::CompiledInstruction, pubkey::Pubkey, transaction::VersionedTransaction,
};
use solana_transaction_status::{
EncodedTransactionWithStatusMeta, UiCompiledInstruction, UiInnerInstructions, UiInstruction,
};
use solana_transaction_status::{InnerInstructions, TransactionWithStatusMeta};
use std::collections::HashMap;
use std::fmt::Debug;
use std::{collections::HashMap, str::FromStr};
use crate::streaming::event_parser::common::{
parse_transfer_datas_from_next_instructions, SwapData, TransferData,
};
use crate::streaming::event_parser::common::{parse_swap_data_from_next_instructions, SwapData};
use crate::streaming::event_parser::protocols::pumpswap::{PumpSwapBuyEvent, PumpSwapSellEvent};
use crate::streaming::event_parser::{
common::{utils::*, EventMetadata, EventType, ProtocolType},
@@ -20,12 +17,16 @@ use crate::streaming::event_parser::{
pumpfun::{PumpFunCreateTokenEvent, PumpFunTradeEvent},
},
};
use crate::streaming::shred::MetricsEventType;
/// Unified Event Interface - All protocol events must implement this trait
pub trait UnifiedEvent: Debug + Send + Sync {
/// Get event ID
fn id(&self) -> &str;
/// Set event ID
fn clear_id(&mut self);
/// Get event type
fn event_type(&self) -> EventType;
@@ -36,13 +37,13 @@ pub trait UnifiedEvent: Debug + Send + Sync {
fn slot(&self) -> u64;
/// Get program received timestamp (milliseconds)
fn program_received_time_ms(&self) -> i64;
fn program_received_time_us(&self) -> i64;
/// Processing time consumption (milliseconds)
fn program_handle_time_consuming_ms(&self) -> i64;
fn program_handle_time_consuming_us(&self) -> i64;
/// Set processing time consumption (milliseconds)
fn set_program_handle_time_consuming_ms(&mut self, program_handle_time_consuming_ms: i64);
fn set_program_handle_time_consuming_us(&mut self, program_handle_time_consuming_us: i64);
/// Convert event to Any for downcasting
fn as_any(&self) -> &dyn std::any::Any;
@@ -58,12 +59,8 @@ pub trait UnifiedEvent: Debug + Send + Sync {
// Default implementation: no merging operation
}
/// Set transfer datas
fn set_transfer_datas(
&mut self,
transfer_datas: Vec<TransferData>,
swap_data: Option<SwapData>,
);
/// Set swap data
fn set_swap_data(&mut self, swap_data: SwapData);
/// Get index
fn index(&self) -> String;
@@ -80,11 +77,11 @@ pub trait EventParser: Send + Sync {
#[allow(clippy::too_many_arguments)]
fn parse_events_from_inner_instruction(
&self,
inner_instruction: &UiCompiledInstruction,
signature: &str,
inner_instruction: &CompiledInstruction,
signature: Signature,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
program_received_time_us: i64,
index: String,
) -> Vec<Box<dyn UnifiedEvent>>;
@@ -94,10 +91,10 @@ pub trait EventParser: Send + Sync {
&self,
instruction: &CompiledInstruction,
accounts: &[Pubkey],
signature: &str,
signature: Signature,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
program_received_time_us: i64,
index: String,
) -> Vec<Box<dyn UnifiedEvent>>;
@@ -106,12 +103,12 @@ pub trait EventParser: Send + Sync {
async fn parse_instruction_events_from_versioned_transaction(
&self,
transaction: &VersionedTransaction,
signature: &str,
signature: Signature,
slot: Option<u64>,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
program_received_time_us: i64,
accounts: &[Pubkey],
inner_instructions: &[UiInnerInstructions],
inner_instructions: &[InnerInstructions],
) -> Result<Vec<Box<dyn UnifiedEvent>>> {
// 预分配容量，避免动态扩容
let mut instruction_events = Vec::with_capacity(16);
@@ -140,7 +137,7 @@ pub trait EventParser: Send + Sync {
signature,
slot,
block_time,
program_received_time_ms,
program_received_time_us,
format!("{index}"),
)
.await
@@ -152,14 +149,15 @@ pub trait EventParser: Send + Sync {
})
{
events.iter_mut().for_each(|event| {
let (transfer_datas, swap_data) =
parse_transfer_datas_from_next_instructions(
event.clone_boxed(),
inn,
-1_i8,
&accounts,
);
event.set_transfer_datas(transfer_datas, swap_data);
let swap_data = parse_swap_data_from_next_instructions(
event.clone_boxed(),
inn,
-1_i8,
&accounts,
);
if let Some(swap_data) = swap_data {
event.set_swap_data(swap_data);
}
});
}
instruction_events.extend(events);
@@ -175,10 +173,10 @@ pub trait EventParser: Send + Sync {
async fn parse_versioned_transaction(
&self,
versioned_tx: &VersionedTransaction,
signature: &str,
signature: Signature,
slot: Option<u64>,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
program_received_time_us: i64,
bot_wallet: Option<Pubkey>,
) -> Result<Vec<Box<dyn UnifiedEvent>>> {
let accounts: Vec<Pubkey> = versioned_tx.message.static_account_keys().to_vec();
@@ -188,7 +186,7 @@ pub trait EventParser: Send + Sync {
signature,
slot,
block_time,
program_received_time_ms,
program_received_time_us,
&accounts,
&[],
)
@@ -199,146 +197,108 @@ pub trait EventParser: Send + Sync {
async fn parse_transaction(
&self,
tx: EncodedTransactionWithStatusMeta,
signature: &str,
tx: TransactionWithStatusMeta,
signature: Signature,
slot: Option<u64>,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
program_received_time_us: i64,
bot_wallet: Option<Pubkey>,
) -> Result<Vec<Box<dyn UnifiedEvent>>> {
// TODO: bug - 待优化
// // 生成缓存键
// let cache_key = format!("{}_{}_{}", signature, slot.unwrap_or(0), program_received_time_ms);
// // 尝试从缓存获取
// if let Some(cached_events) = PARSE_CACHE.get(&cache_key).await {
// return Ok(cached_events);
// }
let transaction = tx.transaction;
// 检查交易元数据
let meta =
tx.meta.as_ref().ok_or_else(|| anyhow::anyhow!("Missing transaction metadata"))?;
let versioned_tx = tx.get_transaction();
let meta = tx.get_status_meta();
let mut address_table_lookups: Vec<Pubkey> = vec![];
let mut inner_instructions: Vec<UiInnerInstructions> = vec![];
if meta.err.is_none() {
// 正确处理OptionSerializer类型
if let solana_transaction_status::option_serializer::OptionSerializer::Some(
meta_inner_instructions,
) = &meta.inner_instructions
{
inner_instructions = meta_inner_instructions.clone();
let mut inner_instructions: Vec<InnerInstructions> = vec![];
if let Some(meta) = meta {
inner_instructions = meta.inner_instructions.unwrap_or_default();
for loopup in meta.loaded_addresses.writable {
address_table_lookups.push(loopup);
}
if let solana_transaction_status::option_serializer::OptionSerializer::Some(
loaded_addresses,
) = &meta.loaded_addresses
{
for lookup in &loaded_addresses.writable {
if let Ok(pubkey) = Pubkey::from_str(lookup) {
address_table_lookups.push(pubkey);
}
}
for lookup in &loaded_addresses.readonly {
if let Ok(pubkey) = Pubkey::from_str(lookup) {
address_table_lookups.push(pubkey);
}
}
for loopup in meta.loaded_addresses.readonly {
address_table_lookups.push(loopup);
}
}
let mut accounts: Vec<Pubkey> = vec![];
// 预分配容量，避免动态扩容
let mut instruction_events = Vec::with_capacity(16);
let mut instruction_events: Vec<Box<dyn UnifiedEvent>> = Vec::with_capacity(16);
// 解析指令事件
if let Some(versioned_tx) = transaction.decode() {
accounts = versioned_tx.message.static_account_keys().to_vec();
accounts.extend(address_table_lookups.clone());
accounts = versioned_tx.message.static_account_keys().to_vec();
accounts.extend(address_table_lookups.clone());
instruction_events = self
.parse_instruction_events_from_versioned_transaction(
&versioned_tx,
signature,
slot,
block_time,
program_received_time_ms,
&accounts,
&inner_instructions,
)
.await
.unwrap_or_else(|_e| vec![]);
} else {
accounts.extend(address_table_lookups.clone());
}
instruction_events = self
.parse_instruction_events_from_versioned_transaction(
&versioned_tx,
signature,
slot,
block_time,
program_received_time_us,
&accounts,
&inner_instructions,
)
.await
.unwrap_or_else(|_e| vec![]);
// 解析内联指令事件
// 预分配容量，避免动态扩容
let mut inner_instruction_events = Vec::with_capacity(8);
let mut inner_instruction_events: Vec<Box<dyn UnifiedEvent>> = Vec::with_capacity(8);
// 检查交易是否成功
if meta.err.is_none() {
for inner_instruction in inner_instructions {
for (index, instruction) in inner_instruction.instructions.iter().enumerate() {
if let UiInstruction::Compiled(compiled) = instruction {
// 解析嵌套指令
let compiled_instruction = CompiledInstruction {
program_id_index: compiled.program_id_index,
accounts: compiled.accounts.clone(),
data: bs58::decode(compiled.data.clone())
.into_vec()
.unwrap_or_else(|_| vec![]),
};
if let Ok(mut events) = self
.parse_instruction(
&compiled_instruction,
for inner_instruction in inner_instructions {
for (index, instruction) in inner_instruction.instructions.iter().enumerate() {
// 解析嵌套指令
let compiled_instruction = instruction.instruction.clone();
if let Ok(mut events) = self
.parse_instruction(
&compiled_instruction,
&accounts,
signature,
slot,
block_time,
program_received_time_us,
format!("{}.{}", inner_instruction.index, index),
)
.await
{
if !events.is_empty() {
events.iter_mut().for_each(|event| {
let swap_data = parse_swap_data_from_next_instructions(
event.clone_boxed(),
&inner_instruction,
index as i8,
&accounts,
signature,
slot,
block_time,
program_received_time_ms,
format!("{}.{}", inner_instruction.index, index),
)
.await
{
if !events.is_empty() {
events.iter_mut().for_each(|event| {
let (transfer_datas, swap_data) =
parse_transfer_datas_from_next_instructions(
event.clone_boxed(),
&inner_instruction,
index as i8,
&accounts,
);
event.set_transfer_datas(transfer_datas, swap_data);
});
instruction_events.extend(events);
);
if let Some(swap_data) = swap_data {
event.set_swap_data(swap_data);
}
}
if let Ok(mut events) = self
.parse_inner_instruction(
compiled,
signature,
slot,
block_time,
program_received_time_ms,
format!("{}.{}", inner_instruction.index, index),
)
.await
{
if !events.is_empty() {
events.iter_mut().for_each(|event| {
let (transfer_datas, swap_data) =
parse_transfer_datas_from_next_instructions(
event.clone_boxed(),
&inner_instruction,
index as i8,
&accounts,
);
event.set_transfer_datas(transfer_datas, swap_data);
});
inner_instruction_events.extend(events);
});
instruction_events.extend(events);
}
}
if let Ok(mut events) = self
.parse_inner_instruction(
&compiled_instruction,
signature,
slot,
block_time,
program_received_time_us,
format!("{}.{}", inner_instruction.index, index),
)
.await
{
if !events.is_empty() {
events.iter_mut().for_each(|event| {
let swap_data = parse_swap_data_from_next_instructions(
event.clone_boxed(),
&inner_instruction,
index as i8,
&accounts,
);
if let Some(swap_data) = swap_data {
event.set_swap_data(swap_data);
}
}
});
inner_instruction_events.extend(events);
}
}
}
@@ -386,9 +346,6 @@ pub trait EventParser: Send + Sync {
let result = self.process_events(instruction_events, bot_wallet);
// 缓存结果
// PARSE_CACHE.set(cache_key, result.clone()).await;
Ok(result)
}
@@ -397,7 +354,6 @@ pub trait EventParser: Send + Sync {
mut events: Vec<Box<dyn UnifiedEvent>>,
bot_wallet: Option<Pubkey>,
) -> Vec<Box<dyn UnifiedEvent>> {
let start_time = std::time::Instant::now();
let mut dev_address = vec![];
let mut bonk_dev_address = None;
for event in &mut events {
@@ -458,18 +414,7 @@ pub trait EventParser: Send + Sync {
trade_info.is_dev_create_token_trade = false;
}
}
let now = chrono::Utc::now().timestamp_millis();
event.set_program_handle_time_consuming_ms(now - event.program_received_time_ms());
}
// 记录处理时间
let processing_time = start_time.elapsed();
if processing_time.as_millis() > 10 {
log::warn!(
"Event processing took {}ms for {} events",
processing_time.as_millis(),
events.len()
);
event.clear_id();
}
events
@@ -477,11 +422,11 @@ pub trait EventParser: Send + Sync {
async fn parse_inner_instruction(
&self,
instruction: &UiCompiledInstruction,
signature: &str,
instruction: &CompiledInstruction,
signature: Signature,
slot: Option<u64>,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
program_received_time_us: i64,
index: String,
) -> Result<Vec<Box<dyn UnifiedEvent>>> {
let slot = slot.unwrap_or(0);
@@ -490,7 +435,7 @@ pub trait EventParser: Send + Sync {
signature,
slot,
block_time,
program_received_time_ms,
program_received_time_us,
index,
);
Ok(events)
@@ -501,10 +446,10 @@ pub trait EventParser: Send + Sync {
&self,
instruction: &CompiledInstruction,
accounts: &[Pubkey],
signature: &str,
signature: Signature,
slot: Option<u64>,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
program_received_time_us: i64,
index: String,
) -> Result<Vec<Box<dyn UnifiedEvent>>> {
let slot = slot.unwrap_or(0);
@@ -514,7 +459,7 @@ pub trait EventParser: Send + Sync {
signature,
slot,
block_time,
program_received_time_ms,
program_received_time_us,
index,
);
Ok(events)
@@ -588,10 +533,10 @@ impl GenericEventParser {
&self,
config: &GenericEventParseConfig,
data: &[u8],
signature: &str,
signature: Signature,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
program_received_time_us: i64,
index: String,
) -> Option<Box<dyn UnifiedEvent>> {
if let Some(parser) = config.inner_instruction_parser {
@@ -607,7 +552,7 @@ impl GenericEventParser {
config.event_type.clone(),
config.program_id,
index,
program_received_time_ms,
program_received_time_us,
);
parser(data, metadata)
} else {
@@ -622,10 +567,10 @@ impl GenericEventParser {
config: &GenericEventParseConfig,
data: &[u8],
account_pubkeys: &[Pubkey],
signature: &str,
signature: Signature,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
program_received_time_us: i64,
index: String,
) -> Option<Box<dyn UnifiedEvent>> {
if let Some(parser) = config.instruction_parser {
@@ -641,7 +586,7 @@ impl GenericEventParser {
config.event_type.clone(),
config.program_id,
index,
program_received_time_ms,
program_received_time_us,
);
parser(data, account_pubkeys, metadata)
} else {
@@ -662,16 +607,14 @@ impl EventParser for GenericEventParser {
#[allow(clippy::too_many_arguments)]
fn parse_events_from_inner_instruction(
&self,
inner_instruction: &UiCompiledInstruction,
signature: &str,
inner_instruction: &CompiledInstruction,
signature: Signature,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
program_received_time_us: i64,
index: String,
) -> Vec<Box<dyn UnifiedEvent>> {
let inner_instruction_data = inner_instruction.data.clone();
let inner_instruction_data_decoded =
bs58::decode(inner_instruction_data).into_vec().unwrap_or_else(|_| vec![]);
let inner_instruction_data_decoded = inner_instruction.data.clone();
if inner_instruction_data_decoded.len() < 16 {
return Vec::new();
}
@@ -688,7 +631,7 @@ impl EventParser for GenericEventParser {
signature,
slot,
block_time,
program_received_time_ms,
program_received_time_us,
index.clone(),
) {
events.push(event);
@@ -705,10 +648,10 @@ impl EventParser for GenericEventParser {
&self,
instruction: &CompiledInstruction,
accounts: &[Pubkey],
signature: &str,
signature: Signature,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
program_received_time_us: i64,
index: String,
) -> Vec<Box<dyn UnifiedEvent>> {
let program_id = accounts[instruction.program_id_index as usize];
@@ -741,7 +684,7 @@ impl EventParser for GenericEventParser {
signature,
slot,
block_time,
program_received_time_ms,
program_received_time_us,
index.clone(),
) {
events.push(event);
src/streaming/event_parser/protocols/block/block_meta_event.rs
+7 -2
View File
@@ -13,7 +13,12 @@ pub struct BlockMetaEvent {
}
impl BlockMetaEvent {
pub fn new(slot: u64, block_hash: String, block_time_ms: i64) -> Self {
pub fn new(
slot: u64,
block_hash: String,
block_time_ms: i64,
program_received_time_us: i64,
) -> Self {
let metadata = EventMetadata::new(
format!("block_{}_{}", slot, block_hash),
"".to_string(),
@@ -24,7 +29,7 @@ impl BlockMetaEvent {
EventType::BlockMeta,
solana_sdk::pubkey::Pubkey::default(),
"".to_string(),
chrono::Utc::now().timestamp_millis(),
program_received_time_us,
);
Self { metadata, slot, block_hash }
}
src/streaming/event_parser/protocols/bonk/parser.rs
+12 -10
View File
@@ -1,14 +1,16 @@
use std::collections::HashMap;
use prost_types::Timestamp;
use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey};
use solana_transaction_status::UiCompiledInstruction;
use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey, signature::Signature};
use crate::streaming::event_parser::{
common::{utils::*, EventMetadata, EventType, ProtocolType},
core::traits::{EventParser, GenericEventParseConfig, GenericEventParser, UnifiedEvent},
protocols::bonk::{
bonk_pool_create_event_log_decode, bonk_trade_event_log_decode, discriminators, AmmFeeOn, BonkMigrateToAmmEvent, BonkMigrateToCpswapEvent, BonkPoolCreateEvent, BonkTradeEvent, ConstantCurve, CurveParams, FixedCurve, LinearCurve, MintParams, TradeDirection, VestingParams
bonk_pool_create_event_log_decode, bonk_trade_event_log_decode, discriminators, AmmFeeOn,
BonkMigrateToAmmEvent, BonkMigrateToCpswapEvent, BonkPoolCreateEvent, BonkTradeEvent,
ConstantCurve, CurveParams, FixedCurve, LinearCurve, MintParams, TradeDirection,
VestingParams,
},
};
@@ -611,11 +613,11 @@ impl EventParser for BonkEventParser {
}
fn parse_events_from_inner_instruction(
&self,
inner_instruction: &UiCompiledInstruction,
signature: &str,
inner_instruction: &CompiledInstruction,
signature: Signature,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
program_received_time_us: i64,
index: String,
) -> Vec<Box<dyn UnifiedEvent>> {
self.inner.parse_events_from_inner_instruction(
@@ -623,7 +625,7 @@ impl EventParser for BonkEventParser {
signature,
slot,
block_time,
program_received_time_ms,
program_received_time_us,
index,
)
}
@@ -632,10 +634,10 @@ impl EventParser for BonkEventParser {
&self,
instruction: &CompiledInstruction,
accounts: &[Pubkey],
signature: &str,
signature: Signature,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
program_received_time_us: i64,
index: String,
) -> Vec<Box<dyn UnifiedEvent>> {
self.inner.parse_events_from_instruction(
@@ -644,7 +646,7 @@ impl EventParser for BonkEventParser {
signature,
slot,
block_time,
program_received_time_ms,
program_received_time_us,
index,
)
}
src/streaming/event_parser/protocols/mutil/parser.rs
+36 -16
View File
@@ -1,8 +1,8 @@
use std::collections::HashMap;
use prost_types::Timestamp;
use solana_sdk::signature::Signature;
use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey};
use solana_transaction_status::UiCompiledInstruction;
use crate::streaming::event_parser::common::filter::EventTypeFilter;
use crate::streaming::event_parser::{
@@ -23,18 +23,38 @@ impl MutilEventParser {
// Merge inner_instruction_configs, append configurations to existing Vec
for (key, configs) in parse.inner_instruction_configs() {
let filtered_configs: Vec<GenericEventParseConfig> = configs.into_iter().filter(|config| {
event_type_filter.as_ref().map(|filter| filter.include.contains(&config.event_type)).unwrap_or(true)
}).collect();
inner.inner_instruction_configs.entry(key).or_insert_with(Vec::new).extend(filtered_configs);
let filtered_configs: Vec<GenericEventParseConfig> = configs
.into_iter()
.filter(|config| {
event_type_filter
.as_ref()
.map(|filter| filter.include.contains(&config.event_type))
.unwrap_or(true)
})
.collect();
inner
.inner_instruction_configs
.entry(key)
.or_insert_with(Vec::new)
.extend(filtered_configs);
}
// Merge instruction_configs, append configurations to existing Vec
for (key, configs) in parse.instruction_configs() {
let filtered_configs: Vec<GenericEventParseConfig> = configs.into_iter().filter(|config| {
event_type_filter.as_ref().map(|filter| filter.include.contains(&config.event_type)).unwrap_or(true)
}).collect();
inner.instruction_configs.entry(key).or_insert_with(Vec::new).extend(filtered_configs);
let filtered_configs: Vec<GenericEventParseConfig> = configs
.into_iter()
.filter(|config| {
event_type_filter
.as_ref()
.map(|filter| filter.include.contains(&config.event_type))
.unwrap_or(true)
})
.collect();
inner
.instruction_configs
.entry(key)
.or_insert_with(Vec::new)
.extend(filtered_configs);
}
// Append program_ids (this is already appending)
@@ -54,11 +74,11 @@ impl EventParser for MutilEventParser {
}
fn parse_events_from_inner_instruction(
&self,
inner_instruction: &UiCompiledInstruction,
signature: &str,
inner_instruction: &CompiledInstruction,
signature: Signature,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
program_received_time_us: i64,
index: String,
) -> Vec<Box<dyn UnifiedEvent>> {
self.inner.parse_events_from_inner_instruction(
@@ -66,7 +86,7 @@ impl EventParser for MutilEventParser {
signature,
slot,
block_time,
program_received_time_ms,
program_received_time_us,
index,
)
}
@@ -75,10 +95,10 @@ impl EventParser for MutilEventParser {
&self,
instruction: &CompiledInstruction,
accounts: &[Pubkey],
signature: &str,
signature: Signature,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
program_received_time_us: i64,
index: String,
) -> Vec<Box<dyn UnifiedEvent>> {
self.inner.parse_events_from_instruction(
@@ -87,7 +107,7 @@ impl EventParser for MutilEventParser {
signature,
slot,
block_time,
program_received_time_ms,
program_received_time_us,
index,
)
}
src/streaming/event_parser/protocols/pumpfun/parser.rs
+8 -9
View File
@@ -1,8 +1,7 @@
use std::collections::HashMap;
use prost_types::Timestamp;
use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey};
use solana_transaction_status::UiCompiledInstruction;
use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey, signature::Signature};
use crate::streaming::event_parser::{
common::{EventMetadata, EventType, ProtocolType},
@@ -295,11 +294,11 @@ impl EventParser for PumpFunEventParser {
}
fn parse_events_from_inner_instruction(
&self,
inner_instruction: &UiCompiledInstruction,
signature: &str,
inner_instruction: &CompiledInstruction,
signature: Signature,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
program_received_time_us: i64,
index: String,
) -> Vec<Box<dyn UnifiedEvent>> {
self.inner.parse_events_from_inner_instruction(
@@ -307,7 +306,7 @@ impl EventParser for PumpFunEventParser {
signature,
slot,
block_time,
program_received_time_ms,
program_received_time_us,
index,
)
}
@@ -316,10 +315,10 @@ impl EventParser for PumpFunEventParser {
&self,
instruction: &CompiledInstruction,
accounts: &[Pubkey],
signature: &str,
signature: Signature,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
program_received_time_us: i64,
index: String,
) -> Vec<Box<dyn UnifiedEvent>> {
self.inner.parse_events_from_instruction(
@@ -328,7 +327,7 @@ impl EventParser for PumpFunEventParser {
signature,
slot,
block_time,
program_received_time_ms,
program_received_time_us,
index,
)
}
src/streaming/event_parser/protocols/pumpswap/parser.rs
+8 -9
View File
@@ -1,8 +1,7 @@
use std::collections::HashMap;
use prost_types::Timestamp;
use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey};
use solana_transaction_status::UiCompiledInstruction;
use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey, signature::Signature};
use crate::streaming::event_parser::{
common::{read_u64_le, EventMetadata, EventType, ProtocolType},
@@ -385,11 +384,11 @@ impl EventParser for PumpSwapEventParser {
}
fn parse_events_from_inner_instruction(
&self,
inner_instruction: &UiCompiledInstruction,
signature: &str,
inner_instruction: &CompiledInstruction,
signature: Signature,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
program_received_time_us: i64,
index: String,
) -> Vec<Box<dyn UnifiedEvent>> {
self.inner.parse_events_from_inner_instruction(
@@ -397,7 +396,7 @@ impl EventParser for PumpSwapEventParser {
signature,
slot,
block_time,
program_received_time_ms,
program_received_time_us,
index,
)
}
@@ -406,10 +405,10 @@ impl EventParser for PumpSwapEventParser {
&self,
instruction: &CompiledInstruction,
accounts: &[Pubkey],
signature: &str,
signature: Signature,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
program_received_time_us: i64,
index: String,
) -> Vec<Box<dyn UnifiedEvent>> {
self.inner.parse_events_from_instruction(
@@ -418,7 +417,7 @@ impl EventParser for PumpSwapEventParser {
signature,
slot,
block_time,
program_received_time_ms,
program_received_time_us,
index,
)
}
src/streaming/event_parser/protocols/raydium_amm_v4/parser.rs
+8 -9
View File
@@ -1,8 +1,7 @@
use std::collections::HashMap;
use prost_types::Timestamp;
use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey};
use solana_transaction_status::UiCompiledInstruction;
use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey, signature::Signature};
use crate::streaming::event_parser::{
common::{read_u64_le, EventMetadata, EventType, ProtocolType},
@@ -387,11 +386,11 @@ impl EventParser for RaydiumAmmV4EventParser {
}
fn parse_events_from_inner_instruction(
&self,
inner_instruction: &UiCompiledInstruction,
signature: &str,
inner_instruction: &CompiledInstruction,
signature: Signature,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
program_received_time_us: i64,
index: String,
) -> Vec<Box<dyn UnifiedEvent>> {
self.inner.parse_events_from_inner_instruction(
@@ -399,7 +398,7 @@ impl EventParser for RaydiumAmmV4EventParser {
signature,
slot,
block_time,
program_received_time_ms,
program_received_time_us,
index,
)
}
@@ -408,10 +407,10 @@ impl EventParser for RaydiumAmmV4EventParser {
&self,
instruction: &CompiledInstruction,
accounts: &[Pubkey],
signature: &str,
signature: Signature,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
program_received_time_us: i64,
index: String,
) -> Vec<Box<dyn UnifiedEvent>> {
self.inner.parse_events_from_instruction(
@@ -420,7 +419,7 @@ impl EventParser for RaydiumAmmV4EventParser {
signature,
slot,
block_time,
program_received_time_ms,
program_received_time_us,
index,
)
}
src/streaming/event_parser/protocols/raydium_clmm/parser.rs
+8 -9
View File
@@ -1,8 +1,7 @@
use std::collections::HashMap;
use prost_types::Timestamp;
use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey};
use solana_transaction_status::UiCompiledInstruction;
use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey, signature::Signature};
use crate::streaming::event_parser::{
common::{
@@ -428,11 +427,11 @@ impl EventParser for RaydiumClmmEventParser {
}
fn parse_events_from_inner_instruction(
&self,
inner_instruction: &UiCompiledInstruction,
signature: &str,
inner_instruction: &CompiledInstruction,
signature: Signature,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
program_received_time_us: i64,
index: String,
) -> Vec<Box<dyn UnifiedEvent>> {
self.inner.parse_events_from_inner_instruction(
@@ -440,7 +439,7 @@ impl EventParser for RaydiumClmmEventParser {
signature,
slot,
block_time,
program_received_time_ms,
program_received_time_us,
index,
)
}
@@ -449,10 +448,10 @@ impl EventParser for RaydiumClmmEventParser {
&self,
instruction: &CompiledInstruction,
accounts: &[Pubkey],
signature: &str,
signature: Signature,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
program_received_time_us: i64,
index: String,
) -> Vec<Box<dyn UnifiedEvent>> {
self.inner.parse_events_from_instruction(
@@ -461,7 +460,7 @@ impl EventParser for RaydiumClmmEventParser {
signature,
slot,
block_time,
program_received_time_ms,
program_received_time_us,
index,
)
}
src/streaming/event_parser/protocols/raydium_cpmm/parser.rs
+8 -9
View File
@@ -1,8 +1,7 @@
use std::collections::HashMap;
use prost_types::Timestamp;
use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey};
use solana_transaction_status::UiCompiledInstruction;
use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey, signature::Signature};
use crate::streaming::event_parser::{
common::{read_u64_le, EventMetadata, EventType, ProtocolType},
@@ -278,11 +277,11 @@ impl EventParser for RaydiumCpmmEventParser {
}
fn parse_events_from_inner_instruction(
&self,
inner_instruction: &UiCompiledInstruction,
signature: &str,
inner_instruction: &CompiledInstruction,
signature: Signature,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
program_received_time_us: i64,
index: String,
) -> Vec<Box<dyn UnifiedEvent>> {
self.inner.parse_events_from_inner_instruction(
@@ -290,7 +289,7 @@ impl EventParser for RaydiumCpmmEventParser {
signature,
slot,
block_time,
program_received_time_ms,
program_received_time_us,
index,
)
}
@@ -299,10 +298,10 @@ impl EventParser for RaydiumCpmmEventParser {
&self,
instruction: &CompiledInstruction,
accounts: &[Pubkey],
signature: &str,
signature: Signature,
slot: u64,
block_time: Option<Timestamp>,
program_received_time_ms: i64,
program_received_time_us: i64,
index: String,
) -> Vec<Box<dyn UnifiedEvent>> {
self.inner.parse_events_from_instruction(
@@ -311,7 +310,7 @@ impl EventParser for RaydiumCpmmEventParser {
signature,
slot,
block_time,
program_received_time_ms,
program_received_time_us,
index,
)
}