feat: refactor to multi-protocol event streaming system

Major architectural refactor, upgrading from simple logging system to comprehensive multi-protocol Solana DEX event streaming system:

 New Features:
- Support for 5 DEX protocols: PumpFun, PumpSwap, Bonk, Raydium CPMM, Raydium CLMM
- Implement unified event interface (UnifiedEvent trait) and event factory pattern
- Add dual streaming support: Yellowstone gRPC and ShredStream
- Add Chinese documentation (README_CN.md)

🏗️ Architectural Improvements:
- Refactor event parsing system with modular design
- Implement protocol-specific parsers and event types
- Optimize dependency management, update Cargo.toml
- Remove legacy logging modules, clean up redundant code

📊 Statistics:
- Added 46 files, 4511 lines of code
- Removed 1381 lines of legacy code
- Net addition of 3130 lines of code

Tech Stack:
- Rust async/await for asynchronous processing
- Protocol Buffers support
- Multi-protocol event parsing
- High-performance event stream subscription
This commit is contained in:
ysq
2025-07-19 23:46:42 +08:00
parent a7c9721877
commit 9e34a01874
46 changed files with 4511 additions and 1381 deletions
+120
View File
@@ -0,0 +1,120 @@
use std::sync::Arc;
use futures::{channel::mpsc, StreamExt};
use solana_entry::entry::Entry;
use tonic::transport::Channel;
use log::error;
use solana_sdk::transaction::VersionedTransaction;
use crate::common::AnyResult;
use crate::streaming::event_parser::{EventParserFactory, Protocol, UnifiedEvent};
use crate::protos::shredstream::shredstream_proxy_client::ShredstreamProxyClient;
use crate::protos::shredstream::SubscribeEntriesRequest;
use solana_sdk::pubkey::Pubkey;
const CHANNEL_SIZE: usize = 1000;
pub struct ShredStreamGrpc {
shredstream_client: Arc<ShredstreamProxyClient<Channel>>,
}
struct TransactionWithSlot {
transaction: VersionedTransaction,
slot: u64,
}
impl ShredStreamGrpc {
pub async fn new(endpoint: String) -> AnyResult<Self> {
let shredstream_client = ShredstreamProxyClient::connect(endpoint.clone()).await?;
Ok(Self {
shredstream_client: Arc::new(shredstream_client),
})
}
pub async fn shredstream_subscribe<F>(
&self,
protocols: Vec<Protocol>,
bot_wallet: Option<Pubkey>,
callback: F,
) -> AnyResult<()>
where
F: Fn(Box<dyn UnifiedEvent>) + Send + Sync + 'static,
{
let request = tonic::Request::new(SubscribeEntriesRequest {});
let mut client = (*self.shredstream_client).clone();
let mut stream = client.subscribe_entries(request).await?.into_inner();
let (mut tx, mut rx) = mpsc::channel::<TransactionWithSlot>(CHANNEL_SIZE);
let callback = Box::new(callback);
tokio::spawn(async move {
while let Some(message) = stream.next().await {
match message {
Ok(msg) => {
if let Ok(entries) = bincode::deserialize::<Vec<Entry>>(&msg.entries) {
for entry in entries {
for transaction in entry.transactions {
let _ = tx.try_send(TransactionWithSlot {
transaction: transaction.clone(),
slot: msg.slot,
});
}
}
}
}
Err(error) => {
error!("Stream error: {error:?}");
break;
}
}
}
});
while let Some(transaction_with_slot) = rx.next().await {
if let Err(e) = Self::process_transaction(
transaction_with_slot,
protocols.clone(),
bot_wallet,
&*callback,
)
.await
{
error!("Error processing transaction: {:?}", e);
}
}
Ok(())
}
async fn process_transaction<F>(
transaction_with_slot: TransactionWithSlot,
protocols: Vec<Protocol>,
bot_wallet: Option<Pubkey>,
callback: &F,
) -> AnyResult<()>
where
F: Fn(Box<dyn UnifiedEvent>) + Send + Sync,
{
let slot = transaction_with_slot.slot;
let versioned_tx = transaction_with_slot.transaction;
let signature = versioned_tx.signatures[0];
for protocol in protocols {
let parser = EventParserFactory::create_parser(protocol.clone());
let events = parser
.parse_versioned_transaction(
&versioned_tx,
&signature.to_string(),
Some(slot),
bot_wallet.clone(),
)
.await
.unwrap_or_else(|_e| vec![]);
for event in events {
callback(event);
}
}
Ok(())
}
}