feat: refactor to multi-protocol event streaming system

Major architectural refactor, upgrading from simple logging system to comprehensive multi-protocol Solana DEX event streaming system:

 New Features:
- Support for 5 DEX protocols: PumpFun, PumpSwap, Bonk, Raydium CPMM, Raydium CLMM
- Implement unified event interface (UnifiedEvent trait) and event factory pattern
- Add dual streaming support: Yellowstone gRPC and ShredStream
- Add Chinese documentation (README_CN.md)

🏗️ Architectural Improvements:
- Refactor event parsing system with modular design
- Implement protocol-specific parsers and event types
- Optimize dependency management, update Cargo.toml
- Remove legacy logging modules, clean up redundant code

📊 Statistics:
- Added 46 files, 4511 lines of code
- Removed 1381 lines of legacy code
- Net addition of 3130 lines of code

Tech Stack:
- Rust async/await for asynchronous processing
- Protocol Buffers support
- Multi-protocol event parsing
- High-performance event stream subscription
This commit is contained in:
ysq
2025-07-19 23:46:42 +08:00
parent a7c9721877
commit 9e34a01874
46 changed files with 4511 additions and 1381 deletions
+2
View File
@@ -0,0 +1,2 @@
pub mod shared;
pub mod shredstream;
+18
View File
@@ -0,0 +1,18 @@
// This file is @generated by prost-build.
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
pub struct Header {
#[prost(message, optional, tag = "1")]
pub ts: ::core::option::Option<::prost_types::Timestamp>,
}
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
pub struct Heartbeat {
#[prost(uint64, tag = "1")]
pub count: u64,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct Socket {
#[prost(string, tag = "1")]
pub ip: ::prost::alloc::string::String,
#[prost(int64, tag = "2")]
pub port: i64,
}
+279
View File
@@ -0,0 +1,279 @@
// This file is @generated by prost-build.
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct Heartbeat {
/// don't trust IP:PORT from tcp header since it can be tampered over the wire
/// `socket.ip` must match incoming packet's ip. this prevents spamming an unwitting destination
#[prost(message, optional, tag = "1")]
pub socket: ::core::option::Option<super::shared::Socket>,
/// regions for shredstream proxy to receive shreds from
/// list of valid regions: <https://docs.jito.wtf/lowlatencytxnsend/#api>
#[prost(string, repeated, tag = "2")]
pub regions: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
}
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
pub struct HeartbeatResponse {
/// client must respond within `ttl_ms` to keep stream alive
#[prost(uint32, tag = "1")]
pub ttl_ms: u32,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct TraceShred {
/// source region, one of: <https://docs.jito.wtf/lowlatencytxnsend/#api>
#[prost(string, tag = "1")]
pub region: ::prost::alloc::string::String,
/// timestamp of creation
#[prost(message, optional, tag = "2")]
pub created_at: ::core::option::Option<::prost_types::Timestamp>,
/// monotonically increases, resets upon service restart
#[prost(uint32, tag = "3")]
pub seq_num: u32,
}
/// tbd: we may want to add filters here
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
pub struct SubscribeEntriesRequest {}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct Entry {
/// the slot that the entry is from
#[prost(uint64, tag = "1")]
pub slot: u64,
/// Serialized bytes of Vec<Entry>: <https://docs.rs/solana-entry/latest/solana_entry/entry/struct.Entry.html>
#[prost(bytes = "vec", tag = "2")]
pub entries: ::prost::alloc::vec::Vec<u8>,
}
/// Generated client implementations.
pub mod shredstream_client {
#![allow(
unused_variables,
dead_code,
missing_docs,
clippy::wildcard_imports,
clippy::let_unit_value,
)]
use tonic::codegen::*;
use tonic::codegen::http::Uri;
#[derive(Debug, Clone)]
pub struct ShredstreamClient<T> {
inner: tonic::client::Grpc<T>,
}
impl ShredstreamClient<tonic::transport::Channel> {
/// Attempt to create a new client by connecting to a given endpoint.
pub async fn connect<D>(dst: D) -> Result<Self, tonic::transport::Error>
where
D: TryInto<tonic::transport::Endpoint>,
D::Error: Into<StdError>,
{
let conn = tonic::transport::Endpoint::new(dst)?.connect().await?;
Ok(Self::new(conn))
}
}
impl<T> ShredstreamClient<T>
where
T: tonic::client::GrpcService<tonic::body::BoxBody>,
T::Error: Into<StdError>,
T::ResponseBody: Body<Data = Bytes> + std::marker::Send + 'static,
<T::ResponseBody as Body>::Error: Into<StdError> + std::marker::Send,
{
pub fn new(inner: T) -> Self {
let inner = tonic::client::Grpc::new(inner);
Self { inner }
}
pub fn with_origin(inner: T, origin: Uri) -> Self {
let inner = tonic::client::Grpc::with_origin(inner, origin);
Self { inner }
}
pub fn with_interceptor<F>(
inner: T,
interceptor: F,
) -> ShredstreamClient<InterceptedService<T, F>>
where
F: tonic::service::Interceptor,
T::ResponseBody: Default,
T: tonic::codegen::Service<
http::Request<tonic::body::BoxBody>,
Response = http::Response<
<T as tonic::client::GrpcService<tonic::body::BoxBody>>::ResponseBody,
>,
>,
<T as tonic::codegen::Service<
http::Request<tonic::body::BoxBody>,
>>::Error: Into<StdError> + std::marker::Send + std::marker::Sync,
{
ShredstreamClient::new(InterceptedService::new(inner, interceptor))
}
/// Compress requests with the given encoding.
///
/// This requires the server to support it otherwise it might respond with an
/// error.
#[must_use]
pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self {
self.inner = self.inner.send_compressed(encoding);
self
}
/// Enable decompressing responses.
#[must_use]
pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self {
self.inner = self.inner.accept_compressed(encoding);
self
}
/// Limits the maximum size of a decoded message.
///
/// Default: `4MB`
#[must_use]
pub fn max_decoding_message_size(mut self, limit: usize) -> Self {
self.inner = self.inner.max_decoding_message_size(limit);
self
}
/// Limits the maximum size of an encoded message.
///
/// Default: `usize::MAX`
#[must_use]
pub fn max_encoding_message_size(mut self, limit: usize) -> Self {
self.inner = self.inner.max_encoding_message_size(limit);
self
}
/// RPC endpoint to send heartbeats to keep shreds flowing
pub async fn send_heartbeat(
&mut self,
request: impl tonic::IntoRequest<super::Heartbeat>,
) -> std::result::Result<
tonic::Response<super::HeartbeatResponse>,
tonic::Status,
> {
self.inner
.ready()
.await
.map_err(|e| {
tonic::Status::unknown(
format!("Service was not ready: {}", e.into()),
)
})?;
let codec = tonic::codec::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static(
"/shredstream.Shredstream/SendHeartbeat",
);
let mut req = request.into_request();
req.extensions_mut()
.insert(GrpcMethod::new("shredstream.Shredstream", "SendHeartbeat"));
self.inner.unary(req, path, codec).await
}
}
}
/// Generated client implementations.
pub mod shredstream_proxy_client {
#![allow(
unused_variables,
dead_code,
missing_docs,
clippy::wildcard_imports,
clippy::let_unit_value,
)]
use tonic::codegen::*;
use tonic::codegen::http::Uri;
#[derive(Debug, Clone)]
pub struct ShredstreamProxyClient<T> {
inner: tonic::client::Grpc<T>,
}
impl ShredstreamProxyClient<tonic::transport::Channel> {
/// Attempt to create a new client by connecting to a given endpoint.
pub async fn connect<D>(dst: D) -> Result<Self, tonic::transport::Error>
where
D: TryInto<tonic::transport::Endpoint>,
D::Error: Into<StdError>,
{
let conn = tonic::transport::Endpoint::new(dst)?.connect().await?;
Ok(Self::new(conn))
}
}
impl<T> ShredstreamProxyClient<T>
where
T: tonic::client::GrpcService<tonic::body::BoxBody>,
T::Error: Into<StdError>,
T::ResponseBody: Body<Data = Bytes> + std::marker::Send + 'static,
<T::ResponseBody as Body>::Error: Into<StdError> + std::marker::Send,
{
pub fn new(inner: T) -> Self {
let inner = tonic::client::Grpc::new(inner);
Self { inner }
}
pub fn with_origin(inner: T, origin: Uri) -> Self {
let inner = tonic::client::Grpc::with_origin(inner, origin);
Self { inner }
}
pub fn with_interceptor<F>(
inner: T,
interceptor: F,
) -> ShredstreamProxyClient<InterceptedService<T, F>>
where
F: tonic::service::Interceptor,
T::ResponseBody: Default,
T: tonic::codegen::Service<
http::Request<tonic::body::BoxBody>,
Response = http::Response<
<T as tonic::client::GrpcService<tonic::body::BoxBody>>::ResponseBody,
>,
>,
<T as tonic::codegen::Service<
http::Request<tonic::body::BoxBody>,
>>::Error: Into<StdError> + std::marker::Send + std::marker::Sync,
{
ShredstreamProxyClient::new(InterceptedService::new(inner, interceptor))
}
/// Compress requests with the given encoding.
///
/// This requires the server to support it otherwise it might respond with an
/// error.
#[must_use]
pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self {
self.inner = self.inner.send_compressed(encoding);
self
}
/// Enable decompressing responses.
#[must_use]
pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self {
self.inner = self.inner.accept_compressed(encoding);
self
}
/// Limits the maximum size of a decoded message.
///
/// Default: `4MB`
#[must_use]
pub fn max_decoding_message_size(mut self, limit: usize) -> Self {
self.inner = self.inner.max_decoding_message_size(limit);
self
}
/// Limits the maximum size of an encoded message.
///
/// Default: `usize::MAX`
#[must_use]
pub fn max_encoding_message_size(mut self, limit: usize) -> Self {
self.inner = self.inner.max_encoding_message_size(limit);
self
}
pub async fn subscribe_entries(
&mut self,
request: impl tonic::IntoRequest<super::SubscribeEntriesRequest>,
) -> std::result::Result<
tonic::Response<tonic::codec::Streaming<super::Entry>>,
tonic::Status,
> {
self.inner
.ready()
.await
.map_err(|e| {
tonic::Status::unknown(
format!("Service was not ready: {}", e.into()),
)
})?;
let codec = tonic::codec::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static(
"/shredstream.ShredstreamProxy/SubscribeEntries",
);
let mut req = request.into_request();
req.extensions_mut()
.insert(
GrpcMethod::new("shredstream.ShredstreamProxy", "SubscribeEntries"),
);
self.inner.server_streaming(req, path, codec).await
}
}
}