mirror of
https://github.com/0xfnzero/solana-streamer.git
synced 2026-08-16 18:38:05 +00:00
feat: refactor to multi-protocol event streaming system
Major architectural refactor, upgrading from simple logging system to comprehensive multi-protocol Solana DEX event streaming system: ✨ New Features: - Support for 5 DEX protocols: PumpFun, PumpSwap, Bonk, Raydium CPMM, Raydium CLMM - Implement unified event interface (UnifiedEvent trait) and event factory pattern - Add dual streaming support: Yellowstone gRPC and ShredStream - Add Chinese documentation (README_CN.md) 🏗️ Architectural Improvements: - Refactor event parsing system with modular design - Implement protocol-specific parsers and event types - Optimize dependency management, update Cargo.toml - Remove legacy logging modules, clean up redundant code 📊 Statistics: - Added 46 files, 4511 lines of code - Removed 1381 lines of legacy code - Net addition of 3130 lines of code Tech Stack: - Rust async/await for asynchronous processing - Protocol Buffers support - Multi-protocol event parsing - High-performance event stream subscription
This commit is contained in:
Executable
+2
@@ -0,0 +1,2 @@
|
||||
pub mod shared;
|
||||
pub mod shredstream;
|
||||
Executable
+18
@@ -0,0 +1,18 @@
|
||||
// This file is @generated by prost-build.
|
||||
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
|
||||
pub struct Header {
|
||||
#[prost(message, optional, tag = "1")]
|
||||
pub ts: ::core::option::Option<::prost_types::Timestamp>,
|
||||
}
|
||||
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
|
||||
pub struct Heartbeat {
|
||||
#[prost(uint64, tag = "1")]
|
||||
pub count: u64,
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct Socket {
|
||||
#[prost(string, tag = "1")]
|
||||
pub ip: ::prost::alloc::string::String,
|
||||
#[prost(int64, tag = "2")]
|
||||
pub port: i64,
|
||||
}
|
||||
Executable
+279
@@ -0,0 +1,279 @@
|
||||
// This file is @generated by prost-build.
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct Heartbeat {
|
||||
/// don't trust IP:PORT from tcp header since it can be tampered over the wire
|
||||
/// `socket.ip` must match incoming packet's ip. this prevents spamming an unwitting destination
|
||||
#[prost(message, optional, tag = "1")]
|
||||
pub socket: ::core::option::Option<super::shared::Socket>,
|
||||
/// regions for shredstream proxy to receive shreds from
|
||||
/// list of valid regions: <https://docs.jito.wtf/lowlatencytxnsend/#api>
|
||||
#[prost(string, repeated, tag = "2")]
|
||||
pub regions: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
|
||||
}
|
||||
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
|
||||
pub struct HeartbeatResponse {
|
||||
/// client must respond within `ttl_ms` to keep stream alive
|
||||
#[prost(uint32, tag = "1")]
|
||||
pub ttl_ms: u32,
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct TraceShred {
|
||||
/// source region, one of: <https://docs.jito.wtf/lowlatencytxnsend/#api>
|
||||
#[prost(string, tag = "1")]
|
||||
pub region: ::prost::alloc::string::String,
|
||||
/// timestamp of creation
|
||||
#[prost(message, optional, tag = "2")]
|
||||
pub created_at: ::core::option::Option<::prost_types::Timestamp>,
|
||||
/// monotonically increases, resets upon service restart
|
||||
#[prost(uint32, tag = "3")]
|
||||
pub seq_num: u32,
|
||||
}
|
||||
/// tbd: we may want to add filters here
|
||||
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
|
||||
pub struct SubscribeEntriesRequest {}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct Entry {
|
||||
/// the slot that the entry is from
|
||||
#[prost(uint64, tag = "1")]
|
||||
pub slot: u64,
|
||||
/// Serialized bytes of Vec<Entry>: <https://docs.rs/solana-entry/latest/solana_entry/entry/struct.Entry.html>
|
||||
#[prost(bytes = "vec", tag = "2")]
|
||||
pub entries: ::prost::alloc::vec::Vec<u8>,
|
||||
}
|
||||
/// Generated client implementations.
|
||||
pub mod shredstream_client {
|
||||
#![allow(
|
||||
unused_variables,
|
||||
dead_code,
|
||||
missing_docs,
|
||||
clippy::wildcard_imports,
|
||||
clippy::let_unit_value,
|
||||
)]
|
||||
use tonic::codegen::*;
|
||||
use tonic::codegen::http::Uri;
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ShredstreamClient<T> {
|
||||
inner: tonic::client::Grpc<T>,
|
||||
}
|
||||
impl ShredstreamClient<tonic::transport::Channel> {
|
||||
/// Attempt to create a new client by connecting to a given endpoint.
|
||||
pub async fn connect<D>(dst: D) -> Result<Self, tonic::transport::Error>
|
||||
where
|
||||
D: TryInto<tonic::transport::Endpoint>,
|
||||
D::Error: Into<StdError>,
|
||||
{
|
||||
let conn = tonic::transport::Endpoint::new(dst)?.connect().await?;
|
||||
Ok(Self::new(conn))
|
||||
}
|
||||
}
|
||||
impl<T> ShredstreamClient<T>
|
||||
where
|
||||
T: tonic::client::GrpcService<tonic::body::BoxBody>,
|
||||
T::Error: Into<StdError>,
|
||||
T::ResponseBody: Body<Data = Bytes> + std::marker::Send + 'static,
|
||||
<T::ResponseBody as Body>::Error: Into<StdError> + std::marker::Send,
|
||||
{
|
||||
pub fn new(inner: T) -> Self {
|
||||
let inner = tonic::client::Grpc::new(inner);
|
||||
Self { inner }
|
||||
}
|
||||
pub fn with_origin(inner: T, origin: Uri) -> Self {
|
||||
let inner = tonic::client::Grpc::with_origin(inner, origin);
|
||||
Self { inner }
|
||||
}
|
||||
pub fn with_interceptor<F>(
|
||||
inner: T,
|
||||
interceptor: F,
|
||||
) -> ShredstreamClient<InterceptedService<T, F>>
|
||||
where
|
||||
F: tonic::service::Interceptor,
|
||||
T::ResponseBody: Default,
|
||||
T: tonic::codegen::Service<
|
||||
http::Request<tonic::body::BoxBody>,
|
||||
Response = http::Response<
|
||||
<T as tonic::client::GrpcService<tonic::body::BoxBody>>::ResponseBody,
|
||||
>,
|
||||
>,
|
||||
<T as tonic::codegen::Service<
|
||||
http::Request<tonic::body::BoxBody>,
|
||||
>>::Error: Into<StdError> + std::marker::Send + std::marker::Sync,
|
||||
{
|
||||
ShredstreamClient::new(InterceptedService::new(inner, interceptor))
|
||||
}
|
||||
/// Compress requests with the given encoding.
|
||||
///
|
||||
/// This requires the server to support it otherwise it might respond with an
|
||||
/// error.
|
||||
#[must_use]
|
||||
pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self {
|
||||
self.inner = self.inner.send_compressed(encoding);
|
||||
self
|
||||
}
|
||||
/// Enable decompressing responses.
|
||||
#[must_use]
|
||||
pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self {
|
||||
self.inner = self.inner.accept_compressed(encoding);
|
||||
self
|
||||
}
|
||||
/// Limits the maximum size of a decoded message.
|
||||
///
|
||||
/// Default: `4MB`
|
||||
#[must_use]
|
||||
pub fn max_decoding_message_size(mut self, limit: usize) -> Self {
|
||||
self.inner = self.inner.max_decoding_message_size(limit);
|
||||
self
|
||||
}
|
||||
/// Limits the maximum size of an encoded message.
|
||||
///
|
||||
/// Default: `usize::MAX`
|
||||
#[must_use]
|
||||
pub fn max_encoding_message_size(mut self, limit: usize) -> Self {
|
||||
self.inner = self.inner.max_encoding_message_size(limit);
|
||||
self
|
||||
}
|
||||
/// RPC endpoint to send heartbeats to keep shreds flowing
|
||||
pub async fn send_heartbeat(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::Heartbeat>,
|
||||
) -> std::result::Result<
|
||||
tonic::Response<super::HeartbeatResponse>,
|
||||
tonic::Status,
|
||||
> {
|
||||
self.inner
|
||||
.ready()
|
||||
.await
|
||||
.map_err(|e| {
|
||||
tonic::Status::unknown(
|
||||
format!("Service was not ready: {}", e.into()),
|
||||
)
|
||||
})?;
|
||||
let codec = tonic::codec::ProstCodec::default();
|
||||
let path = http::uri::PathAndQuery::from_static(
|
||||
"/shredstream.Shredstream/SendHeartbeat",
|
||||
);
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(GrpcMethod::new("shredstream.Shredstream", "SendHeartbeat"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
}
|
||||
}
|
||||
/// Generated client implementations.
|
||||
pub mod shredstream_proxy_client {
|
||||
#![allow(
|
||||
unused_variables,
|
||||
dead_code,
|
||||
missing_docs,
|
||||
clippy::wildcard_imports,
|
||||
clippy::let_unit_value,
|
||||
)]
|
||||
use tonic::codegen::*;
|
||||
use tonic::codegen::http::Uri;
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ShredstreamProxyClient<T> {
|
||||
inner: tonic::client::Grpc<T>,
|
||||
}
|
||||
impl ShredstreamProxyClient<tonic::transport::Channel> {
|
||||
/// Attempt to create a new client by connecting to a given endpoint.
|
||||
pub async fn connect<D>(dst: D) -> Result<Self, tonic::transport::Error>
|
||||
where
|
||||
D: TryInto<tonic::transport::Endpoint>,
|
||||
D::Error: Into<StdError>,
|
||||
{
|
||||
let conn = tonic::transport::Endpoint::new(dst)?.connect().await?;
|
||||
Ok(Self::new(conn))
|
||||
}
|
||||
}
|
||||
impl<T> ShredstreamProxyClient<T>
|
||||
where
|
||||
T: tonic::client::GrpcService<tonic::body::BoxBody>,
|
||||
T::Error: Into<StdError>,
|
||||
T::ResponseBody: Body<Data = Bytes> + std::marker::Send + 'static,
|
||||
<T::ResponseBody as Body>::Error: Into<StdError> + std::marker::Send,
|
||||
{
|
||||
pub fn new(inner: T) -> Self {
|
||||
let inner = tonic::client::Grpc::new(inner);
|
||||
Self { inner }
|
||||
}
|
||||
pub fn with_origin(inner: T, origin: Uri) -> Self {
|
||||
let inner = tonic::client::Grpc::with_origin(inner, origin);
|
||||
Self { inner }
|
||||
}
|
||||
pub fn with_interceptor<F>(
|
||||
inner: T,
|
||||
interceptor: F,
|
||||
) -> ShredstreamProxyClient<InterceptedService<T, F>>
|
||||
where
|
||||
F: tonic::service::Interceptor,
|
||||
T::ResponseBody: Default,
|
||||
T: tonic::codegen::Service<
|
||||
http::Request<tonic::body::BoxBody>,
|
||||
Response = http::Response<
|
||||
<T as tonic::client::GrpcService<tonic::body::BoxBody>>::ResponseBody,
|
||||
>,
|
||||
>,
|
||||
<T as tonic::codegen::Service<
|
||||
http::Request<tonic::body::BoxBody>,
|
||||
>>::Error: Into<StdError> + std::marker::Send + std::marker::Sync,
|
||||
{
|
||||
ShredstreamProxyClient::new(InterceptedService::new(inner, interceptor))
|
||||
}
|
||||
/// Compress requests with the given encoding.
|
||||
///
|
||||
/// This requires the server to support it otherwise it might respond with an
|
||||
/// error.
|
||||
#[must_use]
|
||||
pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self {
|
||||
self.inner = self.inner.send_compressed(encoding);
|
||||
self
|
||||
}
|
||||
/// Enable decompressing responses.
|
||||
#[must_use]
|
||||
pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self {
|
||||
self.inner = self.inner.accept_compressed(encoding);
|
||||
self
|
||||
}
|
||||
/// Limits the maximum size of a decoded message.
|
||||
///
|
||||
/// Default: `4MB`
|
||||
#[must_use]
|
||||
pub fn max_decoding_message_size(mut self, limit: usize) -> Self {
|
||||
self.inner = self.inner.max_decoding_message_size(limit);
|
||||
self
|
||||
}
|
||||
/// Limits the maximum size of an encoded message.
|
||||
///
|
||||
/// Default: `usize::MAX`
|
||||
#[must_use]
|
||||
pub fn max_encoding_message_size(mut self, limit: usize) -> Self {
|
||||
self.inner = self.inner.max_encoding_message_size(limit);
|
||||
self
|
||||
}
|
||||
pub async fn subscribe_entries(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::SubscribeEntriesRequest>,
|
||||
) -> std::result::Result<
|
||||
tonic::Response<tonic::codec::Streaming<super::Entry>>,
|
||||
tonic::Status,
|
||||
> {
|
||||
self.inner
|
||||
.ready()
|
||||
.await
|
||||
.map_err(|e| {
|
||||
tonic::Status::unknown(
|
||||
format!("Service was not ready: {}", e.into()),
|
||||
)
|
||||
})?;
|
||||
let codec = tonic::codec::ProstCodec::default();
|
||||
let path = http::uri::PathAndQuery::from_static(
|
||||
"/shredstream.ShredstreamProxy/SubscribeEntries",
|
||||
);
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(
|
||||
GrpcMethod::new("shredstream.ShredstreamProxy", "SubscribeEntries"),
|
||||
);
|
||||
self.inner.server_streaming(req, path, codec).await
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user