From aa145dc01801ca4ef5ea3dbff24b37b37ca80962 Mon Sep 17 00:00:00 2001 From: smypmsa Date: Tue, 6 May 2025 08:39:09 +0000 Subject: [PATCH] feat: add geyser auth type, compare listeners for new tokens --- .env.example | 9 +- bots/bot-sniper-1-geyser.yaml | 1 + bots/bot-sniper-2-logs.yaml | 1 + .../listen-new-tokens/compare_listeners.py | 637 ++++++++++++++++++ .../listen-new-tokens/listen_geyser.py | 18 +- src/bot_runner.py | 1 + src/monitoring/geyser_listener.py | 15 +- src/trading/trader.py | 5 +- 8 files changed, 674 insertions(+), 13 deletions(-) create mode 100644 learning-examples/listen-new-tokens/compare_listeners.py diff --git a/.env.example b/.env.example index b3d084c..6d5aa2c 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,8 @@ SOLANA_NODE_RPC_ENDPOINT=... SOLANA_NODE_WSS_ENDPOINT=... -GEYSER_ENDPOINT=... -GEYSER_API_TOKEN=... -SOLANA_PRIVATE_KEY=... \ No newline at end of file + +GEYSER_ENDPOINT= +GEYSER_API_TOKEN= +GEYSER_AUTH_TYPE=x-token or basic + +SOLANA_PRIVATE_KEY= \ No newline at end of file diff --git a/bots/bot-sniper-1-geyser.yaml b/bots/bot-sniper-1-geyser.yaml index 65b38c2..cdb010a 100644 --- a/bots/bot-sniper-1-geyser.yaml +++ b/bots/bot-sniper-1-geyser.yaml @@ -15,6 +15,7 @@ separate_process: true geyser: endpoint: "${GEYSER_ENDPOINT}" api_token: "${GEYSER_API_TOKEN}" + auth_type: "x-token" # or "basic" # Trading parameters # Control trade execution: amount of SOL per trade and acceptable price deviation diff --git a/bots/bot-sniper-2-logs.yaml b/bots/bot-sniper-2-logs.yaml index 08c24b2..e50ea58 100644 --- a/bots/bot-sniper-2-logs.yaml +++ b/bots/bot-sniper-2-logs.yaml @@ -15,6 +15,7 @@ separate_process: true geyser: endpoint: "${GEYSER_ENDPOINT}" api_token: "${GEYSER_API_TOKEN}" + auth_type: "basic" # or "x-token" # Trading parameters # Control trade execution: amount of SOL per trade and acceptable price deviation diff --git a/learning-examples/listen-new-tokens/compare_listeners.py b/learning-examples/listen-new-tokens/compare_listeners.py new file mode 100644 index 0000000..3adc316 --- /dev/null +++ b/learning-examples/listen-new-tokens/compare_listeners.py @@ -0,0 +1,637 @@ +""" +This script compares four methods of detecting new Pump.fun tokens: +1. Block subscription listener - listens for blocks containing Pump.fun program +2. Geyser gRPC listener - uses Geyser gRPC API to get transactions containing Pump.fun program +3. Logs subscription listener - listens for logs containing Pump.fun program +4. PumpPortal WebSocket listener - connects to PumpPortal WebSocket and listens for token events + +The script tracks which method detects new tokens first and provides detailed performance statistics. + +Note: multiple endpoints available. Scroll down to change providers which you want to test. +""" + +import asyncio +import base64 +import json +import os +import struct +import time + +import base58 +import grpc +import websockets +from dotenv import load_dotenv +from solders.pubkey import Pubkey +from solders.transaction import VersionedTransaction + +load_dotenv(override=True) + +# Constants +PUMP_PROGRAM_ID = Pubkey.from_string("6EF8rrecthR5Dkzon8Nwu78hRvfCKubJ14M5uBEwF6P") +PUMP_CREATE_PREFIX = struct.pack(" 0 else 0 + print(f"{provider:<22} | {count:<16} | {percentage:.1f}%") + + # Calculate average latency between providers + self._print_provider_latency_matrix() + + def _print_provider_latency_matrix(self): + """Print a matrix of average latency between providers""" + # Get unique providers + all_providers = set() + for token_data in self.tokens.values(): + all_providers.update(token_data['detections'].keys()) + + if len(all_providers) <= 1: + return + + providers_list = sorted(all_providers) + + print("\nAverage Latency Matrix (ms):") + # Print header + header = " |" + for provider in providers_list: + header += f" {provider[:8]:>8} |" + print(header) + print("-" * len(header)) + + # Calculate and print latency matrix + for provider1 in providers_list: + row = f"{provider1[:8]:>8} |" + for provider2 in providers_list: + if provider1 == provider2: + row += " — |" + continue + + # Calculate average latency + latencies = [] + for token_data in self.tokens.values(): + detections = token_data['detections'] + if provider1 in detections and provider2 in detections: + latency_ms = (detections[provider2] - detections[provider1]) * 1000 + latencies.append(latency_ms) + + if latencies: + avg_latency = sum(latencies) / len(latencies) + row += f" {avg_latency:>+7.1f} |" + else: + row += " ? |" + print(row) + + +# ============ TOKEN DETECTION METHODS ============ + +async def fetch_existing_token_mints(): + """ + Fetch existing token mints to avoid duplicate detections + """ + # You could implement this by querying a known database or API + # For simplicity, we'll return an empty set + return set() + + +def parse_create_instruction(data): + """ + Parse binary create instruction data into a structured format + """ + if len(data) < 8: + return None + + offset = 8 # Skip discriminator + parsed_data = {} + + try: + # Parse name (string) + length = struct.unpack(" 0: + try: + mint = str(transaction.message.account_keys[ix.accounts[0]]) # First account is usually the mint + + if mint in known_tokens: + continue + + ts = time.time() + tracker.add_token(mint, parsed["name"], parsed["symbol"], + f"{provider_name}_block", ts) + known_tokens.add(mint) + except Exception as e: + print(f"[ERROR] Failed to process block instruction: {e}") + except Exception as e: + print(f"[ERROR] Failed to process transaction: {e}") + + except Exception as e: + print(f"[ERROR] Block listener for {provider_name}: {e}") + + except Exception as e: + print(f"[ERROR] Connection error in block listener for {provider_name}: {e}") + print("[INFO] Reconnecting in 5 seconds...") + await asyncio.sleep(5) + + +async def listen_logs_subscription(wss_url, provider_name, tracker, known_tokens=None): + """ + Listen for new tokens via logs subscription + """ + if known_tokens is None: + known_tokens = set() + + while True: + try: + print(f"[INFO] Connecting logs listener to {provider_name}...") + async with websockets.connect(wss_url) as websocket: + subscription_message = json.dumps({ + "jsonrpc": "2.0", + "id": 1, + "method": "logsSubscribe", + "params": [ + {"mentions": [str(PUMP_PROGRAM_ID)]}, + {"commitment": "processed"}, + ], + }) + await websocket.send(subscription_message) + await websocket.recv() + print(f"[INFO] Logs listener active for {provider_name}") + + while True: + try: + response = await websocket.recv() + data = json.loads(response) + tracker.increment_messages(provider_name) + + if data.get("method") != "logsNotification": + continue + + log_data = data["params"]["result"]["value"] + logs = log_data.get("logs", []) + + if not any("Program log: Instruction: Create" in log for log in logs): + continue + + for log in logs: + if "Program data:" in log: + try: + encoded_data = log.split(": ")[1] + data_bytes = base64.b64decode(encoded_data) + + parsed = parse_create_instruction(data_bytes) + if not parsed: + continue + + mint = parsed.get("mint") + if not mint: + continue + if mint in known_tokens: + continue + + ts = time.time() + tracker.add_token( + mint, + parsed.get("name", "Unknown"), + parsed.get("symbol", "UNK"), + f"{provider_name}_logs", + ts + ) + known_tokens.add(mint) + break + except Exception as e: + print(f"[ERROR] Failed to decode Program data: {e}") + + except Exception as e: + print(f"[ERROR] Logs listener for {provider_name}: {e}") + break + + except Exception as e: + print(f"[ERROR] Connection error in logs listener for {provider_name}: {e}") + print("[INFO] Reconnecting in 5 seconds...") + await asyncio.sleep(5) + + +async def listen_geyser_grpc(endpoint, api_token, provider_name, tracker, known_tokens=None): + """ + Listen for new tokens via Geyser gRPC API + """ + try: + # Import the generated protobuf modules + from generated import geyser_pb2, geyser_pb2_grpc + except ImportError: + print("[ERROR] Could not import geyser_pb2 or geyser_pb2_grpc. Make sure to generate from .proto files") + return + + if known_tokens is None: + known_tokens = set() + + while True: + try: + print(f"[INFO] Connecting Geyser gRPC listener to {provider_name}...") + + if GEYSER_AUTH_TYPE == "x-token": + auth = grpc.metadata_call_credentials( + lambda context, callback: callback((("x-token", api_token),), None) + ) + else: + auth = grpc.metadata_call_credentials( + lambda context, callback: callback((("authorization", f"Basic {api_token}"),), None) + ) + + creds = grpc.composite_channel_credentials(grpc.ssl_channel_credentials(), auth) + channel = grpc.aio.secure_channel(endpoint, creds) + stub = geyser_pb2_grpc.GeyserStub(channel) + + request = geyser_pb2.SubscribeRequest() + request.transactions["pump_filter"].account_include.append(str(PUMP_PROGRAM_ID)) + request.transactions["pump_filter"].failed = False + request.commitment = geyser_pb2.CommitmentLevel.PROCESSED + + print(f"[INFO] Geyser gRPC listener active for {provider_name}") + + async for update in stub.Subscribe(iter([request])): + tracker.increment_messages(provider_name) + + # Skip non-transaction updates + if not update.HasField("transaction"): + continue + + tx = update.transaction.transaction.transaction + msg = getattr(tx, "message", None) + if msg is None: + continue + + for ix in msg.instructions: + if not ix.data.startswith(PUMP_CREATE_PREFIX): + continue + + parsed = parse_create_instruction(ix.data) + if not parsed: + continue + + if len(ix.accounts) == 0 or ix.accounts[0] >= len(msg.account_keys): + continue + + mint = base58.b58encode(bytes(msg.account_keys[ix.accounts[0]])).decode() + + if mint in known_tokens: + continue + + ts = time.time() + tracker.add_token(mint, parsed["name"], parsed["symbol"], + f"{provider_name}_geyser", ts) + known_tokens.add(mint) + + except Exception as e: + print(f"[ERROR] Connection error in Geyser gRPC listener for {provider_name}: {e}") + print("[INFO] Reconnecting in 5 seconds...") + await asyncio.sleep(5) + + +async def listen_pumpportal(provider_name, tracker, known_tokens=None): + """ + Listen for new tokens via PumpPortal WebSocket + """ + if known_tokens is None: + known_tokens = set() + + while True: + try: + print("[INFO] Connecting to PumpPortal WebSocket...") + async with websockets.connect(PUMPPORTAL_WS_URL) as websocket: + # Subscribe to new token events + await websocket.send(json.dumps({"method": "subscribeNewToken", "params": []})) + print(f"[INFO] PumpPortal listener active for {provider_name}") + + while True: + try: + # Receive WebSocket message + message = await websocket.recv() + data = json.loads(message) + tracker.increment_messages(provider_name) + + # Extract token information + token_info = None + if "method" in data and data["method"] == "newToken": + token_info = data.get("params", [{}])[0] + elif "signature" in data and "mint" in data: + token_info = data + + if not token_info: + continue + + # Get token details + mint = token_info.get("mint") + name = token_info.get("name", "Unknown") + symbol = token_info.get("symbol", "UNK") + + if not mint: + continue + + # Skip known tokens + if mint in known_tokens: + continue + + # Record the token detection + ts = time.time() + tracker.add_token(mint, name, symbol, + f"{provider_name}_pumpportal", ts) + known_tokens.add(mint) + + except Exception as e: + print(f"[ERROR] PumpPortal listener for {provider_name}: {e}") + break + + except Exception as e: + print(f"[ERROR] Connection error in PumpPortal listener for {provider_name}: {e}") + print("[INFO] Reconnecting in 5 seconds...") + await asyncio.sleep(5) + + +# ============ MAIN TEST RUNNER ============ + +async def run_comparison_test(providers, test_duration=600): + """ + Run the comparison test with multiple WebSocket endpoints + + Args: + providers: Dict of {provider_name: {'wss': wss_url, 'geyser': (endpoint, api_token)}} + test_duration: How long to run the test in seconds (default: 10 minutes) + """ + # Initialize our tracker and fetch existing tokens to avoid duplicates + tracker = DetectionTracker() + known_tokens = await fetch_existing_token_mints() + print(f"[INFO] Loaded {len(known_tokens)} existing tokens") + + tasks = [] + + # Start all listeners for each provider + for provider_name, urls in providers.items(): + if urls.get('wss'): + print(f"[INFO] Starting block listener for {provider_name}") + task = asyncio.create_task( + listen_block_subscription(urls['wss'], provider_name, tracker, known_tokens.copy()) + ) + tasks.append(task) + + if urls.get('wss'): + print(f"[INFO] Starting logs listener for {provider_name}") + task = asyncio.create_task( + listen_logs_subscription(urls['wss'], provider_name, tracker, known_tokens.copy()) + ) + tasks.append(task) + + if urls.get('geyser'): + endpoint, api_token = urls['geyser'] + if endpoint and api_token: + print(f"[INFO] Starting Geyser gRPC listener for {provider_name}") + task = asyncio.create_task( + listen_geyser_grpc(endpoint, api_token, provider_name, tracker, known_tokens.copy()) + ) + tasks.append(task) + + # Start PumpPortal listener (only once, not per provider) + print("[INFO] Starting PumpPortal listener") + task = asyncio.create_task( + listen_pumpportal("pumpportal", tracker, known_tokens.copy()) + ) + tasks.append(task) + + print(f"[INFO] Test running for {test_duration} seconds...") + await asyncio.sleep(test_duration) + + for task in tasks: + task.cancel() + + await asyncio.gather(*tasks, return_exceptions=True) + tracker.print_summary() + + +if __name__ == "__main__": + # Read providers from environment variables + providers = { + "provider_1": { + 'wss': os.environ.get("SOLANA_NODE_WSS_ENDPOINT"), + 'geyser': ( + os.environ.get("GEYSER_ENDPOINT"), + os.environ.get("GEYSER_API_TOKEN") + ) + }, + # Add more providers to .env as needed + } + + # Filter out any providers with missing endpoints + providers = {name: urls for name, urls in providers.items() + if (urls.get('wss')) or + ('geyser' in urls and urls['geyser'][0] and urls['geyser'][1])} + + print(f"[INFO] Starting Pump.fun token detector comparison test for {TEST_DURATION} seconds") + print(f"[INFO] Providers: {', '.join(providers.keys())}") + + asyncio.run(run_comparison_test(providers, test_duration=TEST_DURATION)) \ No newline at end of file diff --git a/learning-examples/listen-new-tokens/listen_geyser.py b/learning-examples/listen-new-tokens/listen_geyser.py index 4290c6c..5e617cb 100644 --- a/learning-examples/listen-new-tokens/listen_geyser.py +++ b/learning-examples/listen-new-tokens/listen_geyser.py @@ -2,6 +2,7 @@ Monitors Solana for new Pump.fun token creations using Geyser gRPC. Decodes 'create' instructions to extract and display token details (name, symbol, mint, bonding curve). Requires a Geyser API token for access. +Supports both Basic and X-Token authentication methods. It is proven to be the fastest listener. """ @@ -21,16 +22,24 @@ load_dotenv() GEYSER_ENDPOINT = os.getenv("GEYSER_ENDPOINT") GEYSER_API_TOKEN = os.getenv("GEYSER_API_TOKEN") +# Default to x-token auth, can be set to "basic" +AUTH_TYPE = "x-token" PUMP_PROGRAM_ID = Pubkey.from_string("6EF8rrecthR5Dkzon8Nwu78hRvfCKubJ14M5uBEwF6P") PUMP_CREATE_PREFIX = struct.pack("