diff --git a/CLAUDE.md b/CLAUDE.md index 90c0d96..7449c68 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -151,11 +151,13 @@ user-supplied and post-2026-04-28 may differ from the canonical `BC.creator` **not** set the flag; the geyser parser prefers `meta.log_messages` over instruction decoding for exactly this reason. `trade.trust_create_event: false` is the escape hatch back to always-refresh. PumpPortal payloads carry -none of these fields and always refresh. Related pitfall: the strict IDL -instruction decoder rejects `create_v2` transactions that omit the trailing -`is_cashback_enabled` OptionBool (a legal wire form), so the instruction path -alone silently misses those coins — one more reason the log/event path is -preferred everywhere. +none of these fields and always refresh. Related pitfall (fixed in #184): the +IDL instruction decoder used to reject `create_v2` transactions that omit the +trailing `is_cashback_enabled` OptionBool (a legal wire form), silently +dropping those coins from the instruction path. It now reports omitted +trailing option-typed args as unset — `verify_create_v2_optional_args.py` +machine-checks that, and that mandatory args still fail the decode. The +log/event path stays preferred for the canonical-creator reason above. ### Verifying transaction-status handling @@ -315,7 +317,9 @@ The IDLs under `idl/` are vendored verbatim from `github.com/pump-fun/pump-publi `create_v2` instructions carry `0001` and `00` after `creator`: one sends both trailing args, the other omits the last. A decoder that reads a fixed number of trailing bytes raises `IndexError` on roughly half of all coins. Decode - trailing args defensively and report a missing one as unset. + trailing args defensively and report a missing one as unset — + `utils/idl_parser.py` does this for trailing option-typed args since #184 + (`uv run learning-examples/verify_create_v2_optional_args.py` checks it). - `create_v2` accounts 1-16 are in the IDL; accounts **17-19 are optional remaining accounts** (`quote_mint`, `associated_quote_bonding_curve`, `quote_token_program`). All three or none. This is the only way to read a new diff --git a/learning-examples/verify_create_v2_optional_args.py b/learning-examples/verify_create_v2_optional_args.py new file mode 100644 index 0000000..993a844 --- /dev/null +++ b/learning-examples/verify_create_v2_optional_args.py @@ -0,0 +1,188 @@ +"""Verify the IDL instruction decoder accepts omitted trailing optional args. + +create_v2's trailing `is_cashback_enabled` OptionBool can legally be absent +from the wire (issue #184): the committed blocksubscribe fixture carries a +145-byte create_v2 whose args end right after `is_mayhem_mode`. A decoder +that insists on the byte silently drops roughly half of all coins for every +consumer of `parse_token_creation_from_instruction`. + +Offline machine checks, no network and no funds moved: + + 1. The raw fixture create_v2 (trailing OptionBool absent) decodes, with + `is_cashback_enabled` reported as unset (None). + 2. The same data with the byte present still decodes to the OptionBool + struct form ({"field_0": bool}) consumers already handle. + 3. Mandatory args stay enforced: truncating `is_mayhem_mode` as well must + fail the decode, not fabricate a default. + 4. Native `option` types decode (update_buyback_config's Option): + None tag, Some tag, and omitted-trailing forms. + 5. The pump.fun event parser turns the raw fixture instruction into a + TokenInfo with is_cashback_coin=False and state_from_event=False. + +Usage: + uv run learning-examples/verify_create_v2_optional_args.py +""" + +import base64 +import json +import struct +import sys +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(PROJECT_ROOT / "src")) + +from solders.transaction import VersionedTransaction # noqa: E402 + +from interfaces.core import Platform # noqa: E402 +from platforms.pumpfun.event_parser import PumpFunEventParser # noqa: E402 +from utils.idl_manager import get_idl_manager # noqa: E402 +from utils.idl_parser import IDLParser # noqa: E402 + +FIXTURE = ( + PROJECT_ROOT + / "learning-examples" + / "blocksubscribe-transactions" + / "raw_create_tx_from_blocksubscribe.json" +) +IDL_PATH = PROJECT_ROOT / "idl" / "pump_fun_idl.json" + +CREATE_V2_DISCRIMINATOR = bytes.fromhex("d6904cec5f8b31b4") +UPDATE_BUYBACK_CONFIG_DISCRIMINATOR = bytes([251, 224, 171, 146, 160, 26, 113, 233]) + + +def _parser() -> IDLParser: + return IDLParser(str(IDL_PATH)) + + +def _fixture_create_v2() -> tuple[bytes, list[int], list[bytes]]: + """Raw create_v2 data, account indices and keys from the committed fixture.""" + fixture = json.loads(FIXTURE.read_text()) + raw = base64.b64decode(fixture["transaction"][0]) + message = VersionedTransaction.from_bytes(raw).message + account_keys = [bytes(k) for k in message.account_keys] + for ix in message.instructions: + data = bytes(ix.data) + if data.startswith(CREATE_V2_DISCRIMINATOR): + return data, list(ix.accounts), account_keys + raise ValueError("fixture has no create_v2 instruction") # noqa: TRY003 + + +def check_omitted_trailing_option_decodes() -> bool: + """The 145-byte wire form (no is_cashback_enabled byte) must decode.""" + data, accounts, keys = _fixture_create_v2() + decoded = _parser().decode_instruction(data, keys, accounts) + if decoded is None: + print(f" decode_instruction returned None for {len(data)}-byte create_v2") + return False + args = decoded["args"] + ok = ( + decoded["instruction_name"] == "create_v2" + and args.get("is_cashback_enabled") is None + and args.get("is_mayhem_mode") is False + and bool(args.get("name")) + and bool(args.get("creator")) + ) + if not ok: + print(f" unexpected args: {args}") + return ok + + +def check_present_trailing_option_unchanged() -> bool: + """With the byte on the wire, the OptionBool struct form must survive.""" + data, accounts, keys = _fixture_create_v2() + parser = _parser() + for byte, expected in ((b"\x00", False), (b"\x01", True)): + decoded = parser.decode_instruction(data + byte, keys, accounts) + if decoded is None: + print(f" decode failed with trailing byte {byte.hex()}") + return False + value = decoded["args"].get("is_cashback_enabled") + if not (isinstance(value, dict) and value.get("field_0") is expected): + print(f" trailing byte {byte.hex()} decoded to {value}") + return False + return True + + +def check_mandatory_args_still_enforced() -> bool: + """Dropping is_mayhem_mode (a plain bool) must fail, not default.""" + data, accounts, keys = _fixture_create_v2() + decoded = _parser().decode_instruction(data[:-1], keys, accounts) + if decoded is not None: + print(f" truncated create_v2 decoded anyway: {decoded['args']}") + return False + return True + + +def check_native_option_decodes() -> bool: + """update_buyback_config carries Option: None, Some and omitted forms.""" + parser = _parser() + cases = ( + (UPDATE_BUYBACK_CONFIG_DISCRIMINATOR + b"\x00", None), + (UPDATE_BUYBACK_CONFIG_DISCRIMINATOR + b"\x01" + struct.pack(" bool: + """parse_token_creation_from_instruction must not need the appended byte.""" + data, accounts, keys = _fixture_create_v2() + parser = PumpFunEventParser( + idl_parser=get_idl_manager().get_parser(Platform.PUMP_FUN) + ) + token_info = parser.parse_token_creation_from_instruction(data, accounts, keys) + if token_info is None: + print(" parse_token_creation_from_instruction returned None") + return False + ok = ( + token_info.is_cashback_coin is False + and getattr(token_info, "state_from_event", False) is False + ) + if not ok: + print( + f" is_cashback_coin={token_info.is_cashback_coin} " + f"state_from_event={getattr(token_info, 'state_from_event', None)}" + ) + return ok + + +def main() -> int: + checks = [ + ( + "omitted trailing OptionBool decodes as unset", + check_omitted_trailing_option_decodes, + ), + ( + "present trailing OptionBool keeps struct form", + check_present_trailing_option_unchanged, + ), + ("mandatory args still enforced", check_mandatory_args_still_enforced), + ("native Option decodes", check_native_option_decodes), + ("event parser reads the raw fixture", check_event_parser_reads_raw_fixture), + ] + failed = 0 + for label, check in checks: + try: + ok = check() + except Exception as error: # noqa: BLE001 - report and continue + print(f"FAIL {label}: {type(error).__name__}: {error}") + failed += 1 + continue + print(f"{'PASS' if ok else 'FAIL'} {label}") + failed += 0 if ok else 1 + print(f"\n{len(checks) - failed}/{len(checks)} checks passed") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/learning-examples/verify_extreme_fast_zero_rpc.py b/learning-examples/verify_extreme_fast_zero_rpc.py index cc6171d..d083f9d 100644 --- a/learning-examples/verify_extreme_fast_zero_rpc.py +++ b/learning-examples/verify_extreme_fast_zero_rpc.py @@ -212,15 +212,11 @@ def check_instruction_parser_stays_conservative() -> bool: account_keys = [bytes(k) for k in msg.account_keys] parser = _event_parser() for ix in msg.instructions: - data = bytes(ix.data) # The fixture's create_v2 omits the trailing is_cashback_enabled - # OptionBool (a legal wire form the strict IDL decoder rejects — see - # the "decode trailing args defensively" gotcha in CLAUDE.md), so - # append the byte to exercise the parser's flag behaviour. - if data.startswith(bytes.fromhex("d6904cec5f8b31b4")): - data += b"\x00" + # OptionBool — a legal wire form the decoder accepts since #184 + # (verify_create_v2_optional_args.py covers the decode itself). token_info = parser.parse_token_creation_from_instruction( - data, list(ix.accounts), account_keys + bytes(ix.data), list(ix.accounts), account_keys ) if token_info is not None: ok = getattr(token_info, "state_from_event", False) is False diff --git a/src/utils/idl_parser.py b/src/utils/idl_parser.py index fff1878..9a73b72 100644 --- a/src/utils/idl_parser.py +++ b/src/utils/idl_parser.py @@ -15,6 +15,7 @@ DISCRIMINATOR_SIZE = 8 PUBLIC_KEY_SIZE = 32 STRING_LENGTH_PREFIX_SIZE = 4 ENUM_DISCRIMINATOR_SIZE = 1 +OPTION_PREFIX_SIZE = 1 class IDLParser: @@ -110,19 +111,9 @@ class IDLParser: instruction = self.instructions[discriminator] data_args = ix_data[DISCRIMINATOR_SIZE:] - # Decode instruction arguments - args = {} - decode_offset = 0 - for arg in instruction.get("args", []): - try: - value, decode_offset = self._decode_type( - data_args, decode_offset, arg["type"] - ) - args[arg["name"]] = value - except Exception as e: - if self.verbose: - print(f"❌ Decode error in argument '{arg['name']}': {e}") - return None + args = self._decode_instruction_args(instruction, data_args) + if args is None: + return None # Helper to safely retrieve account public keys def get_account_key(index: int) -> str | None: @@ -144,6 +135,37 @@ class IDLParser: "accounts": account_info, } + def _decode_instruction_args( + self, instruction: dict[str, Any], data_args: bytes + ) -> dict[str, Any] | None: + """Decode instruction arguments, or None if the data is malformed. + + Trailing option-typed args can legally be absent from the wire + (create_v2's is_cashback_enabled, issue #184): when the buffer is + exhausted and every remaining arg is optional, they are reported + as unset instead of failing the whole decode. + """ + args: dict[str, Any] = {} + decode_offset = 0 + arg_defs = instruction.get("args", []) + for i, arg in enumerate(arg_defs): + if decode_offset >= len(data_args) and all( + self._is_optional_type(remaining["type"]) for remaining in arg_defs[i:] + ): + for remaining in arg_defs[i:]: + args[remaining["name"]] = None + break + try: + value, decode_offset = self._decode_type( + data_args, decode_offset, arg["type"] + ) + args[arg["name"]] = value + except Exception as e: + if self.verbose: + print(f"❌ Decode error in argument '{arg['name']}': {e}") + return None + return args + # -------------------------------------------------------------------------- # Public Methods (External API) - Events # -------------------------------------------------------------------------- @@ -361,8 +383,15 @@ class IDLParser: """Calculate minimum data sizes for each instruction.""" for discriminator, instruction in self.instructions.items(): try: + required_args = list(instruction.get("args", [])) + # Trailing option-typed args may be omitted from the wire + # entirely, so they contribute nothing to the minimum size. + while required_args and self._is_optional_type( + required_args[-1]["type"] + ): + required_args.pop() min_size = DISCRIMINATOR_SIZE - for arg in instruction.get("args", []): + for arg in required_args: min_size += self._calculate_type_min_size(arg["type"]) self.instruction_min_sizes[discriminator] = min_size if self.verbose and instruction["name"] == "initialize": @@ -385,11 +414,28 @@ class IDLParser: element_type, array_length = type_def["array"] element_size = self._calculate_type_min_size(element_type) return element_size * array_length + if "option" in type_def: + # The None form is just the tag byte. + return OPTION_PREFIX_SIZE raise ValueError( f"Invalid or unknown type definition for size calculation: {type_def}" ) + def _is_optional_type(self, type_def: str | dict) -> bool: + """Whether a type may legally be absent when it trails instruction data. + + Covers Anchor's native ``option`` wrapper and pump.fun's ``OptionBool`` + defined type — both are observed omitted from the wire when trailing. + """ + if not isinstance(type_def, dict): + return False + if "option" in type_def: + return True + if "defined" in type_def: + return self._get_defined_type_name(type_def) == "OptionBool" + return False + def _get_primitive_size(self, type_name: str) -> int: """Get size in bytes for primitive types from the central map.""" info = self._PRIMITIVE_TYPE_INFO.get(type_name) @@ -449,9 +495,23 @@ class IDLParser: return self._decode_defined_type(data, offset, type_name) if "array" in type_def: return self._decode_array(data, offset, type_def["array"]) + if "option" in type_def: + return self._decode_option(data, offset, type_def["option"]) raise ValueError(f"Invalid or unknown type definition for decoding: {type_def}") + def _decode_option( + self, data: bytes, offset: int, inner_type: str | dict + ) -> tuple[Any, int]: + """Decode Anchor's native option type: a 1-byte tag, then the value if Some.""" + tag = struct.unpack_from(" tuple[list[Any], int]: