From 676b0f58a8d06ea5c908c95570aeeae8ab27ae38 Mon Sep 17 00:00:00 2001 From: Seonpil Date: Fri, 13 Feb 2026 11:52:51 +0000 Subject: [PATCH] docs: add Technical Challenge, TECHNICAL_NOTES; selective section extraction --- app.py | 117 ++++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 83 insertions(+), 34 deletions(-) diff --git a/app.py b/app.py index 5f8f45a..ad0689a 100644 --- a/app.py +++ b/app.py @@ -57,24 +57,73 @@ def extract_text_from_file(file_path: Path) -> str: return "" +# ---------- Selective Section Extraction (pre-filter: only Item 7 & 8, no PART I / ITEM 1–6) ---------- +ITEM7_PATTERNS = [ + r"Item\s+7\s*[.:]\s*Management['\u2019]s\s+Discussion\s+and\s+Analysis", + r"ITEM\s+7\s*[.:]\s*Management['\u2019]s\s+Discussion", + r"Item\s+7\s*[.:]\s*[\w\s]+MD&A", +] +ITEM8_PATTERNS = [ + r"Item\s+8\s*[.:]\s*Financial\s+Statements", + r"ITEM\s+8\s*[.:]\s*Financial\s+Statements", + r"Item\s+8\s*[.:]\s*[\w\s]+Consolidated\s+Financial", +] + + +def _find_section_start(text: str, patterns: list, item_num: int) -> int: + """Return start index of first matching pattern, or -1.""" + for pat in patterns: + m = re.search(pat, text, re.IGNORECASE) + if m: + return m.start() + m = re.search(r"\bItem\s+" + str(item_num) + r"\b", text, re.IGNORECASE) + return m.start() if m else -1 + + +def prefilter_after_item7(full_text: str) -> str: + """Drop PART I, ITEM 1–6; keep only from Item 7 onward to reduce noise and token use.""" + start = _find_section_start(full_text, ITEM7_PATTERNS, 7) + return full_text[start:] if start >= 0 else full_text + + def find_item_section(text: str, item_num: int, title_keywords: list) -> str: - """Find Item N section (e.g. item_num=7 -> Item 7, item_num=8 -> Item 8).""" - pattern = re.compile( - r"\bItem\s+" + str(item_num) + r"\b[.\s]*[^\n]*(" + "|".join(re.escape(k) for k in title_keywords) + r")?", - re.IGNORECASE, - ) - match = pattern.search(text) - if not match: - return "" - start = match.start() - next_item = re.search(r"\n\s*Item\s+\d+\s+", text[start + 50 :], re.IGNORECASE) + """Extract only Item N section (regex-based). Used for Item 7 (MD&A) and Item 8 (Financial Statements).""" + patterns = ITEM7_PATTERNS if item_num == 7 else ITEM8_PATTERNS + start = _find_section_start(text, patterns, item_num) + if start == -1: + pattern = re.compile( + r"\bItem\s+" + str(item_num) + r"\b[.\s]*[^\n]*(" + "|".join(re.escape(k) for k in title_keywords) + r")?", + re.IGNORECASE, + ) + match = pattern.search(text) + if not match: + return "" + start = match.start() + # End at next "Item N" (next major section) + next_item = re.search(r"\n\s*Item\s+\d+\s+", text[start + 100 :], re.IGNORECASE) if next_item: - end = start + 50 + next_item.start() + end = start + 100 + next_item.start() else: - end = min(start + 150000, len(text)) # Cap section size to stay within token limits + end = min(start + 150000, len(text)) return text[start:end].strip() +def smart_chunk(section: str, max_chars: int = 30000, head_ratio: float = 0.5) -> str: + """ + If section exceeds max_chars, keep head and tail (quantitative data often at start/end). + Reduces tokens while preserving high-signal content. + """ + if len(section) <= max_chars: + return section + head_size = int(max_chars * head_ratio) + tail_size = max_chars - head_size - 100 # reserve for separator + return ( + section[:head_size] + + "\n\n[ ... middle omitted to stay within token limit ... ]\n\n" + + section[-tail_size:] + ) + + def find_downloaded_10k_path(download_root: Path, ticker: str) -> Optional[Path]: """Return the path to the latest 10-K folder for the given ticker under download_root.""" ticker_upper = ticker.upper() @@ -162,25 +211,21 @@ def get_ai_summary_and_report( ticker: str, ) -> tuple[str, str]: """ - Use Gemini 1.5 Flash to produce: - 1) A detailed three-part summary (financial health, profitability, key risks). - 2) A CFA Investment Report-style section (Executive Summary, Investment Thesis, Risks, etc.). - Only Item 7 and Item 8 are sent; sections are trimmed to avoid token/rate limits. + Produce detailed analysis and CFA report. Only Item 7 and Item 8 are sent (pre-filtered). + Sections are smart-chunked (head + tail) when long to keep token use low and avoid 429. """ model = get_gemini_model(api_key) - # Keep payload smaller to reduce token usage and avoid 429 rate limits - max_chars_per_section = 40000 - if len(item7_text) > max_chars_per_section: - item7_text = item7_text[:max_chars_per_section] + "\n\n[ ... section truncated ... ]" - if len(item8_text) > max_chars_per_section: - item8_text = item8_text[:max_chars_per_section] + "\n\n[ ... section truncated ... ]" + # Smart chunking: when over limit, keep head + tail (figures often at start/end) + max_chars_per_section = 30000 + item7_text = smart_chunk(item7_text, max_chars=max_chars_per_section) + item8_text = smart_chunk(item8_text, max_chars=max_chars_per_section) - user_prompt = f"""You are a CFA charterholder and senior equity analyst writing for an accounting and finance audience. Your analysis must be evidence-based, cite specific figures from the 10-K where relevant, and follow professional investment report standards. Use British English throughout (e.g. analyse, summarise, colour, favour, organisation). + user_prompt = f"""You are a CFA charterholder and senior equity analyst. Use British English. Omit unnecessary qualifiers and filler; focus on figures, risks, and material facts. -Analyse the following 10-K content for company ticker: {ticker}. +Analyse the following 10-K excerpts for company ticker: {ticker}. The text below contains ONLY Item 7 (MD&A) and Item 8 (Financial Statements)—other sections have been pre-filtered out. -Use the FULL text provided below (Item 7 MD&A and Item 8 Financial Statements) to produce a thorough, detailed analysis. Do not summarise superficially—reference specific numbers, trends, and risk disclosures. +Use the provided text to produce a thorough, evidence-based analysis. Cite specific numbers and risk disclosures where relevant. First, write a "DETAILED ANALYSIS" section with exactly three paragraphs (use subheadings): 1. **Financial Health**: Liquidity (current ratio, cash position, credit facilities), leverage (debt/equity, interest coverage), capital structure, and any covenant or refinancing risks. Cite figures from the statements. @@ -241,12 +286,11 @@ Keep the entire response in British English. Use clear section headers (e.g. ## def get_metrics_table_from_ai(api_key: str, item8_text: str, ticker: str) -> pd.DataFrame: - """Ask Gemini to extract Revenue, Net Income, Operating Cash Flow by year from Item 8 and return a table.""" + """Extract Revenue, Net Income, Operating Cash Flow from Item 8 only (pre-filtered).""" model = get_gemini_model(api_key) - max_item8_chars = 25000 - excerpt = item8_text[:max_item8_chars] if len(item8_text) > max_item8_chars else item8_text + excerpt = smart_chunk(item8_text, max_chars=25000) - prompt = f"""You are a financial analyst. From the 10-K Item 8 excerpt below for company {ticker}, extract the following for the most recent 3–5 fiscal years (if available): + prompt = f"""You are a financial analyst. From the 10-K Item 8 excerpt below for company {ticker}, extract the following for the most recent 3–5 fiscal years. Focus only on figures; omit filler text. - Revenue (or Net sales) - Net Income (or Net earnings attributable to common shareholders) - Cash flows from operating activities (Operating Cash Flow) @@ -299,14 +343,19 @@ def run_analysis(ticker: str, api_key: str, email: str, analysis_only: bool = Fa if not full_text: raise ValueError("Could not extract text from the 10-K.") - item7 = find_item_section(full_text, 7, ["Management's Discussion", "MD&A", "Analysis"]) - item8 = find_item_section(full_text, 8, ["Financial Statements", "Consolidated"]) + # Pre-filter: drop PART I, ITEM 1–6; work only from Item 7 onward (reduces tokens) + text_from_item7 = prefilter_after_item7(full_text) - # Use only extracted Item 7 / Item 8; fallback to trimmed full text if sections not found + # Section extraction: only Item 7 (MD&A) and Item 8 (Financial Statements) + item7 = find_item_section(text_from_item7, 7, ["Management's Discussion", "MD&A", "Analysis"]) + item8 = find_item_section(text_from_item7, 8, ["Financial Statements", "Consolidated"]) + + # Fallback if extraction fails: use trimmed slice (still no full document) if not item7: - item7 = full_text[:80000] + item7 = smart_chunk(text_from_item7[:120000], max_chars=30000) if not item8: - item8 = full_text[80000:160000] if len(full_text) > 80000 else full_text[:80000] + remainder = text_from_item7[100000:220000] if len(text_from_item7) > 100000 else text_from_item7 + item8 = smart_chunk(remainder, max_chars=30000) detailed_summary, cfa_report = get_ai_summary_and_report(api_key, full_text, item7, item8, ticker) if analysis_only: