""" FX RADAR — Data Generation Pipeline Fetches global yields, commodities, ETFs, vol data, FX, and news. Outputs data.json which the HTML dashboard reads. Run locally: python generate.py Run via GitHub Actions: automatic on schedule """ import json import os import sys import time from datetime import datetime, timedelta from pathlib import Path import requests import yfinance as yf # ─── CONFIG ────────────────────────────────────────────────────────────────── FRED_KEY = os.environ.get("FRED_API_KEY", "") FINNHUB_KEY = os.environ.get("FINNHUB_API_KEY", "") OUTPUT_FILE = "data.json" # ─── YIELD CURVE TICKERS (yfinance) ───────────────────────────────────────── # These are Yahoo Finance tickers for government bond yields globally YIELD_TICKERS = { "US": { "1M": "^IRX", # 3-month T-bill (closest free proxy) "2Y": None, # Will use FRED "5Y": "^FVX", "10Y": "^TNX", "30Y": "^TYX", }, "UK": { "2Y": None, "5Y": None, "10Y": None, # We'll scrape or use proxy }, "Germany": { # Proxy for Eurozone "2Y": None, "5Y": None, "10Y": None, }, "Japan": { "2Y": None, "10Y": None, }, } # FRED series for full US yield curve FRED_YIELDS = { "DGS1MO": "1M", "DGS3MO": "3M", "DGS6MO": "6M", "DGS1": "1Y", "DGS2": "2Y", "DGS3": "3Y", "DGS5": "5Y", "DGS7": "7Y", "DGS10": "10Y", "DGS20": "20Y", "DGS30": "30Y", } # FRED series for other countries (where available) FRED_GLOBAL_YIELDS = { "Eurozone": {"IRLTLT01DEM156N": "10Y"}, # Germany 10Y (monthly, delayed) "UK": {"IRLTLT01GBM156N": "10Y"}, "Japan": {"IRLTLT01JPM156N": "10Y"}, "Canada": {"IRLTLT01CAM156N": "10Y"}, "Australia": {"IRLTLT01AUM156N": "10Y"}, } # Bond ETFs as yield proxies (price moves inversely to yields) BOND_ETFS = { "TLT": {"name": "US 20Y+ Treasury", "country": "US", "duration": "long"}, "IEF": {"name": "US 7-10Y Treasury", "country": "US", "duration": "medium"}, "SHY": {"name": "US 1-3Y Treasury", "country": "US", "duration": "short"}, "IGLT.L": {"name": "UK Gilts", "country": "UK", "duration": "mixed"}, "IBGL.L": {"name": "Eurozone Govt Bonds", "country": "Eurozone", "duration": "mixed"}, "JGBS": {"name": "Japan Govt Bonds", "country": "Japan", "duration": "mixed"}, "GOVT": {"name": "US Total Treasury", "country": "US", "duration": "mixed"}, } # ─── COMMODITY TICKERS ─────────────────────────────────────────────────────── COMMODITIES = { "GC=F": {"name": "Gold", "unit": "$/oz"}, "SI=F": {"name": "Silver", "unit": "$/oz"}, "CL=F": {"name": "WTI Crude", "unit": "$/bbl"}, "BZ=F": {"name": "Brent Crude", "unit": "$/bbl"}, "NG=F": {"name": "Natural Gas", "unit": "$/MMBtu"}, "HG=F": {"name": "Copper", "unit": "$/lb"}, "ZW=F": {"name": "Wheat", "unit": "¢/bu"}, "ZC=F": {"name": "Corn", "unit": "¢/bu"}, } # ─── KEY ETFs ──────────────────────────────────────────────────────────────── KEY_ETFS = { "SPY": "S&P 500", "QQQ": "Nasdaq 100", "IWM": "Russell 2000", "EEM": "EM Equities", "FXI": "China Large Cap", "EWJ": "Japan Equities", "EWZ": "Brazil Equities", "EWG": "Germany Equities", "EWU": "UK Equities", "UUP": "US Dollar Index", "FXE": "Euro ETF", "FXY": "Yen ETF", "FXB": "GBP ETF", "FXA": "AUD ETF", "FXC": "CAD ETF", "GLD": "Gold ETF", "USO": "Oil ETF", "DBA": "Agriculture ETF", "XLE": "Energy Sector", "XLF": "Financials Sector", } # ─── VOLATILITY ────────────────────────────────────────────────────────────── VOL_TICKERS = { "^VIX": "VIX (S&P 500 Vol)", "^VXN": "VXN (Nasdaq Vol)", "^MOVE": "MOVE (Bond Vol)", "VIXY": "VIX Short-Term ETF", } # ─── FX PAIRS (yfinance format) ───────────────────────────────────────────── FX_PAIRS = { "EURUSD=X": "EURUSD", "GBPUSD=X": "GBPUSD", "USDJPY=X": "USDJPY", "USDCHF=X": "USDCHF", "AUDUSD=X": "AUDUSD", "NZDUSD=X": "NZDUSD", "USDCAD=X": "USDCAD", "USDSEK=X": "USDSEK", "USDNOK=X": "USDNOK", "USDCNY=X": "USDCNY", "USDMXN=X": "USDMXN", "USDBRL=X": "USDBRL", "USDZAR=X": "USDZAR", "USDINR=X": "USDINR", "USDKRW=X": "USDKRW", "USDTRY=X": "USDTRY", "USDPLN=X": "USDPLN", "USDHUF=X": "USDHUF", "USDCZK=X": "USDCZK", "USDSGD=X": "USDSGD", "USDIDR=X": "USDIDR", "USDTHB=X": "USDTHB", "DX-Y.NYB": "DXY", } def fetch_fred(series_id, limit=5): """Fetch latest value from FRED API.""" if not FRED_KEY: return None try: url = f"https://api.stlouisfed.org/fred/series/observations" params = { "series_id": series_id, "api_key": FRED_KEY, "file_type": "json", "sort_order": "desc", "limit": limit, } r = requests.get(url, params=params, timeout=10) data = r.json() obs = [o for o in data.get("observations", []) if o["value"] != "."] if obs: return float(obs[0]["value"]) except Exception as e: print(f" FRED error ({series_id}): {e}") return None def fetch_yf_quotes(tickers_dict, period="5d"): """Fetch quotes from Yahoo Finance for a dict of ticker: name.""" results = {} ticker_list = list(tickers_dict.keys()) try: # Batch download for efficiency data = yf.download(ticker_list, period=period, progress=False, threads=True) for ticker in ticker_list: name = tickers_dict[ticker] try: if len(ticker_list) == 1: close_series = data["Close"] else: close_series = data["Close"][ticker] closes = close_series.dropna() if len(closes) >= 1: last = round(float(closes.iloc[-1]), 4) prev = round(float(closes.iloc[-2]), 4) if len(closes) >= 2 else None chg = round((last / prev - 1) * 100, 2) if prev and prev > 0 else None # Calculate 1W, 1M changes if we have enough data results[name] = { "price": last, "prev_close": prev, "change_pct": chg, "timestamp": str(closes.index[-1]), } except Exception as e: print(f" yfinance parse error ({ticker}): {e}") except Exception as e: print(f" yfinance batch error: {e}") return results def fetch_yf_history(tickers_dict, period="3mo"): """Fetch longer history for calculating 1W, 1M, YTD changes.""" results = {} ticker_list = list(tickers_dict.keys()) try: data = yf.download(ticker_list, period=period, progress=False, threads=True) for ticker in ticker_list: name = tickers_dict[ticker] try: if len(ticker_list) == 1: closes = data["Close"].dropna() else: closes = data["Close"][ticker].dropna() if len(closes) < 2: continue last = float(closes.iloc[-1]) d1 = float(closes.iloc[-2]) if len(closes) >= 2 else None w1 = float(closes.iloc[-6]) if len(closes) >= 6 else None m1 = float(closes.iloc[-22]) if len(closes) >= 22 else None results[name] = { "price": round(last, 4), "d1_pct": round((last / d1 - 1) * 100, 2) if d1 else None, "w1_pct": round((last / w1 - 1) * 100, 2) if w1 else None, "m1_pct": round((last / m1 - 1) * 100, 2) if m1 else None, } except Exception as e: print(f" History parse error ({ticker}): {e}") except Exception as e: print(f" History batch error: {e}") return results def fetch_finnhub_news(category="forex"): """Fetch news from Finnhub.""" if not FINNHUB_KEY: return [] try: url = f"https://finnhub.io/api/v1/news?category={category}&token={FINNHUB_KEY}" r = requests.get(url, timeout=10) articles = r.json() return [ { "headline": a.get("headline", ""), "source": a.get("source", ""), "url": a.get("url", ""), "datetime": a.get("datetime", 0), "category": category, } for a in articles[:30] ] except Exception as e: print(f" Finnhub news error: {e}") return [] def fetch_finnhub_calendar(): """Fetch economic calendar from Finnhub.""" if not FINNHUB_KEY: return [] try: today = datetime.now().strftime("%Y-%m-%d") future = (datetime.now() + timedelta(days=7)).strftime("%Y-%m-%d") url = f"https://finnhub.io/api/v1/calendar/economic?from={today}&to={future}&token={FINNHUB_KEY}" r = requests.get(url, timeout=10) data = r.json() events = data.get("economicCalendar", []) return [ { "date": e.get("time", "")[:10], "time": e.get("time", "")[11:16], "country": e.get("country", ""), "event": e.get("event", ""), "prev": e.get("prev"), "estimate": e.get("estimate"), "actual": e.get("actual"), "impact": e.get("impact", ""), } for e in events[:40] ] except Exception as e: print(f" Finnhub calendar error: {e}") return [] def fetch_rss_news(): """Fetch news from free RSS feeds (no API key needed).""" import xml.etree.ElementTree as ET feeds = [ ("https://feeds.content.dowjones.io/public/rss/mw_topstories", "MarketWatch"), ("https://feeds.bbci.co.uk/news/business/rss.xml", "BBC Business"), ] articles = [] for url, source in feeds: try: r = requests.get(url, timeout=10, headers={"User-Agent": "FXRadar/1.0"}) root = ET.fromstring(r.content) for item in root.findall(".//item")[:10]: title = item.findtext("title", "") link = item.findtext("link", "") pub = item.findtext("pubDate", "") if title: articles.append({ "headline": title, "source": source, "url": link, "datetime": pub, "category": "general", }) except Exception as e: print(f" RSS error ({source}): {e}") return articles def main(): print("=" * 60) print(f"FX RADAR Data Generation — {datetime.now().strftime('%Y-%m-%d %H:%M:%S UTC')}") print("=" * 60) output = { "generated_at": datetime.utcnow().isoformat() + "Z", "yields": {}, "commodities": {}, "etfs": {}, "fx": {}, "volatility": {}, "bond_etfs": {}, "news": [], "calendar": [], } # 1. US YIELD CURVE from FRED (most reliable) print("\n[1/8] Fetching US yield curve from FRED...") us_yields = {} for series_id, tenor in FRED_YIELDS.items(): val = fetch_fred(series_id) if val is not None: us_yields[tenor] = val print(f" US {tenor}: {val}%") time.sleep(0.2) # Rate limit output["yields"]["US"] = us_yields # 2. Global yields from FRED (limited but free) print("\n[2/8] Fetching global yields from FRED...") for country, series_map in FRED_GLOBAL_YIELDS.items(): country_yields = {} for series_id, tenor in series_map.items(): val = fetch_fred(series_id) if val is not None: country_yields[tenor] = val print(f" {country} {tenor}: {val}%") time.sleep(0.2) if country_yields: output["yields"][country] = country_yields # 3. Yield proxies from Yahoo Finance (US treasuries) print("\n[3/8] Fetching yield data from Yahoo Finance...") yf_yields = fetch_yf_quotes({ "^IRX": "US_3M", "^FVX": "US_5Y", "^TNX": "US_10Y", "^TYX": "US_30Y" }) for name, data in yf_yields.items(): print(f" {name}: {data['price']}") output["yields"]["US_yf"] = yf_yields # 4. Commodities print("\n[4/8] Fetching commodities...") commod_data = fetch_yf_history(COMMODITIES) for name, data in commod_data.items(): info = next((v for k, v in COMMODITIES.items() if v["name"] == name), {}) data["unit"] = info.get("unit", "") print(f" {name}: ${data['price']} ({data.get('d1_pct', '?')}%)") output["commodities"] = commod_data # 5. Key ETFs print("\n[5/8] Fetching ETFs...") etf_data = fetch_yf_history(KEY_ETFS) for name, data in etf_data.items(): print(f" {name}: ${data['price']} ({data.get('d1_pct', '?')}%)") output["etfs"] = etf_data # 6. Bond ETFs print("\n[6/8] Fetching bond ETFs...") bond_data = fetch_yf_history({k: v["name"] for k, v in BOND_ETFS.items()}) for name, data in bond_data.items(): info = next((v for k, v in BOND_ETFS.items() if v["name"] == name), {}) data["country"] = info.get("country", "") data["duration"] = info.get("duration", "") output["bond_etfs"] = bond_data # 7. Volatility print("\n[7/8] Fetching volatility data...") vol_data = fetch_yf_quotes(VOL_TICKERS) for name, data in vol_data.items(): print(f" {name}: {data['price']}") output["volatility"] = vol_data # 8. FX print("\n[8/8] Fetching FX rates...") fx_data = fetch_yf_history(FX_PAIRS) for name, data in fx_data.items(): print(f" {name}: {data['price']} ({data.get('d1_pct', '?')}%)") output["fx"] = fx_data # 9. News (Finnhub + RSS) print("\n[9] Fetching news...") finnhub_news = fetch_finnhub_news("forex") + fetch_finnhub_news("general") rss_news = fetch_rss_news() all_news = finnhub_news + rss_news # Deduplicate by headline seen = set() unique_news = [] for n in all_news: key = n["headline"][:50] if key not in seen: seen.add(key) unique_news.append(n) output["news"] = unique_news[:40] print(f" {len(unique_news)} unique articles") # 10. Economic Calendar print("\n[10] Fetching calendar...") output["calendar"] = fetch_finnhub_calendar() print(f" {len(output['calendar'])} events") # Write output output_path = Path(OUTPUT_FILE) with open(output_path, "w") as f: json.dump(output, f, indent=2, default=str) size_kb = output_path.stat().st_size / 1024 print(f"\n{'=' * 60}") print(f"✓ Generated {OUTPUT_FILE} ({size_kb:.1f} KB)") print(f" Yields: {len(output['yields'])} countries") print(f" Commodities: {len(output['commodities'])} instruments") print(f" ETFs: {len(output['etfs'])} funds") print(f" FX: {len(output['fx'])} pairs") print(f" News: {len(output['news'])} articles") print(f" Calendar: {len(output['calendar'])} events") print(f"{'=' * 60}") if __name__ == "__main__": main()