""" FX RADAR — Data Generation Pipeline Fetches global yields, commodities, ETFs, vol data, FX, and news. Outputs data.json which the HTML dashboard reads. Run locally: python generate.py Run via GitHub Actions: automatic on schedule """ import json import os import sys import time from datetime import datetime, timedelta from pathlib import Path import requests import yfinance as yf # ─── CONFIG ────────────────────────────────────────────────────────────────── FRED_KEY = os.environ.get("FRED_API_KEY", "") FINNHUB_KEY = os.environ.get("FINNHUB_API_KEY", "") GOV_KEY = os.environ.get("GOV_API_KEY", "") OUTPUT_FILE = "data.json" # ─── YIELD CURVE TICKERS (yfinance) ───────────────────────────────────────── # These are Yahoo Finance tickers for government bond yields globally YIELD_TICKERS = { "US": { "1M": "^IRX", # 3-month T-bill (closest free proxy) "2Y": None, # Will use FRED "5Y": "^FVX", "10Y": "^TNX", "30Y": "^TYX", }, "UK": { "2Y": None, "5Y": None, "10Y": None, # We'll scrape or use proxy }, "Germany": { # Proxy for Eurozone "2Y": None, "5Y": None, "10Y": None, }, "Japan": { "2Y": None, "10Y": None, }, } # FRED series for full US yield curve FRED_YIELDS = { "DGS1MO": "1M", "DGS3MO": "3M", "DGS6MO": "6M", "DGS1": "1Y", "DGS2": "2Y", "DGS3": "3Y", "DGS5": "5Y", "DGS7": "7Y", "DGS10": "10Y", "DGS20": "20Y", "DGS30": "30Y", } # FRED series for other countries (where available) FRED_GLOBAL_YIELDS = { "Eurozone": {"IRLTLT01DEM156N": "10Y"}, # Germany 10Y (monthly, delayed) "UK": {"IRLTLT01GBM156N": "10Y"}, "Japan": {"IRLTLT01JPM156N": "10Y"}, "Canada": {"IRLTLT01CAM156N": "10Y"}, "Australia": {"IRLTLT01AUM156N": "10Y"}, } # Bond ETFs as yield proxies (price moves inversely to yields) BOND_ETFS = { "TLT": {"name": "US 20Y+ Treasury", "country": "US", "duration": "long"}, "IEF": {"name": "US 7-10Y Treasury", "country": "US", "duration": "medium"}, "SHY": {"name": "US 1-3Y Treasury", "country": "US", "duration": "short"}, "IGLT.L": {"name": "UK Gilts", "country": "UK", "duration": "mixed"}, "IBGL.L": {"name": "Eurozone Govt Bonds", "country": "Eurozone", "duration": "mixed"}, "JGBS": {"name": "Japan Govt Bonds", "country": "Japan", "duration": "mixed"}, "GOVT": {"name": "US Total Treasury", "country": "US", "duration": "mixed"}, } # ─── EODHD GOVERNMENT BONDS ───────────────────────────────────────────────── EODHD_BONDS = { "US2Y.GBOND": {"key": "us2y", "label": "US 2Y", "country": "US"}, "US10Y.GBOND": {"key": "us10y", "label": "US 10Y", "country": "US"}, "US30Y.GBOND": {"key": "us30y", "label": "US 30Y", "country": "US"}, "UK2Y.GBOND": {"key": "uk2y", "label": "UK 2Y", "country": "UK"}, "UK10Y.GBOND": {"key": "uk10y", "label": "UK 10Y", "country": "UK"}, "UK30Y.GBOND": {"key": "uk30y", "label": "UK 30Y", "country": "UK"}, "DE2Y.GBOND": {"key": "de2y", "label": "Germany 2Y", "country": "Eurozone"}, "DE10Y.GBOND": {"key": "de10y", "label": "Germany 10Y", "country": "Eurozone"}, "DE30Y.GBOND": {"key": "de30y", "label": "Germany 30Y", "country": "Eurozone"}, "JP2Y.GBOND": {"key": "jp2y", "label": "Japan 2Y", "country": "Japan"}, "JP10Y.GBOND": {"key": "jp10y", "label": "Japan 10Y", "country": "Japan"}, "JP30Y.GBOND": {"key": "jp30y", "label": "Japan 30Y", "country": "Japan"}, "IT2Y.GBOND": {"key": "it2y", "label": "Italy 2Y", "country": "Eurozone"}, "IT10Y.GBOND": {"key": "it10y", "label": "Italy 10Y", "country": "Eurozone"}, "FR10Y.GBOND": {"key": "fr10y", "label": "France 10Y", "country": "Eurozone"}, "ES10Y.GBOND": {"key": "es10y", "label": "Spain 10Y", "country": "Eurozone"}, "CA2Y.GBOND": {"key": "ca2y", "label": "Canada 2Y", "country": "Canada"}, "CA10Y.GBOND": {"key": "ca10y", "label": "Canada 10Y", "country": "Canada"}, "AU2Y.GBOND": {"key": "au2y", "label": "Australia 2Y", "country": "Australia"}, "AU10Y.GBOND": {"key": "au10y", "label": "Australia 10Y", "country": "Australia"}, "NZ2Y.GBOND": {"key": "nz2y", "label": "New Zealand 2Y", "country": "New Zealand"}, "NZ10Y.GBOND": {"key": "nz10y", "label": "New Zealand 10Y", "country": "New Zealand"}, "SW10Y.GBOND": {"key": "ch10y", "label": "Switzerland 10Y", "country": "Switzerland"}, "SE10Y.GBOND": {"key": "se10y", "label": "Sweden 10Y", "country": "Sweden"}, "NO10Y.GBOND": {"key": "no10y", "label": "Norway 10Y", "country": "Norway"}, } # ─── CENTRAL BANK POLICY RATES ─────────────────────────────────────────────── # Updated periodically — these change at most ~8 times/year per central bank POLICY_RATES = { "US": 4.50, # Fed Funds upper bound "Eurozone": 2.65, # ECB deposit facility rate "UK": 4.50, # BoE Bank Rate "Japan": 0.50, # BoJ overnight rate "Switzerland": 0.25, # SNB policy rate "Canada": 2.75, # BoC overnight rate "Australia": 3.85, # RBA cash rate "New Zealand": 3.50, # RBNZ OCR "Sweden": 2.25, # Riksbank repo rate "Norway": 4.50, # Norges Bank sight deposit rate } # ─── MACRO SNAPSHOT DATA ───────────────────────────────────────────────────── # Updated periodically — CPI/GDP/unemployment are published monthly/quarterly MACRO_DATA = { "US": {"cpi": "2.8%", "cpiP": "3.0%", "gdp": "2.4%", "gdpT": "Q1 2025", "unemp": "4.2%", "unempT": "Jun 2025", "stance": "Hawkish Hold", "last": "Hold 4.50%", "next": "Jul 30", "pricing": "1 cut by Sep"}, "Eurozone": {"cpi": "2.1%", "cpiP": "2.2%", "gdp": "1.0%", "gdpT": "Q1 2025", "unemp": "6.3%", "unempT": "May 2025", "stance": "Dovish Easing", "last": "Cut to 2.65%", "next": "Jul 17", "pricing": "1 more cut 2025"}, "UK": {"cpi": "3.5%", "cpiP": "3.3%", "gdp": "1.3%", "gdpT": "Q1 2025", "unemp": "4.4%", "unempT": "May 2025", "stance": "Cautious Hold", "last": "Hold 4.50%", "next": "Aug 7", "pricing": "1 cut by Nov"}, "Japan": {"cpi": "3.6%", "cpiP": "3.2%", "gdp": "-0.7%", "gdpT": "Q1 2025", "unemp": "2.5%", "unempT": "May 2025", "stance": "Gradual Tightening", "last": "Hold 0.50%", "next": "Jul 31", "pricing": "Hike to 0.75% H2"}, "Switzerland": {"cpi": "0.6%", "cpiP": "0.3%", "gdp": "1.4%", "gdpT": "Q1 2025", "unemp": "2.8%", "unempT": "Jun 2025", "stance": "Neutral", "last": "Cut to 0.25%", "next": "Sep 18", "pricing": "Hold through 2025"}, "Canada": {"cpi": "2.9%", "cpiP": "2.7%", "gdp": "1.7%", "gdpT": "Q1 2025", "unemp": "6.7%", "unempT": "Jun 2025", "stance": "Easing", "last": "Cut to 2.75%", "next": "Jul 30", "pricing": "1 more cut 2025"}, "Australia": {"cpi": "2.4%", "cpiP": "2.8%", "gdp": "1.3%", "gdpT": "Q1 2025", "unemp": "4.0%", "unempT": "May 2025", "stance": "Easing", "last": "Cut to 3.85%", "next": "Aug 5", "pricing": "1-2 cuts by Dec"}, "New Zealand": {"cpi": "2.5%", "cpiP": "2.2%", "gdp": "0.7%", "gdpT": "Q1 2025", "unemp": "5.1%", "unempT": "Q1 2025", "stance": "Easing Cycle","last": "Cut to 3.50%", "next": "Jul 9", "pricing": "2 more cuts 2025"}, "Sweden": {"cpi": "2.3%", "cpiP": "2.5%", "gdp": "1.1%", "gdpT": "Q1 2025", "unemp": "8.9%", "unempT": "May 2025", "stance": "Dovish Hold", "last": "Hold 2.25%", "next": "Aug 20", "pricing": "Hold through 2025"}, "Norway": {"cpi": "2.7%", "cpiP": "2.8%", "gdp": "1.5%", "gdpT": "Q1 2025", "unemp": "4.1%", "unempT": "May 2025", "stance": "Hawkish Hold","last": "Hold 4.50%", "next": "Aug 14", "pricing": "Cut Q4 2025"}, } # ─── COMMODITY TICKERS ─────────────────────────────────────────────────────── COMMODITIES = { "GC=F": {"key": "gold", "name": "Gold", "unit": "$/oz"}, "SI=F": {"key": "silver", "name": "Silver", "unit": "$/oz"}, "CL=F": {"key": "wti", "name": "WTI Crude", "unit": "$/bbl"}, "BZ=F": {"key": "brent", "name": "Brent Crude", "unit": "$/bbl"}, "NG=F": {"key": "natgas", "name": "Natural Gas", "unit": "$/MMBtu"}, "HG=F": {"key": "copper", "name": "Copper", "unit": "$/lb"}, "ZW=F": {"key": "wheat", "name": "Wheat", "unit": "¢/bu"}, "ZC=F": {"key": "corn", "name": "Corn", "unit": "¢/bu"}, } # ─── KEY ETFs ──────────────────────────────────────────────────────────────── KEY_ETFS = { "SPY": "S&P 500", "QQQ": "Nasdaq 100", "IWM": "Russell 2000", "EEM": "EM Equities", "FXI": "China Large Cap", "EWJ": "Japan Equities", "EWZ": "Brazil Equities", "EWG": "Germany Equities", "EWU": "UK Equities", "UUP": "US Dollar Index", "FXE": "Euro ETF", "FXY": "Yen ETF", "FXB": "GBP ETF", "FXA": "AUD ETF", "FXC": "CAD ETF", "GLD": "Gold ETF", "USO": "Oil ETF", "DBA": "Agriculture ETF", "XLE": "Energy Sector", "XLF": "Financials Sector", } # ─── VOLATILITY ────────────────────────────────────────────────────────────── VOL_TICKERS = { "^VIX": "VIX (S&P 500 Vol)", "^VXN": "VXN (Nasdaq Vol)", "^MOVE": "MOVE (Bond Vol)", "VIXY": "VIX Short-Term ETF", } # ─── FX PAIRS (yfinance format) ───────────────────────────────────────────── FX_PAIRS = { "EURUSD=X": "EURUSD", "GBPUSD=X": "GBPUSD", "USDJPY=X": "USDJPY", "USDCHF=X": "USDCHF", "AUDUSD=X": "AUDUSD", "NZDUSD=X": "NZDUSD", "USDCAD=X": "USDCAD", "USDSEK=X": "USDSEK", "USDNOK=X": "USDNOK", "USDCNY=X": "USDCNY", "USDMXN=X": "USDMXN", "USDBRL=X": "USDBRL", "USDZAR=X": "USDZAR", "USDINR=X": "USDINR", "USDKRW=X": "USDKRW", "USDTRY=X": "USDTRY", "USDPLN=X": "USDPLN", "USDHUF=X": "USDHUF", "USDCZK=X": "USDCZK", "USDSGD=X": "USDSGD", "USDIDR=X": "USDIDR", "USDTHB=X": "USDTHB", "DX-Y.NYB": "DXY", } def fetch_eodhd_eod(symbol): """Fetch latest EOD data from EODHD for a bond/rate symbol.""" if not GOV_KEY: return None try: url = f"https://eodhd.com/api/eod/{symbol}" params = {"api_token": GOV_KEY, "fmt": "json", "order": "d", "limit": 2} r = requests.get(url, params=params, timeout=15) r.raise_for_status() data = r.json() if not isinstance(data, list) or len(data) < 1: return None latest = data[0] prev = data[1] if len(data) >= 2 else None value = latest.get("close") prev_value = prev.get("close") if prev else None change = round(value - prev_value, 4) if value is not None and prev_value is not None else None pct_change = ( round((value - prev_value) / prev_value * 100, 4) if value is not None and prev_value is not None and prev_value != 0 else None ) return { "value": value, "change": change, "pct_change": pct_change, "date": latest.get("date"), } except Exception as e: print(f" EODHD error ({symbol}): {e}") return None def fetch_fred(series_id, limit=5): """Fetch latest value from FRED API.""" if not FRED_KEY: print(f" FRED key missing for {series_id}") return None try: url = "https://api.stlouisfed.org/fred/series/observations" params = { "series_id": series_id, "api_key": FRED_KEY, "file_type": "json", "sort_order": "desc", "limit": limit, } r = requests.get(url, params=params, timeout=10) r.raise_for_status() data = r.json() obs = [o for o in data.get("observations", []) if o["value"] != "."] if obs: return float(obs[0]["value"]) print(f" No valid FRED obs for {series_id}") except Exception as e: print(f" FRED error ({series_id}): {e}") return None def fetch_yf_quotes(tickers_dict, period="5d"): """Fetch quotes from Yahoo Finance for a dict of ticker: name.""" results = {} ticker_list = list(tickers_dict.keys()) try: # Batch download for efficiency data = yf.download(ticker_list, period=period, progress=False, threads=True) for ticker in ticker_list: name = tickers_dict[ticker] try: if len(ticker_list) == 1: close_series = data["Close"] else: close_series = data["Close"][ticker] closes = close_series.dropna() if len(closes) >= 1: last = round(float(closes.iloc[-1]), 4) prev = round(float(closes.iloc[-2]), 4) if len(closes) >= 2 else None chg = round((last / prev - 1) * 100, 2) if prev and prev > 0 else None # Calculate 1W, 1M changes if we have enough data results[name] = { "price": last, "prev_close": prev, "change_pct": chg, "timestamp": str(closes.index[-1]), } except Exception as e: print(f" yfinance parse error ({ticker}): {e}") except Exception as e: print(f" yfinance batch error: {e}") return results def fetch_yf_history(tickers_dict, period="3mo"): """Fetch longer history for calculating 1W, 1M, YTD changes.""" results = {} ticker_list = list(tickers_dict.keys()) try: data = yf.download(ticker_list, period=period, progress=False, threads=True) for ticker in ticker_list: name = tickers_dict[ticker] try: if len(ticker_list) == 1: closes = data["Close"].dropna() else: closes = data["Close"][ticker].dropna() if len(closes) < 2: continue last = float(closes.iloc[-1]) d1 = float(closes.iloc[-2]) if len(closes) >= 2 else None w1 = float(closes.iloc[-6]) if len(closes) >= 6 else None m1 = float(closes.iloc[-22]) if len(closes) >= 22 else None results[name] = { "price": round(last, 4), "d1_pct": round((last / d1 - 1) * 100, 2) if d1 else None, "w1_pct": round((last / w1 - 1) * 100, 2) if w1 else None, "m1_pct": round((last / m1 - 1) * 100, 2) if m1 else None, } except Exception as e: print(f" History parse error ({ticker}): {e}") except Exception as e: print(f" History batch error: {e}") return results def fetch_finnhub_news(category="forex"): """Fetch news from Finnhub.""" if not FINNHUB_KEY: return [] try: url = f"https://finnhub.io/api/v1/news?category={category}&token={FINNHUB_KEY}" r = requests.get(url, timeout=10) articles = r.json() return [ { "headline": a.get("headline", ""), "source": a.get("source", ""), "url": a.get("url", ""), "datetime": a.get("datetime", 0), "category": category, } for a in articles[:30] ] except Exception as e: print(f" Finnhub news error: {e}") return [] def fetch_finnhub_calendar(): """Fetch economic calendar from Finnhub.""" if not FINNHUB_KEY: return [] try: today = datetime.now().strftime("%Y-%m-%d") future = (datetime.now() + timedelta(days=7)).strftime("%Y-%m-%d") url = f"https://finnhub.io/api/v1/calendar/economic?from={today}&to={future}&token={FINNHUB_KEY}" r = requests.get(url, timeout=10) data = r.json() events = data.get("economicCalendar", []) return [ { "date": e.get("time", "")[:10], "time": e.get("time", "")[11:16], "ctry": e.get("country", ""), "ev": e.get("event", ""), "prev": e.get("prev") if e.get("prev") is not None else "\u2014", "fcast": e.get("estimate") if e.get("estimate") is not None else "\u2014", "act": e.get("actual") if e.get("actual") is not None else "\u2014", "imp": "red" if e.get("impact") == "high" else "orange", } for e in events[:40] ] except Exception as e: print(f" Finnhub calendar error: {e}") return [] def fetch_finviz_calendar(): """Fetch economic calendar from Finviz — data is embedded as JSON in a script tag.""" try: from bs4 import BeautifulSoup except ImportError: print(" bs4 not installed — skipping Finviz calendar") return [] url = "https://finviz.com/calendar.ashx" headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", "Accept-Language": "en-GB,en;q=0.9", } try: r = requests.get(url, headers=headers, timeout=20) r.raise_for_status() print(f" Finviz status: {r.status_code}, html length: {len(r.text)}") soup = BeautifulSoup(r.text, "html.parser") # Calendar data is embedded as JSON in a