mirror of
https://github.com/tvishia29-alt/FXandRatesDashboard.git
synced 2026-07-27 16:07:44 +00:00
Tvishi Agarwal
454 lines
16 KiB
Python
454 lines
16 KiB
Python
"""
|
|
FX RADAR — Data Generation Pipeline
|
|
Fetches global yields, commodities, ETFs, vol data, FX, and news.
|
|
Outputs data.json which the HTML dashboard reads.
|
|
Run locally: python generate.py
|
|
Run via GitHub Actions: automatic on schedule
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
from datetime import datetime, timedelta
|
|
from pathlib import Path
|
|
|
|
import requests
|
|
import yfinance as yf
|
|
|
|
# ─── CONFIG ──────────────────────────────────────────────────────────────────
|
|
FRED_KEY = os.environ.get("FRED_API_KEY", "")
|
|
FINNHUB_KEY = os.environ.get("FINNHUB_API_KEY", "")
|
|
OUTPUT_FILE = "data.json"
|
|
|
|
# ─── YIELD CURVE TICKERS (yfinance) ─────────────────────────────────────────
|
|
# These are Yahoo Finance tickers for government bond yields globally
|
|
YIELD_TICKERS = {
|
|
"US": {
|
|
"1M": "^IRX", # 3-month T-bill (closest free proxy)
|
|
"2Y": None, # Will use FRED
|
|
"5Y": "^FVX",
|
|
"10Y": "^TNX",
|
|
"30Y": "^TYX",
|
|
},
|
|
"UK": {
|
|
"2Y": None,
|
|
"5Y": None,
|
|
"10Y": None, # We'll scrape or use proxy
|
|
},
|
|
"Germany": { # Proxy for Eurozone
|
|
"2Y": None,
|
|
"5Y": None,
|
|
"10Y": None,
|
|
},
|
|
"Japan": {
|
|
"2Y": None,
|
|
"10Y": None,
|
|
},
|
|
}
|
|
|
|
# FRED series for full US yield curve
|
|
FRED_YIELDS = {
|
|
"DGS1MO": "1M",
|
|
"DGS3MO": "3M",
|
|
"DGS6MO": "6M",
|
|
"DGS1": "1Y",
|
|
"DGS2": "2Y",
|
|
"DGS3": "3Y",
|
|
"DGS5": "5Y",
|
|
"DGS7": "7Y",
|
|
"DGS10": "10Y",
|
|
"DGS20": "20Y",
|
|
"DGS30": "30Y",
|
|
}
|
|
|
|
# FRED series for other countries (where available)
|
|
FRED_GLOBAL_YIELDS = {
|
|
"Eurozone": {"IRLTLT01DEM156N": "10Y"}, # Germany 10Y (monthly, delayed)
|
|
"UK": {"IRLTLT01GBM156N": "10Y"},
|
|
"Japan": {"IRLTLT01JPM156N": "10Y"},
|
|
"Canada": {"IRLTLT01CAM156N": "10Y"},
|
|
"Australia": {"IRLTLT01AUM156N": "10Y"},
|
|
}
|
|
|
|
# Bond ETFs as yield proxies (price moves inversely to yields)
|
|
BOND_ETFS = {
|
|
"TLT": {"name": "US 20Y+ Treasury", "country": "US", "duration": "long"},
|
|
"IEF": {"name": "US 7-10Y Treasury", "country": "US", "duration": "medium"},
|
|
"SHY": {"name": "US 1-3Y Treasury", "country": "US", "duration": "short"},
|
|
"IGLT.L": {"name": "UK Gilts", "country": "UK", "duration": "mixed"},
|
|
"IBGL.L": {"name": "Eurozone Govt Bonds", "country": "Eurozone", "duration": "mixed"},
|
|
"JGBS": {"name": "Japan Govt Bonds", "country": "Japan", "duration": "mixed"},
|
|
"GOVT": {"name": "US Total Treasury", "country": "US", "duration": "mixed"},
|
|
}
|
|
|
|
# ─── COMMODITY TICKERS ───────────────────────────────────────────────────────
|
|
COMMODITIES = {
|
|
"GC=F": {"name": "Gold", "unit": "$/oz"},
|
|
"SI=F": {"name": "Silver", "unit": "$/oz"},
|
|
"CL=F": {"name": "WTI Crude", "unit": "$/bbl"},
|
|
"BZ=F": {"name": "Brent Crude", "unit": "$/bbl"},
|
|
"NG=F": {"name": "Natural Gas", "unit": "$/MMBtu"},
|
|
"HG=F": {"name": "Copper", "unit": "$/lb"},
|
|
"ZW=F": {"name": "Wheat", "unit": "¢/bu"},
|
|
"ZC=F": {"name": "Corn", "unit": "¢/bu"},
|
|
}
|
|
|
|
# ─── KEY ETFs ────────────────────────────────────────────────────────────────
|
|
KEY_ETFS = {
|
|
"SPY": "S&P 500",
|
|
"QQQ": "Nasdaq 100",
|
|
"IWM": "Russell 2000",
|
|
"EEM": "EM Equities",
|
|
"FXI": "China Large Cap",
|
|
"EWJ": "Japan Equities",
|
|
"EWZ": "Brazil Equities",
|
|
"EWG": "Germany Equities",
|
|
"EWU": "UK Equities",
|
|
"UUP": "US Dollar Index",
|
|
"FXE": "Euro ETF",
|
|
"FXY": "Yen ETF",
|
|
"FXB": "GBP ETF",
|
|
"FXA": "AUD ETF",
|
|
"FXC": "CAD ETF",
|
|
"GLD": "Gold ETF",
|
|
"USO": "Oil ETF",
|
|
"DBA": "Agriculture ETF",
|
|
"XLE": "Energy Sector",
|
|
"XLF": "Financials Sector",
|
|
}
|
|
|
|
# ─── VOLATILITY ──────────────────────────────────────────────────────────────
|
|
VOL_TICKERS = {
|
|
"^VIX": "VIX (S&P 500 Vol)",
|
|
"^VXN": "VXN (Nasdaq Vol)",
|
|
"^MOVE": "MOVE (Bond Vol)",
|
|
"VIXY": "VIX Short-Term ETF",
|
|
}
|
|
|
|
# ─── FX PAIRS (yfinance format) ─────────────────────────────────────────────
|
|
FX_PAIRS = {
|
|
"EURUSD=X": "EURUSD", "GBPUSD=X": "GBPUSD", "USDJPY=X": "USDJPY",
|
|
"USDCHF=X": "USDCHF", "AUDUSD=X": "AUDUSD", "NZDUSD=X": "NZDUSD",
|
|
"USDCAD=X": "USDCAD", "USDSEK=X": "USDSEK", "USDNOK=X": "USDNOK",
|
|
"USDCNY=X": "USDCNY", "USDMXN=X": "USDMXN", "USDBRL=X": "USDBRL",
|
|
"USDZAR=X": "USDZAR", "USDINR=X": "USDINR", "USDKRW=X": "USDKRW",
|
|
"USDTRY=X": "USDTRY", "USDPLN=X": "USDPLN", "USDHUF=X": "USDHUF",
|
|
"USDCZK=X": "USDCZK", "USDSGD=X": "USDSGD", "USDIDR=X": "USDIDR",
|
|
"USDTHB=X": "USDTHB", "DX-Y.NYB": "DXY",
|
|
}
|
|
|
|
|
|
def fetch_fred(series_id, limit=5):
|
|
"""Fetch latest value from FRED API."""
|
|
if not FRED_KEY:
|
|
return None
|
|
try:
|
|
url = f"https://api.stlouisfed.org/fred/series/observations"
|
|
params = {
|
|
"series_id": series_id,
|
|
"api_key": FRED_KEY,
|
|
"file_type": "json",
|
|
"sort_order": "desc",
|
|
"limit": limit,
|
|
}
|
|
r = requests.get(url, params=params, timeout=10)
|
|
data = r.json()
|
|
obs = [o for o in data.get("observations", []) if o["value"] != "."]
|
|
if obs:
|
|
return float(obs[0]["value"])
|
|
except Exception as e:
|
|
print(f" FRED error ({series_id}): {e}")
|
|
return None
|
|
|
|
|
|
def fetch_yf_quotes(tickers_dict, period="5d"):
|
|
"""Fetch quotes from Yahoo Finance for a dict of ticker: name."""
|
|
results = {}
|
|
ticker_list = list(tickers_dict.keys())
|
|
|
|
try:
|
|
# Batch download for efficiency
|
|
data = yf.download(ticker_list, period=period, progress=False, threads=True)
|
|
|
|
for ticker in ticker_list:
|
|
name = tickers_dict[ticker]
|
|
try:
|
|
if len(ticker_list) == 1:
|
|
close_series = data["Close"]
|
|
else:
|
|
close_series = data["Close"][ticker]
|
|
|
|
closes = close_series.dropna()
|
|
if len(closes) >= 1:
|
|
last = round(float(closes.iloc[-1]), 4)
|
|
prev = round(float(closes.iloc[-2]), 4) if len(closes) >= 2 else None
|
|
chg = round((last / prev - 1) * 100, 2) if prev and prev > 0 else None
|
|
|
|
# Calculate 1W, 1M changes if we have enough data
|
|
results[name] = {
|
|
"price": last,
|
|
"prev_close": prev,
|
|
"change_pct": chg,
|
|
"timestamp": str(closes.index[-1]),
|
|
}
|
|
except Exception as e:
|
|
print(f" yfinance parse error ({ticker}): {e}")
|
|
|
|
except Exception as e:
|
|
print(f" yfinance batch error: {e}")
|
|
|
|
return results
|
|
|
|
|
|
def fetch_yf_history(tickers_dict, period="3mo"):
|
|
"""Fetch longer history for calculating 1W, 1M, YTD changes."""
|
|
results = {}
|
|
ticker_list = list(tickers_dict.keys())
|
|
|
|
try:
|
|
data = yf.download(ticker_list, period=period, progress=False, threads=True)
|
|
|
|
for ticker in ticker_list:
|
|
name = tickers_dict[ticker]
|
|
try:
|
|
if len(ticker_list) == 1:
|
|
closes = data["Close"].dropna()
|
|
else:
|
|
closes = data["Close"][ticker].dropna()
|
|
|
|
if len(closes) < 2:
|
|
continue
|
|
|
|
last = float(closes.iloc[-1])
|
|
d1 = float(closes.iloc[-2]) if len(closes) >= 2 else None
|
|
w1 = float(closes.iloc[-6]) if len(closes) >= 6 else None
|
|
m1 = float(closes.iloc[-22]) if len(closes) >= 22 else None
|
|
|
|
results[name] = {
|
|
"price": round(last, 4),
|
|
"d1_pct": round((last / d1 - 1) * 100, 2) if d1 else None,
|
|
"w1_pct": round((last / w1 - 1) * 100, 2) if w1 else None,
|
|
"m1_pct": round((last / m1 - 1) * 100, 2) if m1 else None,
|
|
}
|
|
except Exception as e:
|
|
print(f" History parse error ({ticker}): {e}")
|
|
|
|
except Exception as e:
|
|
print(f" History batch error: {e}")
|
|
|
|
return results
|
|
|
|
|
|
def fetch_finnhub_news(category="forex"):
|
|
"""Fetch news from Finnhub."""
|
|
if not FINNHUB_KEY:
|
|
return []
|
|
try:
|
|
url = f"https://finnhub.io/api/v1/news?category={category}&token={FINNHUB_KEY}"
|
|
r = requests.get(url, timeout=10)
|
|
articles = r.json()
|
|
return [
|
|
{
|
|
"headline": a.get("headline", ""),
|
|
"source": a.get("source", ""),
|
|
"url": a.get("url", ""),
|
|
"datetime": a.get("datetime", 0),
|
|
"category": category,
|
|
}
|
|
for a in articles[:30]
|
|
]
|
|
except Exception as e:
|
|
print(f" Finnhub news error: {e}")
|
|
return []
|
|
|
|
|
|
def fetch_finnhub_calendar():
|
|
"""Fetch economic calendar from Finnhub."""
|
|
if not FINNHUB_KEY:
|
|
return []
|
|
try:
|
|
today = datetime.now().strftime("%Y-%m-%d")
|
|
future = (datetime.now() + timedelta(days=7)).strftime("%Y-%m-%d")
|
|
url = f"https://finnhub.io/api/v1/calendar/economic?from={today}&to={future}&token={FINNHUB_KEY}"
|
|
r = requests.get(url, timeout=10)
|
|
data = r.json()
|
|
events = data.get("economicCalendar", [])
|
|
return [
|
|
{
|
|
"date": e.get("time", "")[:10],
|
|
"time": e.get("time", "")[11:16],
|
|
"country": e.get("country", ""),
|
|
"event": e.get("event", ""),
|
|
"prev": e.get("prev"),
|
|
"estimate": e.get("estimate"),
|
|
"actual": e.get("actual"),
|
|
"impact": e.get("impact", ""),
|
|
}
|
|
for e in events[:40]
|
|
]
|
|
except Exception as e:
|
|
print(f" Finnhub calendar error: {e}")
|
|
return []
|
|
|
|
|
|
def fetch_rss_news():
|
|
"""Fetch news from free RSS feeds (no API key needed)."""
|
|
import xml.etree.ElementTree as ET
|
|
|
|
feeds = [
|
|
("https://feeds.content.dowjones.io/public/rss/mw_topstories", "MarketWatch"),
|
|
("https://feeds.bbci.co.uk/news/business/rss.xml", "BBC Business"),
|
|
]
|
|
articles = []
|
|
for url, source in feeds:
|
|
try:
|
|
r = requests.get(url, timeout=10, headers={"User-Agent": "FXRadar/1.0"})
|
|
root = ET.fromstring(r.content)
|
|
for item in root.findall(".//item")[:10]:
|
|
title = item.findtext("title", "")
|
|
link = item.findtext("link", "")
|
|
pub = item.findtext("pubDate", "")
|
|
if title:
|
|
articles.append({
|
|
"headline": title,
|
|
"source": source,
|
|
"url": link,
|
|
"datetime": pub,
|
|
"category": "general",
|
|
})
|
|
except Exception as e:
|
|
print(f" RSS error ({source}): {e}")
|
|
return articles
|
|
|
|
|
|
def main():
|
|
print("=" * 60)
|
|
print(f"FX RADAR Data Generation — {datetime.now().strftime('%Y-%m-%d %H:%M:%S UTC')}")
|
|
print("=" * 60)
|
|
|
|
output = {
|
|
"generated_at": datetime.utcnow().isoformat() + "Z",
|
|
"yields": {},
|
|
"commodities": {},
|
|
"etfs": {},
|
|
"fx": {},
|
|
"volatility": {},
|
|
"bond_etfs": {},
|
|
"news": [],
|
|
"calendar": [],
|
|
}
|
|
|
|
# 1. US YIELD CURVE from FRED (most reliable)
|
|
print("\n[1/8] Fetching US yield curve from FRED...")
|
|
us_yields = {}
|
|
for series_id, tenor in FRED_YIELDS.items():
|
|
val = fetch_fred(series_id)
|
|
if val is not None:
|
|
us_yields[tenor] = val
|
|
print(f" US {tenor}: {val}%")
|
|
time.sleep(0.2) # Rate limit
|
|
output["yields"]["US"] = us_yields
|
|
|
|
# 2. Global yields from FRED (limited but free)
|
|
print("\n[2/8] Fetching global yields from FRED...")
|
|
for country, series_map in FRED_GLOBAL_YIELDS.items():
|
|
country_yields = {}
|
|
for series_id, tenor in series_map.items():
|
|
val = fetch_fred(series_id)
|
|
if val is not None:
|
|
country_yields[tenor] = val
|
|
print(f" {country} {tenor}: {val}%")
|
|
time.sleep(0.2)
|
|
if country_yields:
|
|
output["yields"][country] = country_yields
|
|
|
|
# 3. Yield proxies from Yahoo Finance (US treasuries)
|
|
print("\n[3/8] Fetching yield data from Yahoo Finance...")
|
|
yf_yields = fetch_yf_quotes({
|
|
"^IRX": "US_3M", "^FVX": "US_5Y", "^TNX": "US_10Y", "^TYX": "US_30Y"
|
|
})
|
|
for name, data in yf_yields.items():
|
|
print(f" {name}: {data['price']}")
|
|
output["yields"]["US_yf"] = yf_yields
|
|
|
|
# 4. Commodities
|
|
print("\n[4/8] Fetching commodities...")
|
|
commod_data = fetch_yf_history(COMMODITIES)
|
|
for name, data in commod_data.items():
|
|
info = next((v for k, v in COMMODITIES.items() if v["name"] == name), {})
|
|
data["unit"] = info.get("unit", "")
|
|
print(f" {name}: ${data['price']} ({data.get('d1_pct', '?')}%)")
|
|
output["commodities"] = commod_data
|
|
|
|
# 5. Key ETFs
|
|
print("\n[5/8] Fetching ETFs...")
|
|
etf_data = fetch_yf_history(KEY_ETFS)
|
|
for name, data in etf_data.items():
|
|
print(f" {name}: ${data['price']} ({data.get('d1_pct', '?')}%)")
|
|
output["etfs"] = etf_data
|
|
|
|
# 6. Bond ETFs
|
|
print("\n[6/8] Fetching bond ETFs...")
|
|
bond_data = fetch_yf_history({k: v["name"] for k, v in BOND_ETFS.items()})
|
|
for name, data in bond_data.items():
|
|
info = next((v for k, v in BOND_ETFS.items() if v["name"] == name), {})
|
|
data["country"] = info.get("country", "")
|
|
data["duration"] = info.get("duration", "")
|
|
output["bond_etfs"] = bond_data
|
|
|
|
# 7. Volatility
|
|
print("\n[7/8] Fetching volatility data...")
|
|
vol_data = fetch_yf_quotes(VOL_TICKERS)
|
|
for name, data in vol_data.items():
|
|
print(f" {name}: {data['price']}")
|
|
output["volatility"] = vol_data
|
|
|
|
# 8. FX
|
|
print("\n[8/8] Fetching FX rates...")
|
|
fx_data = fetch_yf_history(FX_PAIRS)
|
|
for name, data in fx_data.items():
|
|
print(f" {name}: {data['price']} ({data.get('d1_pct', '?')}%)")
|
|
output["fx"] = fx_data
|
|
|
|
# 9. News (Finnhub + RSS)
|
|
print("\n[9] Fetching news...")
|
|
finnhub_news = fetch_finnhub_news("forex") + fetch_finnhub_news("general")
|
|
rss_news = fetch_rss_news()
|
|
all_news = finnhub_news + rss_news
|
|
# Deduplicate by headline
|
|
seen = set()
|
|
unique_news = []
|
|
for n in all_news:
|
|
key = n["headline"][:50]
|
|
if key not in seen:
|
|
seen.add(key)
|
|
unique_news.append(n)
|
|
output["news"] = unique_news[:40]
|
|
print(f" {len(unique_news)} unique articles")
|
|
|
|
# 10. Economic Calendar
|
|
print("\n[10] Fetching calendar...")
|
|
output["calendar"] = fetch_finnhub_calendar()
|
|
print(f" {len(output['calendar'])} events")
|
|
|
|
# Write output
|
|
output_path = Path(OUTPUT_FILE)
|
|
with open(output_path, "w") as f:
|
|
json.dump(output, f, indent=2, default=str)
|
|
|
|
size_kb = output_path.stat().st_size / 1024
|
|
print(f"\n{'=' * 60}")
|
|
print(f"✓ Generated {OUTPUT_FILE} ({size_kb:.1f} KB)")
|
|
print(f" Yields: {len(output['yields'])} countries")
|
|
print(f" Commodities: {len(output['commodities'])} instruments")
|
|
print(f" ETFs: {len(output['etfs'])} funds")
|
|
print(f" FX: {len(output['fx'])} pairs")
|
|
print(f" News: {len(output['news'])} articles")
|
|
print(f" Calendar: {len(output['calendar'])} events")
|
|
print(f"{'=' * 60}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|