gavindiaz/DinQuant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor code for improved readability and consistency

Browse Source 
- Cleaned up whitespace and formatting in various files including http.py, language.py, logger.py, safe_exec.py, and SQL migration scripts.
- Consolidated import statements and removed unnecessary blank lines.
- Updated logging configuration for better clarity.
- Enhanced the safe execution code with improved error handling and logging.
- Removed commented-out code and unnecessary variables in backfill_zero_trades.py and other scripts.
- Added a pyproject.toml for Ruff and Vulture configuration.
- Introduced requirements-dev.txt for development dependencies.
- Removed commented-out stock entries in init.sql for cleaner migration scripts.
This commit is contained in:
dienakdz
2026-04-09 14:30:51 +07:00
parent 103055b3df
commit 87f2845483
157 changed files with 19026 additions and 17773 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend_api_python/app/data_sources/forex.py
+174 -178
View File
@@ -2,15 +2,17 @@
Forex data source
Get Forex Data with Tiingo
"""
from typing import Dict, List, Any, Optional
from datetime import datetime, timedelta
import time
import requests
import threading
from app.data_sources.base import BaseDataSource, TIMEFRAME_SECONDS
import threading
import time
from datetime import datetime, timedelta
from typing import Any, Dict, List, Optional
import requests
from app.config import APIKeys, TiingoConfig
from app.data_sources.base import TIMEFRAME_SECONDS, BaseDataSource
from app.utils.logger import get_logger
from app.config import TiingoConfig, APIKeys
logger = get_logger(__name__)
@@ -22,52 +24,52 @@ _FOREX_CACHE_TTL = 60 # Forex price caching for 60 seconds (Tiingo free API has
class ForexDataSource(BaseDataSource):
"""Forex data source (Tiingo)"""
name = "Forex/Tiingo"
# Tiingo resampleFreq mapping
# Tiingo free account support: 5min, 15min, 30min, 1hour, 4hour, 1day
# Note: 1min requires paid subscription, 1week/1month is not supported by Tiingo FX API
TIMEFRAME_MAP = {
'1m': '1min', # Paid subscription required
'5m': '5min',
'15m': '15min',
'30m': '30min',
'1H': '1hour',
'4H': '4hour',
'1D': '1day',
'1W': None, # Tiingo does not support it and needs to be aggregated.
'1M': None # Tiingo does not support it and needs to be aggregated.
"1m": "1min", # Paid subscription required
"5m": "5min",
"15m": "15min",
"30m": "30min",
"1H": "1hour",
"4H": "4hour",
"1D": "1day",
"1W": None, # Tiingo does not support it and needs to be aggregated.
"1M": None, # Tiingo does not support it and needs to be aggregated.
}
# Forex pair mapping (Tiingo uses standard tickers such as eurusd, audusd)
# Uppercase letters are also acceptable. Tiingo is usually not case-sensitive, but uniformity is recommended.
SYMBOL_MAP = {
# Precious metals (Tiingo does not necessarily support all precious metals in OANDA format, usually XAUUSD)
'XAUUSD': 'xauusd',
'XAGUSD': 'xagusd',
"XAUUSD": "xauusd",
"XAGUSD": "xagusd",
# major currency pairs
'EURUSD': 'eurusd',
'GBPUSD': 'gbpusd',
'USDJPY': 'usdjpy',
'AUDUSD': 'audusd',
'USDCAD': 'usdcad',
'USDCHF': 'usdchf',
'NZDUSD': 'nzdusd',
"EURUSD": "eurusd",
"GBPUSD": "gbpusd",
"USDJPY": "usdjpy",
"AUDUSD": "audusd",
"USDCAD": "usdcad",
"USDCHF": "usdchf",
"NZDUSD": "nzdusd",
}
def __init__(self):
self.base_url = TiingoConfig.BASE_URL
if not APIKeys.TIINGO_API_KEY:
logger.warning("Tiingo API key is not configured; FX data will be unavailable")
logger.warning("Tiingo API key is not configured; FX data will be unavailable")
def get_ticker(self, symbol: str) -> Dict[str, Any]:
"""
Get realtime quotes for foreign exchange
Get realtime quotes using the Tiingo FX Top-of-Book API
Comes with 60 second cache to avoid triggering Tiingo rate limit frequently
Returns:
dict: {
'last': current price (mid price),
@@ -80,42 +82,39 @@ class ForexDataSource(BaseDataSource):
api_key = APIKeys.TIINGO_API_KEY
if not api_key:
logger.warning("Tiingo API key not configured")
return {'last': 0, 'symbol': symbol}
return {"last": 0, "symbol": symbol}
# Check cache
cache_key = f"ticker_{symbol}"
with _forex_cache_lock:
cached = _forex_cache.get(cache_key)
if cached:
cache_time = cached.get('_cache_time', 0)
cache_time = cached.get("_cache_time", 0)
if time.time() - cache_time < _FOREX_CACHE_TTL:
logger.debug(f"Using cached forex ticker for {symbol}")
return cached
try:
# parse symbol
tiingo_symbol = self.SYMBOL_MAP.get(symbol)
if not tiingo_symbol:
tiingo_symbol = symbol.lower()
# Tiingo FX Top-of-Book API
# https://api.tiingo.com/tiingo/fx/top?tickers=eurusd&token=...
url = f"{self.base_url}/fx/top"
params = {
'tickers': tiingo_symbol,
'token': api_key
}
params = {"tickers": tiingo_symbol, "token": api_key}
# Retry logic: Handling 429 rate limiting
for attempt in range(3):
response = requests.get(url, params=params, timeout=TiingoConfig.TIMEOUT)
if response.status_code == 429:
wait_time = 2 * (attempt + 1)
logger.warning(f"Tiingo rate limit (429), waiting {wait_time}s before retry ({attempt+1}/3)")
logger.warning(f"Tiingo rate limit (429), waiting {wait_time}s before retry ({attempt + 1}/3)")
time.sleep(wait_time)
continue
break
if response.status_code == 429:
logger.warning("Tiingo rate limit exceeded for ticker request")
logger.info("Note: Tiingo 1-minute forex data requires a paid subscription")
@@ -124,86 +123,77 @@ class ForexDataSource(BaseDataSource):
if cache_key in _forex_cache:
logger.info(f"Returning stale cache for {symbol} due to rate limit")
return _forex_cache[cache_key]
return {'last': 0, 'symbol': symbol}
return {"last": 0, "symbol": symbol}
response.raise_for_status()
data = response.json()
if data and isinstance(data, list) and len(data) > 0:
item = data[0]
# Tiingo FX top returns: ticker, quoteTimestamp, bidPrice, bidSize, askPrice, askSize, midPrice
bid = float(item.get('bidPrice', 0) or 0)
ask = float(item.get('askPrice', 0) or 0)
mid = float(item.get('midPrice', 0) or 0)
bid = float(item.get("bidPrice", 0) or 0)
ask = float(item.get("askPrice", 0) or 0)
mid = float(item.get("midPrice", 0) or 0)
# If there is no midPrice, calculate the mid price
if not mid and bid and ask:
mid = (bid + ask) / 2
last_price = mid or bid or ask
# Get the closing price of the previous day to calculate the rise and fall (additional request for daily data is required)
prev_close = 0
change = 0
change_pct = 0
try:
# Get yesterday's closing price
yesterday = (datetime.now() - timedelta(days=2)).strftime('%Y-%m-%d')
today = datetime.now().strftime('%Y-%m-%d')
yesterday = (datetime.now() - timedelta(days=2)).strftime("%Y-%m-%d")
today = datetime.now().strftime("%Y-%m-%d")
price_url = f"{self.base_url}/fx/{tiingo_symbol}/prices"
price_params = {
'startDate': yesterday,
'endDate': today,
'resampleFreq': '1day',
'token': api_key
}
price_params = {"startDate": yesterday, "endDate": today, "resampleFreq": "1day", "token": api_key}
price_resp = requests.get(price_url, params=price_params, timeout=TiingoConfig.TIMEOUT)
if price_resp.status_code == 200:
price_data = price_resp.json()
if price_data and len(price_data) > 0:
prev_close = float(price_data[-1].get('close', 0) or 0)
prev_close = float(price_data[-1].get("close", 0) or 0)
if prev_close and last_price:
change = last_price - prev_close
change_pct = (change / prev_close) * 100
except Exception:
pass # Failure to calculate the rise or fall does not affect the main functions
result = {
'last': round(last_price, 5),
'bid': round(bid, 5),
'ask': round(ask, 5),
'change': round(change, 5),
'changePercent': round(change_pct, 2),
'previousClose': round(prev_close, 5) if prev_close else 0,
'_cache_time': time.time()
"last": round(last_price, 5),
"bid": round(bid, 5),
"ask": round(ask, 5),
"change": round(change, 5),
"changePercent": round(change_pct, 2),
"previousClose": round(prev_close, 5) if prev_close else 0,
"_cache_time": time.time(),
}
# cache results
with _forex_cache_lock:
_forex_cache[cache_key] = result
return result
except Exception as e:
logger.error(f"Failed to get forex ticker for {symbol}: {e}")
return {'last': 0, 'symbol': symbol}
return {"last": 0, "symbol": symbol}
def _get_timeframe_seconds(self, timeframe: str) -> int:
"""Get the number of seconds corresponding to the time period"""
return TIMEFRAME_SECONDS.get(timeframe, 86400)
def get_kline(
self,
symbol: str,
timeframe: str,
limit: int,
before_time: Optional[int] = None
self, symbol: str, timeframe: str, limit: int, before_time: Optional[int] = None
) -> List[Dict[str, Any]]:
"""
Get foreign exchange K-line data
Args:
symbol: Forex pair symbol (such as XAUUSD, EURUSD)
timeframe: time period
@@ -215,7 +205,7 @@ class ForexDataSource(BaseDataSource):
if not api_key:
logger.error("Tiingo API key is not configured")
return []
try:
# 1. Parse Symbol
tiingo_symbol = self.SYMBOL_MAP.get(symbol)
@@ -225,15 +215,15 @@ class ForexDataSource(BaseDataSource):
# 2. Analysis Resolution (resampleFreq)
resample_freq = self.TIMEFRAME_MAP.get(timeframe)
# Special treatment: 1W/1M requires daily aggregation
aggregate_to_weekly = (timeframe == '1W')
aggregate_to_monthly = (timeframe == '1M')
aggregate_to_weekly = timeframe == "1W"
aggregate_to_monthly = timeframe == "1M"
original_limit = limit # Save original request quantity
if aggregate_to_weekly or aggregate_to_monthly:
# Aggregate using daily data
resample_freq = '1day'
resample_freq = "1day"
# Limit the maximum number of weekly/monthly requests (Tiingo free API has data volume limit)
# The maximum weekly request is 100 weeks = 700 days ≈ 2 years
# The maximum monthly request is 36 months = 1080 days ≈ 3 years
@@ -241,21 +231,21 @@ class ForexDataSource(BaseDataSource):
original_limit = min(original_limit, max_limit)
# More daily data is needed to aggregate (weekly lines require 7 days, monthly lines require 30 days)
limit = original_limit * (7 if aggregate_to_weekly else 30)
if not resample_freq:
logger.warning(f"Tiingo does not support timeframe: {timeframe}")
return []
# 1 minute data requires paid subscription reminder
if timeframe == '1m':
logger.info(f"Note: Tiingo 1-minute forex data requires a paid subscription")
if timeframe == "1m":
logger.info("Note: Tiingo 1-minute forex data requires a paid subscription")
# 3. Calculation time range
if before_time:
end_dt = datetime.fromtimestamp(before_time)
else:
end_dt = datetime.now()
# Calculate start time based on period and quantity
# Note: Use daily seconds calculation in aggregation mode
if aggregate_to_weekly or aggregate_to_monthly:
@@ -264,71 +254,75 @@ class ForexDataSource(BaseDataSource):
tf_seconds = self._get_timeframe_seconds(timeframe)
# Get more buffer time (1.5 times, foreign exchange does not trade on weekends)
start_dt = end_dt - timedelta(seconds=limit * tf_seconds * 1.5)
# Tiingo free API supports up to about 5 years of data, limiting the maximum time range
max_days = 365 * 3 # up to 3 years
if (end_dt - start_dt).days > max_days:
start_dt = end_dt - timedelta(days=max_days)
logger.info(f"Tiingo: Limited date range to {max_days} days")
# Format the date as YYYY-MM-DD (Tiingo supports this format)
start_date_str = start_dt.strftime('%Y-%m-%d')
end_date_str = end_dt.strftime('%Y-%m-%d')
start_date_str = start_dt.strftime("%Y-%m-%d")
end_date_str = end_dt.strftime("%Y-%m-%d")
# 4. API request (with retry logic)
# URL: https://api.tiingo.com/tiingo/fx/{ticker}/prices
url = f"{self.base_url}/fx/{tiingo_symbol}/prices"
params = {
'startDate': start_date_str,
'endDate': end_date_str,
'resampleFreq': resample_freq,
'token': api_key,
'format': 'json'
"startDate": start_date_str,
"endDate": end_date_str,
"resampleFreq": resample_freq,
"token": api_key,
"format": "json",
}
# logger.info(f"Tiingo Request: {url} params={params}")
# Retry logic: Handling 429 rate limiting
max_retries = 3
retry_delay = 2 # Second
response = None
for attempt in range(max_retries):
try:
response = requests.get(url, params=params, timeout=TiingoConfig.TIMEOUT)
if response.status_code == 429:
# Rate limit, wait and try again
wait_time = retry_delay * (attempt + 1)
logger.warning(f"Tiingo rate limit (429), waiting {wait_time}s before retry ({attempt + 1}/{max_retries})")
logger.warning(
f"Tiingo rate limit (429), waiting {wait_time}s before retry ({attempt + 1}/{max_retries})"
)
time.sleep(wait_time)
continue
break # Success or other errors, exit the retry loop
except requests.exceptions.Timeout:
if attempt < max_retries - 1:
logger.warning(f"Tiingo request timeout, retrying ({attempt + 1}/{max_retries})")
time.sleep(retry_delay)
continue
raise
if response is None:
logger.error("Tiingo API request failed after all retries")
return []
if response.status_code == 429:
logger.error("Tiingo API rate limit exceeded. Please wait a moment before retrying.")
return []
if response.status_code == 403:
logger.error("Tiingo API permission error (403): check whether your API key is valid and has access to this dataset.")
logger.error(
"Tiingo API permission error (403): check whether your API key is valid and has access to this dataset."
)
return []
response.raise_for_status()
data = response.json()
# 5. Process the response
# Tiingo returns a list of dicts:
# [
@@ -343,35 +337,37 @@ class ForexDataSource(BaseDataSource):
# }, ...
# ]
# Note: Tiingo FX prices objects keys: date, open, high, low, close.
if not isinstance(data, list):
logger.warning(f"Tiingo response is not a list: {data}")
return []
klines = []
for item in data:
# Parsing time: "2023-01-01T00:00:00.000Z"
dt_str = item.get('date')
dt_str = item.get("date")
# Tiingo returns UTC time in ISO format and needs to handle the time zone correctly.
# Convert UTC time to local timestamp
if dt_str.endswith('Z'):
dt_str = dt_str[:-1] + '+00:00' # Replace Z with +00:00 for UTC
if dt_str.endswith("Z"):
dt_str = dt_str[:-1] + "+00:00" # Replace Z with +00:00 for UTC
dt = datetime.fromisoformat(dt_str)
ts = int(dt.timestamp()) # UTC time zone is now handled correctly
klines.append({
'time': ts,
'open': float(item.get('open')),
'high': float(item.get('high')),
'low': float(item.get('low')),
'close': float(item.get('close')),
'volume': 0.0 # Tiingo FX usually does not have volume
})
klines.append(
{
"time": ts,
"open": float(item.get("open")),
"high": float(item.get("high")),
"low": float(item.get("low")),
"close": float(item.get("close")),
"volume": 0.0, # Tiingo FX usually does not have volume
}
)
# Sort by time
klines.sort(key=lambda x: x['time'])
klines.sort(key=lambda x: x["time"])
# If you need to aggregate to weekly or monthly lines
if aggregate_to_weekly:
klines = self._aggregate_to_weekly(klines)
@@ -379,36 +375,36 @@ class ForexDataSource(BaseDataSource):
elif aggregate_to_monthly:
klines = self._aggregate_to_monthly(klines)
logger.debug(f"Aggregated {len(klines)} monthly candles from daily data")
# Filter to original request count
if len(klines) > original_limit:
klines = klines[-original_limit:]
# logger.info(f"obtained {len(klines)} pieces of Tiingo foreign exchange data")
return klines
except requests.exceptions.RequestException as e:
logger.error(f"Tiingo API request failed: {e}")
return []
except Exception as e:
logger.error(f"Failed to process Tiingo data: {e}")
return []
def _aggregate_to_weekly(self, daily_klines: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""Aggregate daily data into weekly data"""
if not daily_klines:
return []
weekly_klines = []
current_week = None
week_data = None
for kline in daily_klines:
dt = datetime.fromtimestamp(kline['time'])
dt = datetime.fromtimestamp(kline["time"])
# Get the Monday of the week in which the date is located
week_start = dt - timedelta(days=dt.weekday())
week_key = week_start.strftime('%Y-%W')
week_key = week_start.strftime("%Y-%W")
if week_key != current_week:
# Save data from last week
if week_data:
@@ -416,39 +412,39 @@ class ForexDataSource(BaseDataSource):
# start a new week
current_week = week_key
week_data = {
'time': int(week_start.timestamp()),
'open': kline['open'],
'high': kline['high'],
'low': kline['low'],
'close': kline['close'],
'volume': kline['volume']
"time": int(week_start.timestamp()),
"open": kline["open"],
"high": kline["high"],
"low": kline["low"],
"close": kline["close"],
"volume": kline["volume"],
}
else:
# Update this week's data
week_data['high'] = max(week_data['high'], kline['high'])
week_data['low'] = min(week_data['low'], kline['low'])
week_data['close'] = kline['close']
week_data['volume'] += kline['volume']
week_data["high"] = max(week_data["high"], kline["high"])
week_data["low"] = min(week_data["low"], kline["low"])
week_data["close"] = kline["close"]
week_data["volume"] += kline["volume"]
# Add last week
if week_data:
weekly_klines.append(week_data)
return weekly_klines
def _aggregate_to_monthly(self, daily_klines: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""Aggregate daily data into monthly data"""
if not daily_klines:
return []
monthly_klines = []
current_month = None
month_data = None
for kline in daily_klines:
dt = datetime.fromtimestamp(kline['time'])
month_key = dt.strftime('%Y-%m')
dt = datetime.fromtimestamp(kline["time"])
month_key = dt.strftime("%Y-%m")
if month_key != current_month:
# Save last month’s data
if month_data:
@@ -457,22 +453,22 @@ class ForexDataSource(BaseDataSource):
current_month = month_key
month_start = dt.replace(day=1, hour=0, minute=0, second=0)
month_data = {
'time': int(month_start.timestamp()),
'open': kline['open'],
'high': kline['high'],
'low': kline['low'],
'close': kline['close'],
'volume': kline['volume']
"time": int(month_start.timestamp()),
"open": kline["open"],
"high": kline["high"],
"low": kline["low"],
"close": kline["close"],
"volume": kline["volume"],
}
else:
# Update this month's data
month_data['high'] = max(month_data['high'], kline['high'])
month_data['low'] = min(month_data['low'], kline['low'])
month_data['close'] = kline['close']
month_data['volume'] += kline['volume']
month_data["high"] = max(month_data["high"], kline["high"])
month_data["low"] = min(month_data["low"], kline["low"])
month_data["close"] = kline["close"]
month_data["volume"] += kline["volume"]
# Add last month
if month_data:
monthly_klines.append(month_data)
return monthly_klines