Refactor code for improved readability and consistency
- Cleaned up whitespace and formatting in various files including http.py, language.py, logger.py, safe_exec.py, and SQL migration scripts. - Consolidated import statements and removed unnecessary blank lines. - Updated logging configuration for better clarity. - Enhanced the safe execution code with improved error handling and logging. - Removed commented-out code and unnecessary variables in backfill_zero_trades.py and other scripts. - Added a pyproject.toml for Ruff and Vulture configuration. - Introduced requirements-dev.txt for development dependencies. - Removed commented-out stock entries in init.sql for cleaner migration scripts.
This commit is contained in:
dienakdz
@@ -4,64 +4,61 @@ support:
|
||||
1. Cryptocurrency Futures (Binance Futures via CCXT)
|
||||
2. Traditional futures (Yahoo Finance)
|
||||
"""
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import ccxt
|
||||
import yfinance as yf
|
||||
|
||||
from app.data_sources.base import BaseDataSource, TIMEFRAME_SECONDS
|
||||
from app.config import CCXTConfig
|
||||
from app.data_sources.base import TIMEFRAME_SECONDS, BaseDataSource
|
||||
from app.utils.logger import get_logger
|
||||
from app.config import CCXTConfig, APIKeys
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class FuturesDataSource(BaseDataSource):
|
||||
"""Futures data source"""
|
||||
|
||||
|
||||
name = "Futures"
|
||||
|
||||
|
||||
# Yahoo Finance time period mapping
|
||||
YF_TIMEFRAME_MAP = {
|
||||
'1m': '1m',
|
||||
'5m': '5m',
|
||||
'15m': '15m',
|
||||
'30m': '30m',
|
||||
'1H': '1h',
|
||||
'4H': '4h',
|
||||
'1D': '1d',
|
||||
'1W': '1wk'
|
||||
"1m": "1m",
|
||||
"5m": "5m",
|
||||
"15m": "15m",
|
||||
"30m": "30m",
|
||||
"1H": "1h",
|
||||
"4H": "4h",
|
||||
"1D": "1d",
|
||||
"1W": "1wk",
|
||||
}
|
||||
|
||||
|
||||
# CCXT time period mapping
|
||||
CCXT_TIMEFRAME_MAP = CCXTConfig.TIMEFRAME_MAP
|
||||
|
||||
|
||||
# Traditional futures contract code (Yahoo Finance)
|
||||
YF_SYMBOLS = {
|
||||
'GC': 'GC=F', # gold futures
|
||||
'SI': 'SI=F', # Silver futures
|
||||
'CL': 'CL=F', # Crude oil futures
|
||||
'NG': 'NG=F', # Natural gas futures
|
||||
'ZC': 'ZC=F', # Corn futures
|
||||
'ZW': 'ZW=F', # Wheat futures
|
||||
"GC": "GC=F", # gold futures
|
||||
"SI": "SI=F", # Silver futures
|
||||
"CL": "CL=F", # Crude oil futures
|
||||
"NG": "NG=F", # Natural gas futures
|
||||
"ZC": "ZC=F", # Corn futures
|
||||
"ZW": "ZW=F", # Wheat futures
|
||||
}
|
||||
|
||||
|
||||
def __init__(self):
|
||||
# Initialize CCXT (for cryptocurrency futures)
|
||||
config = {
|
||||
'timeout': CCXTConfig.TIMEOUT,
|
||||
'enableRateLimit': CCXTConfig.ENABLE_RATE_LIMIT,
|
||||
'options': {
|
||||
'defaultType': 'future'
|
||||
}
|
||||
"timeout": CCXTConfig.TIMEOUT,
|
||||
"enableRateLimit": CCXTConfig.ENABLE_RATE_LIMIT,
|
||||
"options": {"defaultType": "future"},
|
||||
}
|
||||
|
||||
|
||||
if CCXTConfig.PROXY:
|
||||
config['proxies'] = {
|
||||
'http': CCXTConfig.PROXY,
|
||||
'https': CCXTConfig.PROXY
|
||||
}
|
||||
|
||||
config["proxies"] = {"http": CCXTConfig.PROXY, "https": CCXTConfig.PROXY}
|
||||
|
||||
self.exchange = ccxt.binance(config)
|
||||
|
||||
def get_ticker(self, symbol: str) -> Dict[str, Any]:
|
||||
@@ -101,21 +98,17 @@ class FuturesDataSource(BaseDataSource):
|
||||
elif sym.endswith("USD") and len(sym) > 3:
|
||||
sym = f"{sym[:-3]}/USD"
|
||||
return self.exchange.fetch_ticker(sym)
|
||||
|
||||
|
||||
def _get_timeframe_seconds(self, timeframe: str) -> int:
|
||||
"""Get the number of seconds corresponding to the time period"""
|
||||
return TIMEFRAME_SECONDS.get(timeframe, 86400)
|
||||
|
||||
|
||||
def get_kline(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int] = None
|
||||
self, symbol: str, timeframe: str, limit: int, before_time: Optional[int] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Get futures K-line data
|
||||
|
||||
|
||||
Args:
|
||||
symbol: futures contract code
|
||||
timeframe: time period
|
||||
@@ -123,124 +116,106 @@ class FuturesDataSource(BaseDataSource):
|
||||
before_time: end timestamp
|
||||
"""
|
||||
# Determine whether it is traditional futures or cryptocurrency futures
|
||||
if symbol in self.YF_SYMBOLS or symbol.endswith('=F'):
|
||||
if symbol in self.YF_SYMBOLS or symbol.endswith("=F"):
|
||||
return self._get_traditional_futures(symbol, timeframe, limit, before_time)
|
||||
else:
|
||||
return self._get_crypto_futures(symbol, timeframe, limit, before_time)
|
||||
|
||||
|
||||
def _get_traditional_futures(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int] = None
|
||||
self, symbol: str, timeframe: str, limit: int, before_time: Optional[int] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Use yfinance to obtain traditional futures data"""
|
||||
try:
|
||||
# Convert symbol format
|
||||
yf_symbol = self.YF_SYMBOLS.get(symbol, symbol)
|
||||
if not yf_symbol.endswith('=F'):
|
||||
yf_symbol = symbol + '=F'
|
||||
|
||||
if not yf_symbol.endswith("=F"):
|
||||
yf_symbol = symbol + "=F"
|
||||
|
||||
# conversion time period
|
||||
yf_interval = self.YF_TIMEFRAME_MAP.get(timeframe, '1d')
|
||||
|
||||
yf_interval = self.YF_TIMEFRAME_MAP.get(timeframe, "1d")
|
||||
|
||||
# logger.info(f"Get traditional futures K-line: {yf_symbol}, period: {yf_interval}, number of bars: {limit}")
|
||||
|
||||
|
||||
# Calculation time range
|
||||
if before_time:
|
||||
end_time = datetime.fromtimestamp(before_time)
|
||||
else:
|
||||
end_time = datetime.now()
|
||||
|
||||
|
||||
tf_seconds = self._get_timeframe_seconds(timeframe)
|
||||
start_time = end_time - timedelta(seconds=tf_seconds * limit * 1.5)
|
||||
|
||||
|
||||
# The end parameter of yfinance is not included (exclusive), and one day needs to be added.
|
||||
end_time_inclusive = end_time + timedelta(days=1)
|
||||
|
||||
|
||||
# Get data
|
||||
ticker = yf.Ticker(yf_symbol)
|
||||
df = ticker.history(
|
||||
start=start_time,
|
||||
end=end_time_inclusive,
|
||||
interval=yf_interval
|
||||
)
|
||||
|
||||
df = ticker.history(start=start_time, end=end_time_inclusive, interval=yf_interval)
|
||||
|
||||
if df.empty:
|
||||
logger.warning(f"No data: {yf_symbol}")
|
||||
return []
|
||||
|
||||
|
||||
# Convert format
|
||||
klines = []
|
||||
for index, row in df.iterrows():
|
||||
klines.append({
|
||||
'time': int(index.timestamp()),
|
||||
'open': float(row['Open']),
|
||||
'high': float(row['High']),
|
||||
'low': float(row['Low']),
|
||||
'close': float(row['Close']),
|
||||
'volume': float(row['Volume'])
|
||||
})
|
||||
|
||||
klines.sort(key=lambda x: x['time'])
|
||||
klines.append(
|
||||
{
|
||||
"time": int(index.timestamp()),
|
||||
"open": float(row["Open"]),
|
||||
"high": float(row["High"]),
|
||||
"low": float(row["Low"]),
|
||||
"close": float(row["Close"]),
|
||||
"volume": float(row["Volume"]),
|
||||
}
|
||||
)
|
||||
|
||||
klines.sort(key=lambda x: x["time"])
|
||||
if len(klines) > limit:
|
||||
klines = klines[-limit:]
|
||||
|
||||
|
||||
# logger.info(f"obtained {len(klines)} pieces of traditional futures data")
|
||||
return klines
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to fetch traditional futures data: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def _get_crypto_futures(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int] = None
|
||||
self, symbol: str, timeframe: str, limit: int, before_time: Optional[int] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Obtain cryptocurrency futures data using CCXT"""
|
||||
try:
|
||||
# Make sure the symbol format is correct
|
||||
ccxt_symbol = symbol if '/' in symbol else f"{symbol}/USDT"
|
||||
ccxt_timeframe = self.CCXT_TIMEFRAME_MAP.get(timeframe, '1d')
|
||||
|
||||
ccxt_symbol = symbol if "/" in symbol else f"{symbol}/USDT"
|
||||
ccxt_timeframe = self.CCXT_TIMEFRAME_MAP.get(timeframe, "1d")
|
||||
|
||||
# logger.info(f"Get cryptocurrency futures K-line: {ccxt_symbol}, period: {ccxt_timeframe}, number of bars: {limit}")
|
||||
|
||||
|
||||
# Get data
|
||||
if before_time:
|
||||
since_time = before_time - limit * self._get_timeframe_seconds(timeframe)
|
||||
ohlcv = self.exchange.fetch_ohlcv(
|
||||
ccxt_symbol,
|
||||
ccxt_timeframe,
|
||||
since=since_time * 1000,
|
||||
limit=limit
|
||||
)
|
||||
ohlcv = self.exchange.fetch_ohlcv(ccxt_symbol, ccxt_timeframe, since=since_time * 1000, limit=limit)
|
||||
else:
|
||||
ohlcv = self.exchange.fetch_ohlcv(
|
||||
ccxt_symbol,
|
||||
ccxt_timeframe,
|
||||
limit=limit
|
||||
)
|
||||
|
||||
ohlcv = self.exchange.fetch_ohlcv(ccxt_symbol, ccxt_timeframe, limit=limit)
|
||||
|
||||
# Convert format
|
||||
klines = []
|
||||
for candle in ohlcv:
|
||||
klines.append({
|
||||
'time': int(candle[0] / 1000),
|
||||
'open': float(candle[1]),
|
||||
'high': float(candle[2]),
|
||||
'low': float(candle[3]),
|
||||
'close': float(candle[4]),
|
||||
'volume': float(candle[5])
|
||||
})
|
||||
|
||||
klines.append(
|
||||
{
|
||||
"time": int(candle[0] / 1000),
|
||||
"open": float(candle[1]),
|
||||
"high": float(candle[2]),
|
||||
"low": float(candle[3]),
|
||||
"close": float(candle[4]),
|
||||
"volume": float(candle[5]),
|
||||
}
|
||||
)
|
||||
|
||||
# logger.info(f"obtained {len(klines)} pieces of cryptocurrency futures data")
|
||||
return klines
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to fetch crypto futures data: {e}")
|
||||
return []
|
||||
|
||||
|
||||
Reference in New Issue
Block a user