gavindiaz/DinQuant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor code for improved readability and consistency

Browse Source 
- Cleaned up whitespace and formatting in various files including http.py, language.py, logger.py, safe_exec.py, and SQL migration scripts.
- Consolidated import statements and removed unnecessary blank lines.
- Updated logging configuration for better clarity.
- Enhanced the safe execution code with improved error handling and logging.
- Removed commented-out code and unnecessary variables in backfill_zero_trades.py and other scripts.
- Added a pyproject.toml for Ruff and Vulture configuration.
- Introduced requirements-dev.txt for development dependencies.
- Removed commented-out stock entries in init.sql for cleaner migration scripts.
This commit is contained in:
dienakdz
2026-04-09 14:30:51 +07:00
parent 103055b3df
commit 87f2845483
157 changed files with 19026 additions and 17773 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend_api_python/app/data_sources/rate_limiter.py
+62 -79
View File
@@ -12,11 +12,11 @@ Provide anti-crawler strategies:
4. Request frequency limit
"""
import time
import random
import logging
from typing import Optional, Callable, Any, Type, Tuple
import random
import time
from functools import wraps
from typing import Any, Callable, Optional, Tuple, Type
logger = logging.getLogger(__name__)
@@ -27,23 +27,23 @@ logger = logging.getLogger(__name__)
USER_AGENTS = [
# Chrome Windows
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
# Chrome Mac
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
# Firefox
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:121.0) Gecko/20100101 Firefox/121.0',
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:121.0) Gecko/20100101 Firefox/121.0",
# Safari
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15',
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15",
# Edge
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0',
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0",
# Linux Chrome
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
]
@@ -55,24 +55,24 @@ def get_random_user_agent() -> str:
def get_request_headers(referer: Optional[str] = None) -> dict:
"""
Get request header with random User-Agent
Args:
referer: optional Referer header
Returns:
Request header dictionary
"""
headers = {
'User-Agent': get_random_user_agent(),
'Accept': 'application/json, text/plain, */*',
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
'Accept-Encoding': 'gzip, deflate',
'Connection': 'keep-alive',
"User-Agent": get_random_user_agent(),
"Accept": "application/json, text/plain, */*",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
"Accept-Encoding": "gzip, deflate",
"Connection": "keep-alive",
}
if referer:
headers['Referer'] = referer
headers["Referer"] = referer
return headers
@@ -80,17 +80,14 @@ def get_request_headers(referer: Optional[str] = None) -> dict:
# Random sleep
# ============================================
def random_sleep(
min_seconds: float = 1.0,
max_seconds: float = 3.0,
log: bool = False
) -> None:
def random_sleep(min_seconds: float = 1.0, max_seconds: float = 3.0, log: bool = False) -> None:
"""
Random sleep (Jitter)
Anti-ban strategy: simulate random delays in human behavior
Incorporate irregular wait times between requests
Args:
min_seconds: Minimum sleep time (seconds)
max_seconds: Maximum sleep time (seconds)
@@ -106,22 +103,18 @@ def random_sleep(
# Request frequency limiter
# ============================================
class RateLimiter:
"""
Request frequency limiter
Ensure there is a minimum amount of time between requests
"""
def __init__(
self,
min_interval: float = 1.0,
jitter_min: float = 0.5,
jitter_max: float = 1.5
):
def __init__(self, min_interval: float = 1.0, jitter_min: float = 0.5, jitter_max: float = 1.5):
"""
Initialize frequency limiter
Args:
min_interval: Minimum request interval (seconds)
jitter_min: minimum random jitter (seconds)
@@ -131,33 +124,33 @@ class RateLimiter:
self.jitter_min = jitter_min
self.jitter_max = jitter_max
self._last_request_time: Optional[float] = None
def wait(self) -> float:
"""
Wait until the next request can be made
Returns:
Actual waiting time (seconds)
"""
wait_time = 0.0
if self._last_request_time is not None:
elapsed = time.time() - self._last_request_time
if elapsed < self.min_interval:
# Supplement sleep to minimum interval
wait_time = self.min_interval - elapsed
time.sleep(wait_time)
# Add random jitter
jitter = random.uniform(self.jitter_min, self.jitter_max)
time.sleep(jitter)
wait_time += jitter
# Record the time of this request
self._last_request_time = time.time()
return wait_time
def reset(self) -> None:
"""reset limiter"""
self._last_request_time = None
@@ -167,17 +160,18 @@ class RateLimiter:
# Exponential backoff retry decorator
# ============================================
def retry_with_backoff(
max_attempts: int = 3,
base_delay: float = 2.0,
max_delay: float = 30.0,
exponential_base: float = 2.0,
exceptions: Tuple[Type[Exception], ...] = (Exception,),
on_retry: Optional[Callable[[int, Exception], None]] = None
on_retry: Optional[Callable[[int, Exception], None]] = None,
):
"""
Exponential backoff retry decorator
Args:
max_attempts: Maximum number of retries
base_delay: base delay time (seconds)
@@ -185,49 +179,50 @@ def retry_with_backoff(
exponential_base: exponential base
exceptions: Exception types that need to be retried
on_retry: callback function when retrying
Usage example:
@retry_with_backoff(max_attempts=3, exceptions=(ConnectionError, TimeoutError))
def fetch_data():
...
"""
def decorator(func: Callable) -> Callable:
@wraps(func)
def wrapper(*args, **kwargs) -> Any:
last_exception = None
for attempt in range(1, max_attempts + 1):
try:
return func(*args, **kwargs)
except exceptions as e:
last_exception = e
if attempt == max_attempts:
logger.error(f"[Retry] {func.__name__} has reached the maximum number of retries ({max_attempts}), giving up")
logger.error(
f"[Retry] {func.__name__} has reached the maximum number of retries ({max_attempts}), giving up"
)
raise
# Calculate the backoff delay: base_delay * (exponential_base ^ (attempt - 1))
delay = min(
base_delay * (exponential_base ** (attempt - 1)),
max_delay
)
delay = min(base_delay * (exponential_base ** (attempt - 1)), max_delay)
# Add random jitter (±20%)
delay *= random.uniform(0.8, 1.2)
logger.warning(
f"[Retry] {func.__name__} failed for the {attempt}/{max_attempts} time: {e}, "
f"waiting {delay:.1f}s before retrying..."
)
if on_retry:
on_retry(attempt, e)
time.sleep(delay)
# Shouldn't have gotten here
raise last_exception
return wrapper
return decorator
@@ -236,25 +231,13 @@ def retry_with_backoff(
# ============================================
# Oriental Fortune interface current limiter (more stringent)
_eastmoney_limiter = RateLimiter(
min_interval=2.0,
jitter_min=1.0,
jitter_max=3.0
)
_eastmoney_limiter = RateLimiter(min_interval=2.0, jitter_min=1.0, jitter_max=3.0)
# Tencent Finance interface current limiter (relatively loose)
_tencent_limiter = RateLimiter(
min_interval=1.0,
jitter_min=0.5,
jitter_max=1.5
)
_tencent_limiter = RateLimiter(min_interval=1.0, jitter_min=0.5, jitter_max=1.5)
# Akshare interface current limiter
_akshare_limiter = RateLimiter(
min_interval=2.0,
jitter_min=1.5,
jitter_max=3.5
)
_akshare_limiter = RateLimiter(min_interval=2.0, jitter_min=1.5, jitter_max=3.5)
def get_eastmoney_limiter() -> RateLimiter: