TIANHE
273 lines
8.1 KiB
Python
273 lines
8.1 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""
|
||
===================================
|
||
防封禁工具模块 (Rate Limiter)
|
||
===================================
|
||
|
||
参考 daily_stock_analysis 项目实现
|
||
提供反爬虫策略：
|
||
1. 随机休眠（Jitter）
|
||
2. 随机 User-Agent 轮换
|
||
3. 指数退避重试
|
||
4. 请求频率限制
|
||
"""
|
||
|
||
import time
|
||
import random
|
||
import logging
|
||
from typing import Optional, Callable, Any, Type, Tuple
|
||
from functools import wraps
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
# ============================================
|
||
# User-Agent 池
|
||
# ============================================
|
||
|
||
USER_AGENTS = [
|
||
# Chrome Windows
|
||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
|
||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
|
||
# Chrome Mac
|
||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
|
||
# Firefox
|
||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0',
|
||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0',
|
||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:121.0) Gecko/20100101 Firefox/121.0',
|
||
# Safari
|
||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15',
|
||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15',
|
||
# Edge
|
||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0',
|
||
# Linux Chrome
|
||
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||
]
|
||
|
||
|
||
def get_random_user_agent() -> str:
|
||
"""获取随机 User-Agent"""
|
||
return random.choice(USER_AGENTS)
|
||
|
||
|
||
def get_request_headers(referer: Optional[str] = None) -> dict:
|
||
"""
|
||
获取带有随机 User-Agent 的请求头
|
||
|
||
Args:
|
||
referer: 可选的 Referer 头
|
||
|
||
Returns:
|
||
请求头字典
|
||
"""
|
||
headers = {
|
||
'User-Agent': get_random_user_agent(),
|
||
'Accept': 'application/json, text/plain, */*',
|
||
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
|
||
'Accept-Encoding': 'gzip, deflate',
|
||
'Connection': 'keep-alive',
|
||
}
|
||
|
||
if referer:
|
||
headers['Referer'] = referer
|
||
|
||
return headers
|
||
|
||
|
||
# ============================================
|
||
# 随机休眠
|
||
# ============================================
|
||
|
||
def random_sleep(
|
||
min_seconds: float = 1.0,
|
||
max_seconds: float = 3.0,
|
||
log: bool = False
|
||
) -> None:
|
||
"""
|
||
随机休眠（Jitter）
|
||
|
||
防封禁策略：模拟人类行为的随机延迟
|
||
在请求之间加入不规则的等待时间
|
||
|
||
Args:
|
||
min_seconds: 最小休眠时间（秒）
|
||
max_seconds: 最大休眠时间（秒）
|
||
log: 是否记录日志
|
||
"""
|
||
sleep_time = random.uniform(min_seconds, max_seconds)
|
||
if log:
|
||
logger.debug(f"随机休眠 {sleep_time:.2f} 秒...")
|
||
time.sleep(sleep_time)
|
||
|
||
|
||
# ============================================
|
||
# 请求频率限制器
|
||
# ============================================
|
||
|
||
class RateLimiter:
|
||
"""
|
||
请求频率限制器
|
||
|
||
确保请求之间有最小间隔时间
|
||
"""
|
||
|
||
def __init__(
|
||
self,
|
||
min_interval: float = 1.0,
|
||
jitter_min: float = 0.5,
|
||
jitter_max: float = 1.5
|
||
):
|
||
"""
|
||
初始化频率限制器
|
||
|
||
Args:
|
||
min_interval: 最小请求间隔（秒）
|
||
jitter_min: 随机抖动最小值（秒）
|
||
jitter_max: 随机抖动最大值（秒）
|
||
"""
|
||
self.min_interval = min_interval
|
||
self.jitter_min = jitter_min
|
||
self.jitter_max = jitter_max
|
||
self._last_request_time: Optional[float] = None
|
||
|
||
def wait(self) -> float:
|
||
"""
|
||
等待直到可以发起下一次请求
|
||
|
||
Returns:
|
||
实际等待的时间（秒）
|
||
"""
|
||
wait_time = 0.0
|
||
|
||
if self._last_request_time is not None:
|
||
elapsed = time.time() - self._last_request_time
|
||
if elapsed < self.min_interval:
|
||
# 补充休眠到最小间隔
|
||
wait_time = self.min_interval - elapsed
|
||
time.sleep(wait_time)
|
||
|
||
# 添加随机抖动
|
||
jitter = random.uniform(self.jitter_min, self.jitter_max)
|
||
time.sleep(jitter)
|
||
wait_time += jitter
|
||
|
||
# 记录本次请求时间
|
||
self._last_request_time = time.time()
|
||
|
||
return wait_time
|
||
|
||
def reset(self) -> None:
|
||
"""重置限制器"""
|
||
self._last_request_time = None
|
||
|
||
|
||
# ============================================
|
||
# 指数退避重试装饰器
|
||
# ============================================
|
||
|
||
def retry_with_backoff(
|
||
max_attempts: int = 3,
|
||
base_delay: float = 2.0,
|
||
max_delay: float = 30.0,
|
||
exponential_base: float = 2.0,
|
||
exceptions: Tuple[Type[Exception], ...] = (Exception,),
|
||
on_retry: Optional[Callable[[int, Exception], None]] = None
|
||
):
|
||
"""
|
||
指数退避重试装饰器
|
||
|
||
Args:
|
||
max_attempts: 最大重试次数
|
||
base_delay: 基础延迟时间（秒）
|
||
max_delay: 最大延迟时间（秒）
|
||
exponential_base: 指数基数
|
||
exceptions: 需要重试的异常类型
|
||
on_retry: 重试时的回调函数
|
||
|
||
使用示例:
|
||
@retry_with_backoff(max_attempts=3, exceptions=(ConnectionError, TimeoutError))
|
||
def fetch_data():
|
||
...
|
||
"""
|
||
def decorator(func: Callable) -> Callable:
|
||
@wraps(func)
|
||
def wrapper(*args, **kwargs) -> Any:
|
||
last_exception = None
|
||
|
||
for attempt in range(1, max_attempts + 1):
|
||
try:
|
||
return func(*args, **kwargs)
|
||
except exceptions as e:
|
||
last_exception = e
|
||
|
||
if attempt == max_attempts:
|
||
logger.error(f"[重试] {func.__name__} 已达最大重试次数 ({max_attempts})，放弃")
|
||
raise
|
||
|
||
# 计算退避延迟: base_delay * (exponential_base ^ (attempt - 1))
|
||
delay = min(
|
||
base_delay * (exponential_base ** (attempt - 1)),
|
||
max_delay
|
||
)
|
||
# 添加随机抖动 (±20%)
|
||
delay *= random.uniform(0.8, 1.2)
|
||
|
||
logger.warning(
|
||
f"[重试] {func.__name__} 第 {attempt}/{max_attempts} 次失败: {e}, "
|
||
f"等待 {delay:.1f}s 后重试..."
|
||
)
|
||
|
||
if on_retry:
|
||
on_retry(attempt, e)
|
||
|
||
time.sleep(delay)
|
||
|
||
# 不应该到达这里
|
||
raise last_exception
|
||
|
||
return wrapper
|
||
return decorator
|
||
|
||
|
||
# ============================================
|
||
# 全局限流器实例
|
||
# ============================================
|
||
|
||
# 东方财富接口限流器（较严格）
|
||
_eastmoney_limiter = RateLimiter(
|
||
min_interval=2.0,
|
||
jitter_min=1.0,
|
||
jitter_max=3.0
|
||
)
|
||
|
||
# 腾讯财经接口限流器（较宽松）
|
||
_tencent_limiter = RateLimiter(
|
||
min_interval=1.0,
|
||
jitter_min=0.5,
|
||
jitter_max=1.5
|
||
)
|
||
|
||
# Akshare 接口限流器
|
||
_akshare_limiter = RateLimiter(
|
||
min_interval=2.0,
|
||
jitter_min=1.5,
|
||
jitter_max=3.5
|
||
)
|
||
|
||
|
||
def get_eastmoney_limiter() -> RateLimiter:
|
||
"""获取东方财富限流器"""
|
||
return _eastmoney_limiter
|
||
|
||
|
||
def get_tencent_limiter() -> RateLimiter:
|
||
"""获取腾讯财经限流器"""
|
||
return _tencent_limiter
|
||
|
||
|
||
def get_akshare_limiter() -> RateLimiter:
|
||
"""获取 Akshare 限流器"""
|
||
return _akshare_limiter
|