Refactor and translate comments and docstrings in utility modules to English for better clarity and maintainability. Update Gunicorn and application startup messages for consistency in language. Enhance documentation with English translations for better accessibility.
This commit is contained in:
dienakdz
@@ -1,15 +1,15 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
===================================
|
||||
防封禁工具模块 (Rate Limiter)
|
||||
Anti-ban tool module (Rate Limiter)
|
||||
===================================
|
||||
|
||||
参考 daily_stock_analysis 项目实现
|
||||
提供反爬虫策略：
|
||||
1. 随机休眠（Jitter）
|
||||
2. 随机 User-Agent 轮换
|
||||
3. 指数退避重试
|
||||
4. 请求频率限制
|
||||
Refer to daily_stock_analysis project implementation
|
||||
Provide anti-crawler strategies:
|
||||
1. Random sleep (Jitter)
|
||||
2. Random User-Agent rotation
|
||||
3. Exponential backoff retry
|
||||
4. Request frequency limit
|
||||
"""
|
||||
|
||||
import time
|
||||
@@ -22,7 +22,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ============================================
|
||||
# User-Agent 池
|
||||
# User-Agent Pond
|
||||
# ============================================
|
||||
|
||||
USER_AGENTS = [
|
||||
@@ -48,19 +48,19 @@ USER_AGENTS = [
|
||||
|
||||
|
||||
def get_random_user_agent() -> str:
|
||||
"""获取随机 User-Agent"""
|
||||
"""Get a random User-Agent"""
|
||||
return random.choice(USER_AGENTS)
|
||||
|
||||
|
||||
def get_request_headers(referer: Optional[str] = None) -> dict:
|
||||
"""
|
||||
获取带有随机 User-Agent 的请求头
|
||||
Get request header with random User-Agent
|
||||
|
||||
Args:
|
||||
referer: 可选的 Referer 头
|
||||
referer: optional Referer header
|
||||
|
||||
Returns:
|
||||
请求头字典
|
||||
Request header dictionary
|
||||
"""
|
||||
headers = {
|
||||
'User-Agent': get_random_user_agent(),
|
||||
@@ -77,7 +77,7 @@ def get_request_headers(referer: Optional[str] = None) -> dict:
|
||||
|
||||
|
||||
# ============================================
|
||||
# 随机休眠
|
||||
# Random sleep
|
||||
# ============================================
|
||||
|
||||
def random_sleep(
|
||||
@@ -86,31 +86,31 @@ def random_sleep(
|
||||
log: bool = False
|
||||
) -> None:
|
||||
"""
|
||||
随机休眠（Jitter）
|
||||
Random sleep (Jitter)
|
||||
|
||||
防封禁策略：模拟人类行为的随机延迟
|
||||
在请求之间加入不规则的等待时间
|
||||
Anti-ban strategy: simulate random delays in human behavior
|
||||
Incorporate irregular wait times between requests
|
||||
|
||||
Args:
|
||||
min_seconds: 最小休眠时间（秒）
|
||||
max_seconds: 最大休眠时间（秒）
|
||||
log: 是否记录日志
|
||||
min_seconds: Minimum sleep time (seconds)
|
||||
max_seconds: Maximum sleep time (seconds)
|
||||
log: whether to log
|
||||
"""
|
||||
sleep_time = random.uniform(min_seconds, max_seconds)
|
||||
if log:
|
||||
logger.debug(f"随机休眠 {sleep_time:.2f} 秒...")
|
||||
logger.debug(f"Random hibernation {sleep_time:.2f} seconds...")
|
||||
time.sleep(sleep_time)
|
||||
|
||||
|
||||
# ============================================
|
||||
# 请求频率限制器
|
||||
# Request frequency limiter
|
||||
# ============================================
|
||||
|
||||
class RateLimiter:
|
||||
"""
|
||||
请求频率限制器
|
||||
Request frequency limiter
|
||||
|
||||
确保请求之间有最小间隔时间
|
||||
Ensure there is a minimum amount of time between requests
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -120,12 +120,12 @@ class RateLimiter:
|
||||
jitter_max: float = 1.5
|
||||
):
|
||||
"""
|
||||
初始化频率限制器
|
||||
Initialize frequency limiter
|
||||
|
||||
Args:
|
||||
min_interval: 最小请求间隔（秒）
|
||||
jitter_min: 随机抖动最小值（秒）
|
||||
jitter_max: 随机抖动最大值（秒）
|
||||
min_interval: Minimum request interval (seconds)
|
||||
jitter_min: minimum random jitter (seconds)
|
||||
jitter_max: maximum random jitter (seconds)
|
||||
"""
|
||||
self.min_interval = min_interval
|
||||
self.jitter_min = jitter_min
|
||||
@@ -134,37 +134,37 @@ class RateLimiter:
|
||||
|
||||
def wait(self) -> float:
|
||||
"""
|
||||
等待直到可以发起下一次请求
|
||||
Wait until the next request can be made
|
||||
|
||||
Returns:
|
||||
实际等待的时间（秒）
|
||||
Actual waiting time (seconds)
|
||||
"""
|
||||
wait_time = 0.0
|
||||
|
||||
if self._last_request_time is not None:
|
||||
elapsed = time.time() - self._last_request_time
|
||||
if elapsed < self.min_interval:
|
||||
# 补充休眠到最小间隔
|
||||
# Supplement sleep to minimum interval
|
||||
wait_time = self.min_interval - elapsed
|
||||
time.sleep(wait_time)
|
||||
|
||||
# 添加随机抖动
|
||||
# Add random jitter
|
||||
jitter = random.uniform(self.jitter_min, self.jitter_max)
|
||||
time.sleep(jitter)
|
||||
wait_time += jitter
|
||||
|
||||
# 记录本次请求时间
|
||||
# Record the time of this request
|
||||
self._last_request_time = time.time()
|
||||
|
||||
return wait_time
|
||||
|
||||
def reset(self) -> None:
|
||||
"""重置限制器"""
|
||||
"""reset limiter"""
|
||||
self._last_request_time = None
|
||||
|
||||
|
||||
# ============================================
|
||||
# 指数退避重试装饰器
|
||||
# Exponential backoff retry decorator
|
||||
# ============================================
|
||||
|
||||
def retry_with_backoff(
|
||||
@@ -176,17 +176,17 @@ def retry_with_backoff(
|
||||
on_retry: Optional[Callable[[int, Exception], None]] = None
|
||||
):
|
||||
"""
|
||||
指数退避重试装饰器
|
||||
Exponential backoff retry decorator
|
||||
|
||||
Args:
|
||||
max_attempts: 最大重试次数
|
||||
base_delay: 基础延迟时间（秒）
|
||||
max_delay: 最大延迟时间（秒）
|
||||
exponential_base: 指数基数
|
||||
exceptions: 需要重试的异常类型
|
||||
on_retry: 重试时的回调函数
|
||||
max_attempts: Maximum number of retries
|
||||
base_delay: base delay time (seconds)
|
||||
max_delay: maximum delay time (seconds)
|
||||
exponential_base: exponential base
|
||||
exceptions: Exception types that need to be retried
|
||||
on_retry: callback function when retrying
|
||||
|
||||
使用示例:
|
||||
Usage example:
|
||||
@retry_with_backoff(max_attempts=3, exceptions=(ConnectionError, TimeoutError))
|
||||
def fetch_data():
|
||||
...
|
||||
@@ -203,20 +203,20 @@ def retry_with_backoff(
|
||||
last_exception = e
|
||||
|
||||
if attempt == max_attempts:
|
||||
logger.error(f"[重试] {func.__name__} 已达最大重试次数 ({max_attempts})，放弃")
|
||||
logger.error(f"[Retry] {func.__name__} has reached the maximum number of retries ({max_attempts}), giving up")
|
||||
raise
|
||||
|
||||
# 计算退避延迟: base_delay * (exponential_base ^ (attempt - 1))
|
||||
# Calculate the backoff delay: base_delay * (exponential_base ^ (attempt - 1))
|
||||
delay = min(
|
||||
base_delay * (exponential_base ** (attempt - 1)),
|
||||
max_delay
|
||||
)
|
||||
# 添加随机抖动 (±20%)
|
||||
# Add random jitter (±20%)
|
||||
delay *= random.uniform(0.8, 1.2)
|
||||
|
||||
logger.warning(
|
||||
f"[重试] {func.__name__} 第 {attempt}/{max_attempts} 次失败: {e}, "
|
||||
f"等待 {delay:.1f}s 后重试..."
|
||||
f"[Retry] {func.__name__} failed for the {attempt}/{max_attempts} time: {e}, "
|
||||
f"waiting {delay:.1f}s before retrying..."
|
||||
)
|
||||
|
||||
if on_retry:
|
||||
@@ -224,7 +224,7 @@ def retry_with_backoff(
|
||||
|
||||
time.sleep(delay)
|
||||
|
||||
# 不应该到达这里
|
||||
# Shouldn't have gotten here
|
||||
raise last_exception
|
||||
|
||||
return wrapper
|
||||
@@ -232,24 +232,24 @@ def retry_with_backoff(
|
||||
|
||||
|
||||
# ============================================
|
||||
# 全局限流器实例
|
||||
# Global current limiter example
|
||||
# ============================================
|
||||
|
||||
# 东方财富接口限流器（较严格）
|
||||
# Oriental Fortune interface current limiter (more stringent)
|
||||
_eastmoney_limiter = RateLimiter(
|
||||
min_interval=2.0,
|
||||
jitter_min=1.0,
|
||||
jitter_max=3.0
|
||||
)
|
||||
|
||||
# 腾讯财经接口限流器（较宽松）
|
||||
# Tencent Finance interface current limiter (relatively loose)
|
||||
_tencent_limiter = RateLimiter(
|
||||
min_interval=1.0,
|
||||
jitter_min=0.5,
|
||||
jitter_max=1.5
|
||||
)
|
||||
|
||||
# Akshare 接口限流器
|
||||
# Akshare interface current limiter
|
||||
_akshare_limiter = RateLimiter(
|
||||
min_interval=2.0,
|
||||
jitter_min=1.5,
|
||||
@@ -258,15 +258,15 @@ _akshare_limiter = RateLimiter(
|
||||
|
||||
|
||||
def get_eastmoney_limiter() -> RateLimiter:
|
||||
"""获取东方财富限流器"""
|
||||
"""Get Oriental Wealth Current Limiter"""
|
||||
return _eastmoney_limiter
|
||||
|
||||
|
||||
def get_tencent_limiter() -> RateLimiter:
|
||||
"""获取腾讯财经限流器"""
|
||||
"""Get Tencent Finance current limiter"""
|
||||
return _tencent_limiter
|
||||
|
||||
|
||||
def get_akshare_limiter() -> RateLimiter:
|
||||
"""获取 Akshare 限流器"""
|
||||
"""Get Akshare Throttler"""
|
||||
return _akshare_limiter
|
||||
|
||||
Reference in New Issue
Block a user