TIANHE
1047 lines
35 KiB
Python
1047 lines
35 KiB
Python
"""
|
||
Search service v2.0 - 增强版搜索服务
|
||
整合多个搜索引擎，支持 API Key 轮换和故障转移
|
||
|
||
支持的搜索引擎（按优先级）：
|
||
1. Bocha (博查) - 搜索优化
|
||
2. Tavily - 专为AI设计，免费1000次/月
|
||
3. SerpAPI - Google/Bing 结果抓取
|
||
4. Google CSE - 自定义搜索引擎
|
||
5. Bing Search API
|
||
6. DuckDuckGo - 免费兜底
|
||
|
||
参考：daily_stock_analysis-main/src/search_service.py
|
||
"""
|
||
import requests
|
||
import json
|
||
import time
|
||
import re
|
||
from abc import ABC, abstractmethod
|
||
from dataclasses import dataclass, field
|
||
from datetime import datetime
|
||
from typing import List, Dict, Any, Optional
|
||
from itertools import cycle
|
||
from urllib.parse import urlparse
|
||
|
||
from app.utils.logger import get_logger
|
||
from app.utils.config_loader import load_addon_config
|
||
|
||
logger = get_logger(__name__)
|
||
|
||
# Track Google API quota status
|
||
_google_quota_exhausted = False
|
||
_google_quota_reset_time = 0
|
||
|
||
|
||
@dataclass
|
||
class SearchResult:
|
||
"""搜索结果数据类"""
|
||
title: str
|
||
snippet: str # 摘要
|
||
url: str
|
||
source: str # 来源网站
|
||
published_date: Optional[str] = None
|
||
sentiment: str = 'neutral' # 情绪标签
|
||
|
||
def to_text(self) -> str:
|
||
"""转换为文本格式"""
|
||
date_str = f" ({self.published_date})" if self.published_date else ""
|
||
return f"【{self.source}】{self.title}{date_str}\n{self.snippet}"
|
||
|
||
def to_dict(self) -> Dict[str, Any]:
|
||
"""转换为字典"""
|
||
return {
|
||
'title': self.title,
|
||
'link': self.url,
|
||
'snippet': self.snippet,
|
||
'source': self.source,
|
||
'published': self.published_date or '',
|
||
'sentiment': self.sentiment,
|
||
}
|
||
|
||
|
||
@dataclass
|
||
class SearchResponse:
|
||
"""搜索响应"""
|
||
query: str
|
||
results: List[SearchResult]
|
||
provider: str # 使用的搜索引擎
|
||
success: bool = True
|
||
error_message: Optional[str] = None
|
||
search_time: float = 0.0 # 搜索耗时（秒）
|
||
|
||
def to_context(self, max_results: int = 5) -> str:
|
||
"""将搜索结果转换为可用于 AI 分析的上下文"""
|
||
if not self.success or not self.results:
|
||
return f"搜索 '{self.query}' 未找到相关结果。"
|
||
|
||
lines = [f"【{self.query} 搜索结果】（来源：{self.provider}）"]
|
||
for i, result in enumerate(self.results[:max_results], 1):
|
||
lines.append(f"\n{i}. {result.to_text()}")
|
||
|
||
return "\n".join(lines)
|
||
|
||
def to_list(self) -> List[Dict[str, Any]]:
|
||
"""转换为列表格式（兼容旧接口）"""
|
||
return [r.to_dict() for r in self.results]
|
||
|
||
|
||
class BaseSearchProvider(ABC):
|
||
"""搜索引擎基类"""
|
||
|
||
def __init__(self, api_keys: List[str], name: str):
|
||
"""
|
||
初始化搜索引擎
|
||
|
||
Args:
|
||
api_keys: API Key 列表（支持多个 key 负载均衡）
|
||
name: 搜索引擎名称
|
||
"""
|
||
self._api_keys = api_keys
|
||
self._name = name
|
||
self._key_cycle = cycle(api_keys) if api_keys else None
|
||
self._key_usage: Dict[str, int] = {key: 0 for key in api_keys}
|
||
self._key_errors: Dict[str, int] = {key: 0 for key in api_keys}
|
||
|
||
@property
|
||
def name(self) -> str:
|
||
return self._name
|
||
|
||
@property
|
||
def is_available(self) -> bool:
|
||
"""检查是否有可用的 API Key"""
|
||
return bool(self._api_keys)
|
||
|
||
def _get_next_key(self) -> Optional[str]:
|
||
"""
|
||
获取下一个可用的 API Key（负载均衡）
|
||
|
||
策略：轮询 + 跳过错误过多的 key
|
||
"""
|
||
if not self._key_cycle:
|
||
return None
|
||
|
||
# 最多尝试所有 key
|
||
for _ in range(len(self._api_keys)):
|
||
key = next(self._key_cycle)
|
||
# 跳过错误次数过多的 key（超过 3 次）
|
||
if self._key_errors.get(key, 0) < 3:
|
||
return key
|
||
|
||
# 所有 key 都有问题，重置错误计数并返回第一个
|
||
logger.warning(f"[{self._name}] 所有 API Key 都有错误记录，重置错误计数")
|
||
self._key_errors = {key: 0 for key in self._api_keys}
|
||
return self._api_keys[0] if self._api_keys else None
|
||
|
||
def _record_success(self, key: str) -> None:
|
||
"""记录成功使用"""
|
||
self._key_usage[key] = self._key_usage.get(key, 0) + 1
|
||
# 成功后减少错误计数
|
||
if key in self._key_errors and self._key_errors[key] > 0:
|
||
self._key_errors[key] -= 1
|
||
|
||
def _record_error(self, key: str) -> None:
|
||
"""记录错误"""
|
||
self._key_errors[key] = self._key_errors.get(key, 0) + 1
|
||
logger.warning(f"[{self._name}] API Key {key[:8]}... 错误计数: {self._key_errors[key]}")
|
||
|
||
@abstractmethod
|
||
def _do_search(self, query: str, api_key: str, max_results: int, days: int = 7) -> SearchResponse:
|
||
"""执行搜索（子类实现）"""
|
||
pass
|
||
|
||
def search(self, query: str, max_results: int = 5, days: int = 7) -> SearchResponse:
|
||
"""
|
||
执行搜索
|
||
|
||
Args:
|
||
query: 搜索关键词
|
||
max_results: 最大返回结果数
|
||
days: 搜索最近几天的时间范围（默认7天）
|
||
|
||
Returns:
|
||
SearchResponse 对象
|
||
"""
|
||
api_key = self._get_next_key()
|
||
if not api_key:
|
||
return SearchResponse(
|
||
query=query,
|
||
results=[],
|
||
provider=self._name,
|
||
success=False,
|
||
error_message=f"{self._name} 未配置 API Key"
|
||
)
|
||
|
||
start_time = time.time()
|
||
try:
|
||
response = self._do_search(query, api_key, max_results, days=days)
|
||
response.search_time = time.time() - start_time
|
||
|
||
if response.success:
|
||
self._record_success(api_key)
|
||
logger.info(f"[{self._name}] 搜索 '{query}' 成功，返回 {len(response.results)} 条结果，耗时 {response.search_time:.2f}s")
|
||
else:
|
||
self._record_error(api_key)
|
||
|
||
return response
|
||
|
||
except Exception as e:
|
||
self._record_error(api_key)
|
||
elapsed = time.time() - start_time
|
||
logger.error(f"[{self._name}] 搜索 '{query}' 失败: {e}")
|
||
return SearchResponse(
|
||
query=query,
|
||
results=[],
|
||
provider=self._name,
|
||
success=False,
|
||
error_message=str(e),
|
||
search_time=elapsed
|
||
)
|
||
|
||
@staticmethod
|
||
def _extract_domain(url: str) -> str:
|
||
"""从 URL 提取域名作为来源"""
|
||
try:
|
||
parsed = urlparse(url)
|
||
domain = parsed.netloc.replace('www.', '')
|
||
return domain or '未知来源'
|
||
except:
|
||
return '未知来源'
|
||
|
||
|
||
class TavilySearchProvider(BaseSearchProvider):
|
||
"""
|
||
Tavily 搜索引擎
|
||
|
||
特点：
|
||
- 专为 AI/LLM 优化的搜索 API
|
||
- 免费版每月 1000 次请求
|
||
- 返回结构化的搜索结果
|
||
|
||
文档：https://docs.tavily.com/
|
||
"""
|
||
|
||
def __init__(self, api_keys: List[str]):
|
||
super().__init__(api_keys, "Tavily")
|
||
|
||
def _do_search(self, query: str, api_key: str, max_results: int, days: int = 7) -> SearchResponse:
|
||
"""执行 Tavily 搜索"""
|
||
try:
|
||
from tavily import TavilyClient
|
||
except ImportError:
|
||
# 如果未安装 tavily-python，使用 REST API
|
||
return self._do_search_rest(query, api_key, max_results, days)
|
||
|
||
try:
|
||
client = TavilyClient(api_key=api_key)
|
||
|
||
# 执行搜索
|
||
response = client.search(
|
||
query=query,
|
||
search_depth="advanced",
|
||
max_results=max_results,
|
||
include_answer=False,
|
||
include_raw_content=False,
|
||
days=days,
|
||
)
|
||
|
||
# 解析结果
|
||
results = []
|
||
for item in response.get('results', []):
|
||
results.append(SearchResult(
|
||
title=item.get('title', ''),
|
||
snippet=item.get('content', '')[:500],
|
||
url=item.get('url', ''),
|
||
source=self._extract_domain(item.get('url', '')),
|
||
published_date=item.get('published_date'),
|
||
))
|
||
|
||
return SearchResponse(
|
||
query=query,
|
||
results=results,
|
||
provider=self.name,
|
||
success=True,
|
||
)
|
||
|
||
except Exception as e:
|
||
error_msg = str(e)
|
||
if 'rate limit' in error_msg.lower() or 'quota' in error_msg.lower():
|
||
error_msg = f"API 配额已用尽: {error_msg}"
|
||
|
||
return SearchResponse(
|
||
query=query,
|
||
results=[],
|
||
provider=self.name,
|
||
success=False,
|
||
error_message=error_msg
|
||
)
|
||
|
||
def _do_search_rest(self, query: str, api_key: str, max_results: int, days: int = 7) -> SearchResponse:
|
||
"""使用 REST API 执行 Tavily 搜索（备选方案）"""
|
||
try:
|
||
url = "https://api.tavily.com/search"
|
||
headers = {
|
||
'Content-Type': 'application/json',
|
||
}
|
||
payload = {
|
||
'api_key': api_key,
|
||
'query': query,
|
||
'search_depth': 'advanced',
|
||
'max_results': max_results,
|
||
'include_answer': False,
|
||
'include_raw_content': False,
|
||
}
|
||
|
||
response = requests.post(url, headers=headers, json=payload, timeout=15)
|
||
|
||
if response.status_code != 200:
|
||
return SearchResponse(
|
||
query=query,
|
||
results=[],
|
||
provider=self.name,
|
||
success=False,
|
||
error_message=f"HTTP {response.status_code}: {response.text}"
|
||
)
|
||
|
||
data = response.json()
|
||
results = []
|
||
for item in data.get('results', []):
|
||
results.append(SearchResult(
|
||
title=item.get('title', ''),
|
||
snippet=item.get('content', '')[:500],
|
||
url=item.get('url', ''),
|
||
source=self._extract_domain(item.get('url', '')),
|
||
published_date=item.get('published_date'),
|
||
))
|
||
|
||
return SearchResponse(
|
||
query=query,
|
||
results=results,
|
||
provider=self.name,
|
||
success=True,
|
||
)
|
||
|
||
except Exception as e:
|
||
return SearchResponse(
|
||
query=query,
|
||
results=[],
|
||
provider=self.name,
|
||
success=False,
|
||
error_message=str(e)
|
||
)
|
||
|
||
|
||
class BochaSearchProvider(BaseSearchProvider):
|
||
"""
|
||
博查搜索引擎
|
||
|
||
特点：
|
||
- 专为AI优化的中文搜索API
|
||
- 结果准确、摘要完整
|
||
- 支持时间范围过滤和AI摘要
|
||
|
||
文档：https://bocha-ai.feishu.cn/wiki/RXEOw02rFiwzGSkd9mUcqoeAnNK
|
||
"""
|
||
|
||
def __init__(self, api_keys: List[str]):
|
||
super().__init__(api_keys, "Bocha")
|
||
|
||
def _do_search(self, query: str, api_key: str, max_results: int, days: int = 7) -> SearchResponse:
|
||
"""执行博查搜索"""
|
||
try:
|
||
url = "https://api.bochaai.com/v1/web-search"
|
||
|
||
headers = {
|
||
'Authorization': f'Bearer {api_key}',
|
||
'Content-Type': 'application/json'
|
||
}
|
||
|
||
# 确定时间范围
|
||
freshness = "oneWeek"
|
||
if days <= 1:
|
||
freshness = "oneDay"
|
||
elif days <= 7:
|
||
freshness = "oneWeek"
|
||
elif days <= 30:
|
||
freshness = "oneMonth"
|
||
else:
|
||
freshness = "oneYear"
|
||
|
||
payload = {
|
||
"query": query,
|
||
"freshness": freshness,
|
||
"summary": True,
|
||
"count": min(max_results, 50)
|
||
}
|
||
|
||
response = requests.post(url, headers=headers, json=payload, timeout=15)
|
||
|
||
if response.status_code != 200:
|
||
error_message = response.text
|
||
try:
|
||
if response.headers.get('content-type', '').startswith('application/json'):
|
||
error_data = response.json()
|
||
error_message = error_data.get('message', response.text)
|
||
except:
|
||
pass
|
||
|
||
if response.status_code == 403:
|
||
error_msg = f"余额不足: {error_message}"
|
||
elif response.status_code == 401:
|
||
error_msg = f"API KEY无效: {error_message}"
|
||
elif response.status_code == 429:
|
||
error_msg = f"请求频率达到限制: {error_message}"
|
||
else:
|
||
error_msg = f"HTTP {response.status_code}: {error_message}"
|
||
|
||
return SearchResponse(
|
||
query=query,
|
||
results=[],
|
||
provider=self.name,
|
||
success=False,
|
||
error_message=error_msg
|
||
)
|
||
|
||
data = response.json()
|
||
|
||
if data.get('code') != 200:
|
||
return SearchResponse(
|
||
query=query,
|
||
results=[],
|
||
provider=self.name,
|
||
success=False,
|
||
error_message=data.get('msg') or f"API返回错误码: {data.get('code')}"
|
||
)
|
||
|
||
results = []
|
||
web_pages = data.get('data', {}).get('webPages', {})
|
||
value_list = web_pages.get('value', [])
|
||
|
||
for item in value_list[:max_results]:
|
||
snippet = item.get('summary') or item.get('snippet', '')
|
||
if snippet:
|
||
snippet = snippet[:500]
|
||
|
||
results.append(SearchResult(
|
||
title=item.get('name', ''),
|
||
snippet=snippet,
|
||
url=item.get('url', ''),
|
||
source=item.get('siteName') or self._extract_domain(item.get('url', '')),
|
||
published_date=item.get('datePublished'),
|
||
))
|
||
|
||
return SearchResponse(
|
||
query=query,
|
||
results=results,
|
||
provider=self.name,
|
||
success=True,
|
||
)
|
||
|
||
except requests.exceptions.Timeout:
|
||
return SearchResponse(
|
||
query=query,
|
||
results=[],
|
||
provider=self.name,
|
||
success=False,
|
||
error_message="请求超时"
|
||
)
|
||
except Exception as e:
|
||
return SearchResponse(
|
||
query=query,
|
||
results=[],
|
||
provider=self.name,
|
||
success=False,
|
||
error_message=str(e)
|
||
)
|
||
|
||
|
||
class SerpAPISearchProvider(BaseSearchProvider):
|
||
"""
|
||
SerpAPI 搜索引擎
|
||
|
||
特点：
|
||
- 支持 Google、Bing、百度等多种搜索引擎
|
||
- 免费版每月 100 次请求
|
||
|
||
文档：https://serpapi.com/
|
||
"""
|
||
|
||
def __init__(self, api_keys: List[str]):
|
||
super().__init__(api_keys, "SerpAPI")
|
||
|
||
def _do_search(self, query: str, api_key: str, max_results: int, days: int = 7) -> SearchResponse:
|
||
"""执行 SerpAPI 搜索"""
|
||
try:
|
||
from serpapi import GoogleSearch
|
||
except ImportError:
|
||
return self._do_search_rest(query, api_key, max_results, days)
|
||
|
||
try:
|
||
tbs = "qdr:w"
|
||
if days <= 1:
|
||
tbs = "qdr:d"
|
||
elif days <= 7:
|
||
tbs = "qdr:w"
|
||
elif days <= 30:
|
||
tbs = "qdr:m"
|
||
else:
|
||
tbs = "qdr:y"
|
||
|
||
params = {
|
||
"engine": "google",
|
||
"q": query,
|
||
"api_key": api_key,
|
||
"google_domain": "google.com.hk",
|
||
"hl": "zh-cn",
|
||
"gl": "cn",
|
||
"tbs": tbs,
|
||
"num": max_results
|
||
}
|
||
|
||
search = GoogleSearch(params)
|
||
response = search.get_dict()
|
||
|
||
results = []
|
||
organic_results = response.get('organic_results', [])
|
||
|
||
for item in organic_results[:max_results]:
|
||
results.append(SearchResult(
|
||
title=item.get('title', ''),
|
||
snippet=item.get('snippet', '')[:500],
|
||
url=item.get('link', ''),
|
||
source=item.get('source', self._extract_domain(item.get('link', ''))),
|
||
published_date=item.get('date'),
|
||
))
|
||
|
||
return SearchResponse(
|
||
query=query,
|
||
results=results,
|
||
provider=self.name,
|
||
success=True,
|
||
)
|
||
|
||
except Exception as e:
|
||
return SearchResponse(
|
||
query=query,
|
||
results=[],
|
||
provider=self.name,
|
||
success=False,
|
||
error_message=str(e)
|
||
)
|
||
|
||
def _do_search_rest(self, query: str, api_key: str, max_results: int, days: int = 7) -> SearchResponse:
|
||
"""使用 REST API 执行 SerpAPI 搜索"""
|
||
try:
|
||
tbs = "qdr:w"
|
||
if days <= 1:
|
||
tbs = "qdr:d"
|
||
elif days <= 7:
|
||
tbs = "qdr:w"
|
||
elif days <= 30:
|
||
tbs = "qdr:m"
|
||
|
||
url = "https://serpapi.com/search"
|
||
params = {
|
||
"engine": "google",
|
||
"q": query,
|
||
"api_key": api_key,
|
||
"hl": "zh-cn",
|
||
"gl": "cn",
|
||
"tbs": tbs,
|
||
"num": max_results
|
||
}
|
||
|
||
response = requests.get(url, params=params, timeout=15)
|
||
|
||
if response.status_code != 200:
|
||
return SearchResponse(
|
||
query=query,
|
||
results=[],
|
||
provider=self.name,
|
||
success=False,
|
||
error_message=f"HTTP {response.status_code}"
|
||
)
|
||
|
||
data = response.json()
|
||
results = []
|
||
|
||
for item in data.get('organic_results', [])[:max_results]:
|
||
results.append(SearchResult(
|
||
title=item.get('title', ''),
|
||
snippet=item.get('snippet', '')[:500],
|
||
url=item.get('link', ''),
|
||
source=self._extract_domain(item.get('link', '')),
|
||
published_date=item.get('date'),
|
||
))
|
||
|
||
return SearchResponse(
|
||
query=query,
|
||
results=results,
|
||
provider=self.name,
|
||
success=True,
|
||
)
|
||
|
||
except Exception as e:
|
||
return SearchResponse(
|
||
query=query,
|
||
results=[],
|
||
provider=self.name,
|
||
success=False,
|
||
error_message=str(e)
|
||
)
|
||
|
||
|
||
class GoogleSearchProvider(BaseSearchProvider):
|
||
"""Google Custom Search (CSE) 搜索引擎"""
|
||
|
||
def __init__(self, api_key: str, cx: str):
|
||
super().__init__([api_key] if api_key else [], "Google")
|
||
self._cx = cx
|
||
|
||
def _do_search(self, query: str, api_key: str, max_results: int, days: int = 7) -> SearchResponse:
|
||
"""执行 Google 搜索"""
|
||
global _google_quota_exhausted, _google_quota_reset_time
|
||
|
||
if not self._cx:
|
||
return SearchResponse(
|
||
query=query,
|
||
results=[],
|
||
provider=self.name,
|
||
success=False,
|
||
error_message="Google Search 未配置 CX"
|
||
)
|
||
|
||
try:
|
||
url = "https://www.googleapis.com/customsearch/v1"
|
||
params = {
|
||
'key': api_key,
|
||
'cx': self._cx,
|
||
'q': query,
|
||
'num': min(max_results, 10),
|
||
}
|
||
|
||
# 添加时间限制
|
||
if days <= 1:
|
||
params['dateRestrict'] = 'd1'
|
||
elif days <= 7:
|
||
params['dateRestrict'] = 'w1'
|
||
elif days <= 30:
|
||
params['dateRestrict'] = 'm1'
|
||
|
||
response = requests.get(url, params=params, timeout=10)
|
||
|
||
if response.status_code == 429:
|
||
_google_quota_exhausted = True
|
||
import datetime
|
||
tomorrow = datetime.datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0) + datetime.timedelta(days=1)
|
||
_google_quota_reset_time = tomorrow.timestamp()
|
||
return SearchResponse(
|
||
query=query,
|
||
results=[],
|
||
provider=self.name,
|
||
success=False,
|
||
error_message="Google API 配额已用尽"
|
||
)
|
||
|
||
response.raise_for_status()
|
||
data = response.json()
|
||
|
||
results = []
|
||
if 'items' in data:
|
||
for item in data['items']:
|
||
results.append(SearchResult(
|
||
title=item.get('title', ''),
|
||
snippet=item.get('snippet', ''),
|
||
url=item.get('link', ''),
|
||
source='Google',
|
||
published_date=item.get('pagemap', {}).get('metatags', [{}])[0].get('article:published_time', ''),
|
||
))
|
||
|
||
return SearchResponse(
|
||
query=query,
|
||
results=results,
|
||
provider=self.name,
|
||
success=True,
|
||
)
|
||
|
||
except Exception as e:
|
||
return SearchResponse(
|
||
query=query,
|
||
results=[],
|
||
provider=self.name,
|
||
success=False,
|
||
error_message=str(e)
|
||
)
|
||
|
||
|
||
class BingSearchProvider(BaseSearchProvider):
|
||
"""Bing Search API 搜索引擎"""
|
||
|
||
def __init__(self, api_key: str):
|
||
super().__init__([api_key] if api_key else [], "Bing")
|
||
|
||
def _do_search(self, query: str, api_key: str, max_results: int, days: int = 7) -> SearchResponse:
|
||
"""执行 Bing 搜索"""
|
||
try:
|
||
url = "https://api.bing.microsoft.com/v7.0/search"
|
||
headers = {"Ocp-Apim-Subscription-Key": api_key}
|
||
params = {
|
||
"q": query,
|
||
"count": max_results,
|
||
"textDecorations": True,
|
||
"textFormat": "HTML"
|
||
}
|
||
|
||
response = requests.get(url, headers=headers, params=params, timeout=10)
|
||
response.raise_for_status()
|
||
data = response.json()
|
||
|
||
results = []
|
||
if 'webPages' in data and 'value' in data['webPages']:
|
||
for item in data['webPages']['value']:
|
||
results.append(SearchResult(
|
||
title=item.get('name', ''),
|
||
snippet=item.get('snippet', ''),
|
||
url=item.get('url', ''),
|
||
source='Bing',
|
||
published_date=item.get('datePublished', ''),
|
||
))
|
||
|
||
return SearchResponse(
|
||
query=query,
|
||
results=results,
|
||
provider=self.name,
|
||
success=True,
|
||
)
|
||
|
||
except Exception as e:
|
||
return SearchResponse(
|
||
query=query,
|
||
results=[],
|
||
provider=self.name,
|
||
success=False,
|
||
error_message=str(e)
|
||
)
|
||
|
||
|
||
class DuckDuckGoSearchProvider(BaseSearchProvider):
|
||
"""DuckDuckGo 搜索引擎（免费，无需 API Key）"""
|
||
|
||
def __init__(self):
|
||
super().__init__(['free'], "DuckDuckGo")
|
||
|
||
def _do_search(self, query: str, api_key: str, max_results: int, days: int = 7) -> SearchResponse:
|
||
"""执行 DuckDuckGo 搜索"""
|
||
try:
|
||
# 使用 DuckDuckGo Instant Answer API
|
||
url = "https://api.duckduckgo.com/"
|
||
params = {
|
||
'q': query,
|
||
'format': 'json',
|
||
'no_html': 1,
|
||
'skip_disambig': 1
|
||
}
|
||
|
||
response = requests.get(url, params=params, timeout=10)
|
||
response.raise_for_status()
|
||
data = response.json()
|
||
|
||
results = []
|
||
|
||
# 获取 RelatedTopics
|
||
related_topics = data.get('RelatedTopics', [])
|
||
for topic in related_topics[:max_results]:
|
||
if isinstance(topic, dict):
|
||
if 'FirstURL' in topic:
|
||
results.append(SearchResult(
|
||
title=topic.get('Text', '')[:100],
|
||
snippet=topic.get('Text', ''),
|
||
url=topic.get('FirstURL', ''),
|
||
source='DuckDuckGo',
|
||
))
|
||
elif 'Topics' in topic:
|
||
for sub_topic in topic['Topics']:
|
||
if len(results) >= max_results:
|
||
break
|
||
if 'FirstURL' in sub_topic:
|
||
results.append(SearchResult(
|
||
title=sub_topic.get('Text', '')[:100],
|
||
snippet=sub_topic.get('Text', ''),
|
||
url=sub_topic.get('FirstURL', ''),
|
||
source='DuckDuckGo',
|
||
))
|
||
|
||
# 检查 AbstractURL
|
||
if data.get('AbstractURL') and len(results) < max_results:
|
||
results.insert(0, SearchResult(
|
||
title=data.get('Heading', query),
|
||
snippet=data.get('AbstractText', ''),
|
||
url=data.get('AbstractURL', ''),
|
||
source='DuckDuckGo',
|
||
))
|
||
|
||
# 如果没有结果，尝试 HTML 版本
|
||
if not results:
|
||
results = self._search_html(query, max_results)
|
||
|
||
return SearchResponse(
|
||
query=query,
|
||
results=results[:max_results],
|
||
provider=self.name,
|
||
success=len(results) > 0,
|
||
)
|
||
|
||
except Exception as e:
|
||
return SearchResponse(
|
||
query=query,
|
||
results=[],
|
||
provider=self.name,
|
||
success=False,
|
||
error_message=str(e)
|
||
)
|
||
|
||
def _search_html(self, query: str, max_results: int) -> List[SearchResult]:
|
||
"""DuckDuckGo HTML 搜索备选"""
|
||
try:
|
||
url = "https://lite.duckduckgo.com/lite/"
|
||
headers = {
|
||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
|
||
}
|
||
data = {'q': query}
|
||
|
||
response = requests.post(url, headers=headers, data=data, timeout=10)
|
||
response.raise_for_status()
|
||
|
||
results = []
|
||
html = response.text
|
||
|
||
link_pattern = r'<a[^>]*class="result-link"[^>]*href="([^"]*)"[^>]*>([^<]*)</a>'
|
||
snippet_pattern = r'<td[^>]*class="result-snippet"[^>]*>([^<]*)</td>'
|
||
|
||
links = re.findall(link_pattern, html)
|
||
snippets = re.findall(snippet_pattern, html)
|
||
|
||
for i, (link, title) in enumerate(links[:max_results]):
|
||
snippet = snippets[i] if i < len(snippets) else ''
|
||
if link and title:
|
||
results.append(SearchResult(
|
||
title=title.strip(),
|
||
snippet=snippet.strip(),
|
||
url=link,
|
||
source='DuckDuckGo',
|
||
))
|
||
|
||
return results
|
||
|
||
except Exception as e:
|
||
logger.debug(f"DuckDuckGo HTML search failed: {e}")
|
||
return []
|
||
|
||
|
||
class SearchService:
|
||
"""
|
||
搜索服务
|
||
|
||
功能：
|
||
1. 管理多个搜索引擎
|
||
2. 自动故障转移
|
||
3. 结果聚合和格式化
|
||
"""
|
||
|
||
def __init__(self):
|
||
self._providers: List[BaseSearchProvider] = []
|
||
self._config = {}
|
||
self._load_config()
|
||
self._init_providers()
|
||
|
||
def _load_config(self):
|
||
"""加载配置"""
|
||
config = load_addon_config()
|
||
self._config = config.get('search', {})
|
||
self.provider = self._config.get('provider', 'google')
|
||
self.max_results = int(self._config.get('max_results', 10))
|
||
|
||
def _init_providers(self):
|
||
"""初始化搜索引擎（按优先级排序）"""
|
||
from app.config import APIKeys
|
||
|
||
# 1. Bocha 优先（国内搜索优化）
|
||
bocha_keys = APIKeys.BOCHA_API_KEYS
|
||
if bocha_keys:
|
||
self._providers.append(BochaSearchProvider(bocha_keys))
|
||
logger.info(f"已配置 Bocha 搜索，共 {len(bocha_keys)} 个 API Key")
|
||
|
||
# 2. Tavily（AI优化搜索）
|
||
tavily_keys = APIKeys.TAVILY_API_KEYS
|
||
if tavily_keys:
|
||
self._providers.append(TavilySearchProvider(tavily_keys))
|
||
logger.info(f"已配置 Tavily 搜索，共 {len(tavily_keys)} 个 API Key")
|
||
|
||
# 3. SerpAPI
|
||
serpapi_keys = APIKeys.SERPAPI_KEYS
|
||
if serpapi_keys:
|
||
self._providers.append(SerpAPISearchProvider(serpapi_keys))
|
||
logger.info(f"已配置 SerpAPI 搜索，共 {len(serpapi_keys)} 个 API Key")
|
||
|
||
# 4. Google CSE
|
||
google_api_key = self._config.get('google', {}).get('api_key')
|
||
google_cx = self._config.get('google', {}).get('cx')
|
||
if google_api_key and google_cx:
|
||
self._providers.append(GoogleSearchProvider(google_api_key, google_cx))
|
||
logger.info("已配置 Google CSE 搜索")
|
||
|
||
# 5. Bing
|
||
bing_api_key = self._config.get('bing', {}).get('api_key')
|
||
if bing_api_key:
|
||
self._providers.append(BingSearchProvider(bing_api_key))
|
||
logger.info("已配置 Bing 搜索")
|
||
|
||
# 6. DuckDuckGo（免费兜底）
|
||
self._providers.append(DuckDuckGoSearchProvider())
|
||
logger.info("已配置 DuckDuckGo 搜索（免费兜底）")
|
||
|
||
if len(self._providers) == 1:
|
||
logger.warning("仅有 DuckDuckGo 可用，建议配置更多搜索引擎 API Key")
|
||
|
||
@property
|
||
def is_available(self) -> bool:
|
||
"""检查是否有可用的搜索引擎"""
|
||
return any(p.is_available for p in self._providers)
|
||
|
||
def search(self, query: str, num_results: int = None, date_restrict: str = None, days: int = 7) -> List[Dict[str, Any]]:
|
||
"""
|
||
执行搜索（兼容旧接口）
|
||
|
||
Args:
|
||
query: 搜索关键词
|
||
num_results: 最大返回结果数
|
||
date_restrict: 时间限制（Google 格式，如 'd7'）
|
||
days: 搜索最近几天（优先级高于 date_restrict）
|
||
|
||
Returns:
|
||
搜索结果列表
|
||
"""
|
||
limit = num_results if num_results else self.max_results
|
||
|
||
# 解析 date_restrict 为 days
|
||
if date_restrict and not days:
|
||
if date_restrict.startswith('d'):
|
||
days = int(date_restrict[1:])
|
||
elif date_restrict.startswith('w'):
|
||
days = int(date_restrict[1:]) * 7
|
||
elif date_restrict.startswith('m'):
|
||
days = int(date_restrict[1:]) * 30
|
||
|
||
response = self.search_with_fallback(query, limit, days)
|
||
return response.to_list()
|
||
|
||
def search_with_fallback(self, query: str, max_results: int = 5, days: int = 7) -> SearchResponse:
|
||
"""
|
||
执行搜索（带自动故障转移）
|
||
|
||
Args:
|
||
query: 搜索关键词
|
||
max_results: 最大返回结果数
|
||
days: 搜索最近几天
|
||
|
||
Returns:
|
||
SearchResponse 对象
|
||
"""
|
||
# 依次尝试各个搜索引擎
|
||
for provider in self._providers:
|
||
if not provider.is_available:
|
||
continue
|
||
|
||
response = provider.search(query, max_results, days)
|
||
|
||
if response.success and response.results:
|
||
return response
|
||
else:
|
||
logger.warning(f"{provider.name} 搜索失败: {response.error_message}，尝试下一个引擎")
|
||
|
||
# 所有引擎都失败
|
||
return SearchResponse(
|
||
query=query,
|
||
results=[],
|
||
provider="None",
|
||
success=False,
|
||
error_message="所有搜索引擎都不可用或搜索失败"
|
||
)
|
||
|
||
def search_stock_news(
|
||
self,
|
||
stock_code: str,
|
||
stock_name: str,
|
||
market: str = "USStock",
|
||
max_results: int = 5
|
||
) -> SearchResponse:
|
||
"""
|
||
搜索股票相关新闻
|
||
|
||
Args:
|
||
stock_code: 股票代码
|
||
stock_name: 股票名称
|
||
market: 市场类型
|
||
max_results: 最大返回结果数
|
||
|
||
Returns:
|
||
SearchResponse 对象
|
||
"""
|
||
# 智能确定搜索时间范围
|
||
today_weekday = datetime.now().weekday()
|
||
if today_weekday == 0: # 周一
|
||
search_days = 3
|
||
elif today_weekday >= 5: # 周末
|
||
search_days = 2
|
||
else:
|
||
search_days = 1
|
||
|
||
# 根据市场类型构建搜索查询
|
||
if market == "USStock":
|
||
query = f"{stock_name} {stock_code} stock news latest"
|
||
elif market == "Crypto":
|
||
query = f"{stock_name} crypto news price analysis"
|
||
elif market == "Forex":
|
||
query = f"{stock_name} {stock_code} forex news analysis"
|
||
else:
|
||
query = f"{stock_name} {stock_code} latest news"
|
||
|
||
logger.info(f"搜索股票新闻: {stock_name}({stock_code}), market={market}, days={search_days}")
|
||
|
||
return self.search_with_fallback(query, max_results, search_days)
|
||
|
||
def search_stock_events(
|
||
self,
|
||
stock_code: str,
|
||
stock_name: str,
|
||
event_types: Optional[List[str]] = None
|
||
) -> SearchResponse:
|
||
"""
|
||
搜索股票特定事件（年报预告、减持等）
|
||
"""
|
||
if event_types is None:
|
||
event_types = ["年报预告", "减持公告", "业绩快报"]
|
||
|
||
event_query = " OR ".join(event_types)
|
||
query = f"{stock_name} ({event_query})"
|
||
|
||
return self.search_with_fallback(query, max_results=5, days=30)
|
||
|
||
|
||
# 单例实例
|
||
_search_service: Optional[SearchService] = None
|
||
|
||
|
||
def get_search_service() -> SearchService:
|
||
"""获取搜索服务单例"""
|
||
global _search_service
|
||
if _search_service is None:
|
||
_search_service = SearchService()
|
||
return _search_service
|
||
|
||
|
||
def reset_search_service() -> None:
|
||
"""重置搜索服务（用于测试或配置更新后）"""
|
||
global _search_service
|
||
_search_service = None
|