""" Search service v2.0 - 增强版搜索服务 整合多个搜索引擎,支持 API Key 轮换和故障转移 支持的搜索引擎(按优先级): 1. Bocha (博查) - 搜索优化 2. Tavily - 专为AI设计,免费1000次/月 3. SerpAPI - Google/Bing 结果抓取 4. Google CSE - 自定义搜索引擎 5. Bing Search API 6. DuckDuckGo - 免费兜底 参考:daily_stock_analysis-main/src/search_service.py """ import requests import json import time import re from abc import ABC, abstractmethod from dataclasses import dataclass, field from datetime import datetime from typing import List, Dict, Any, Optional from itertools import cycle from urllib.parse import urlparse from app.utils.logger import get_logger from app.utils.config_loader import load_addon_config logger = get_logger(__name__) # Track Google API quota status _google_quota_exhausted = False _google_quota_reset_time = 0 @dataclass class SearchResult: """搜索结果数据类""" title: str snippet: str # 摘要 url: str source: str # 来源网站 published_date: Optional[str] = None sentiment: str = 'neutral' # 情绪标签 def to_text(self) -> str: """转换为文本格式""" date_str = f" ({self.published_date})" if self.published_date else "" return f"【{self.source}】{self.title}{date_str}\n{self.snippet}" def to_dict(self) -> Dict[str, Any]: """转换为字典""" return { 'title': self.title, 'link': self.url, 'snippet': self.snippet, 'source': self.source, 'published': self.published_date or '', 'sentiment': self.sentiment, } @dataclass class SearchResponse: """搜索响应""" query: str results: List[SearchResult] provider: str # 使用的搜索引擎 success: bool = True error_message: Optional[str] = None search_time: float = 0.0 # 搜索耗时(秒) def to_context(self, max_results: int = 5) -> str: """将搜索结果转换为可用于 AI 分析的上下文""" if not self.success or not self.results: return f"搜索 '{self.query}' 未找到相关结果。" lines = [f"【{self.query} 搜索结果】(来源:{self.provider})"] for i, result in enumerate(self.results[:max_results], 1): lines.append(f"\n{i}. {result.to_text()}") return "\n".join(lines) def to_list(self) -> List[Dict[str, Any]]: """转换为列表格式(兼容旧接口)""" return [r.to_dict() for r in self.results] class BaseSearchProvider(ABC): """搜索引擎基类""" def __init__(self, api_keys: List[str], name: str): """ 初始化搜索引擎 Args: api_keys: API Key 列表(支持多个 key 负载均衡) name: 搜索引擎名称 """ self._api_keys = api_keys self._name = name self._key_cycle = cycle(api_keys) if api_keys else None self._key_usage: Dict[str, int] = {key: 0 for key in api_keys} self._key_errors: Dict[str, int] = {key: 0 for key in api_keys} @property def name(self) -> str: return self._name @property def is_available(self) -> bool: """检查是否有可用的 API Key""" return bool(self._api_keys) def _get_next_key(self) -> Optional[str]: """ 获取下一个可用的 API Key(负载均衡) 策略:轮询 + 跳过错误过多的 key """ if not self._key_cycle: return None # 最多尝试所有 key for _ in range(len(self._api_keys)): key = next(self._key_cycle) # 跳过错误次数过多的 key(超过 3 次) if self._key_errors.get(key, 0) < 3: return key # 所有 key 都有问题,重置错误计数并返回第一个 logger.warning(f"[{self._name}] 所有 API Key 都有错误记录,重置错误计数") self._key_errors = {key: 0 for key in self._api_keys} return self._api_keys[0] if self._api_keys else None def _record_success(self, key: str) -> None: """记录成功使用""" self._key_usage[key] = self._key_usage.get(key, 0) + 1 # 成功后减少错误计数 if key in self._key_errors and self._key_errors[key] > 0: self._key_errors[key] -= 1 def _record_error(self, key: str) -> None: """记录错误""" self._key_errors[key] = self._key_errors.get(key, 0) + 1 logger.warning(f"[{self._name}] API Key {key[:8]}... 错误计数: {self._key_errors[key]}") @abstractmethod def _do_search(self, query: str, api_key: str, max_results: int, days: int = 7) -> SearchResponse: """执行搜索(子类实现)""" pass def search(self, query: str, max_results: int = 5, days: int = 7) -> SearchResponse: """ 执行搜索 Args: query: 搜索关键词 max_results: 最大返回结果数 days: 搜索最近几天的时间范围(默认7天) Returns: SearchResponse 对象 """ api_key = self._get_next_key() if not api_key: return SearchResponse( query=query, results=[], provider=self._name, success=False, error_message=f"{self._name} 未配置 API Key" ) start_time = time.time() try: response = self._do_search(query, api_key, max_results, days=days) response.search_time = time.time() - start_time if response.success: self._record_success(api_key) logger.info(f"[{self._name}] 搜索 '{query}' 成功,返回 {len(response.results)} 条结果,耗时 {response.search_time:.2f}s") else: self._record_error(api_key) return response except Exception as e: self._record_error(api_key) elapsed = time.time() - start_time logger.error(f"[{self._name}] 搜索 '{query}' 失败: {e}") return SearchResponse( query=query, results=[], provider=self._name, success=False, error_message=str(e), search_time=elapsed ) @staticmethod def _extract_domain(url: str) -> str: """从 URL 提取域名作为来源""" try: parsed = urlparse(url) domain = parsed.netloc.replace('www.', '') return domain or '未知来源' except: return '未知来源' class TavilySearchProvider(BaseSearchProvider): """ Tavily 搜索引擎 特点: - 专为 AI/LLM 优化的搜索 API - 免费版每月 1000 次请求 - 返回结构化的搜索结果 文档:https://docs.tavily.com/ """ def __init__(self, api_keys: List[str]): super().__init__(api_keys, "Tavily") def _do_search(self, query: str, api_key: str, max_results: int, days: int = 7) -> SearchResponse: """执行 Tavily 搜索""" try: from tavily import TavilyClient except ImportError: # 如果未安装 tavily-python,使用 REST API return self._do_search_rest(query, api_key, max_results, days) try: client = TavilyClient(api_key=api_key) # 执行搜索 response = client.search( query=query, search_depth="advanced", max_results=max_results, include_answer=False, include_raw_content=False, days=days, ) # 解析结果 results = [] for item in response.get('results', []): results.append(SearchResult( title=item.get('title', ''), snippet=item.get('content', '')[:500], url=item.get('url', ''), source=self._extract_domain(item.get('url', '')), published_date=item.get('published_date'), )) return SearchResponse( query=query, results=results, provider=self.name, success=True, ) except Exception as e: error_msg = str(e) if 'rate limit' in error_msg.lower() or 'quota' in error_msg.lower(): error_msg = f"API 配额已用尽: {error_msg}" return SearchResponse( query=query, results=[], provider=self.name, success=False, error_message=error_msg ) def _do_search_rest(self, query: str, api_key: str, max_results: int, days: int = 7) -> SearchResponse: """使用 REST API 执行 Tavily 搜索(备选方案)""" try: url = "https://api.tavily.com/search" headers = { 'Content-Type': 'application/json', } payload = { 'api_key': api_key, 'query': query, 'search_depth': 'advanced', 'max_results': max_results, 'include_answer': False, 'include_raw_content': False, } response = requests.post(url, headers=headers, json=payload, timeout=15) if response.status_code != 200: return SearchResponse( query=query, results=[], provider=self.name, success=False, error_message=f"HTTP {response.status_code}: {response.text}" ) data = response.json() results = [] for item in data.get('results', []): results.append(SearchResult( title=item.get('title', ''), snippet=item.get('content', '')[:500], url=item.get('url', ''), source=self._extract_domain(item.get('url', '')), published_date=item.get('published_date'), )) return SearchResponse( query=query, results=results, provider=self.name, success=True, ) except Exception as e: return SearchResponse( query=query, results=[], provider=self.name, success=False, error_message=str(e) ) class BochaSearchProvider(BaseSearchProvider): """ 博查搜索引擎 特点: - 专为AI优化的中文搜索API - 结果准确、摘要完整 - 支持时间范围过滤和AI摘要 文档:https://bocha-ai.feishu.cn/wiki/RXEOw02rFiwzGSkd9mUcqoeAnNK """ def __init__(self, api_keys: List[str]): super().__init__(api_keys, "Bocha") def _do_search(self, query: str, api_key: str, max_results: int, days: int = 7) -> SearchResponse: """执行博查搜索""" try: url = "https://api.bochaai.com/v1/web-search" headers = { 'Authorization': f'Bearer {api_key}', 'Content-Type': 'application/json' } # 确定时间范围 freshness = "oneWeek" if days <= 1: freshness = "oneDay" elif days <= 7: freshness = "oneWeek" elif days <= 30: freshness = "oneMonth" else: freshness = "oneYear" payload = { "query": query, "freshness": freshness, "summary": True, "count": min(max_results, 50) } response = requests.post(url, headers=headers, json=payload, timeout=15) if response.status_code != 200: error_message = response.text try: if response.headers.get('content-type', '').startswith('application/json'): error_data = response.json() error_message = error_data.get('message', response.text) except: pass if response.status_code == 403: error_msg = f"余额不足: {error_message}" elif response.status_code == 401: error_msg = f"API KEY无效: {error_message}" elif response.status_code == 429: error_msg = f"请求频率达到限制: {error_message}" else: error_msg = f"HTTP {response.status_code}: {error_message}" return SearchResponse( query=query, results=[], provider=self.name, success=False, error_message=error_msg ) data = response.json() if data.get('code') != 200: return SearchResponse( query=query, results=[], provider=self.name, success=False, error_message=data.get('msg') or f"API返回错误码: {data.get('code')}" ) results = [] web_pages = data.get('data', {}).get('webPages', {}) value_list = web_pages.get('value', []) for item in value_list[:max_results]: snippet = item.get('summary') or item.get('snippet', '') if snippet: snippet = snippet[:500] results.append(SearchResult( title=item.get('name', ''), snippet=snippet, url=item.get('url', ''), source=item.get('siteName') or self._extract_domain(item.get('url', '')), published_date=item.get('datePublished'), )) return SearchResponse( query=query, results=results, provider=self.name, success=True, ) except requests.exceptions.Timeout: return SearchResponse( query=query, results=[], provider=self.name, success=False, error_message="请求超时" ) except Exception as e: return SearchResponse( query=query, results=[], provider=self.name, success=False, error_message=str(e) ) class SerpAPISearchProvider(BaseSearchProvider): """ SerpAPI 搜索引擎 特点: - 支持 Google、Bing、百度等多种搜索引擎 - 免费版每月 100 次请求 文档:https://serpapi.com/ """ def __init__(self, api_keys: List[str]): super().__init__(api_keys, "SerpAPI") def _do_search(self, query: str, api_key: str, max_results: int, days: int = 7) -> SearchResponse: """执行 SerpAPI 搜索""" try: from serpapi import GoogleSearch except ImportError: return self._do_search_rest(query, api_key, max_results, days) try: tbs = "qdr:w" if days <= 1: tbs = "qdr:d" elif days <= 7: tbs = "qdr:w" elif days <= 30: tbs = "qdr:m" else: tbs = "qdr:y" params = { "engine": "google", "q": query, "api_key": api_key, "google_domain": "google.com.hk", "hl": "zh-cn", "gl": "cn", "tbs": tbs, "num": max_results } search = GoogleSearch(params) response = search.get_dict() results = [] organic_results = response.get('organic_results', []) for item in organic_results[:max_results]: results.append(SearchResult( title=item.get('title', ''), snippet=item.get('snippet', '')[:500], url=item.get('link', ''), source=item.get('source', self._extract_domain(item.get('link', ''))), published_date=item.get('date'), )) return SearchResponse( query=query, results=results, provider=self.name, success=True, ) except Exception as e: return SearchResponse( query=query, results=[], provider=self.name, success=False, error_message=str(e) ) def _do_search_rest(self, query: str, api_key: str, max_results: int, days: int = 7) -> SearchResponse: """使用 REST API 执行 SerpAPI 搜索""" try: tbs = "qdr:w" if days <= 1: tbs = "qdr:d" elif days <= 7: tbs = "qdr:w" elif days <= 30: tbs = "qdr:m" url = "https://serpapi.com/search" params = { "engine": "google", "q": query, "api_key": api_key, "hl": "zh-cn", "gl": "cn", "tbs": tbs, "num": max_results } response = requests.get(url, params=params, timeout=15) if response.status_code != 200: return SearchResponse( query=query, results=[], provider=self.name, success=False, error_message=f"HTTP {response.status_code}" ) data = response.json() results = [] for item in data.get('organic_results', [])[:max_results]: results.append(SearchResult( title=item.get('title', ''), snippet=item.get('snippet', '')[:500], url=item.get('link', ''), source=self._extract_domain(item.get('link', '')), published_date=item.get('date'), )) return SearchResponse( query=query, results=results, provider=self.name, success=True, ) except Exception as e: return SearchResponse( query=query, results=[], provider=self.name, success=False, error_message=str(e) ) class GoogleSearchProvider(BaseSearchProvider): """Google Custom Search (CSE) 搜索引擎""" def __init__(self, api_key: str, cx: str): super().__init__([api_key] if api_key else [], "Google") self._cx = cx def _do_search(self, query: str, api_key: str, max_results: int, days: int = 7) -> SearchResponse: """执行 Google 搜索""" global _google_quota_exhausted, _google_quota_reset_time if not self._cx: return SearchResponse( query=query, results=[], provider=self.name, success=False, error_message="Google Search 未配置 CX" ) try: url = "https://www.googleapis.com/customsearch/v1" params = { 'key': api_key, 'cx': self._cx, 'q': query, 'num': min(max_results, 10), } # 添加时间限制 if days <= 1: params['dateRestrict'] = 'd1' elif days <= 7: params['dateRestrict'] = 'w1' elif days <= 30: params['dateRestrict'] = 'm1' response = requests.get(url, params=params, timeout=10) if response.status_code == 429: _google_quota_exhausted = True import datetime tomorrow = datetime.datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0) + datetime.timedelta(days=1) _google_quota_reset_time = tomorrow.timestamp() return SearchResponse( query=query, results=[], provider=self.name, success=False, error_message="Google API 配额已用尽" ) response.raise_for_status() data = response.json() results = [] if 'items' in data: for item in data['items']: results.append(SearchResult( title=item.get('title', ''), snippet=item.get('snippet', ''), url=item.get('link', ''), source='Google', published_date=item.get('pagemap', {}).get('metatags', [{}])[0].get('article:published_time', ''), )) return SearchResponse( query=query, results=results, provider=self.name, success=True, ) except Exception as e: return SearchResponse( query=query, results=[], provider=self.name, success=False, error_message=str(e) ) class BingSearchProvider(BaseSearchProvider): """Bing Search API 搜索引擎""" def __init__(self, api_key: str): super().__init__([api_key] if api_key else [], "Bing") def _do_search(self, query: str, api_key: str, max_results: int, days: int = 7) -> SearchResponse: """执行 Bing 搜索""" try: url = "https://api.bing.microsoft.com/v7.0/search" headers = {"Ocp-Apim-Subscription-Key": api_key} params = { "q": query, "count": max_results, "textDecorations": True, "textFormat": "HTML" } response = requests.get(url, headers=headers, params=params, timeout=10) response.raise_for_status() data = response.json() results = [] if 'webPages' in data and 'value' in data['webPages']: for item in data['webPages']['value']: results.append(SearchResult( title=item.get('name', ''), snippet=item.get('snippet', ''), url=item.get('url', ''), source='Bing', published_date=item.get('datePublished', ''), )) return SearchResponse( query=query, results=results, provider=self.name, success=True, ) except Exception as e: return SearchResponse( query=query, results=[], provider=self.name, success=False, error_message=str(e) ) class DuckDuckGoSearchProvider(BaseSearchProvider): """DuckDuckGo 搜索引擎(免费,无需 API Key)""" def __init__(self): super().__init__(['free'], "DuckDuckGo") def _do_search(self, query: str, api_key: str, max_results: int, days: int = 7) -> SearchResponse: """执行 DuckDuckGo 搜索""" try: # 使用 DuckDuckGo Instant Answer API url = "https://api.duckduckgo.com/" params = { 'q': query, 'format': 'json', 'no_html': 1, 'skip_disambig': 1 } response = requests.get(url, params=params, timeout=10) response.raise_for_status() data = response.json() results = [] # 获取 RelatedTopics related_topics = data.get('RelatedTopics', []) for topic in related_topics[:max_results]: if isinstance(topic, dict): if 'FirstURL' in topic: results.append(SearchResult( title=topic.get('Text', '')[:100], snippet=topic.get('Text', ''), url=topic.get('FirstURL', ''), source='DuckDuckGo', )) elif 'Topics' in topic: for sub_topic in topic['Topics']: if len(results) >= max_results: break if 'FirstURL' in sub_topic: results.append(SearchResult( title=sub_topic.get('Text', '')[:100], snippet=sub_topic.get('Text', ''), url=sub_topic.get('FirstURL', ''), source='DuckDuckGo', )) # 检查 AbstractURL if data.get('AbstractURL') and len(results) < max_results: results.insert(0, SearchResult( title=data.get('Heading', query), snippet=data.get('AbstractText', ''), url=data.get('AbstractURL', ''), source='DuckDuckGo', )) # 如果没有结果,尝试 HTML 版本 if not results: results = self._search_html(query, max_results) return SearchResponse( query=query, results=results[:max_results], provider=self.name, success=len(results) > 0, ) except Exception as e: return SearchResponse( query=query, results=[], provider=self.name, success=False, error_message=str(e) ) def _search_html(self, query: str, max_results: int) -> List[SearchResult]: """DuckDuckGo HTML 搜索备选""" try: url = "https://lite.duckduckgo.com/lite/" headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36' } data = {'q': query} response = requests.post(url, headers=headers, data=data, timeout=10) response.raise_for_status() results = [] html = response.text link_pattern = r']*class="result-link"[^>]*href="([^"]*)"[^>]*>([^<]*)' snippet_pattern = r'