gavindiaz/PolyWeather
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(ci): format code and fix ruff lintings issues

Browse Source
This commit is contained in:
2569718930@qq.com
2026-03-03 00:12:16 +08:00
parent e423eaeb77
commit c8a7fe8548
8 changed files with 872 additions and 469 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/analysis/ai_analyzer.py
+17 -15
View File
@@ -9,6 +9,7 @@ MODELS = [
"llama-3.1-8b-instant",
]
def get_ai_analysis(weather_insights: str, city_name: str, temp_symbol: str) -> str:
"""
通过 Groq API (LLaMA 3.3 70B) 对天气态势进行极速交易分析
@@ -18,13 +19,10 @@ def get_ai_analysis(weather_insights: str, city_name: str, temp_symbol: str) ->
if not api_key:
logger.warning("GROQ_API_KEY 未配置，跳过 AI 分析")
return ""
url = "https://api.groq.com/openai/v1/chat/completions"
headers = {
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json"
}
headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
prompt = f"""
你是一个专业的天气衍生品（如 Polymarket）交易员。你的任务是分析当前天气特征，判断今日实测最高温是否能达到或超过预报中的【最高值】。
@@ -69,23 +67,26 @@ P4 **预报背景**（最低优先级）：
payload = {
"model": model,
"messages": [
{"role": "system", "content": "你是不讲废话、只看数据的专业气象分析师。"},
{"role": "user", "content": prompt}
{
"role": "system",
"content": "你是不讲废话、只看数据的专业气象分析师。",
},
{"role": "user", "content": prompt},
],
"temperature": 0.5,
"max_tokens": 250
"max_tokens": 250,
}
response = requests.post(url, json=payload, headers=headers, timeout=15)
response.raise_for_status()
result = response.json()
content = result['choices'][0]['message']['content'].strip()
content = result["choices"][0]["message"]["content"].strip()
if model != MODELS[0]:
logger.info(f"Groq 降级到备用模型 {model} 成功")
return content
except requests.exceptions.HTTPError as e:
status = e.response.status_code if e.response is not None else 0
if status in (500, 502, 503) and attempt == 0:
@@ -93,7 +94,9 @@ P4 **预报背景**（最低优先级）：
time.sleep(1.5)
continue
else:
logger.warning(f"Groq {model} 失败 (HTTP {status})，尝试下一个模型...")
logger.warning(
f"Groq {model} 失败 (HTTP {status})，尝试下一个模型..."
)
break # 换下一个模型
except Exception as e:
logger.warning(f"Groq {model} 异常: {e}，尝试下一个模型...")
@@ -101,4 +104,3 @@ P4 **预报背景**（最低优先级）：
logger.error("所有 Groq 模型均不可用")
return "\n⚠️ Groq AI 暂时不可用，请稍后再试"
src/analysis/deb_algorithm.py
+82 -61
View File
@@ -1,6 +1,5 @@
import os
import json
import logging
from datetime import datetime, timedelta
import fcntl
@@ -9,23 +8,24 @@ import fcntl
_history_cache = {}
_history_mtime = 0
def load_history(filepath):
global _history_cache, _history_mtime
if not os.path.exists(filepath):
return {}
try:
current_mtime = os.path.getmtime(filepath)
if current_mtime == _history_mtime and _history_cache:
return _history_cache
with open(filepath, 'r', encoding='utf-8') as f:
with open(filepath, "r", encoding="utf-8") as f:
# We don't strictly need a lock for reading in Python if the write is atomic,
# but using one prevents reading half-written JSONs.
fcntl.flock(f, fcntl.LOCK_SH)
data = json.load(f)
fcntl.flock(f, fcntl.LOCK_UN)
_history_cache = data
_history_mtime = current_mtime
return data
@@ -33,11 +33,12 @@ def load_history(filepath):
print(f"Error loading history: {e}")
return _history_cache if _history_cache else {}
def save_history(filepath, data):
global _history_cache, _history_mtime
_history_cache = data
try:
with open(filepath, 'w', encoding='utf-8') as f:
with open(filepath, "w", encoding="utf-8") as f:
fcntl.flock(f, fcntl.LOCK_EX)
json.dump(data, f, ensure_ascii=False, indent=2)
fcntl.flock(f, fcntl.LOCK_UN)
@@ -45,94 +46,106 @@ def save_history(filepath, data):
except Exception as e:
print(f"Error saving history: {e}")
def update_daily_record(city_name, date_str, forecasts, actual_high, deb_prediction=None):
def update_daily_record(
city_name, date_str, forecasts, actual_high, deb_prediction=None
):
"""
保存/更新某城市某天的各个模型预报与最终实测值
forecasts: dict, 例如 {"ECMWF": 28.5, "GFS": 30.0, ...}
actual_high: float, 最终实测最高温
deb_prediction: float, DEB 融合预测值（用于准确率追踪）
"""
project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
history_file = os.path.join(project_root, 'data', 'daily_records.json')
project_root = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
)
history_file = os.path.join(project_root, "data", "daily_records.json")
data = load_history(history_file)
if city_name not in data:
data[city_name] = {}
if date_str not in data[city_name]:
data[city_name][date_str] = {}
# 避免无意义的频繁磁盘写入
old_actual = data[city_name][date_str].get('actual_high')
if old_actual == actual_high and data[city_name][date_str].get('forecasts') == forecasts:
old_actual = data[city_name][date_str].get("actual_high")
if (
old_actual == actual_high
and data[city_name][date_str].get("forecasts") == forecasts
):
return
data[city_name][date_str]['forecasts'] = forecasts
data[city_name][date_str]['actual_high'] = actual_high
data[city_name][date_str]["forecasts"] = forecasts
data[city_name][date_str]["actual_high"] = actual_high
if deb_prediction is not None:
data[city_name][date_str]['deb_prediction'] = deb_prediction
data[city_name][date_str]["deb_prediction"] = deb_prediction
# 自动清理：只保留最近 14 天的记录（DEB 只用 7 天，14 天留足余量）
cutoff = (datetime.now() - timedelta(days=14)).strftime("%Y-%m-%d")
for city in list(data.keys()):
old_dates = [d for d in data[city] if d < cutoff]
for d in old_dates:
del data[city][d]
save_history(history_file, data)
def calculate_dynamic_weights(city_name, current_forecasts, lookback_days=7):
"""
计算动态权重融合 (Dynamic Ensemble Blending, DEB)
根据过去 N 天各模型的 Mean Absolute Error (MAE) 计算倒数权重
返回: blended_high (融合预报值), weights_info (权重展示字符串)
"""
project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
history_file = os.path.join(project_root, 'data', 'daily_records.json')
project_root = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
)
history_file = os.path.join(project_root, "data", "daily_records.json")
data = load_history(history_file)
if city_name not in data or not data[city_name]:
# 没有历史数据，返回简单的平均/中位数
valid_vals = [v for v in current_forecasts.values() if v is not None]
if not valid_vals: return None, "暂无模型数据"
if not valid_vals:
return None, "暂无模型数据"
avg = sum(valid_vals) / len(valid_vals)
return round(avg, 1), "等权平均(历史数据不足)"
# 获取过去 lookback_days 天的有 actual_high 的记录
city_data = data[city_name]
sorted_dates = sorted(city_data.keys(), reverse=True)
# 我们只用真正结清（或者有比较准确最高温）的历史来算误差
# 这边简化：凡是有 actual_high 的都算进去
errors = {model: [] for model in current_forecasts.keys()}
days_used = 0
for date_str in sorted_dates:
# 跳过今天，今天还没出最终结果
if date_str == datetime.now().strftime("%Y-%m-%d"):
continue
record = city_data[date_str]
actual = record.get('actual_high')
past_forecasts = record.get('forecasts', {})
actual = record.get("actual_high")
past_forecasts = record.get("forecasts", {})
if actual is None:
continue
for model in current_forecasts.keys():
if model in past_forecasts and past_forecasts[model] is not None:
errors[model].append(abs(past_forecasts[model] - actual))
days_used += 1
if days_used >= lookback_days:
break
# 如果有效历史天数 < 2 天，还是使用等权
if days_used < 2:
valid_vals = [v for v in current_forecasts.values() if v is not None]
avg = sum(valid_vals) / len(valid_vals)
return round(avg, 1), f"等权平均(由于仅{days_used}天历史)"
# 计算 MAE
maes = {}
for model, err_list in errors.items():
@@ -140,28 +153,32 @@ def calculate_dynamic_weights(city_name, current_forecasts, lookback_days=7):
maes[model] = sum(err_list) / len(err_list)
else:
# 如果某个新模型没有历史数据，给它一个平均误差
maes[model] = 2.0
maes[model] = 2.0
# 计算权重（用 MAE 的倒数，误差越小权重越大；加 0.1 防止除以0）
inverse_errors = {m: 1.0 / (mae + 0.1) for m, mae in maes.items() if current_forecasts.get(m) is not None}
inverse_errors = {
m: 1.0 / (mae + 0.1)
for m, mae in maes.items()
if current_forecasts.get(m) is not None
}
total_inv = sum(inverse_errors.values())
if total_inv == 0:
return None, "权重计算异常"
weights = {m: inv / total_inv for m, inv in inverse_errors.items()}
# 计算加权最高温
blended_high = 0.0
for m in weights.keys():
blended_high += current_forecasts[m] * weights[m]
# 格式化权重信息，挑选前权重最高的2-3个模型展示
sorted_models = sorted(weights.items(), key=lambda x: x[1], reverse=True)
weight_str_parts = []
for m, w in sorted_models[:3]:
weight_str_parts.append(f"{m}({w*100:.0f}%,MAE:{maes[m]:.1f}°)")
weight_str_parts.append(f"{m}({w * 100:.0f}%,MAE:{maes[m]:.1f}°)")
return round(blended_high, 1), " | ".join(weight_str_parts)
@@ -174,49 +191,53 @@ def get_deb_accuracy(city_name):
- total_days: 有效天数
- details_str: 格式化的展示字符串
"""
project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
history_file = os.path.join(project_root, 'data', 'daily_records.json')
project_root = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
)
history_file = os.path.join(project_root, "data", "daily_records.json")
data = load_history(history_file)
if city_name not in data:
return None
city_data = data[city_name]
today_str = datetime.now().strftime("%Y-%m-%d")
hits = 0
total = 0
errors = []
for date_str in sorted(city_data.keys()):
if date_str == today_str:
continue # 跳过今天，还没结算
record = city_data[date_str]
deb_pred = record.get('deb_prediction')
actual = record.get('actual_high')
deb_pred = record.get("deb_prediction")
actual = record.get("actual_high")
if deb_pred is None or actual is None:
continue
try:
deb_pred = float(deb_pred)
actual = float(actual)
except:
except Exception:
continue
total += 1
deb_wu = round(deb_pred)
actual_wu = round(actual)
if deb_wu == actual_wu:
hits += 1
errors.append(abs(deb_pred - actual))
if total == 0:
return None
hit_rate = hits / total * 100
mae = sum(errors) / len(errors)
details_str = f"过去{total}天 WU命中 {hits}/{total} ({hit_rate:.0f}%) | MAE: {mae:.1f}°"
details_str = (
f"过去{total}天 WU命中 {hits}/{total} ({hit_rate:.0f}%) | MAE: {mae:.1f}°"
)
return hit_rate, mae, total, details_str
src/data_collection/city_risk_profiles.py
+9 -15
View File
@@ -27,7 +27,6 @@ CITY_RISK_PROFILES = {
"warning": "冬天温差最不稳定",
"season_notes": "冬季",
},
# 🟡 中危城市 - 存在系统偏差，需注意
"ankara": {
"risk_level": "medium",
@@ -90,7 +89,6 @@ CITY_RISK_PROFILES = {
"warning": "机场在北郊，冬季北风时比市区更冷",
"season_notes": "夏季热浪期间偏差最大",
},
# 🟢 低危城市 - 数据相对靠谱
"toronto": {
"risk_level": "low",
@@ -170,7 +168,7 @@ CITY_RISK_PROFILES = {
def get_city_risk_profile(city_name: str) -> dict:
"""获取城市的风险档案"""
city_lower = city_name.lower().strip()
city_key = city_lower
return CITY_RISK_PROFILES.get(city_key)
@@ -179,32 +177,28 @@ def format_risk_warning(profile: dict, temp_symbol: str) -> str:
"""格式化风险警告信息"""
if not profile:
return ""
lines = []
# 风险等级标题
risk_labels = {
"high": "高危",
"medium": "中危",
"low": "低危"
}
risk_labels = {"high": "高危", "medium": "中危", "low": "低危"}
risk_label = risk_labels.get(profile["risk_level"], "未知")
lines.append(f"⚠️ <b>数据偏差风险</b>: {profile['risk_emoji']} {risk_label}")
# 机场信息
lines.append(f" 📍 机场: {profile['airport_name']} ({profile['icao']})")
lines.append(f" 📏 距市区: {profile['distance_km']}km")
# 典型偏差
if profile["typical_bias_f"] >= 1.0:
lines.append(f" 📊 偏差: ±{profile['typical_bias_f']}{temp_symbol}")
# 偏差方向说明
if profile["bias_direction"]:
lines.append(f" 💡 {profile['bias_direction']}")
# 特别警告
if profile["warning"]:
lines.append(f" 🚨 {profile['warning']}")
return "\n".join(lines)
src/data_collection/weather_sources.py
+247 -126
View File
@@ -207,7 +207,9 @@ class WeatherDataCollector:
return None
def fetch_metar(self, city: str, use_fahrenheit: bool = False, utc_offset: int = 0) -> Optional[Dict]:
def fetch_metar(
self, city: str, use_fahrenheit: bool = False, utc_offset: int = 0
) -> Optional[Dict]:
"""
从 NOAA Aviation Weather Center 获取 METAR 航空气象数据
@@ -236,10 +238,10 @@ class WeatherDataCollector:
}
response = self.session.get(
url,
params=params,
url,
params=params,
headers={"Cache-Control": "no-cache", "Pragma": "no-cache"},
timeout=self.timeout
timeout=self.timeout,
)
response.raise_for_status()
@@ -251,46 +253,60 @@ class WeatherDataCollector:
latest = data[0]
temp_c = latest.get("temp")
dewp_c = latest.get("dewp")
# 从 rawOb 中提取真实观测时间（比 reportTime 更准确，reportTime 会被取整）
# rawOb 格式: "METAR EGLC 271150Z AUTO ..." → "271150Z" → 27日11:50 UTC
def _parse_rawob_time(obs):
"""从 rawOb 中提取精确的 UTC 观测时间"""
raw = obs.get("rawOb", "")
import re as _re
m = _re.search(r'\b(\d{2})(\d{2})(\d{2})Z\b', raw)
m = _re.search(r"\b(\d{2})(\d{2})(\d{2})Z\b", raw)
if m:
day, hour, minute = int(m.group(1)), int(m.group(2)), int(m.group(3))
_day, hour, minute = (
int(m.group(1)),
int(m.group(2)),
int(m.group(3)),
)
# 用 reportTime 的日期部分 + rawOb 的时分
fallback = obs.get("reportTime", "")
try:
clean = fallback.replace(" ", "T")
if not clean.endswith("Z"): clean += "Z"
if not clean.endswith("Z"):
clean += "Z"
base_dt = datetime.fromisoformat(clean.replace("Z", "+00:00"))
result = base_dt.replace(hour=hour, minute=minute, second=0)
# 处理跨日（如 rawOb 是23:50但 reportTime 已经是次日00:00）
if result > base_dt + timedelta(hours=2):
result -= timedelta(days=1)
return result
except:
except Exception:
pass
# fallback 到 reportTime
fallback = obs.get("reportTime", "")
try:
clean = fallback.replace(" ", "T")
if not clean.endswith("Z"): clean += "Z"
if not clean.endswith("Z"):
clean += "Z"
return datetime.fromisoformat(clean.replace("Z", "+00:00"))
except:
except Exception:
return None
obs_dt = _parse_rawob_time(latest)
obs_time = obs_dt.strftime("%Y-%m-%dT%H:%M:%S.000Z") if obs_dt else latest.get("reportTime", "")
obs_time = (
obs_dt.strftime("%Y-%m-%dT%H:%M:%S.000Z")
if obs_dt
else latest.get("reportTime", "")
)
# 2. 精确计算"当地今天"的最高温
from datetime import timezone, timedelta
now_utc = datetime.now(timezone.utc)
local_now = now_utc + timedelta(seconds=utc_offset)
local_midnight = local_now.replace(hour=0, minute=0, second=0, microsecond=0)
local_midnight = local_now.replace(
hour=0, minute=0, second=0, microsecond=0
)
utc_midnight = local_midnight - timedelta(seconds=utc_offset)
max_so_far_c = -999
@@ -306,12 +322,12 @@ class WeatherDataCollector:
max_so_far_c = t
local_report = obs_dt_iter + timedelta(seconds=utc_offset)
max_temp_time = local_report.strftime("%H:%M")
except:
except Exception:
continue
# 3. 提取最近 4 条报文的多维数据（温度 + 风/云/压强，用于趋势和 shock_score）
recent_temps_raw = [] # [(local_time_str, temp_c), ...]
recent_obs_raw = [] # [{time, temp, wdir, wspd, clouds, altim}, ...]
recent_obs_raw = [] # [{time, temp, wdir, wspd, clouds, altim}, ...]
for obs in data[:4]: # data 已按时间倒序
obs_temp = obs.get("temp")
obs_dt_iter = _parse_rawob_time(obs)
@@ -319,21 +335,30 @@ class WeatherDataCollector:
local_rt = obs_dt_iter + timedelta(seconds=utc_offset)
recent_temps_raw.append((local_rt.strftime("%H:%M"), obs_temp))
# 云量码映射: CLR=0, FEW=1, SCT=2, BKN=3, OVC=4
cloud_rank_map = {"CLR": 0, "SKC": 0, "FEW": 1, "SCT": 2, "BKN": 3, "OVC": 4}
cloud_rank_map = {
"CLR": 0,
"SKC": 0,
"FEW": 1,
"SCT": 2,
"BKN": 3,
"OVC": 4,
}
clouds = obs.get("clouds", [])
max_cloud_rank = 0
for c in clouds:
rank = cloud_rank_map.get(c.get("cover", ""), 0)
if rank > max_cloud_rank:
max_cloud_rank = rank
recent_obs_raw.append({
"time": local_rt.strftime("%H:%M"),
"temp": obs_temp,
"wdir": obs.get("wdir"),
"wspd": obs.get("wspd"),
"cloud_rank": max_cloud_rank, # 0~4
"altim": obs.get("altim"),
})
recent_obs_raw.append(
{
"time": local_rt.strftime("%H:%M"),
"temp": obs_temp,
"wdir": obs.get("wdir"),
"wspd": obs.get("wspd"),
"cloud_rank": max_cloud_rank, # 0~4
"altim": obs.get("altim"),
}
)
# 转换为单位
if use_fahrenheit:
@@ -342,7 +367,9 @@ class WeatherDataCollector:
dewp = dewp_c * 9 / 5 + 32 if dewp_c is not None else None
unit = "fahrenheit"
# 转换最近温度
recent_temps = [(t, round(v * 9 / 5 + 32, 1)) for t, v in recent_temps_raw]
recent_temps = [
(t, round(v * 9 / 5 + 32, 1)) for t, v in recent_temps_raw
]
else:
temp = temp_c
max_so_far = max_so_far_c if max_so_far_c > -900 else None
@@ -358,7 +385,9 @@ class WeatherDataCollector:
"observation_time": obs_time,
"current": {
"temp": round(temp, 1) if temp is not None else None,
"max_temp_so_far": round(max_so_far, 1) if max_so_far is not None else None,
"max_temp_so_far": round(max_so_far, 1)
if max_so_far is not None
else None,
"max_temp_time": max_temp_time,
"dewpoint": round(dewp, 1) if dewp is not None else None,
"humidity": latest.get("rh"),
@@ -396,17 +425,18 @@ class WeatherDataCollector:
# 必须带 Origin，否则会被反爬拦截
headers = {
"Origin": "https://www.mgm.gov.tr",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
}
results = {}
try:
# 1. 实时数据 (添加时间戳防止 CDN 缓存)
import time
obs_resp = self.session.get(
f"{base_url}/sondurumlar?istno={istno}&_={int(time.time()*1000)}",
headers=headers,
timeout=self.timeout
f"{base_url}/sondurumlar?istno={istno}&_={int(time.time() * 1000)}",
headers=headers,
timeout=self.timeout,
)
if obs_resp.status_code == 200:
data = obs_resp.json()
@@ -415,26 +445,47 @@ class WeatherDataCollector:
# MGM 数据字段映射
# ruzgarHiz 实测为 km/h，转为 m/s 需要除以 3.6
ruz_hiz_kmh = latest.get("ruzgarHiz", 0)
# MGM 返回 -9999 表示数据缺失，需要过滤
def _valid(v):
return v is not None and v > -9000
results["current"] = {
"temp": latest.get("sicaklik") if _valid(latest.get("sicaklik")) else None,
"feels_like": latest.get("hissedilenSicaklik") if _valid(latest.get("hissedilenSicaklik")) else None,
"humidity": latest.get("nem") if _valid(latest.get("nem")) else None,
"wind_speed_ms": round(ruz_hiz_kmh / 3.6, 1) if _valid(ruz_hiz_kmh) else None,
"wind_speed_kt": round(ruz_hiz_kmh / 1.852, 1) if _valid(ruz_hiz_kmh) else None,
"wind_dir": latest.get("ruzgarYon") if _valid(latest.get("ruzgarYon")) else None,
"rain_24h": latest.get("toplamYagis") if _valid(latest.get("toplamYagis")) else None,
"pressure": latest.get("aktuelBasinc") if _valid(latest.get("aktuelBasinc")) else None,
"temp": latest.get("sicaklik")
if _valid(latest.get("sicaklik"))
else None,
"feels_like": latest.get("hissedilenSicaklik")
if _valid(latest.get("hissedilenSicaklik"))
else None,
"humidity": latest.get("nem")
if _valid(latest.get("nem"))
else None,
"wind_speed_ms": round(ruz_hiz_kmh / 3.6, 1)
if _valid(ruz_hiz_kmh)
else None,
"wind_speed_kt": round(ruz_hiz_kmh / 1.852, 1)
if _valid(ruz_hiz_kmh)
else None,
"wind_dir": latest.get("ruzgarYon")
if _valid(latest.get("ruzgarYon"))
else None,
"rain_24h": latest.get("toplamYagis")
if _valid(latest.get("toplamYagis"))
else None,
"pressure": latest.get("aktuelBasinc")
if _valid(latest.get("aktuelBasinc"))
else None,
"cloud_cover": latest.get("kapalilik"), # 0-8 八分位云量
"mgm_max_temp": latest.get("maxSicaklik") if _valid(latest.get("maxSicaklik")) else None,
"mgm_max_temp": latest.get("maxSicaklik")
if _valid(latest.get("maxSicaklik"))
else None,
"time": latest.get("veriZamani"),
"station_name": latest.get("istasyonAd") or latest.get("adi") or latest.get("merkezAd") or "Ankara Esenboğa"
"station_name": latest.get("istasyonAd")
or latest.get("adi")
or latest.get("merkezAd")
or "Ankara Esenboğa",
}
# 2. 每日预报（尝试两个可能的 API 路径）
forecast_urls = [
f"{base_url}/tahminler/gunluk?istno={istno}",
@@ -442,7 +493,9 @@ class WeatherDataCollector:
]
for forecast_url in forecast_urls:
try:
daily_resp = self.session.get(forecast_url, headers=headers, timeout=self.timeout)
daily_resp = self.session.get(
forecast_url, headers=headers, timeout=self.timeout
)
if daily_resp.status_code == 200:
forecasts = daily_resp.json()
if forecasts and isinstance(forecasts, list):
@@ -452,17 +505,29 @@ class WeatherDataCollector:
if high_val is not None:
results["today_high"] = high_val
results["today_low"] = low_val
logger.info(f"📋 MGM 每日预报: 最高 {high_val}°C, 最低 {low_val}°C (from {forecast_url})")
logger.info(
f"📋 MGM 每日预报: 最高 {high_val}°C, 最低 {low_val}°C (from {forecast_url})"
)
break
else:
# 记录所有可用字段，方便调试
available_keys = [k for k in today.keys() if "yuksek" in k.lower() or "sicaklik" in k.lower() or "gun" in k.lower()]
logger.warning(f"MGM 每日预报: enYuksekGun1 为空，可用字段: {available_keys}")
available_keys = [
k
for k in today.keys()
if "yuksek" in k.lower()
or "sicaklik" in k.lower()
or "gun" in k.lower()
]
logger.warning(
f"MGM 每日预报: enYuksekGun1 为空，可用字段: {available_keys}"
)
else:
logger.debug(f"MGM forecast URL {forecast_url} returned {daily_resp.status_code}")
logger.debug(
f"MGM forecast URL {forecast_url} returned {daily_resp.status_code}"
)
except Exception as e:
logger.debug(f"MGM forecast URL {forecast_url} failed: {e}")
return results if "current" in results else None
except Exception as e:
logger.error(f"MGM API 请求失败 ({istno}): {e}")
@@ -477,24 +542,28 @@ class WeatherDataCollector:
# 1. 获取网格点
points_url = f"https://api.weather.gov/points/{lat},{lon}"
headers = {"User-Agent": "PolyWeather/1.0 (weather-bot)"}
points_resp = self.session.get(points_url, headers=headers, timeout=self.timeout)
points_resp = self.session.get(
points_url, headers=headers, timeout=self.timeout
)
points_resp.raise_for_status()
points_data = points_resp.json()
forecast_url = points_data.get("properties", {}).get("forecast")
if not forecast_url:
return None
# 2. 获取预报
forecast_resp = self.session.get(forecast_url, headers=headers, timeout=self.timeout)
forecast_resp = self.session.get(
forecast_url, headers=headers, timeout=self.timeout
)
forecast_resp.raise_for_status()
forecast_data = forecast_resp.json()
periods = forecast_data.get("properties", {}).get("periods", [])
if not periods:
return None
# 3. 提取今日最高温（找 isDaytime=True 的第一个）
today_high = None
for p in periods:
@@ -507,7 +576,7 @@ class WeatherDataCollector:
if p.get("isDaytime"):
today_high = p.get("temperature")
break
return {
"source": "nws",
"today_high": today_high,
@@ -570,19 +639,19 @@ class WeatherDataCollector:
if "temperature_2m_max_ecmwf_ifs04" in daily_data:
ecmwf_max = daily_data.get("temperature_2m_max_ecmwf_ifs04", [])
hrrr_max = daily_data.get("temperature_2m_max_ncep_hrrr_conus", [])
# 记录今日模型分歧
daily_data["model_split"] = {
"ecmwf": ecmwf_max[0] if ecmwf_max else None,
"hrrr": hrrr_max[0] if hrrr_max else None
"hrrr": hrrr_max[0] if hrrr_max else None,
}
# 智能合并：HRRR 仅覆盖 48 小时，远期用 ECMWF 补全
merged_max = []
for i in range(len(ecmwf_max)):
hrrr_val = hrrr_max[i] if i < len(hrrr_max) else None
ecmwf_val = ecmwf_max[i] if i < len(ecmwf_max) else None
# 优先 HRRR，其次 ECMWF，都没有就跳过
if hrrr_val is not None:
merged_max.append(hrrr_val)
@@ -596,7 +665,9 @@ class WeatherDataCollector:
# 映射逐小时数据
hourly_data = data.get("hourly", {})
if "temperature_2m_ncep_hrrr_conus" in hourly_data:
hourly_data["temperature_2m"] = hourly_data["temperature_2m_ncep_hrrr_conus"]
hourly_data["temperature_2m"] = hourly_data[
"temperature_2m_ncep_hrrr_conus"
]
# 计算精确的当地时间
now_utc = datetime.utcnow()
@@ -662,7 +733,7 @@ class WeatherDataCollector:
if key.startswith("temperature_2m_max") and key != "temperature_2m_max":
if values and values[0] is not None:
today_highs.append(values[0])
# 也检查非成员键（有些返回格式不同）
if not today_highs:
raw_max = daily.get("temperature_2m_max", [])
@@ -712,14 +783,14 @@ class WeatherDataCollector:
"""
从 Open-Meteo 获取多个独立 NWP 模型的预报
用于真正的多模型共识评分
模型列表:
- ECMWF IFS (欧洲中期天气预报中心)
- GFS (美国 NOAA)
- ICON (德国气象局 DWD)
- GEM (加拿大气象局)
- JMA (日本气象厅)
返回 3 天的预报数据，支持今日+明日共识分析
"""
try:
@@ -748,7 +819,7 @@ class WeatherDataCollector:
daily = data.get("daily", {})
dates = daily.get("time", [])
model_labels = {
"ecmwf_ifs025": "ECMWF",
"gfs_seamless": "GFS",
@@ -756,7 +827,7 @@ class WeatherDataCollector:
"gem_seamless": "GEM",
"jma_seamless": "JMA",
}
# 按天提取每个模型的预报
daily_forecasts = {} # {"2026-02-23": {"ECMWF": 7.9, "GFS": 6.5, ...}, ...}
for day_idx, date_str in enumerate(dates):
@@ -778,8 +849,10 @@ class WeatherDataCollector:
forecasts = daily_forecasts.get(today_date, {})
labels_str = ", ".join([f"{k}={v}" for k, v in forecasts.items()])
logger.info(f"🔬 Multi-model ({len(forecasts)}个, {len(daily_forecasts)}天): {labels_str}")
logger.info(
f"🔬 Multi-model ({len(forecasts)}个, {len(daily_forecasts)}天): {labels_str}"
)
return {
"source": "multi_model",
"forecasts": forecasts, # 今天 {"ECMWF": 12.3, "GFS": 11.8, ...} (向后兼容)
@@ -814,47 +887,47 @@ class WeatherDataCollector:
"lat": lat,
"lon": lon,
"format": "json",
"as_daylight": "true"
"as_daylight": "true",
}
response = self.session.get(
url,
params=params,
timeout=self.timeout
)
response = self.session.get(url, params=params, timeout=self.timeout)
response.raise_for_status()
data = response.json()
day_data = data.get("data_day", {})
max_temps = day_data.get("temperature_max", [])
if not max_temps:
logger.warning(f"Meteoblue API 返回数据中找不到最高温 (坐标: {lat},{lon})")
logger.warning(
f"Meteoblue API 返回数据中找不到最高温 (坐标: {lat},{lon})"
)
return None
# 2. 转换单位
def c_to_f(c):
return round((c * 9/5) + 32, 1)
return round((c * 9 / 5) + 32, 1)
result = {
"source": "meteoblue",
"today_high": None,
"daily_highs": [],
"unit": "fahrenheit" if use_fahrenheit else "celsius",
"url": f"https://www.meteoblue.com/en/weather/week/{lat}N{lon}E" # 仅供参考
"url": f"https://www.meteoblue.com/en/weather/week/{lat}N{lon}E", # 仅供参考
}
# 提取今日最高
mb_today_c = max_temps[0]
result["today_high"] = c_to_f(mb_today_c) if use_fahrenheit else mb_today_c
# 提取接下来几天的最高温
if use_fahrenheit:
result["daily_highs"] = [c_to_f(t) for t in max_temps]
else:
result["daily_highs"] = max_temps
logger.info(f"✅ Meteoblue API 获取成功 ({lat},{lon}): 今天 {result['today_high']}{result['unit']}")
logger.info(
f"✅ Meteoblue API 获取成功 ({lat},{lon}): 今天 {result['today_high']}{result['unit']}"
)
return result
except Exception as e:
logger.error(f"Meteoblue API fetch failed: {e}")
@@ -867,9 +940,18 @@ class WeatherDataCollector:
"""
# 1. 尝试英文月份
months = {
"January": "01", "February": "02", "March": "03", "April": "04",
"May": "05", "June": "06", "July": "07", "August": "08",
"September": "09", "October": "10", "November": "11", "December": "12",
"January": "01",
"February": "02",
"March": "03",
"April": "04",
"May": "05",
"June": "06",
"July": "07",
"August": "08",
"September": "09",
"October": "10",
"November": "11",
"December": "12",
}
for month_name, month_val in months.items():
if month_name in title:
@@ -886,7 +968,7 @@ class WeatherDataCollector:
day = int(zh_match.group(2))
year = datetime.now().year
return f"{year}-{month:02d}-{day:02d}"
# 3. 尝试 ISO 格式 YYYY-MM-DD
iso_match = re.search(r"(\d{4})-(\d{2})-(\d{2})", title)
if iso_match:
@@ -900,21 +982,21 @@ class WeatherDataCollector:
"""
# 坐标使用 METAR 机场位置（Polymarket 以机场数据结算）
static_coords = {
"london": {"lat": 51.5053, "lon": 0.0553}, # EGLC London City
"paris": {"lat": 49.0097, "lon": 2.5478}, # LFPG Charles de Gaulle
"new york": {"lat": 40.7750, "lon": -73.8750}, # KLGA LaGuardia
"london": {"lat": 51.5053, "lon": 0.0553}, # EGLC London City
"paris": {"lat": 49.0097, "lon": 2.5478}, # LFPG Charles de Gaulle
"new york": {"lat": 40.7750, "lon": -73.8750}, # KLGA LaGuardia
"new york's central park": {"lat": 40.7812, "lon": -73.9665},
"nyc": {"lat": 40.7750, "lon": -73.8750}, # KLGA LaGuardia
"seattle": {"lat": 47.4499, "lon": -122.3118}, # KSEA Sea-Tac
"chicago": {"lat": 41.9769, "lon": -87.9081}, # KORD O'Hare
"dallas": {"lat": 32.8459, "lon": -96.8509}, # KDAL Love Field
"miami": {"lat": 25.7933, "lon": -80.2906}, # KMIA International
"atlanta": {"lat": 33.6367, "lon": -84.4281}, # KATL Hartsfield-Jackson
"seoul": {"lat": 37.4691, "lon": 126.4510}, # RKSI Incheon
"toronto": {"lat": 43.6759, "lon": -79.6294}, # CYYZ Pearson
"ankara": {"lat": 40.1281, "lon": 32.9950}, # LTAC Esenboğa
"wellington": {"lat": -41.3272, "lon": 174.8053}, # NZWN Wellington
"buenos aires": {"lat": -34.8222, "lon": -58.5358}, # SAEZ Ezeiza
"nyc": {"lat": 40.7750, "lon": -73.8750}, # KLGA LaGuardia
"seattle": {"lat": 47.4499, "lon": -122.3118}, # KSEA Sea-Tac
"chicago": {"lat": 41.9769, "lon": -87.9081}, # KORD O'Hare
"dallas": {"lat": 32.8459, "lon": -96.8509}, # KDAL Love Field
"miami": {"lat": 25.7933, "lon": -80.2906}, # KMIA International
"atlanta": {"lat": 33.6367, "lon": -84.4281}, # KATL Hartsfield-Jackson
"seoul": {"lat": 37.4691, "lon": 126.4510}, # RKSI Incheon
"toronto": {"lat": 43.6759, "lon": -79.6294}, # CYYZ Pearson
"ankara": {"lat": 40.1281, "lon": 32.9950}, # LTAC Esenboğa
"wellington": {"lat": -41.3272, "lon": 174.8053}, # NZWN Wellington
"buenos aires": {"lat": -34.8222, "lon": -58.5358}, # SAEZ Ezeiza
}
normalized_city = city.lower().strip()
@@ -956,30 +1038,64 @@ class WeatherDataCollector:
# 1. 优先尝试已知城市列表 (硬编码匹配)
known_cities = {
"london": "London", "伦敦": "London",
"new york": "New York", "new york's central park": "New York", "nyc": "New York", "纽约": "New York",
"seattle": "Seattle", "西雅图": "Seattle",
"chicago": "Chicago", "芝加哥": "Chicago",
"dallas": "Dallas", "达拉斯": "Dallas",
"miami": "Miami", "迈阿密": "Miami",
"atlanta": "Atlanta", "亚特兰大": "Atlanta",
"seoul": "Seoul", "首尔": "Seoul",
"toronto": "Toronto", "多伦多": "Toronto",
"ankara": "Ankara", "安卡拉": "Ankara",
"wellington": "Wellington", "惠灵顿": "Wellington",
"buenos aires": "Buenos Aires", "布宜诺斯艾利斯": "Buenos Aires"
"london": "London",
"伦敦": "London",
"new york": "New York",
"new york's central park": "New York",
"nyc": "New York",
"纽约": "New York",
"seattle": "Seattle",
"西雅图": "Seattle",
"chicago": "Chicago",
"芝加哥": "Chicago",
"dallas": "Dallas",
"达拉斯": "Dallas",
"miami": "Miami",
"迈阿密": "Miami",
"atlanta": "Atlanta",
"亚特兰大": "Atlanta",
"seoul": "Seoul",
"首尔": "Seoul",
"toronto": "Toronto",
"多伦多": "Toronto",
"ankara": "Ankara",
"安卡拉": "Ankara",
"wellington": "Wellington",
"惠灵顿": "Wellington",
"buenos aires": "Buenos Aires",
"布宜诺斯艾利斯": "Buenos Aires",
}
for key, val in known_cities.items():
if key in q:
return val
# 2. 从英文模板中提取
triggers = ["temperature in ", "temp in ", "weather in ", "highest-temperature-in-", "temperature-in-"]
triggers = [
"temperature in ",
"temp in ",
"weather in ",
"highest-temperature-in-",
"temperature-in-",
]
for trigger in triggers:
if trigger in q:
part = q.split(trigger)[1]
delimiters = [" on ", " at ", " above ", " below ", " be ", " is ", " will ", " has ", " reached ", "?", " (", ", ", "-"]
delimiters = [
" on ",
" at ",
" above ",
" below ",
" be ",
" is ",
" will ",
" has ",
" reached ",
"?",
" (",
", ",
"-",
]
city = part
for d in delimiters:
if d in city:
@@ -1039,22 +1155,25 @@ class WeatherDataCollector:
results["open-meteo"] = open_meteo
# 获取时区偏移以过滤 METAR
utc_offset = open_meteo.get("utc_offset", 0)
metar_data = self.fetch_metar(city, use_fahrenheit=use_fahrenheit, utc_offset=utc_offset)
metar_data = self.fetch_metar(
city, use_fahrenheit=use_fahrenheit, utc_offset=utc_offset
)
if metar_data:
results["metar"] = metar_data
# 对安卡拉，额外获取 MGM 官方数据
if city_lower == "ankara":
mgm_data = self.fetch_from_mgm("17128")
if mgm_data:
results["mgm"] = mgm_data
# 对伦敦，获取 Meteoblue 预测 (公认最准)
if city_lower == "london":
mb_data = self.fetch_from_meteoblue(
lat, lon,
lat,
lon,
timezone_name=open_meteo.get("timezone", "UTC"),
use_fahrenheit=use_fahrenheit
use_fahrenheit=use_fahrenheit,
)
if mb_data:
results["meteoblue"] = mb_data
@@ -1064,14 +1183,16 @@ class WeatherDataCollector:
nws_data = self.fetch_nws(lat, lon)
if nws_data:
results["nws"] = nws_data
# 集合预报 (所有城市通用，用于不确定性分析)
ens_data = self.fetch_ensemble(lat, lon, use_fahrenheit=use_fahrenheit)
if ens_data:
results["ensemble"] = ens_data
# 多模型预报 (所有城市通用，用于共识评分)
mm_data = self.fetch_multi_model(lat, lon, use_fahrenheit=use_fahrenheit)
mm_data = self.fetch_multi_model(
lat, lon, use_fahrenheit=use_fahrenheit
)
if mm_data:
results["multi_model"] = mm_data
else:
src/data_mining/fetch_history.py
+42 -30
View File
@@ -1,27 +1,28 @@
import os
import sys
import json
import logging
from datetime import datetime
import pandas as pd
import requests
# Set up logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logging.basicConfig(
level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
)
def fetch_historical_data_for_city(city_info, output_dir):
city_name = city_info['city']
lat = city_info['latitude']
lon = city_info['longitude']
city_name = city_info["city"]
lat = city_info["latitude"]
lon = city_info["longitude"]
# We will fetch data from Jan 1, 2023 to yesterday (or to latest available)
start_date = "2023-01-01"
end_date = "2025-12-31" # API handles dates in the future up to latest available archive usually
# For safety let's use a dynamic yesterday end_date
import datetime
today = datetime.datetime.now()
yesterday = (today - datetime.timedelta(days=2)).strftime("%Y-%m-%d")
url = (
f"https://archive-api.open-meteo.com/v1/archive?latitude={lat}&longitude={lon}"
f"&start_date={start_date}&end_date={yesterday}"
@@ -29,57 +30,68 @@ def fetch_historical_data_for_city(city_info, output_dir):
"cloud_cover,shortwave_radiation,precipitation,surface_pressure"
"&timezone=auto"
)
logging.info(f"Downloading historical data for {city_name} (Lat: {lat}, Lon: {lon})...")
logging.info(
f"Downloading historical data for {city_name} (Lat: {lat}, Lon: {lon})..."
)
try:
response = requests.get(url, timeout=60)
response.raise_for_status()
data = response.json()
if "hourly" not in data:
logging.error(f"Failed to find 'hourly' data for {city_name}.")
return
hourly_data = data["hourly"]
# Convert to pandas DataFrame
df = pd.DataFrame(hourly_data)
# Save to CSV
output_path = os.path.join(output_dir, f"{city_name.replace(' ', '_').lower()}_historical.csv")
output_path = os.path.join(
output_dir, f"{city_name.replace(' ', '_').lower()}_historical.csv"
)
df.to_csv(output_path, index=False)
logging.info(f"✅ Successfully saved historical data for {city_name} to {output_path}. Shape: {df.shape}")
logging.info(
f"✅ Successfully saved historical data for {city_name} to {output_path}. Shape: {df.shape}"
)
except requests.exceptions.RequestException as e:
logging.error(f"❌ Network error while fetching data for {city_name}: {e}")
except Exception as e:
logging.error(f"❌ Error processing data for {city_name}: {e}")
def main():
project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
config_path = os.path.join(project_root, 'config', 'config.yaml')
output_dir = os.path.join(project_root, 'data', 'historical')
project_root = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
)
config_path = os.path.join(project_root, "config", "config.yaml")
output_dir = os.path.join(project_root, "data", "historical")
os.makedirs(output_dir, exist_ok=True)
# Load config
try:
import yaml
with open(config_path, 'r', encoding='utf-8') as f:
with open(config_path, "r", encoding="utf-8") as f:
config = yaml.safe_load(f)
except Exception as e:
logging.error(f"Failed to load {config_path}: {e}")
sys.exit(1)
cities = config.get('cities', [])
cities = config.get("cities", [])
if not cities:
logging.warning("No cities found in config.yaml")
return
for city_info in cities:
fetch_historical_data_for_city(city_info, output_dir)
if __name__ == "__main__":
main()
src/utils/config_loader.py
+5 -4
View File
@@ -1,12 +1,13 @@
import os
from dotenv import load_dotenv
def load_config():
"""
Load configuration from environment variables and config files
"""
load_dotenv()
def get_env_or_none(key):
val = os.getenv(key)
if not val or "your_" in val.lower() or val.strip() == "":
@@ -40,14 +41,14 @@ def load_config():
"market_volume_signal": 0.15,
"orderbook_analysis": 0.10,
"technical_indicators": 0.05,
"onchain_whale_signal": 0.05
"onchain_whale_signal": 0.05,
}
},
"app": {
"log_level": os.getenv("LOG_LEVEL", "INFO"),
"env": os.getenv("ENV", "development"),
"proxy": os.getenv("HTTPS_PROXY") or os.getenv("HTTP_PROXY"),
}
},
}
return config
src/utils/logger.py
+5 -4
View File
@@ -1,19 +1,20 @@
import sys
from loguru import logger
def setup_logger(level="DEBUG"):
"""
Configure loguru logger
"""
logger.remove() # Remove default handler
# 控制台输出 - 使用支持中文的格式
logger.add(
sys.stderr,
format="<green>{time:YYYY-MM-DD HH:mm:ss}</green> | <level>{level: <8}</level> | <level>{message}</level>",
level=level
level=level,
)
# 文件输出
logger.add(
"data/logs/trading_system.log",
@@ -21,7 +22,7 @@ def setup_logger(level="DEBUG"):
retention="10 days",
level=level,
encoding="utf-8",
compression="zip"
compression="zip",
)
logger.info("日志系统初始化完成。")