gavindiaz/PolyWeather
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

将商业核心数据源 URL 从源码迁移到环境变量，VPS .env 已补全

Browse Source
This commit is contained in:
2569718930@qq.com
2026-05-19 15:42:03 +08:00
parent 7e0d955d22
commit 3b8098051e
11 changed files with 57 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+16 -2
View File
@@ -63,8 +63,22 @@ POLYWEATHER_METAR_TIMEOUT_SEC=4
POLYWEATHER_METAR_CLUSTER_TIMEOUT_SEC=3.5
METAR_CACHE_TTL_SEC=600
JMA_AMEDAS_CACHE_TTL_SEC=120
# Recommended on VPS: do not train there; pull the SQLite DB to a local machine.
# Optional: set this to a writable runtime path if you manually deploy a
# ── Country-specific data source URLs ──
# These are kept in .env to avoid exposing competitive data-source discovery
# work on the public GitHub repository. Leave empty to use built-in defaults.
# AMSC_AWOS_BASE_URL=https://www.amsc.net.cn/gateway/api/saas/rest/amc/AwosController/getWindPlate
# NMC_FORECAST_BASE_URL=https://m.nmc.cn/publish/forecast
# NMC_REALTIME_BASE_URL=https://www.nmc.cn/rest/real
# KMA_BASE_URL=https://www.weather.go.kr
# AMOS_BASE_URL=https://global.amo.go.kr/amosobsnew/AmosRealTimeImage.do
# JMA_AMEDAS_BASE_URL=https://www.jma.go.jp
# MGM_BASE_URL=https://servis.mgm.gov.tr/web
# MGM_ORIGIN_URL=https://www.mgm.gov.tr
# FMI_BASE_URL=https://opendata.fmi.fi/wfs
# HKO_BASE_URL=https://data.weather.gov.hk/weatherAPI/hko_data/regional-weather
# SINGAPORE_MSS_BASE_URL=https://api.data.gov.sg/v1/environment/air-temperature
# RUSSIA_POGODAIKLIMAT_BASE_URL=https://www.pogodaiklimat.ru
########################################
# 4) Auth / entitlement
src/data_collection/amos_station_sources.py
+2 -1
View File
@@ -7,6 +7,7 @@ Provides per-runway wind, temperature, pressure, visibility, RVR, cloud data.
from __future__ import annotations
import re
import os
import time
from html import unescape
from datetime import datetime
@@ -16,7 +17,7 @@ from loguru import logger
from src.utils.metrics import record_source_call
AMOS_BASE_URL = "https://global.amo.go.kr/amosobsnew/AmosRealTimeImage.do"
AMOS_BASE_URL = os.getenv("AMOS_BASE_URL", "").strip() or "https://global.amo.go.kr/amosobsnew/AmosRealTimeImage.do"
AMOS_AIRPORT_QUERY_KEYS = (
"stnCd",
"icao",
src/data_collection/amsc_awos_sources.py
+2 -5
View File
@@ -21,10 +21,7 @@ from loguru import logger
from src.utils.metrics import record_source_call
AMSC_AWOS_BASE_URL = (
"https://www.amsc.net.cn/gateway/api/saas/rest/amc/"
"AwosController/getWindPlate"
)
AMSC_AWOS_BASE_URL = os.getenv("AMSC_AWOS_BASE_URL", "").strip()
AMSC_AWOS_AIRPORTS: Dict[str, Dict[str, str]] = {
"shanghai": {"icao": "ZSPD", "label": "Shanghai Pudong"},
@@ -176,7 +173,7 @@ class AmscAwosSourceMixin:
def _amsc_headers(self) -> Dict[str, str]:
headers = {
"Accept": "application/json, text/plain, */*",
"Referer": "https://www.amsc.net.cn/",
"Referer": os.getenv("AMSC_AWOS_REFERER", "https://www.amsc.net.cn/"),
"User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36"
src/data_collection/fmi_sources.py
+2 -1
View File
@@ -7,6 +7,7 @@ for Helsinki-Vantaa airport (EFHK, WMO 2974, FMISID 100968).
from __future__ import annotations
import re
import os
import time
from datetime import datetime
from typing import Any, Dict, Optional
@@ -15,7 +16,7 @@ from loguru import logger
from src.utils.metrics import record_source_call
FMI_BASE_URL = "https://opendata.fmi.fi/wfs"
FMI_BASE_URL = os.getenv("FMI_BASE_URL", "").strip() or "https://opendata.fmi.fi/wfs"
FMI_STATION = {
"helsinki": {
"fmisid": "100968",
src/data_collection/hko_obs_sources.py
+2 -1
View File
@@ -8,6 +8,7 @@ No API key required.
from __future__ import annotations
import csv
import os
import io
import time
from datetime import datetime
@@ -17,7 +18,7 @@ from loguru import logger
from src.utils.metrics import record_source_call
HKO_BASE_URL = "https://data.weather.gov.hk/weatherAPI/hko_data/regional-weather"
HKO_BASE_URL = os.getenv("HKO_BASE_URL", "").strip() or "https://data.weather.gov.hk/weatherAPI/hko_data/regional-weather"
HKO_STATIONS = {
"hong kong": {
"code": "HK Observatory",
src/data_collection/jma_amedas_sources.py
+5 -2
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
import os
import time
from datetime import datetime
from typing import Any, Dict, List, Optional
@@ -8,6 +9,8 @@ from loguru import logger
from src.utils.metrics import record_source_call
_JMA_BASE = os.getenv("JMA_AMEDAS_BASE_URL", "").strip() or "https://www.jma.go.jp"
JMA_AMEDAS_STATIONS: Dict[str, Dict[str, Any]] = {
"tokyo": {
@@ -60,13 +63,13 @@ class JmaAmedasSourceMixin:
try:
latest_time_text = self._jma_http_get_text(
"https://www.jma.go.jp/bosai/amedas/data/latest_time.txt"
f"{_JMA_BASE}/bosai/amedas/data/latest_time.txt"
).strip()
latest_dt = datetime.fromisoformat(latest_time_text)
bucket_hour = (latest_dt.hour // 3) * 3
bucket_key = f"{latest_dt.strftime('%Y%m%d')}_{bucket_hour:02d}"
station_code = str(meta.get("station_code") or "").strip()
url = f"https://www.jma.go.jp/bosai/amedas/data/point/{station_code}/{bucket_key}.json"
url = f"{_JMA_BASE}/bosai/amedas/data/point/{station_code}/{bucket_key}.json"
getter = getattr(self, "_http_get_json", None)
if callable(getter):
src/data_collection/kma_station_sources.py
+8 -6
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
import math
import os
import time
from datetime import datetime
from typing import Any, Dict, List, Optional
@@ -9,6 +10,7 @@ from loguru import logger
from src.utils.metrics import record_source_call
_KMA_BASE = os.getenv("KMA_BASE_URL", "").strip() or "https://www.weather.go.kr"
KMA_CITY_STATIONS: Dict[str, Dict[str, Any]] = {
"busan": {
@@ -24,15 +26,15 @@ KMA_CITY_STATIONS: Dict[str, Dict[str, Any]] = {
}
KMA_GEOJSON_URLS: Dict[str, str] = {
"air": "https://www.weather.go.kr/wgis-nuri/js/info/air.geojson",
"sfc": "https://www.weather.go.kr/wgis-nuri/js/info/sfc.geojson",
"aws": "https://www.weather.go.kr/wgis-nuri/js/info/aws.geojson",
"air": f"{_KMA_BASE}/wgis-nuri/js/info/air.geojson",
"sfc": f"{_KMA_BASE}/wgis-nuri/js/info/sfc.geojson",
"aws": f"{_KMA_BASE}/wgis-nuri/js/info/aws.geojson",
}
KMA_OBSERVATION_URLS: Dict[str, str] = {
"air": "https://www.weather.go.kr/wgis-nuri/aws/air?date=",
"sfc": "https://www.weather.go.kr/wgis-nuri/aws/sfc?date=",
"aws": "https://www.weather.go.kr/wgis-nuri/aws/aws?date=",
"air": f"{_KMA_BASE}/wgis-nuri/aws/air?date=",
"sfc": f"{_KMA_BASE}/wgis-nuri/aws/sfc?date=",
"aws": f"{_KMA_BASE}/wgis-nuri/aws/aws?date=",
}
src/data_collection/mgm_sources.py
+5 -5
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
import os
import unicodedata
from datetime import datetime, timedelta
from typing import Dict, Optional
@@ -20,10 +21,9 @@ class MgmSourceMixin:
从土耳其气象局 (MGM) 获取实时数据和预测 (由用户提供其内部 API)
"""
started = datetime.now()
base_url = "https://servis.mgm.gov.tr/web"
# 必须带 Origin，否则会被反爬拦截
base_url = os.getenv("MGM_BASE_URL", "").strip() or "https://servis.mgm.gov.tr/web"
headers = {
"Origin": "https://www.mgm.gov.tr",
"Origin": os.getenv("MGM_ORIGIN_URL", "").strip() or "https://www.mgm.gov.tr",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
}
results = {}
@@ -234,9 +234,9 @@ class MgmSourceMixin:
使用多线程辅助抓取，因为直接通过 il={province} 往往只返回 1 个站。
"""
started = datetime.now()
base_url = "https://servis.mgm.gov.tr/web"
base_url = os.getenv("MGM_BASE_URL", "").strip() or "https://servis.mgm.gov.tr/web"
headers = {
"Origin": "https://www.mgm.gov.tr",
"Origin": os.getenv("MGM_ORIGIN_URL", "").strip() or "https://www.mgm.gov.tr",
"User-Agent": "Mozilla/5.0",
}
import time
src/data_collection/nmc_sources.py
+10 -7
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
import os
import re
import time
from datetime import datetime
@@ -9,36 +10,38 @@ from loguru import logger
from src.utils.metrics import record_source_call
_NMC_FORECAST_BASE = os.getenv("NMC_FORECAST_BASE_URL", "").strip()
_NMC_REALTIME_BASE = os.getenv("NMC_REALTIME_BASE_URL", "").strip()
NMC_CITY_REFERENCES: Dict[str, Dict[str, Any]] = {
"shanghai": {
"region_label": "浦东",
"page_url": "https://m.nmc.cn/publish/forecast/ASH/pudong.html",
"page_url": f"{_NMC_FORECAST_BASE}/ASH/pudong.html" if _NMC_FORECAST_BASE else "",
"station_code": "atcMf",
},
"beijing": {
"region_label": "顺义",
"page_url": "https://m.nmc.cn/publish/forecast/ABJ/shunyi.html",
"page_url": f"{_NMC_FORECAST_BASE}/ABJ/shunyi.html" if _NMC_FORECAST_BASE else "",
"station_code": "MKoqG",
},
"chongqing": {
"region_label": "渝北",
"page_url": "https://m.nmc.cn/publish/forecast/ACQ/yubei.html",
"page_url": f"{_NMC_FORECAST_BASE}/ACQ/yubei.html" if _NMC_FORECAST_BASE else "",
"station_code": "xFVYU",
},
"chengdu": {
"region_label": "双流",
"page_url": "https://m.nmc.cn/publish/forecast/ASC/shuangliu.html",
"page_url": f"{_NMC_FORECAST_BASE}/ASC/shuangliu.html" if _NMC_FORECAST_BASE else "",
"station_code": "grFhZ",
},
"wuhan": {
"region_label": "武汉",
"page_url": "https://m.nmc.cn/publish/forecast/AHB/wuhan.html",
"page_url": f"{_NMC_FORECAST_BASE}/AHB/wuhan.html" if _NMC_FORECAST_BASE else "",
"station_code": "bSpCz",
},
"shenzhen": {
"region_label": "深圳",
"page_url": "https://m.nmc.cn/publish/forecast/AGD/shenzuo.html",
"page_url": f"{_NMC_FORECAST_BASE}/AGD/shenzuo.html" if _NMC_FORECAST_BASE else "",
"station_code": "AhpEU",
},
}
@@ -129,7 +132,7 @@ class NmcSourceMixin:
return None
try:
url = f"https://www.nmc.cn/rest/real/{station_code}"
url = f"{_NMC_REALTIME_BASE}/{station_code}"
payload = self._nmc_http_get_json(url)
if not isinstance(payload, dict) or not isinstance(payload.get("weather"), dict):
record_source_call("nmc", "current", "empty", (time.perf_counter() - started) * 1000.0)
src/data_collection/russia_station_sources.py
+3 -2
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import html
import math
import re
import os
import time
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional
@@ -207,7 +208,7 @@ class RussiaStationSourceMixin:
started = time.perf_counter()
try:
url = f"https://www.pogodaiklimat.ru/weather.php?id={station_code}"
url = f"{os.getenv("RUSSIA_POGODAIKLIMAT_BASE_URL", "").strip() or "https://www.pogodaiklimat.ru"}/weather.php?id={station_code}"
html_text = self._ru_http_get_text(url)
parsed = self._ru_parse_station_current_from_weather_html(html_text)
if not parsed:
@@ -267,7 +268,7 @@ class RussiaStationSourceMixin:
"is_official": True,
"is_airport_station": station_code == "27524",
"is_settlement_anchor": False,
"page_url": f"https://www.pogodaiklimat.ru/weather.php?id={station_code}",
"page_url": f"{os.getenv("RUSSIA_POGODAIKLIMAT_BASE_URL", "").strip() or "https://www.pogodaiklimat.ru"}/weather.php?id={station_code}",
}
with self._ru_station_cache_lock:
self._ru_station_cache[cache_key] = {"d": result, "t": now_ts}
src/data_collection/singapore_mss_sources.py
+2 -1
View File
@@ -10,6 +10,7 @@ URL: https://api.data.gov.sg/v1/environment/air-temperature
from __future__ import annotations
import os
import time
from datetime import datetime, timezone
from typing import Any, Dict, Optional
@@ -19,7 +20,7 @@ from loguru import logger
from src.utils.metrics import record_source_call
SINGAPORE_MSS_TEMP_URL = (
"https://api.data.gov.sg/v1/environment/air-temperature"
os.getenv("SINGAPORE_MSS_BASE_URL", "").strip() or "https://api.data.gov.sg/v1/environment/air-temperature"
)
# Preferred station: closest to Changi Airport (WSSS)