gavindiaz/PolyWeather
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

将商业核心数据源 URL 从源码迁移到环境变量，VPS .env 已补全

Browse Source
This commit is contained in:
2569718930@qq.com
2026-05-19 15:42:03 +08:00
parent 7e0d955d22
commit 3b8098051e
11 changed files with 57 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+16 -2
View File
@@ -63,8 +63,22 @@ POLYWEATHER_METAR_TIMEOUT_SEC=4
POLYWEATHER_METAR_CLUSTER_TIMEOUT_SEC=3.5 POLYWEATHER_METAR_CLUSTER_TIMEOUT_SEC=3.5
METAR_CACHE_TTL_SEC=600 METAR_CACHE_TTL_SEC=600
JMA_AMEDAS_CACHE_TTL_SEC=120 JMA_AMEDAS_CACHE_TTL_SEC=120
# Recommended on VPS: do not train there; pull the SQLite DB to a local machine.
# Optional: set this to a writable runtime path if you manually deploy a # ── Country-specific data source URLs ──
# These are kept in .env to avoid exposing competitive data-source discovery
# work on the public GitHub repository. Leave empty to use built-in defaults.
# AMSC_AWOS_BASE_URL=https://www.amsc.net.cn/gateway/api/saas/rest/amc/AwosController/getWindPlate
# NMC_FORECAST_BASE_URL=https://m.nmc.cn/publish/forecast
# NMC_REALTIME_BASE_URL=https://www.nmc.cn/rest/real
# KMA_BASE_URL=https://www.weather.go.kr
# AMOS_BASE_URL=https://global.amo.go.kr/amosobsnew/AmosRealTimeImage.do
# JMA_AMEDAS_BASE_URL=https://www.jma.go.jp
# MGM_BASE_URL=https://servis.mgm.gov.tr/web
# MGM_ORIGIN_URL=https://www.mgm.gov.tr
# FMI_BASE_URL=https://opendata.fmi.fi/wfs
# HKO_BASE_URL=https://data.weather.gov.hk/weatherAPI/hko_data/regional-weather
# SINGAPORE_MSS_BASE_URL=https://api.data.gov.sg/v1/environment/air-temperature
# RUSSIA_POGODAIKLIMAT_BASE_URL=https://www.pogodaiklimat.ru
######################################## ########################################
# 4) Auth / entitlement # 4) Auth / entitlement
src/data_collection/amos_station_sources.py
+2 -1
View File
@@ -7,6 +7,7 @@ Provides per-runway wind, temperature, pressure, visibility, RVR, cloud data.
from __future__ import annotations from __future__ import annotations
import re import re
import os
import time import time
from html import unescape from html import unescape
from datetime import datetime from datetime import datetime
@@ -16,7 +17,7 @@ from loguru import logger
from src.utils.metrics import record_source_call from src.utils.metrics import record_source_call
AMOS_BASE_URL = "https://global.amo.go.kr/amosobsnew/AmosRealTimeImage.do" AMOS_BASE_URL = os.getenv("AMOS_BASE_URL", "").strip() or "https://global.amo.go.kr/amosobsnew/AmosRealTimeImage.do"
AMOS_AIRPORT_QUERY_KEYS = ( AMOS_AIRPORT_QUERY_KEYS = (
"stnCd", "stnCd",
"icao", "icao",
src/data_collection/amsc_awos_sources.py
+2 -5
View File
@@ -21,10 +21,7 @@ from loguru import logger
from src.utils.metrics import record_source_call from src.utils.metrics import record_source_call
AMSC_AWOS_BASE_URL = ( AMSC_AWOS_BASE_URL = os.getenv("AMSC_AWOS_BASE_URL", "").strip()
"https://www.amsc.net.cn/gateway/api/saas/rest/amc/"
"AwosController/getWindPlate"
)
AMSC_AWOS_AIRPORTS: Dict[str, Dict[str, str]] = { AMSC_AWOS_AIRPORTS: Dict[str, Dict[str, str]] = {
"shanghai": {"icao": "ZSPD", "label": "Shanghai Pudong"}, "shanghai": {"icao": "ZSPD", "label": "Shanghai Pudong"},
@@ -176,7 +173,7 @@ class AmscAwosSourceMixin:
def _amsc_headers(self) -> Dict[str, str]: def _amsc_headers(self) -> Dict[str, str]:
headers = { headers = {
"Accept": "application/json, text/plain, */*", "Accept": "application/json, text/plain, */*",
"Referer": "https://www.amsc.net.cn/", "Referer": os.getenv("AMSC_AWOS_REFERER", "https://www.amsc.net.cn/"),
"User-Agent": ( "User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36" "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36"
src/data_collection/fmi_sources.py
+2 -1
View File
@@ -7,6 +7,7 @@ for Helsinki-Vantaa airport (EFHK, WMO 2974, FMISID 100968).
from __future__ import annotations from __future__ import annotations
import re import re
import os
import time import time
from datetime import datetime from datetime import datetime
from typing import Any, Dict, Optional from typing import Any, Dict, Optional
@@ -15,7 +16,7 @@ from loguru import logger
from src.utils.metrics import record_source_call from src.utils.metrics import record_source_call
FMI_BASE_URL = "https://opendata.fmi.fi/wfs" FMI_BASE_URL = os.getenv("FMI_BASE_URL", "").strip() or "https://opendata.fmi.fi/wfs"
FMI_STATION = { FMI_STATION = {
"helsinki": { "helsinki": {
"fmisid": "100968", "fmisid": "100968",
src/data_collection/hko_obs_sources.py
+2 -1
View File
@@ -8,6 +8,7 @@ No API key required.
from __future__ import annotations from __future__ import annotations
import csv import csv
import os
import io import io
import time import time
from datetime import datetime from datetime import datetime
@@ -17,7 +18,7 @@ from loguru import logger
from src.utils.metrics import record_source_call from src.utils.metrics import record_source_call
HKO_BASE_URL = "https://data.weather.gov.hk/weatherAPI/hko_data/regional-weather" HKO_BASE_URL = os.getenv("HKO_BASE_URL", "").strip() or "https://data.weather.gov.hk/weatherAPI/hko_data/regional-weather"
HKO_STATIONS = { HKO_STATIONS = {
"hong kong": { "hong kong": {
"code": "HK Observatory", "code": "HK Observatory",
src/data_collection/jma_amedas_sources.py
+5 -2
View File
@@ -1,5 +1,6 @@
from __future__ import annotations from __future__ import annotations
import os
import time import time
from datetime import datetime from datetime import datetime
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional
@@ -8,6 +9,8 @@ from loguru import logger
from src.utils.metrics import record_source_call from src.utils.metrics import record_source_call
_JMA_BASE = os.getenv("JMA_AMEDAS_BASE_URL", "").strip() or "https://www.jma.go.jp"
JMA_AMEDAS_STATIONS: Dict[str, Dict[str, Any]] = { JMA_AMEDAS_STATIONS: Dict[str, Dict[str, Any]] = {
"tokyo": { "tokyo": {
@@ -60,13 +63,13 @@ class JmaAmedasSourceMixin:
try: try:
latest_time_text = self._jma_http_get_text( latest_time_text = self._jma_http_get_text(
"https://www.jma.go.jp/bosai/amedas/data/latest_time.txt" f"{_JMA_BASE}/bosai/amedas/data/latest_time.txt"
).strip() ).strip()
latest_dt = datetime.fromisoformat(latest_time_text) latest_dt = datetime.fromisoformat(latest_time_text)
bucket_hour = (latest_dt.hour // 3) * 3 bucket_hour = (latest_dt.hour // 3) * 3
bucket_key = f"{latest_dt.strftime('%Y%m%d')}_{bucket_hour:02d}" bucket_key = f"{latest_dt.strftime('%Y%m%d')}_{bucket_hour:02d}"
station_code = str(meta.get("station_code") or "").strip() station_code = str(meta.get("station_code") or "").strip()
url = f"https://www.jma.go.jp/bosai/amedas/data/point/{station_code}/{bucket_key}.json" url = f"{_JMA_BASE}/bosai/amedas/data/point/{station_code}/{bucket_key}.json"
getter = getattr(self, "_http_get_json", None) getter = getattr(self, "_http_get_json", None)
if callable(getter): if callable(getter):
src/data_collection/kma_station_sources.py
+8 -6
View File
@@ -1,6 +1,7 @@
from __future__ import annotations from __future__ import annotations
import math import math
import os
import time import time
from datetime import datetime from datetime import datetime
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional
@@ -9,6 +10,7 @@ from loguru import logger
from src.utils.metrics import record_source_call from src.utils.metrics import record_source_call
_KMA_BASE = os.getenv("KMA_BASE_URL", "").strip() or "https://www.weather.go.kr"
KMA_CITY_STATIONS: Dict[str, Dict[str, Any]] = { KMA_CITY_STATIONS: Dict[str, Dict[str, Any]] = {
"busan": { "busan": {
@@ -24,15 +26,15 @@ KMA_CITY_STATIONS: Dict[str, Dict[str, Any]] = {
} }
KMA_GEOJSON_URLS: Dict[str, str] = { KMA_GEOJSON_URLS: Dict[str, str] = {
"air": "https://www.weather.go.kr/wgis-nuri/js/info/air.geojson", "air": f"{_KMA_BASE}/wgis-nuri/js/info/air.geojson",
"sfc": "https://www.weather.go.kr/wgis-nuri/js/info/sfc.geojson", "sfc": f"{_KMA_BASE}/wgis-nuri/js/info/sfc.geojson",
"aws": "https://www.weather.go.kr/wgis-nuri/js/info/aws.geojson", "aws": f"{_KMA_BASE}/wgis-nuri/js/info/aws.geojson",
} }
KMA_OBSERVATION_URLS: Dict[str, str] = { KMA_OBSERVATION_URLS: Dict[str, str] = {
"air": "https://www.weather.go.kr/wgis-nuri/aws/air?date=", "air": f"{_KMA_BASE}/wgis-nuri/aws/air?date=",
"sfc": "https://www.weather.go.kr/wgis-nuri/aws/sfc?date=", "sfc": f"{_KMA_BASE}/wgis-nuri/aws/sfc?date=",
"aws": "https://www.weather.go.kr/wgis-nuri/aws/aws?date=", "aws": f"{_KMA_BASE}/wgis-nuri/aws/aws?date=",
} }
src/data_collection/mgm_sources.py
+5 -5
View File
@@ -1,5 +1,6 @@
from __future__ import annotations from __future__ import annotations
import os
import unicodedata import unicodedata
from datetime import datetime, timedelta from datetime import datetime, timedelta
from typing import Dict, Optional from typing import Dict, Optional
@@ -20,10 +21,9 @@ class MgmSourceMixin:
从土耳其气象局 (MGM) 获取实时数据和预测 (由用户提供其内部 API) 从土耳其气象局 (MGM) 获取实时数据和预测 (由用户提供其内部 API)
""" """
started = datetime.now() started = datetime.now()
base_url = "https://servis.mgm.gov.tr/web" base_url = os.getenv("MGM_BASE_URL", "").strip() or "https://servis.mgm.gov.tr/web"
# 必须带 Origin，否则会被反爬拦截
headers = { headers = {
"Origin": "https://www.mgm.gov.tr", "Origin": os.getenv("MGM_ORIGIN_URL", "").strip() or "https://www.mgm.gov.tr",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
} }
results = {} results = {}
@@ -234,9 +234,9 @@ class MgmSourceMixin:
使用多线程辅助抓取，因为直接通过 il={province} 往往只返回 1 个站。 使用多线程辅助抓取，因为直接通过 il={province} 往往只返回 1 个站。
""" """
started = datetime.now() started = datetime.now()
base_url = "https://servis.mgm.gov.tr/web" base_url = os.getenv("MGM_BASE_URL", "").strip() or "https://servis.mgm.gov.tr/web"
headers = { headers = {
"Origin": "https://www.mgm.gov.tr", "Origin": os.getenv("MGM_ORIGIN_URL", "").strip() or "https://www.mgm.gov.tr",
"User-Agent": "Mozilla/5.0", "User-Agent": "Mozilla/5.0",
} }
import time import time
src/data_collection/nmc_sources.py
+10 -7
View File
@@ -1,5 +1,6 @@
from __future__ import annotations from __future__ import annotations
import os
import re import re
import time import time
from datetime import datetime from datetime import datetime
@@ -9,36 +10,38 @@ from loguru import logger
from src.utils.metrics import record_source_call from src.utils.metrics import record_source_call
_NMC_FORECAST_BASE = os.getenv("NMC_FORECAST_BASE_URL", "").strip()
_NMC_REALTIME_BASE = os.getenv("NMC_REALTIME_BASE_URL", "").strip()
NMC_CITY_REFERENCES: Dict[str, Dict[str, Any]] = { NMC_CITY_REFERENCES: Dict[str, Dict[str, Any]] = {
"shanghai": { "shanghai": {
"region_label": "浦东", "region_label": "浦东",
"page_url": "https://m.nmc.cn/publish/forecast/ASH/pudong.html", "page_url": f"{_NMC_FORECAST_BASE}/ASH/pudong.html" if _NMC_FORECAST_BASE else "",
"station_code": "atcMf", "station_code": "atcMf",
}, },
"beijing": { "beijing": {
"region_label": "顺义", "region_label": "顺义",
"page_url": "https://m.nmc.cn/publish/forecast/ABJ/shunyi.html", "page_url": f"{_NMC_FORECAST_BASE}/ABJ/shunyi.html" if _NMC_FORECAST_BASE else "",
"station_code": "MKoqG", "station_code": "MKoqG",
}, },
"chongqing": { "chongqing": {
"region_label": "渝北", "region_label": "渝北",
"page_url": "https://m.nmc.cn/publish/forecast/ACQ/yubei.html", "page_url": f"{_NMC_FORECAST_BASE}/ACQ/yubei.html" if _NMC_FORECAST_BASE else "",
"station_code": "xFVYU", "station_code": "xFVYU",
}, },
"chengdu": { "chengdu": {
"region_label": "双流", "region_label": "双流",
"page_url": "https://m.nmc.cn/publish/forecast/ASC/shuangliu.html", "page_url": f"{_NMC_FORECAST_BASE}/ASC/shuangliu.html" if _NMC_FORECAST_BASE else "",
"station_code": "grFhZ", "station_code": "grFhZ",
}, },
"wuhan": { "wuhan": {
"region_label": "武汉", "region_label": "武汉",
"page_url": "https://m.nmc.cn/publish/forecast/AHB/wuhan.html", "page_url": f"{_NMC_FORECAST_BASE}/AHB/wuhan.html" if _NMC_FORECAST_BASE else "",
"station_code": "bSpCz", "station_code": "bSpCz",
}, },
"shenzhen": { "shenzhen": {
"region_label": "深圳", "region_label": "深圳",
"page_url": "https://m.nmc.cn/publish/forecast/AGD/shenzuo.html", "page_url": f"{_NMC_FORECAST_BASE}/AGD/shenzuo.html" if _NMC_FORECAST_BASE else "",
"station_code": "AhpEU", "station_code": "AhpEU",
}, },
} }
@@ -129,7 +132,7 @@ class NmcSourceMixin:
return None return None
try: try:
url = f"https://www.nmc.cn/rest/real/{station_code}" url = f"{_NMC_REALTIME_BASE}/{station_code}"
payload = self._nmc_http_get_json(url) payload = self._nmc_http_get_json(url)
if not isinstance(payload, dict) or not isinstance(payload.get("weather"), dict): if not isinstance(payload, dict) or not isinstance(payload.get("weather"), dict):
record_source_call("nmc", "current", "empty", (time.perf_counter() - started) * 1000.0) record_source_call("nmc", "current", "empty", (time.perf_counter() - started) * 1000.0)
src/data_collection/russia_station_sources.py
+3 -2
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import html import html
import math import math
import re import re
import os
import time import time
from datetime import datetime, timezone from datetime import datetime, timezone
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional
@@ -207,7 +208,7 @@ class RussiaStationSourceMixin:
started = time.perf_counter() started = time.perf_counter()
try: try:
url = f"https://www.pogodaiklimat.ru/weather.php?id={station_code}" url = f"{os.getenv("RUSSIA_POGODAIKLIMAT_BASE_URL", "").strip() or "https://www.pogodaiklimat.ru"}/weather.php?id={station_code}"
html_text = self._ru_http_get_text(url) html_text = self._ru_http_get_text(url)
parsed = self._ru_parse_station_current_from_weather_html(html_text) parsed = self._ru_parse_station_current_from_weather_html(html_text)
if not parsed: if not parsed:
@@ -267,7 +268,7 @@ class RussiaStationSourceMixin:
"is_official": True, "is_official": True,
"is_airport_station": station_code == "27524", "is_airport_station": station_code == "27524",
"is_settlement_anchor": False, "is_settlement_anchor": False,
"page_url": f"https://www.pogodaiklimat.ru/weather.php?id={station_code}", "page_url": f"{os.getenv("RUSSIA_POGODAIKLIMAT_BASE_URL", "").strip() or "https://www.pogodaiklimat.ru"}/weather.php?id={station_code}",
} }
with self._ru_station_cache_lock: with self._ru_station_cache_lock:
self._ru_station_cache[cache_key] = {"d": result, "t": now_ts} self._ru_station_cache[cache_key] = {"d": result, "t": now_ts}
src/data_collection/singapore_mss_sources.py
+2 -1
View File
@@ -10,6 +10,7 @@ URL: https://api.data.gov.sg/v1/environment/air-temperature
from __future__ import annotations from __future__ import annotations
import os
import time import time
from datetime import datetime, timezone from datetime import datetime, timezone
from typing import Any, Dict, Optional from typing import Any, Dict, Optional
@@ -19,7 +20,7 @@ from loguru import logger
from src.utils.metrics import record_source_call from src.utils.metrics import record_source_call
SINGAPORE_MSS_TEMP_URL = ( SINGAPORE_MSS_TEMP_URL = (
"https://api.data.gov.sg/v1/environment/air-temperature" os.getenv("SINGAPORE_MSS_BASE_URL", "").strip() or "https://api.data.gov.sg/v1/environment/air-temperature"
) )
# Preferred station: closest to Changi Airport (WSSS) # Preferred station: closest to Changi Airport (WSSS)