From 3b8098051e34626fcc09a79d3ed976034833bb26 Mon Sep 17 00:00:00 2001 From: "2569718930@qq.com" <2569718930@qq.com> Date: Tue, 19 May 2026 15:42:03 +0800 Subject: [PATCH] =?UTF-8?q?=E5=B0=86=E5=95=86=E4=B8=9A=E6=A0=B8=E5=BF=83?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E6=BA=90=20URL=20=E4=BB=8E=E6=BA=90=E7=A0=81?= =?UTF-8?q?=E8=BF=81=E7=A7=BB=E5=88=B0=E7=8E=AF=E5=A2=83=E5=8F=98=E9=87=8F?= =?UTF-8?q?=EF=BC=8CVPS=20.env=20=E5=B7=B2=E8=A1=A5=E5=85=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 18 ++++++++++++++++-- src/data_collection/amos_station_sources.py | 3 ++- src/data_collection/amsc_awos_sources.py | 7 ++----- src/data_collection/fmi_sources.py | 3 ++- src/data_collection/hko_obs_sources.py | 3 ++- src/data_collection/jma_amedas_sources.py | 7 +++++-- src/data_collection/kma_station_sources.py | 14 ++++++++------ src/data_collection/mgm_sources.py | 10 +++++----- src/data_collection/nmc_sources.py | 17 ++++++++++------- src/data_collection/russia_station_sources.py | 5 +++-- src/data_collection/singapore_mss_sources.py | 3 ++- 11 files changed, 57 insertions(+), 33 deletions(-) diff --git a/.env.example b/.env.example index 1bdb56aa..2affa2d1 100644 --- a/.env.example +++ b/.env.example @@ -63,8 +63,22 @@ POLYWEATHER_METAR_TIMEOUT_SEC=4 POLYWEATHER_METAR_CLUSTER_TIMEOUT_SEC=3.5 METAR_CACHE_TTL_SEC=600 JMA_AMEDAS_CACHE_TTL_SEC=120 -# Recommended on VPS: do not train there; pull the SQLite DB to a local machine. -# Optional: set this to a writable runtime path if you manually deploy a + +# ── Country-specific data source URLs ── +# These are kept in .env to avoid exposing competitive data-source discovery +# work on the public GitHub repository. Leave empty to use built-in defaults. +# AMSC_AWOS_BASE_URL=https://www.amsc.net.cn/gateway/api/saas/rest/amc/AwosController/getWindPlate +# NMC_FORECAST_BASE_URL=https://m.nmc.cn/publish/forecast +# NMC_REALTIME_BASE_URL=https://www.nmc.cn/rest/real +# KMA_BASE_URL=https://www.weather.go.kr +# AMOS_BASE_URL=https://global.amo.go.kr/amosobsnew/AmosRealTimeImage.do +# JMA_AMEDAS_BASE_URL=https://www.jma.go.jp +# MGM_BASE_URL=https://servis.mgm.gov.tr/web +# MGM_ORIGIN_URL=https://www.mgm.gov.tr +# FMI_BASE_URL=https://opendata.fmi.fi/wfs +# HKO_BASE_URL=https://data.weather.gov.hk/weatherAPI/hko_data/regional-weather +# SINGAPORE_MSS_BASE_URL=https://api.data.gov.sg/v1/environment/air-temperature +# RUSSIA_POGODAIKLIMAT_BASE_URL=https://www.pogodaiklimat.ru ######################################## # 4) Auth / entitlement diff --git a/src/data_collection/amos_station_sources.py b/src/data_collection/amos_station_sources.py index a28fabf6..e73c4ad3 100644 --- a/src/data_collection/amos_station_sources.py +++ b/src/data_collection/amos_station_sources.py @@ -7,6 +7,7 @@ Provides per-runway wind, temperature, pressure, visibility, RVR, cloud data. from __future__ import annotations import re +import os import time from html import unescape from datetime import datetime @@ -16,7 +17,7 @@ from loguru import logger from src.utils.metrics import record_source_call -AMOS_BASE_URL = "https://global.amo.go.kr/amosobsnew/AmosRealTimeImage.do" +AMOS_BASE_URL = os.getenv("AMOS_BASE_URL", "").strip() or "https://global.amo.go.kr/amosobsnew/AmosRealTimeImage.do" AMOS_AIRPORT_QUERY_KEYS = ( "stnCd", "icao", diff --git a/src/data_collection/amsc_awos_sources.py b/src/data_collection/amsc_awos_sources.py index 87904b9c..e3164ce6 100644 --- a/src/data_collection/amsc_awos_sources.py +++ b/src/data_collection/amsc_awos_sources.py @@ -21,10 +21,7 @@ from loguru import logger from src.utils.metrics import record_source_call -AMSC_AWOS_BASE_URL = ( - "https://www.amsc.net.cn/gateway/api/saas/rest/amc/" - "AwosController/getWindPlate" -) +AMSC_AWOS_BASE_URL = os.getenv("AMSC_AWOS_BASE_URL", "").strip() AMSC_AWOS_AIRPORTS: Dict[str, Dict[str, str]] = { "shanghai": {"icao": "ZSPD", "label": "Shanghai Pudong"}, @@ -176,7 +173,7 @@ class AmscAwosSourceMixin: def _amsc_headers(self) -> Dict[str, str]: headers = { "Accept": "application/json, text/plain, */*", - "Referer": "https://www.amsc.net.cn/", + "Referer": os.getenv("AMSC_AWOS_REFERER", "https://www.amsc.net.cn/"), "User-Agent": ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36" diff --git a/src/data_collection/fmi_sources.py b/src/data_collection/fmi_sources.py index a64318d8..a46459c0 100644 --- a/src/data_collection/fmi_sources.py +++ b/src/data_collection/fmi_sources.py @@ -7,6 +7,7 @@ for Helsinki-Vantaa airport (EFHK, WMO 2974, FMISID 100968). from __future__ import annotations import re +import os import time from datetime import datetime from typing import Any, Dict, Optional @@ -15,7 +16,7 @@ from loguru import logger from src.utils.metrics import record_source_call -FMI_BASE_URL = "https://opendata.fmi.fi/wfs" +FMI_BASE_URL = os.getenv("FMI_BASE_URL", "").strip() or "https://opendata.fmi.fi/wfs" FMI_STATION = { "helsinki": { "fmisid": "100968", diff --git a/src/data_collection/hko_obs_sources.py b/src/data_collection/hko_obs_sources.py index 0ad52fe0..5371c0a4 100644 --- a/src/data_collection/hko_obs_sources.py +++ b/src/data_collection/hko_obs_sources.py @@ -8,6 +8,7 @@ No API key required. from __future__ import annotations import csv +import os import io import time from datetime import datetime @@ -17,7 +18,7 @@ from loguru import logger from src.utils.metrics import record_source_call -HKO_BASE_URL = "https://data.weather.gov.hk/weatherAPI/hko_data/regional-weather" +HKO_BASE_URL = os.getenv("HKO_BASE_URL", "").strip() or "https://data.weather.gov.hk/weatherAPI/hko_data/regional-weather" HKO_STATIONS = { "hong kong": { "code": "HK Observatory", diff --git a/src/data_collection/jma_amedas_sources.py b/src/data_collection/jma_amedas_sources.py index 65927ed5..4c33d095 100644 --- a/src/data_collection/jma_amedas_sources.py +++ b/src/data_collection/jma_amedas_sources.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os import time from datetime import datetime from typing import Any, Dict, List, Optional @@ -8,6 +9,8 @@ from loguru import logger from src.utils.metrics import record_source_call +_JMA_BASE = os.getenv("JMA_AMEDAS_BASE_URL", "").strip() or "https://www.jma.go.jp" + JMA_AMEDAS_STATIONS: Dict[str, Dict[str, Any]] = { "tokyo": { @@ -60,13 +63,13 @@ class JmaAmedasSourceMixin: try: latest_time_text = self._jma_http_get_text( - "https://www.jma.go.jp/bosai/amedas/data/latest_time.txt" + f"{_JMA_BASE}/bosai/amedas/data/latest_time.txt" ).strip() latest_dt = datetime.fromisoformat(latest_time_text) bucket_hour = (latest_dt.hour // 3) * 3 bucket_key = f"{latest_dt.strftime('%Y%m%d')}_{bucket_hour:02d}" station_code = str(meta.get("station_code") or "").strip() - url = f"https://www.jma.go.jp/bosai/amedas/data/point/{station_code}/{bucket_key}.json" + url = f"{_JMA_BASE}/bosai/amedas/data/point/{station_code}/{bucket_key}.json" getter = getattr(self, "_http_get_json", None) if callable(getter): diff --git a/src/data_collection/kma_station_sources.py b/src/data_collection/kma_station_sources.py index 25195de0..6078a7bf 100644 --- a/src/data_collection/kma_station_sources.py +++ b/src/data_collection/kma_station_sources.py @@ -1,6 +1,7 @@ from __future__ import annotations import math +import os import time from datetime import datetime from typing import Any, Dict, List, Optional @@ -9,6 +10,7 @@ from loguru import logger from src.utils.metrics import record_source_call +_KMA_BASE = os.getenv("KMA_BASE_URL", "").strip() or "https://www.weather.go.kr" KMA_CITY_STATIONS: Dict[str, Dict[str, Any]] = { "busan": { @@ -24,15 +26,15 @@ KMA_CITY_STATIONS: Dict[str, Dict[str, Any]] = { } KMA_GEOJSON_URLS: Dict[str, str] = { - "air": "https://www.weather.go.kr/wgis-nuri/js/info/air.geojson", - "sfc": "https://www.weather.go.kr/wgis-nuri/js/info/sfc.geojson", - "aws": "https://www.weather.go.kr/wgis-nuri/js/info/aws.geojson", + "air": f"{_KMA_BASE}/wgis-nuri/js/info/air.geojson", + "sfc": f"{_KMA_BASE}/wgis-nuri/js/info/sfc.geojson", + "aws": f"{_KMA_BASE}/wgis-nuri/js/info/aws.geojson", } KMA_OBSERVATION_URLS: Dict[str, str] = { - "air": "https://www.weather.go.kr/wgis-nuri/aws/air?date=", - "sfc": "https://www.weather.go.kr/wgis-nuri/aws/sfc?date=", - "aws": "https://www.weather.go.kr/wgis-nuri/aws/aws?date=", + "air": f"{_KMA_BASE}/wgis-nuri/aws/air?date=", + "sfc": f"{_KMA_BASE}/wgis-nuri/aws/sfc?date=", + "aws": f"{_KMA_BASE}/wgis-nuri/aws/aws?date=", } diff --git a/src/data_collection/mgm_sources.py b/src/data_collection/mgm_sources.py index 447815ee..67e42b48 100644 --- a/src/data_collection/mgm_sources.py +++ b/src/data_collection/mgm_sources.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os import unicodedata from datetime import datetime, timedelta from typing import Dict, Optional @@ -20,10 +21,9 @@ class MgmSourceMixin: 从土耳其气象局 (MGM) 获取实时数据和预测 (由用户提供其内部 API) """ started = datetime.now() - base_url = "https://servis.mgm.gov.tr/web" - # 必须带 Origin，否则会被反爬拦截 + base_url = os.getenv("MGM_BASE_URL", "").strip() or "https://servis.mgm.gov.tr/web" headers = { - "Origin": "https://www.mgm.gov.tr", + "Origin": os.getenv("MGM_ORIGIN_URL", "").strip() or "https://www.mgm.gov.tr", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", } results = {} @@ -234,9 +234,9 @@ class MgmSourceMixin: 使用多线程辅助抓取，因为直接通过 il={province} 往往只返回 1 个站。 """ started = datetime.now() - base_url = "https://servis.mgm.gov.tr/web" + base_url = os.getenv("MGM_BASE_URL", "").strip() or "https://servis.mgm.gov.tr/web" headers = { - "Origin": "https://www.mgm.gov.tr", + "Origin": os.getenv("MGM_ORIGIN_URL", "").strip() or "https://www.mgm.gov.tr", "User-Agent": "Mozilla/5.0", } import time diff --git a/src/data_collection/nmc_sources.py b/src/data_collection/nmc_sources.py index 22390a23..c0f46e47 100644 --- a/src/data_collection/nmc_sources.py +++ b/src/data_collection/nmc_sources.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os import re import time from datetime import datetime @@ -9,36 +10,38 @@ from loguru import logger from src.utils.metrics import record_source_call +_NMC_FORECAST_BASE = os.getenv("NMC_FORECAST_BASE_URL", "").strip() +_NMC_REALTIME_BASE = os.getenv("NMC_REALTIME_BASE_URL", "").strip() NMC_CITY_REFERENCES: Dict[str, Dict[str, Any]] = { "shanghai": { "region_label": "浦东", - "page_url": "https://m.nmc.cn/publish/forecast/ASH/pudong.html", + "page_url": f"{_NMC_FORECAST_BASE}/ASH/pudong.html" if _NMC_FORECAST_BASE else "", "station_code": "atcMf", }, "beijing": { "region_label": "顺义", - "page_url": "https://m.nmc.cn/publish/forecast/ABJ/shunyi.html", + "page_url": f"{_NMC_FORECAST_BASE}/ABJ/shunyi.html" if _NMC_FORECAST_BASE else "", "station_code": "MKoqG", }, "chongqing": { "region_label": "渝北", - "page_url": "https://m.nmc.cn/publish/forecast/ACQ/yubei.html", + "page_url": f"{_NMC_FORECAST_BASE}/ACQ/yubei.html" if _NMC_FORECAST_BASE else "", "station_code": "xFVYU", }, "chengdu": { "region_label": "双流", - "page_url": "https://m.nmc.cn/publish/forecast/ASC/shuangliu.html", + "page_url": f"{_NMC_FORECAST_BASE}/ASC/shuangliu.html" if _NMC_FORECAST_BASE else "", "station_code": "grFhZ", }, "wuhan": { "region_label": "武汉", - "page_url": "https://m.nmc.cn/publish/forecast/AHB/wuhan.html", + "page_url": f"{_NMC_FORECAST_BASE}/AHB/wuhan.html" if _NMC_FORECAST_BASE else "", "station_code": "bSpCz", }, "shenzhen": { "region_label": "深圳", - "page_url": "https://m.nmc.cn/publish/forecast/AGD/shenzuo.html", + "page_url": f"{_NMC_FORECAST_BASE}/AGD/shenzuo.html" if _NMC_FORECAST_BASE else "", "station_code": "AhpEU", }, } @@ -129,7 +132,7 @@ class NmcSourceMixin: return None try: - url = f"https://www.nmc.cn/rest/real/{station_code}" + url = f"{_NMC_REALTIME_BASE}/{station_code}" payload = self._nmc_http_get_json(url) if not isinstance(payload, dict) or not isinstance(payload.get("weather"), dict): record_source_call("nmc", "current", "empty", (time.perf_counter() - started) * 1000.0) diff --git a/src/data_collection/russia_station_sources.py b/src/data_collection/russia_station_sources.py index f300d74f..a8a49a0c 100644 --- a/src/data_collection/russia_station_sources.py +++ b/src/data_collection/russia_station_sources.py @@ -3,6 +3,7 @@ from __future__ import annotations import html import math import re +import os import time from datetime import datetime, timezone from typing import Any, Dict, List, Optional @@ -207,7 +208,7 @@ class RussiaStationSourceMixin: started = time.perf_counter() try: - url = f"https://www.pogodaiklimat.ru/weather.php?id={station_code}" + url = f"{os.getenv("RUSSIA_POGODAIKLIMAT_BASE_URL", "").strip() or "https://www.pogodaiklimat.ru"}/weather.php?id={station_code}" html_text = self._ru_http_get_text(url) parsed = self._ru_parse_station_current_from_weather_html(html_text) if not parsed: @@ -267,7 +268,7 @@ class RussiaStationSourceMixin: "is_official": True, "is_airport_station": station_code == "27524", "is_settlement_anchor": False, - "page_url": f"https://www.pogodaiklimat.ru/weather.php?id={station_code}", + "page_url": f"{os.getenv("RUSSIA_POGODAIKLIMAT_BASE_URL", "").strip() or "https://www.pogodaiklimat.ru"}/weather.php?id={station_code}", } with self._ru_station_cache_lock: self._ru_station_cache[cache_key] = {"d": result, "t": now_ts} diff --git a/src/data_collection/singapore_mss_sources.py b/src/data_collection/singapore_mss_sources.py index 760a106c..57f9b50b 100644 --- a/src/data_collection/singapore_mss_sources.py +++ b/src/data_collection/singapore_mss_sources.py @@ -10,6 +10,7 @@ URL: https://api.data.gov.sg/v1/environment/air-temperature from __future__ import annotations +import os import time from datetime import datetime, timezone from typing import Any, Dict, Optional @@ -19,7 +20,7 @@ from loguru import logger from src.utils.metrics import record_source_call SINGAPORE_MSS_TEMP_URL = ( - "https://api.data.gov.sg/v1/environment/air-temperature" + os.getenv("SINGAPORE_MSS_BASE_URL", "").strip() or "https://api.data.gov.sg/v1/environment/air-temperature" ) # Preferred station: closest to Changi Airport (WSSS)