mirror of
https://github.com/shy3130/tick-stock-panel.git
synced 2026-11-02 10:26:13 +00:00
shy3130
a40c0780d7
用户反馈资金流扩展表点「手动拉取」报错、板块轮动的资金流下拉为空。 排查出三个独立缺陷: 1. 日期一致性校验用错判据 (ext_pull.py) 报错: 「接口返回的日期与请求日期不一致, 已拒绝写入该分区」。 _assert_rows_date 无条件校验响应 date 必须等于请求日期, 但该契约的 目的只是防「接口忽略 ?date= 却把当日值写进历史分区」, 仅对按日查询 接口有意义。资金流/人气等当日快照接口只有最新一个交易日的数据, 周末/节假日/数据源 T+1 时响应日期本就落后于请求日期, 被误判成 「接口忽略日期参数」而整表拒写。 修复: 新增 _assert_rows_date_if_dated, 仅在 pull.date_param 已配置 时校验; 历史回补的 fail-closed 防污染契约不变。 2. schema 端点一直抛异常走兜底 (api/ext_data.py) 「SELECT column_name, data_type FROM (DESCRIBE ...)」在 DuckDB 1.5 抛 BinderException —— 该版本输出列名是 column_type (1.4 及以前才是 data_type)。异常被吞后回退到配置字段, 那些对象只有 dtype 没有 type, 前端按 type 判断数值列的资金流下拉因此永远为空。 修复: 按实际输出列名取值, 两个版本都可用; 兜底分支键名统一为 type。 同时关掉 hive 分区推断 —— date=YYYY-MM-DD 目录会让 DuckDB 凭空多出 一个 date 列, 污染列选择器。 前端 NUMERIC_TYPES 由写死的 5 个简写改为类型族匹配, 覆盖 DOUBLE / BIGINT / DECIMAL 等 Parquet 真实类型名。 3. 分页配置保存后被静默丢弃 (api/ext_data.py) PullConfigReq 缺少 page_param / page_size_param / page_size / page_start / max_pages, 前端「拉取设置」提交这些值后 Pydantic 直接 忽略, 保存后再打开退回默认的单次请求, 且不报错。 修复: 补齐请求模型字段并透传到 PullConfig。 验证: 三处均先写复现用例 (未修复代码上 7 个用例失败), 修复后全绿; 扩展相关后端测试 202 passed; 前端 SectorRotationCard 7 passed; pnpm build 通过; 真实接口验证 —— 修复前 400「日期不一致」, 修复后 200 写入 5580 行, schema-all 返回 DOUBLE/BIGINT/VARCHAR 真实类型。 Signed-off-by: shy3130 <shy3130@qq.com>
421 lines
19 KiB
Python
421 lines
19 KiB
Python
"""扩展数据历史回补测试 — date_param 按日拉取 / 日期一致性防污染 / 幂等回补。
|
|
|
|
背景: 人气排行等时序扩展表历史只能从开启拉取之日起累积; backfill_history
|
|
按本地交易日逐日回补。金融契约重点: 接口忽略日期参数返回当日数据时
|
|
必须拒写历史分区 (实测确有此类接口), 否则整个时序口径错乱。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from datetime import date, datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
from typing import ClassVar
|
|
|
|
import httpx
|
|
import polars as pl
|
|
import pytest
|
|
|
|
from app.services import ext_pull
|
|
from app.services.ext_data import ExtConfig, ExtConfigStore, ExtField, PullConfig
|
|
from app.services.ext_pull import (
|
|
_assert_rows_date,
|
|
_with_date_param,
|
|
backfill_history,
|
|
fetch_rows_for_date,
|
|
)
|
|
|
|
|
|
def _cfg(mode: str = "timeseries", date_param: str | None = "date") -> ExtConfig:
|
|
return ExtConfig(
|
|
id="hot", label="人气", mode=mode,
|
|
fields=[
|
|
ExtField("symbol", "string"), ExtField("rank", "int"),
|
|
ExtField("date", "string"), ExtField("heat", "float"),
|
|
],
|
|
pull=PullConfig(url="https://example.test/rank", date_param=date_param),
|
|
)
|
|
|
|
|
|
def _row(sym: str, day: str) -> dict:
|
|
return {"symbol": sym, "rank": 1, "date": day, "heat": 9.9}
|
|
|
|
|
|
class _FakeResp:
|
|
def __init__(self, payload):
|
|
self._payload = payload
|
|
|
|
def raise_for_status(self) -> None:
|
|
pass
|
|
|
|
def json(self):
|
|
return self._payload
|
|
|
|
|
|
class _FakeClient:
|
|
"""按请求 URL 中的日期参数返回预置数据; 记录全部请求 URL。
|
|
|
|
responses/calls 用函数属性 (class-annotation) 而非实例属性:
|
|
ext_pull 以 ``httpx.AsyncClient(**kw)`` 工厂方式构造, fixture 借类属性
|
|
注入预置数据, RUF012 mutable-default 面由此声明为 ClassVar 语义。
|
|
"""
|
|
|
|
responses: ClassVar[dict[str, list]] = {}
|
|
calls: ClassVar[list[str]] = []
|
|
header_calls: ClassVar[list[dict]] = [] # 每次请求实际发送的 headers
|
|
errors: ClassVar[dict[str, Exception]] = {}
|
|
fail_times: ClassVar[dict[str, int]] = {} # url -> 还需失败的次数
|
|
error_sequence: ClassVar[dict[str, list[Exception]]] = {} # url -> 按序抛出后耗尽
|
|
|
|
def __init__(self, **kwargs):
|
|
pass
|
|
|
|
async def __aenter__(self):
|
|
return self
|
|
|
|
async def __aexit__(self, *exc):
|
|
return False
|
|
|
|
async def request(self, method: str, url: str, **kwargs):
|
|
_FakeClient.calls.append(url)
|
|
_FakeClient.header_calls.append(dict(kwargs.get("headers") or {}))
|
|
seq = _FakeClient.error_sequence.get(url)
|
|
if seq:
|
|
raise seq.pop(0)
|
|
if url in _FakeClient.errors and _FakeClient.fail_times.get(url, 1) > 0:
|
|
_FakeClient.fail_times[url] = _FakeClient.fail_times.get(url, 1) - 1
|
|
raise _FakeClient.errors[url]
|
|
return _FakeResp(_FakeClient.responses.get(url, []))
|
|
|
|
|
|
@pytest.fixture()
|
|
def fake_http(monkeypatch):
|
|
_FakeClient.responses = {}
|
|
_FakeClient.calls = []
|
|
_FakeClient.header_calls = []
|
|
_FakeClient.errors = {}
|
|
_FakeClient.fail_times = {}
|
|
_FakeClient.error_sequence = {}
|
|
monkeypatch.setattr(ext_pull.httpx, "AsyncClient", _FakeClient)
|
|
return _FakeClient
|
|
|
|
|
|
# ── 纯函数 ────────────────────────────────────────────────
|
|
|
|
def test_outbound_headers_default_and_override():
|
|
"""出站标识头: 默认带 tsp UA + X-TSP-Client; 用户同名头优先 (大小写不敏感)。"""
|
|
from app.services.ext_pull import outbound_headers
|
|
|
|
h = outbound_headers()
|
|
assert h["User-Agent"].startswith("tsp/")
|
|
assert h["X-TSP-Client"] == "tick-stock-panel"
|
|
|
|
h2 = outbound_headers({"user-agent": "my-ua", "X-Custom": "1"})
|
|
assert h2["user-agent"] == "my-ua" # 小写同名覆盖默认 UA
|
|
assert "User-Agent" not in h2 # 不重复发送
|
|
assert h2["X-TSP-Client"] == "tick-stock-panel" # 未覆盖的标识头保留
|
|
assert h2["X-Custom"] == "1"
|
|
|
|
|
|
async def test_fetch_rows_carries_tsp_identity(fake_http):
|
|
fake_http.responses["https://example.test/rank?date=2026-01-05"] = [_row("A", "2026-01-05")]
|
|
await fetch_rows_for_date(_cfg(), date(2026, 1, 5))
|
|
headers = fake_http.header_calls[-1]
|
|
assert headers["User-Agent"].startswith("tsp/")
|
|
assert headers["X-TSP-Client"] == "tick-stock-panel"
|
|
|
|
|
|
async def test_fetch_rows_user_headers_take_precedence(fake_http):
|
|
cfg = ExtConfig(
|
|
id="hot", label="人气", mode="timeseries",
|
|
fields=_cfg().fields,
|
|
pull=PullConfig(
|
|
url="https://example.test/rank",
|
|
headers={"User-Agent": "custom-ua"},
|
|
date_param="date",
|
|
),
|
|
)
|
|
fake_http.responses["https://example.test/rank?date=2026-01-05"] = [_row("A", "2026-01-05")]
|
|
await fetch_rows_for_date(cfg, date(2026, 1, 5))
|
|
headers = fake_http.header_calls[-1]
|
|
assert headers["User-Agent"] == "custom-ua"
|
|
assert headers["X-TSP-Client"] == "tick-stock-panel"
|
|
|
|
|
|
def test_with_date_param_url_building():
|
|
d = date(2026, 1, 5)
|
|
assert _with_date_param("https://x/api", "date", d) == "https://x/api?date=2026-01-05"
|
|
assert _with_date_param("https://x/api?a=1", "date", d) == "https://x/api?a=1&date=2026-01-05"
|
|
assert _with_date_param("https://x/api", None, d) == "https://x/api"
|
|
|
|
|
|
def test_with_date_param_formats():
|
|
"""date_format 分支: compact=YYYYMMDD; ts_s/ts_ms = 该交易日北京时间 00:00:00 时间戳。"""
|
|
d = date(2026, 1, 5)
|
|
assert _with_date_param("https://x/api", "date", d, "compact") == "https://x/api?date=20260105"
|
|
ts_s = int(datetime(2026, 1, 5, tzinfo=timezone(timedelta(hours=8))).timestamp())
|
|
assert _with_date_param("https://x/api", "ts", d, "ts_s") == f"https://x/api?ts={ts_s}"
|
|
assert _with_date_param("https://x/api", "ts", d, "ts_ms") == f"https://x/api?ts={ts_s * 1000}"
|
|
# 未知格式回退 iso
|
|
assert _with_date_param("https://x/api", "date", d, "bogus") == "https://x/api?date=2026-01-05"
|
|
|
|
|
|
def test_pull_config_date_format_roundtrip_and_normalize():
|
|
"""date_format 配置往返保留; 手改 config.json 写入非法值时归一为 iso。"""
|
|
cfg = PullConfig(url="https://x/api", date_param="ts", date_format="ts_ms")
|
|
assert cfg.date_format == "ts_ms"
|
|
restored = PullConfig.from_dict(cfg.to_dict())
|
|
assert restored.date_format == "ts_ms"
|
|
assert PullConfig(date_param="date", date_format="bogus").date_format == "iso"
|
|
assert PullConfig.from_dict({"date_param": "date"}).date_format == "iso" # 旧配置缺字段
|
|
|
|
|
|
def test_assert_rows_date_contract():
|
|
d = date(2026, 1, 5)
|
|
_assert_rows_date([_row("A", "2026-01-05")], d) # 一致
|
|
_assert_rows_date([_row("A", "2026-01-05 00:00:00")], d) # 带时间前缀
|
|
_assert_rows_date([{"symbol": "A"}], d) # 无 date 字段: 不校验
|
|
with pytest.raises(ValueError, match="不一致"):
|
|
_assert_rows_date([_row("A", "2026-09-06")], d) # 接口忽略参数返回当日
|
|
|
|
|
|
async def test_snapshot_api_without_date_param_accepts_previous_trading_day(fake_http):
|
|
"""当日快照接口 (未配 date_param) 在周末返回上一交易日数据, 不得拒写。
|
|
|
|
回归: 资金流/人气类接口只有当日快照, 周末/节假日/数据源 T+1 时响应日期
|
|
落后于请求日期是正常现象。无 date_param 却做日期一致性校验会把整表拉取
|
|
全部拒掉 (报「接口可能不支持日期参数」)。
|
|
"""
|
|
cfg = ExtConfig(
|
|
id="flow", label="资金流", mode="timeseries",
|
|
fields=[ExtField("symbol", "string"), ExtField("date", "string"),
|
|
ExtField("net", "float")],
|
|
pull=PullConfig(url="https://example.test/flow"), # date_param 未配置
|
|
)
|
|
# 请求 2026-10-24 (周六), 接口返回 2026-10-23 (周五) 的快照
|
|
fake_http.responses["https://example.test/flow"] = [
|
|
{"symbol": "600000.SH", "date": "2026-10-23", "net": 1.0},
|
|
]
|
|
rows = await fetch_rows_for_date(cfg, date(2026, 10, 24))
|
|
assert [r["symbol"] for r in rows] == ["600000.SH"]
|
|
|
|
|
|
async def test_dated_api_still_rejects_mismatched_date(fake_http):
|
|
"""配了 date_param 的接口保持 fail-closed: 历史分区防污染契约不能松。"""
|
|
fake_http.responses["https://example.test/rank?date=2026-01-05"] = [_row("A", "2026-09-06")]
|
|
with pytest.raises(ValueError, match="不一致"):
|
|
await fetch_rows_for_date(_cfg(), date(2026, 1, 5))
|
|
|
|
|
|
async def test_market_level_snapshot_without_date_param_accepts_stale_date(fake_http):
|
|
"""市场级表同样按 date_param 判定, 不因 market_level 分支绕开修复。"""
|
|
cfg = ExtConfig(
|
|
id="regime", label="情绪", mode="timeseries",
|
|
fields=[ExtField("date", "string"), ExtField("score", "float")],
|
|
pull=PullConfig(url="https://example.test/regime"),
|
|
market_level=True,
|
|
)
|
|
fake_http.responses["https://example.test/regime"] = [
|
|
{"date": "2026-10-23", "score": 55.0},
|
|
]
|
|
rows = await fetch_rows_for_date(cfg, date(2026, 10, 24))
|
|
assert rows == [{"date": "2026-10-23", "score": 55.0}]
|
|
|
|
|
|
def test_pull_config_date_param_roundtrip():
|
|
p = PullConfig(url="u", date_param="date")
|
|
assert p.to_dict()["date_param"] == "date"
|
|
legacy = PullConfig.from_dict({"url": "u"}) # 旧 JSON 无该键
|
|
assert legacy.date_param is None
|
|
|
|
|
|
# ── fetch_rows_for_date ───────────────────────────────────
|
|
|
|
async def test_fetch_rows_builds_dated_url(fake_http):
|
|
fake_http.responses["https://example.test/rank?date=2026-01-05"] = [_row("A", "2026-01-05")]
|
|
rows = await fetch_rows_for_date(_cfg(), date(2026, 1, 5))
|
|
assert rows and rows[0]["symbol"] == "A"
|
|
assert fake_http.calls == ["https://example.test/rank?date=2026-01-05"]
|
|
|
|
|
|
async def test_fetch_rows_rejects_mismatched_date(fake_http):
|
|
# 接口忽略 ?date= 返回当日数据 → 必须拒收, 不给 backfill 写历史分区的机会
|
|
fake_http.responses["https://example.test/rank?date=2026-01-05"] = [_row("A", "2026-09-06")]
|
|
with pytest.raises(ValueError, match="不一致"):
|
|
await fetch_rows_for_date(_cfg(), date(2026, 1, 5))
|
|
|
|
|
|
async def test_fetch_rows_empty_returns_empty_list(fake_http):
|
|
assert await fetch_rows_for_date(_cfg(), date(2026, 1, 5)) == []
|
|
|
|
|
|
async def test_fetch_rows_requires_symbol_or_code(fake_http):
|
|
fake_http.responses["https://example.test/rank?date=2026-01-05"] = [{"rank": 1}]
|
|
with pytest.raises(ValueError, match="symbol/code"):
|
|
await fetch_rows_for_date(_cfg(), date(2026, 1, 5))
|
|
|
|
|
|
# ── backfill_history ──────────────────────────────────────
|
|
|
|
def _seed_trading_days(data_dir: Path, *days: str) -> None:
|
|
root = data_dir / "kline_daily"
|
|
for d in days:
|
|
(root / f"date={d}").mkdir(parents=True, exist_ok=True)
|
|
|
|
|
|
async def test_backfill_rejects_snapshot_and_missing_date_param(tmp_path):
|
|
with pytest.raises(ValueError, match="timeseries"):
|
|
await backfill_history(_cfg(mode="snapshot"), tmp_path, date(2026, 1, 5), date(2026, 1, 9))
|
|
with pytest.raises(ValueError, match="date_param"):
|
|
await backfill_history(_cfg(date_param=None), tmp_path, date(2026, 1, 5), date(2026, 1, 9))
|
|
|
|
|
|
async def test_backfill_rejects_bad_range_and_no_trading_days(tmp_path):
|
|
with pytest.raises(ValueError, match="晚于"):
|
|
await backfill_history(_cfg(), tmp_path, date(2026, 1, 9), date(2026, 1, 5))
|
|
with pytest.raises(ValueError, match="上限"):
|
|
await backfill_history(_cfg(), tmp_path, date(2025, 1, 1), date(2026, 9, 1))
|
|
with pytest.raises(ValueError, match="交易日"):
|
|
await backfill_history(_cfg(), tmp_path, date(2026, 1, 5), date(2026, 1, 9))
|
|
|
|
|
|
async def test_backfill_writes_partitions_and_is_idempotent(tmp_path, fake_http):
|
|
_seed_trading_days(tmp_path, "2026-01-02", "2026-01-05", "2026-01-06")
|
|
# 01-02: 已有分区 → 跳过; 01-05: 接口有数据 → 写入; 01-06: 接口空 → empty
|
|
part_102 = tmp_path / "ext_data" / "hot" / "timeseries" / "date=2026-01-02" / "part.parquet"
|
|
part_102.parent.mkdir(parents=True)
|
|
part_102.write_bytes(b"x")
|
|
fake_http.responses["https://example.test/rank?date=2026-01-05"] = [
|
|
_row("600000.SH", "2026-01-05"), _row("000001.SZ", "2026-01-05"),
|
|
]
|
|
|
|
result = await backfill_history(_cfg(), tmp_path, date(2026, 1, 1), date(2026, 1, 9))
|
|
assert result["total_days"] == 3
|
|
assert result["fetched"] == 1 and result["rows_written"] == 2
|
|
assert result["skipped_existing"] == 1
|
|
assert result["empty"] == 1 and result["failed"] == []
|
|
written = tmp_path / "ext_data" / "hot" / "timeseries" / "date=2026-01-05" / "part.parquet"
|
|
assert written.exists()
|
|
|
|
# 幂等: 已写入的分区不再请求; empty 日 (无分区文件) 允许重试
|
|
fake_http.calls.clear()
|
|
again = await backfill_history(_cfg(), tmp_path, date(2026, 1, 1), date(2026, 1, 9))
|
|
assert again["skipped_existing"] == 2 and again["fetched"] == 0 and again["empty"] == 1
|
|
assert fake_http.calls == ["https://example.test/rank?date=2026-01-06"]
|
|
|
|
|
|
async def test_backfill_collects_failures_without_abort(tmp_path, fake_http):
|
|
_seed_trading_days(tmp_path, "2026-01-05", "2026-01-06")
|
|
# 01-05: 接口忽略参数返回当日 → 拒写 (failed); 01-06: 正常写入
|
|
fake_http.responses["https://example.test/rank?date=2026-01-05"] = [_row("A", "2026-09-06")]
|
|
fake_http.responses["https://example.test/rank?date=2026-01-06"] = [_row("A", "2026-01-06")]
|
|
|
|
result = await backfill_history(_cfg(), tmp_path, date(2026, 1, 5), date(2026, 1, 6))
|
|
assert result["fetched"] == 1
|
|
assert [f["date"] for f in result["failed"]] == ["2026-01-05"]
|
|
assert "不一致" in result["failed"][0]["reason"]
|
|
assert not (tmp_path / "ext_data/hot/timeseries/date=2026-01-05").exists()
|
|
|
|
|
|
# ── 与扩展消费链路的衔接 ─────────────────────────────────
|
|
|
|
async def test_backfilled_partition_feeds_signal_frame(tmp_path, fake_http):
|
|
"""回补落盘的历史分区, ext_factors 按日对齐立即可用 (PIT)。"""
|
|
from app.factors import ext_factors
|
|
|
|
ExtConfigStore(tmp_path).upsert(_cfg())
|
|
_seed_trading_days(tmp_path, "2026-01-05", "2026-01-06")
|
|
fake_http.responses["https://example.test/rank?date=2026-01-05"] = [
|
|
{"symbol": "600000.SH", "rank": 3, "date": "2026-01-05", "heat": 88.0},
|
|
]
|
|
fake_http.responses["https://example.test/rank?date=2026-01-06"] = [
|
|
{"symbol": "600000.SH", "rank": 1, "date": "2026-01-06", "heat": 99.0},
|
|
]
|
|
await backfill_history(_cfg(), tmp_path, date(2026, 1, 5), date(2026, 1, 6))
|
|
|
|
frame = pl.DataFrame(
|
|
{"symbol": ["600000.SH", "600000.SH"], "date": ["2026-01-05", "2026-01-06"]},
|
|
schema={"symbol": pl.Utf8, "date": pl.Utf8},
|
|
)
|
|
out = ext_factors.attach_ext_columns(frame, include_snapshot=False, data_dir=tmp_path)
|
|
assert out["ext_hot_rank"].to_list() == [3.0, 1.0] # rank int → Float64
|
|
assert out["ext_hot_heat"].to_list() == [88.0, 99.0] # 每日各自的值, 无串日
|
|
|
|
|
|
# ── 429 限流退避与中止 ─────────────────────
|
|
|
|
|
|
def _err_429(url: str) -> httpx.HTTPStatusError:
|
|
req = httpx.Request("GET", url)
|
|
return httpx.HTTPStatusError("429", request=req, response=httpx.Response(429, request=req))
|
|
|
|
|
|
async def test_backfill_429_retries_once_and_succeeds(tmp_path, fake_http, monkeypatch):
|
|
monkeypatch.setattr(ext_pull, "_BACKFILL_429_WAIT_S", 0)
|
|
monkeypatch.setattr(ext_pull, "_BACKFILL_DAY_INTERVAL_S", 0)
|
|
_seed_trading_days(tmp_path, "2026-01-05")
|
|
url = "https://example.test/rank?date=2026-01-05"
|
|
fake_http.errors[url] = _err_429(url)
|
|
fake_http.fail_times[url] = 1 # 仅首请 429, 重试成功
|
|
fake_http.responses[url] = [_row("A", "2026-01-05")]
|
|
|
|
result = await backfill_history(_cfg(), tmp_path, date(2026, 1, 5), date(2026, 1, 5))
|
|
assert result["fetched"] == 1 and result["failed"] == []
|
|
assert fake_http.calls.count(url) == 2 # 首请 429 + 退避重试
|
|
|
|
|
|
async def test_backfill_aborts_after_consecutive_429(tmp_path, fake_http, monkeypatch):
|
|
monkeypatch.setattr(ext_pull, "_BACKFILL_429_WAIT_S", 0)
|
|
monkeypatch.setattr(ext_pull, "_BACKFILL_DAY_INTERVAL_S", 0)
|
|
days = ["2026-01-05", "2026-01-06", "2026-01-07", "2026-01-08", "2026-01-09"]
|
|
_seed_trading_days(tmp_path, *days)
|
|
for d in days:
|
|
u = f"https://example.test/rank?date={d}"
|
|
fake_http.errors[u] = _err_429(u)
|
|
fake_http.fail_times[u] = 2 # 首请 + 重试均 429
|
|
|
|
result = await backfill_history(_cfg(), tmp_path, date(2026, 1, 5), date(2026, 1, 9))
|
|
assert result["fetched"] == 0
|
|
# 连续计数达到 3 即中止: 前两日各 2 次请求, 第 3 日首个 429 立即中止
|
|
assert len(fake_http.calls) == 5
|
|
aborts = [f for f in result["failed"] if f["reason"].startswith("限流中止")]
|
|
assert len(result["failed"]) == 5 and len(aborts) == 3 # 剩余未请求的日标记为可续补
|
|
|
|
|
|
# ── 404 无快照日 (tickflow-hub 契约) ──────────────────────
|
|
|
|
|
|
def _err_404(url: str) -> httpx.HTTPStatusError:
|
|
req = httpx.Request("GET", url)
|
|
return httpx.HTTPStatusError("404", request=req, response=httpx.Response(404, request=req))
|
|
|
|
|
|
async def test_backfill_404_counts_as_empty_not_failed(tmp_path, fake_http, monkeypatch):
|
|
"""hub /exports、/fuyao-rank 契约: 该日无快照返回 404 → empty 跳过, 不进失败清单。"""
|
|
monkeypatch.setattr(ext_pull, "_BACKFILL_DAY_INTERVAL_S", 0)
|
|
_seed_trading_days(tmp_path, "2026-01-05", "2026-01-06", "2026-01-07")
|
|
u5, u6, u7 = (f"https://example.test/rank?date={d}" for d in ("2026-01-05", "2026-01-06", "2026-01-07"))
|
|
fake_http.errors[u5] = _err_404(u5)
|
|
fake_http.responses[u6] = [_row("600000.SH", "2026-01-06")]
|
|
fake_http.errors[u7] = _err_404(u7)
|
|
|
|
result = await backfill_history(_cfg(), tmp_path, date(2026, 1, 5), date(2026, 1, 7))
|
|
assert result["fetched"] == 1 and result["empty"] == 2
|
|
assert result["failed"] == [] # 404 不是失败
|
|
assert result["rows_written"] == 1
|
|
assert (tmp_path / "ext_data/hot/timeseries/date=2026-01-06/part.parquet").exists()
|
|
|
|
|
|
async def test_backfill_404_after_429_retry_counts_as_empty(tmp_path, fake_http, monkeypatch):
|
|
"""429 退避重试后返回 404: 同样视为该日无数据, 不进失败清单。"""
|
|
monkeypatch.setattr(ext_pull, "_BACKFILL_429_WAIT_S", 0)
|
|
monkeypatch.setattr(ext_pull, "_BACKFILL_DAY_INTERVAL_S", 0)
|
|
_seed_trading_days(tmp_path, "2026-01-05", "2026-01-06")
|
|
u5 = "https://example.test/rank?date=2026-01-05"
|
|
u6 = "https://example.test/rank?date=2026-01-06"
|
|
fake_http.error_sequence[u5] = [_err_429(u5), _err_404(u5)] # 首请 429 → 重试 404
|
|
fake_http.responses[u6] = [_row("A", "2026-01-06")]
|
|
|
|
result = await backfill_history(_cfg(), tmp_path, date(2026, 1, 5), date(2026, 1, 6))
|
|
assert result["fetched"] == 1 and result["empty"] == 1
|
|
assert result["failed"] == []
|
|
assert fake_http.calls.count(u5) == 2 # 确认重试确实发生
|