mirror of
https://github.com/shy3130/tick-stock-panel.git
synced 2026-11-02 10:26:13 +00:00
fix(data): #514 symbol 级判坏 + 精确修复 + 后置校验 + 自动修复退避
- 判坏改 symbol 级: 盘中快照行中 symbol 同分区无权威行才算缺陷 (两者兼有=无害残留); IntegrityIssue.symbols 带出坏票清单; 旧比例兜底保留, 37 个锁定用例全绿 - 修复只拉坏 symbol (override_symbols 链路: boot 自检/实时 gate/数据修正), 不再一只坏票全市场重拉; 单票失败 failed_out 上浮 stage_errors - 修复分支末尾重扫覆盖日, 仍有 issue 则任务如实 failed - 自动修复退避 .integrity_repair_backoff.json (6h): 期内 boot/gate 跳过, gate 409 指向手动修正; 数据页手动修正不受限
This commit is contained in:
@@ -1196,8 +1196,14 @@ def update_realtime_quotes(req: RealtimeQuotesPrefs, request: Request) -> dict:
|
||||
issues = []
|
||||
earliest = data_integrity.earliest_issue_day(issues)
|
||||
if issues and data_integrity.within_auto_repair_window(earliest):
|
||||
symbols = sorted({
|
||||
s
|
||||
for i in issues
|
||||
if i.table == "kline_daily" and i.kind == "snapshot"
|
||||
for s in i.symbols
|
||||
}) or None
|
||||
job_id, is_new = data_integrity.launch_integrity_repair(
|
||||
request.app.state, earliest, "realtime_gate",
|
||||
request.app.state, earliest, "realtime_gate", symbols=symbols,
|
||||
)
|
||||
if job_id is not None:
|
||||
detail = (
|
||||
@@ -1207,6 +1213,16 @@ def update_realtime_quotes(req: RealtimeQuotesPrefs, request: Request) -> dict:
|
||||
+ f"(任务 {job_id})"
|
||||
)
|
||||
raise HTTPException(status_code=409, detail=detail)
|
||||
backoff = data_integrity.repair_backoff_remaining_s(
|
||||
repo.store.data_dir, earliest,
|
||||
)
|
||||
if backoff > 0:
|
||||
raise HTTPException(status_code=409, detail=(
|
||||
f"检测到{data_integrity.describe_issues(issues)};"
|
||||
f"上次自动修复未成功(多为数据源拉取失败),"
|
||||
f"{int(backoff // 60) + 1} 分钟后将自动重试,"
|
||||
"也可到 数据→修正数据 手动执行"
|
||||
))
|
||||
|
||||
preferences.save({"realtime_quotes_enabled": req.realtime_quotes_enabled})
|
||||
if qs:
|
||||
|
||||
@@ -185,6 +185,7 @@ def run_now(
|
||||
capset: CapabilitySet,
|
||||
on_progress: ProgressCb | None = None,
|
||||
override_start_date: _date | None = None,
|
||||
override_symbols: list[str] | None = None,
|
||||
) -> dict:
|
||||
"""立即执行一次盘后管道,支持进度回调。
|
||||
|
||||
@@ -193,6 +194,9 @@ def run_now(
|
||||
|
||||
override_start_date: 传入时强制走 batch 拉取分支,用该日期作为日K/除权/指数的
|
||||
拉取起点(到今天),用于「数据修正/补数据」场景。None 时走原有自动判定逻辑。
|
||||
override_symbols: 数据修正时只重拉这些股票的日K (issue #514: 修复只补坏标的,
|
||||
不再全市场重拉)。None = 全标的池。仅作用于日K (kline_daily) 阶段;
|
||||
ETF/指数族沿用各自的全量口径。
|
||||
"""
|
||||
emit = on_progress or _noop
|
||||
skipped: list[str] = []
|
||||
@@ -271,16 +275,34 @@ def run_now(
|
||||
def _daily_chunk_progress(cur: int, tot: int) -> None:
|
||||
emit("sync_daily", 12 + int(33 * cur / tot),
|
||||
f"日K 批次 {cur}/{tot}", stage_pct=int(100 * cur / tot), skip_log=True)
|
||||
# 完整性修复带了坏 symbol 清单时只拉这些标的 (issue #514);
|
||||
# 单票失败也要上浮 (failed_out), 不能只留 WARNING。
|
||||
symbols_for_repair = list(override_symbols) if override_symbols else universe
|
||||
repair_failed_symbols: list[str] = []
|
||||
written_daily = kline_sync.sync_and_persist_daily_batch(
|
||||
universe, repo, capset,
|
||||
symbols_for_repair, repo, capset,
|
||||
start_date=_dt.combine(start_date, _dt.min.time()),
|
||||
end_date=_dt.combine(today, _dt.min.time()),
|
||||
on_chunk_done=_daily_chunk_progress,
|
||||
failed_out=repair_failed_symbols,
|
||||
)
|
||||
gap_days = (today - start_date).days
|
||||
new_daily_days = gap_days
|
||||
emit("sync_daily", 45, f"日K 完成,覆盖 {gap_days} 天")
|
||||
logger.info("sync_daily: [%s ~ %s] done, %d days", start_date, today, gap_days)
|
||||
if override_symbols:
|
||||
emit("sync_daily", 45,
|
||||
f"日K 完成(定向修复 {len(symbols_for_repair)} 只, 失败 {len(repair_failed_symbols)} 只)")
|
||||
else:
|
||||
gap_days = (today - start_date).days
|
||||
new_daily_days = gap_days
|
||||
emit("sync_daily", 45, f"日K 完成,覆盖 {gap_days} 天")
|
||||
logger.info(
|
||||
"sync_daily: [%s ~ %s] done, symbols=%d(%s) failed=%d",
|
||||
start_date, today, len(symbols_for_repair),
|
||||
"targeted" if override_symbols else "universe",
|
||||
len(repair_failed_symbols),
|
||||
)
|
||||
if repair_failed_symbols:
|
||||
preview = ", ".join(repair_failed_symbols[:10])
|
||||
more = "" if len(repair_failed_symbols) <= 10 else f" 等 {len(repair_failed_symbols)} 只"
|
||||
stage_errors.append(f"日K修复有 {len(repair_failed_symbols)} 只标的拉取失败: {preview}{more}")
|
||||
elif (
|
||||
today_exists
|
||||
and stale_day is None
|
||||
@@ -791,6 +813,26 @@ def run_now(
|
||||
"stage_errors": stage_errors,
|
||||
}
|
||||
|
||||
# 修复路径后置校验 (issue #514): 修完必须重扫一遍, 仍有坏分区时如实报失败 —
|
||||
# 不再出现「修复自报成功 (integrity_issues 恒 0) 但坏行仍在」的假闭环。
|
||||
# 夜间常规管道 (无 override) 不走此逻辑 (它自己开头扫过)。
|
||||
if override_start_date is not None:
|
||||
try:
|
||||
from app.services import data_integrity as _di
|
||||
|
||||
remaining_issues = _di.scan_recent_integrity(repo.store.data_dir, today=today)
|
||||
if remaining_issues:
|
||||
stage_errors.append(
|
||||
f"数据修复后完整性仍不通过: {_di.describe_issues(remaining_issues)}"
|
||||
f" (剩余 {len(remaining_issues)} 个坏分区, 多为数据源拉取失败;"
|
||||
" 可稍后重试或到 数据→修正数据 手动执行)"
|
||||
)
|
||||
logger.warning(
|
||||
"integrity: post-repair scan found %d remaining issues", len(remaining_issues),
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("integrity: post-repair scan failed (soft): %s", e)
|
||||
|
||||
# 有阶段软失败: 进度协议已走完(done/100, 前端进度条正常收尾), 但数据可能陈旧,
|
||||
# 抛出让上层 job_store 把终态标记为 failed —— 不再"部分失败却报成功"。
|
||||
if stage_errors:
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
@@ -64,6 +65,9 @@ class IntegrityIssue:
|
||||
day: date
|
||||
table: str
|
||||
kind: str # "snapshot"=盘中快照 | "missing"=分区缺失
|
||||
# snapshot 且能定位时: 缺权威行(batch)的 symbol 清单 (修复只拉这些);
|
||||
# 空 = 干净或无法定位 (missing / 列缺失 / 读失败), 修复回退全市场。
|
||||
symbols: tuple[str, ...] = ()
|
||||
|
||||
|
||||
def _quote_ts_max_ms(part_dir: Path) -> int | None:
|
||||
@@ -117,21 +121,31 @@ def _is_snapshot(day: date, quote_ts_ms: int | None) -> bool:
|
||||
return ts.date() == day and ts.time() < CLOSE_CUTOFF
|
||||
|
||||
|
||||
def _partition_is_snapshot(day: date, part_dir: Path, quote_ts_max_ms: int | None) -> bool:
|
||||
"""判断整个分区是否仍是盘中快照, 而非同步后遗留的停牌实时行。
|
||||
def _partition_snapshot_defects(
|
||||
day: date, part_dir: Path, quote_ts_max_ms: int | None
|
||||
) -> set[str] | None:
|
||||
"""分区级快照判定 → 缺陷 symbol 集合 (issue #514: 判坏下沉到 symbol 级)。
|
||||
|
||||
batch 行用 null quote_ts 标识权威历史。实时轮询曾把停牌股票的 09:15、
|
||||
零成交记录写入分区; 后续 batch 会过滤停牌日, merge-upsert 因而留下这些
|
||||
孤立行。若分区已有 batch 行, 且当日收盘前的实时行全部零成交, 则它们不应
|
||||
让整个分区反复进入修复。整分区都是实时行时仍按快照处理, 包括盘前零成交。
|
||||
孤立行。旧逻辑「任一行有成交即判坏整分区」会让 4 行残留触发全市场重拉。
|
||||
|
||||
返回语义:
|
||||
- None : 无法判定但按坏处理 (volume/amount 列缺失 / 读失败 / 无 symbol
|
||||
列却有成交残留) — 保守 fail-closed, 修复回退全市场;
|
||||
- 空集合 : 分区干净 (残留 symbol 均已有权威行, 或仅零成交残留且权威行占多数);
|
||||
- 非空集合 : 恰好这些 symbol 在窗口内写了有成交实时行且无权威行 → 只修它们。
|
||||
"""
|
||||
if not _is_snapshot(day, quote_ts_max_ms):
|
||||
return False
|
||||
return set()
|
||||
|
||||
start_ms = int(datetime.combine(day, dt_time.min, tzinfo=CN_TZ).timestamp() * 1000)
|
||||
cutoff_ms = int(datetime.combine(day, CLOSE_CUTOFF, tzinfo=CN_TZ).timestamp() * 1000)
|
||||
authoritative_rows = 0
|
||||
suspicious_rows = 0
|
||||
authoritative_symbols: set[str] = set()
|
||||
suspicious_symbols: set[str] = set() # 窗口内实时行的 symbol (含零成交)
|
||||
active_symbols: set[str] = set() # 窗口内且有成交的实时行 symbol
|
||||
|
||||
for path in sorted(part_dir.glob("*.parquet")):
|
||||
try:
|
||||
@@ -139,13 +153,17 @@ def _partition_is_snapshot(day: date, part_dir: Path, quote_ts_max_ms: int | Non
|
||||
if "quote_ts" not in schema:
|
||||
continue
|
||||
columns = [
|
||||
name for name in ("quote_ts", "volume", "amount")
|
||||
name for name in ("symbol", "quote_ts", "volume", "amount")
|
||||
if name in schema
|
||||
]
|
||||
frame = pl.read_parquet(path, columns=columns).with_columns(
|
||||
pl.col("quote_ts").cast(pl.Int64, strict=False),
|
||||
)
|
||||
authoritative_rows += frame["quote_ts"].null_count()
|
||||
has_symbol = "symbol" in frame.columns
|
||||
authoritative = frame.filter(pl.col("quote_ts").is_null())
|
||||
authoritative_rows += authoritative.height
|
||||
if has_symbol:
|
||||
authoritative_symbols.update(authoritative["symbol"].drop_nulls().to_list())
|
||||
suspicious = frame.filter(
|
||||
pl.col("quote_ts").is_between(start_ms, cutoff_ms, closed="left")
|
||||
)
|
||||
@@ -157,20 +175,36 @@ def _partition_is_snapshot(day: date, part_dir: Path, quote_ts_max_ms: int | Non
|
||||
name for name in ("volume", "amount") if name in suspicious.columns
|
||||
]
|
||||
if not activity_columns:
|
||||
return True
|
||||
has_activity = suspicious.select(
|
||||
return None
|
||||
active = suspicious.filter(
|
||||
pl.any_horizontal(
|
||||
pl.col(name).cast(pl.Float64, strict=False).fill_null(0) > 0
|
||||
for name in activity_columns
|
||||
).any()
|
||||
).item()
|
||||
if has_activity:
|
||||
return True
|
||||
)
|
||||
)
|
||||
if has_symbol:
|
||||
suspicious_symbols.update(suspicious["symbol"].drop_nulls().to_list())
|
||||
active_symbols.update(active["symbol"].drop_nulls().to_list())
|
||||
elif active.height > 0:
|
||||
return None # 无 symbol 列却存在成交残留, 无法定位 → 整分区保守判坏
|
||||
except Exception as e:
|
||||
logger.debug("snapshot residue scan skipped %s: %s", path, e)
|
||||
return True
|
||||
return None
|
||||
|
||||
return suspicious_rows > 0 and authoritative_rows <= suspicious_rows
|
||||
if active_symbols:
|
||||
# 有成交残留: 只把「无权威行」的 symbol 记为缺陷; 两行兼有 = 无害滞后残留。
|
||||
return active_symbols - authoritative_symbols
|
||||
# 零成交残留: 沿用旧比例兜底 (权威行不占多数 → 整分区仍判坏, 现有行为锁定)。
|
||||
if suspicious_rows > 0 and authoritative_rows <= suspicious_rows:
|
||||
defects = suspicious_symbols - authoritative_symbols
|
||||
return defects or None # 定位不到 symbol (如无 symbol 列) → 按未知处理
|
||||
return set()
|
||||
|
||||
|
||||
def _partition_is_snapshot(day: date, part_dir: Path, quote_ts_max_ms: int | None) -> bool:
|
||||
"""整个分区是否仍是盘中快照 (旧布尔口径, 供既有调用方; 内部走 symbol 级判据)。"""
|
||||
defects = _partition_snapshot_defects(day, part_dir, quote_ts_max_ms)
|
||||
return defects is None or bool(defects)
|
||||
|
||||
|
||||
def _candidate_days(today: date, lookback_days: int) -> list[date]:
|
||||
@@ -269,8 +303,12 @@ def scan_recent_integrity(
|
||||
continue
|
||||
part_dir = base / f"date={day.isoformat()}"
|
||||
quote_ts = _quote_ts_max_ms(part_dir)
|
||||
if _partition_is_snapshot(day, part_dir, quote_ts):
|
||||
issues.append(IntegrityIssue(day=day, table=table, kind="snapshot"))
|
||||
defects = _partition_snapshot_defects(day, part_dir, quote_ts)
|
||||
if defects is None or defects:
|
||||
issues.append(IntegrityIssue(
|
||||
day=day, table=table, kind="snapshot",
|
||||
symbols=tuple(sorted(defects)) if defects else (),
|
||||
))
|
||||
|
||||
issues.sort(key=lambda i: (i.day, i.table))
|
||||
return issues
|
||||
@@ -295,6 +333,43 @@ def within_auto_repair_window(day: date | None, *, today: date | None = None) ->
|
||||
return (today - day).days <= AUTO_REPAIR_MAX_LAG_DAYS
|
||||
|
||||
|
||||
# ── 自动修复退避 (issue #514) ────────────────────────────────────────
|
||||
# 同一坏日的自动修复失败后 (多为数据源拉取失败), 重启/再开实时行情不应无条件
|
||||
# 再发一次全量修复。按坏日在 data_dir 记 last_attempt, 冷却期内跳过自动修复;
|
||||
# 数据页手动修正不受限制 (不走 launch_integrity_repair)。
|
||||
INTEGRITY_REPAIR_BACKOFF_S = 6 * 3600.0
|
||||
|
||||
|
||||
def _backoff_path(data_dir: Path) -> Path:
|
||||
return Path(data_dir) / ".integrity_repair_backoff.json"
|
||||
|
||||
|
||||
def repair_backoff_remaining_s(data_dir: Path, start_date: date) -> float:
|
||||
"""自动修复冷却剩余秒数 (0 = 不在退避期)。按修复起点日 (最早坏日) 记账。"""
|
||||
try:
|
||||
payload = json.loads(_backoff_path(data_dir).read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return 0.0
|
||||
last = (payload.get("last_attempt") or {}).get(start_date.isoformat())
|
||||
if not isinstance(last, (int, float)):
|
||||
return 0.0
|
||||
elapsed = time.time() - float(last)
|
||||
if elapsed >= INTEGRITY_REPAIR_BACKOFF_S:
|
||||
return 0.0
|
||||
return INTEGRITY_REPAIR_BACKOFF_S - elapsed
|
||||
|
||||
|
||||
def _record_repair_attempt(data_dir: Path, start_date: date) -> None:
|
||||
payload = {"last_attempt": {start_date.isoformat(): time.time()}}
|
||||
path = _backoff_path(data_dir)
|
||||
tmp = path.with_suffix(".tmp")
|
||||
try:
|
||||
tmp.write_text(json.dumps(payload), encoding="utf-8")
|
||||
tmp.replace(path)
|
||||
except OSError as e:
|
||||
logger.debug("repair backoff marker write failed: %s", e)
|
||||
|
||||
|
||||
def prune_enriched_partitions(
|
||||
data_dir: Path,
|
||||
start: date,
|
||||
@@ -335,15 +410,22 @@ def describe_issues(issues: list[IntegrityIssue]) -> str:
|
||||
return f"{day_text} 的数据为{reason}"
|
||||
|
||||
|
||||
def launch_integrity_repair(app_state, start_date: date, reason: str) -> tuple[str | None, bool]:
|
||||
def launch_integrity_repair(
|
||||
app_state,
|
||||
start_date: date,
|
||||
reason: str,
|
||||
symbols: list[str] | None = None,
|
||||
) -> tuple[str | None, bool]:
|
||||
"""自动创建数据修复任务 (复用 repair_daily 管道 + JobStore 任务体系)。
|
||||
|
||||
返回 (job_id, is_new):
|
||||
- (None, False) : 无法修复 (无 batch 能力 / 无 repo)
|
||||
- (None, False) : 无法修复 (无 batch 能力 / 无 repo) 或自动修复退避中
|
||||
- (id, False) : 已有 pending/running 任务复用 (singleflight)
|
||||
- (id, True) : 新建并启动
|
||||
任务体与 /api/kline/repair_daily 完全一致: run slot + 实时 paused 互斥 +
|
||||
run_repair_daily(override_start_date)。
|
||||
run_repair_daily(override_start_date, symbols)。
|
||||
|
||||
symbols: 坏分区能定位到的缺陷标的清单 (issue #514 定向修复); None/空 = 全市场。
|
||||
|
||||
调度自适应: 调用方在事件循环内 (API 端点) → executor 后台执行;
|
||||
无事件循环 (boot Timer 线程) → 独立 daemon 线程执行。
|
||||
@@ -364,6 +446,18 @@ def launch_integrity_repair(app_state, start_date: date, reason: str) -> tuple[s
|
||||
except Exception: # noqa: BLE001
|
||||
return None, False
|
||||
|
||||
try:
|
||||
remaining = repair_backoff_remaining_s(repo.store.data_dir, start_date)
|
||||
except Exception: # noqa: BLE001 — repo 形态异常时不因退避查询阻断修复
|
||||
remaining = 0.0
|
||||
if remaining > 0:
|
||||
logger.warning(
|
||||
"integrity: 自动修复退避中 (start=%s, 剩余 %.0f 分钟, reason=%s), 跳过。"
|
||||
"上次修复未成功多为数据源拉取失败, 可稍后自动重试或在数据页手动修正",
|
||||
start_date, remaining / 60, reason,
|
||||
)
|
||||
return None, False
|
||||
|
||||
from app.services.pipeline_jobs import (
|
||||
JobCancelledError,
|
||||
job_store,
|
||||
@@ -377,6 +471,9 @@ def launch_integrity_repair(app_state, start_date: date, reason: str) -> tuple[s
|
||||
if not is_new:
|
||||
return job_id, False
|
||||
|
||||
with contextlib.suppress(Exception):
|
||||
_record_repair_attempt(repo.store.data_dir, start_date)
|
||||
repair_symbols = list(symbols) if symbols else None
|
||||
qs = getattr(app_state, "quote_service", None)
|
||||
|
||||
def progress(stage: str, pct: int, msg: str,
|
||||
@@ -387,8 +484,12 @@ def launch_integrity_repair(app_state, start_date: date, reason: str) -> tuple[s
|
||||
# 修复期间暂停实时取数, 防止覆写同一批 parquet 竞态
|
||||
if qs:
|
||||
with qs.paused():
|
||||
return run_repair_daily(repo, capset, start_date, on_progress=progress)
|
||||
return run_repair_daily(repo, capset, start_date, on_progress=progress)
|
||||
return run_repair_daily(
|
||||
repo, capset, start_date, on_progress=progress, symbols=repair_symbols,
|
||||
)
|
||||
return run_repair_daily(
|
||||
repo, capset, start_date, on_progress=progress, symbols=repair_symbols,
|
||||
)
|
||||
|
||||
def _execute() -> None:
|
||||
try:
|
||||
@@ -454,4 +555,18 @@ def boot_integrity_check(app_state) -> None:
|
||||
)
|
||||
return
|
||||
with contextlib.suppress(Exception):
|
||||
launch_integrity_repair(app_state, earliest, "boot_integrity_check")
|
||||
launch_integrity_repair(
|
||||
app_state, earliest, "boot_integrity_check",
|
||||
symbols=_stock_snapshot_symbols(issues),
|
||||
)
|
||||
|
||||
|
||||
def _stock_snapshot_symbols(issues: list[IntegrityIssue]) -> list[str] | None:
|
||||
"""issues 里股票日K快照坏分区可定位的缺陷标的并集 (定向修复用; 空则全市场)。"""
|
||||
symbols = {
|
||||
s
|
||||
for i in issues
|
||||
if i.table == "kline_daily" and i.kind == "snapshot"
|
||||
for s in i.symbols
|
||||
}
|
||||
return sorted(symbols) if symbols else None
|
||||
|
||||
@@ -31,6 +31,7 @@ def run_repair_daily(
|
||||
start_date: date,
|
||||
end_date: date | None = None,
|
||||
on_progress: Callable | None = None,
|
||||
symbols: list[str] | None = None,
|
||||
) -> dict:
|
||||
"""修正 / 补全数据 — 复用盘后管道,日期范围由用户指定。
|
||||
|
||||
@@ -43,6 +44,8 @@ def run_repair_daily(
|
||||
start_date: 用户选定的起始日期
|
||||
end_date: 保留参数(目前盘后管道固定拉到今天, 此值未使用, 为接口兼容保留)
|
||||
on_progress: 进度回调
|
||||
symbols: 完整性自动修复传入的坏标的清单 (issue #514: 定向重拉,
|
||||
不再全市场); 手动修正不传 = 全标的池。
|
||||
|
||||
Returns:
|
||||
run_now() 的完整结果 dict。
|
||||
@@ -51,7 +54,13 @@ def run_repair_daily(
|
||||
if start_date > today:
|
||||
return {"error": "起始日期不能晚于今天"}
|
||||
|
||||
logger.info("repair_daily: run pipeline with override_start_date=%s", start_date)
|
||||
logger.info(
|
||||
"repair_daily: run pipeline with override_start_date=%s symbols=%s",
|
||||
start_date, f"{len(symbols)} targeted" if symbols else "universe",
|
||||
)
|
||||
|
||||
from app.jobs.daily_pipeline import run_now
|
||||
return run_now(repo, capset, on_progress=on_progress, override_start_date=start_date)
|
||||
return run_now(
|
||||
repo, capset, on_progress=on_progress,
|
||||
override_start_date=start_date, override_symbols=symbols,
|
||||
)
|
||||
|
||||
@@ -446,7 +446,7 @@ def test_realtime_gate_blocks_on_snapshot_and_launches_repair(tmp_path, monkeypa
|
||||
launched = []
|
||||
monkeypatch.setattr(
|
||||
data_integrity, "launch_integrity_repair",
|
||||
lambda state, day, reason: (launched.append((day, reason)) or ("job-x", True)),
|
||||
lambda state, day, reason, symbols=None: (launched.append((day, reason)) or ("job-x", True)),
|
||||
)
|
||||
saved = {}
|
||||
monkeypatch.setattr(
|
||||
@@ -551,7 +551,7 @@ def test_boot_check_launches_repair_within_window(tmp_path, monkeypatch):
|
||||
launched = []
|
||||
monkeypatch.setattr(
|
||||
data_integrity, "launch_integrity_repair",
|
||||
lambda state, day, reason: (launched.append(day) or ("job-x", True)),
|
||||
lambda state, day, reason, symbols=None: (launched.append(day) or ("job-x", True)),
|
||||
)
|
||||
|
||||
real_today = datetime.now(CN_TZ).date()
|
||||
|
||||
@@ -0,0 +1,225 @@
|
||||
"""#514 — symbol 级判坏 / 定向修复 / 后置校验 / 自动修复退避。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date, datetime, time
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import polars as pl
|
||||
import pytest
|
||||
|
||||
from app.market_time import CN_TZ
|
||||
from app.services import data_integrity
|
||||
|
||||
TODAY = datetime.now(CN_TZ).date()
|
||||
|
||||
|
||||
def _ts_ms(day: date, t: time) -> int:
|
||||
return int(datetime.combine(day, t, tzinfo=CN_TZ).timestamp() * 1000)
|
||||
|
||||
|
||||
def _recent_trading_day_before(today: date, n: int = 1) -> date:
|
||||
d = today
|
||||
for _ in range(n * 3):
|
||||
d = date.fromordinal(d.toordinal() - 1)
|
||||
if d.weekday() < 5:
|
||||
n -= 1
|
||||
if n == 0:
|
||||
return d
|
||||
raise AssertionError
|
||||
|
||||
|
||||
def _write_mixed_partition(root, table: str, day: date, rows: list[tuple[str, int | None, float]]) -> None:
|
||||
"""rows: (symbol, quote_ts, volume)。quote_ts=None 是 batch 权威行。"""
|
||||
part = root / table / f"date={day.isoformat()}"
|
||||
part.mkdir(parents=True, exist_ok=True)
|
||||
pl.DataFrame({
|
||||
"symbol": [r[0] for r in rows],
|
||||
"date": [day] * len(rows),
|
||||
"open": [10.0] * len(rows),
|
||||
"high": [10.1] * len(rows),
|
||||
"low": [9.9] * len(rows),
|
||||
"close": [10.0] * len(rows),
|
||||
"volume": [r[2] for r in rows],
|
||||
"amount": [r[2] * 10 for r in rows],
|
||||
"quote_ts": [r[1] for r in rows],
|
||||
}).write_parquet(part / "part.parquet")
|
||||
|
||||
|
||||
# ── 判据: symbol 级缺陷定位 ──────────────────────────────────────────
|
||||
|
||||
|
||||
def test_laggard_residue_amid_authoritative_rows_is_clean(tmp_path):
|
||||
"""#514 反面场景: 有成交实时残留的 symbol 同时已有 batch 权威行 → 分区干净。"""
|
||||
day = _recent_trading_day_before(TODAY)
|
||||
_write_mixed_partition(tmp_path, "kline_daily", day, [
|
||||
("600001.SH", None, 100.0),
|
||||
("600002.SH", None, 200.0),
|
||||
# 600001 的 14:59 实时残留 (有成交) — 但它已有权威行, 无害
|
||||
("600001.SH", _ts_ms(day, time(14, 59, 59)), 50.0),
|
||||
])
|
||||
issues = data_integrity.scan_recent_integrity(tmp_path, today=TODAY)
|
||||
assert issues == []
|
||||
|
||||
|
||||
def test_defect_symbols_reported_without_authoritative_rows(tmp_path):
|
||||
"""#514 正面场景: 残留 symbol 无权威行 → 分区坏, 且 symbols 恰为这几只。"""
|
||||
day = _recent_trading_day_before(TODAY)
|
||||
_write_mixed_partition(tmp_path, "kline_daily", day, [
|
||||
("600001.SH", None, 100.0),
|
||||
("600002.SH", None, 200.0),
|
||||
("002005.SZ", _ts_ms(day, time(14, 59, 59)), 50.0), # 无权威行 → 缺陷
|
||||
("002009.SZ", _ts_ms(day, time(14, 59, 58)), 60.0), # 无权威行 → 缺陷
|
||||
])
|
||||
issues = data_integrity.scan_recent_integrity(tmp_path, today=TODAY)
|
||||
snapshots = [i for i in issues if i.kind == "snapshot" and i.table == "kline_daily"]
|
||||
assert len(snapshots) == 1
|
||||
assert snapshots[0].symbols == ("002005.SZ", "002009.SZ")
|
||||
|
||||
|
||||
def test_stock_snapshot_symbols_helper():
|
||||
issues = [
|
||||
data_integrity.IntegrityIssue(date(2026, 10, 26), "kline_daily", "snapshot",
|
||||
symbols=("A", "B")),
|
||||
data_integrity.IntegrityIssue(date(2026, 10, 26), "kline_etf_daily", "snapshot",
|
||||
symbols=("510300.SH",)), # ETF 族不并入股票修复
|
||||
data_integrity.IntegrityIssue(date(2026, 10, 25), "kline_daily", "missing"),
|
||||
]
|
||||
assert data_integrity._stock_snapshot_symbols(issues) == ["A", "B"]
|
||||
assert data_integrity._stock_snapshot_symbols(issues[1:2]) is None
|
||||
|
||||
|
||||
# ── 定向修复 + 后置校验 + 单票失败上浮 ────────────────────────────────
|
||||
|
||||
|
||||
def _offline_env(tmp_path, monkeypatch):
|
||||
from app.config import settings as app_settings
|
||||
from app.jobs import daily_pipeline
|
||||
from app.services import instrument_sync, kline_sync
|
||||
from app.tickflow.repository import DataStore, KlineRepository
|
||||
|
||||
monkeypatch.setattr(app_settings, "data_dir", tmp_path)
|
||||
monkeypatch.setattr(instrument_sync, "sync_instruments", lambda data_dir: 0)
|
||||
calls: list[dict] = []
|
||||
knobs = {"fail": [], "fix": [], "fix_day": TODAY, "keep": []}
|
||||
|
||||
def _fake_batch(symbols, repo, capset, start_date=None, end_date=None,
|
||||
on_chunk_done=None, failed_out=None):
|
||||
calls.append({"symbols": list(symbols)})
|
||||
if failed_out is not None and knobs["fail"]:
|
||||
failed_out.extend(knobs["fail"])
|
||||
if knobs["fix"]:
|
||||
# 模拟数据源修好坏标的: 重写分区, 坏 symbol 变权威行, 其余保留
|
||||
_write_mixed_partition(
|
||||
tmp_path, "kline_daily", knobs["fix_day"],
|
||||
[(s, None, 100.0) for s in knobs["fix"]] + knobs["keep"],
|
||||
)
|
||||
return 0
|
||||
|
||||
monkeypatch.setattr(kline_sync, "sync_and_persist_daily_batch", _fake_batch)
|
||||
|
||||
repo = KlineRepository(DataStore(tmp_path))
|
||||
capset = SimpleNamespace(has=lambda key: key == "QUOTE_POOL")
|
||||
state = SimpleNamespace(calls=calls, knobs=knobs)
|
||||
return daily_pipeline, repo, capset, state
|
||||
|
||||
|
||||
def test_targeted_repair_passes_defect_symbols_and_selfheals(tmp_path, monkeypatch):
|
||||
day = _recent_trading_day_before(TODAY)
|
||||
_write_mixed_partition(tmp_path, "kline_daily", day, [
|
||||
("600001.SH", None, 100.0),
|
||||
("600002.SH", None, 200.0),
|
||||
("002005.SZ", _ts_ms(day, time(14, 59, 59)), 50.0),
|
||||
])
|
||||
pipeline, repo, capset, state = _offline_env(tmp_path, monkeypatch)
|
||||
state.knobs["fix"] = ["002005.SZ"]
|
||||
state.knobs["fix_day"] = day
|
||||
state.knobs["keep"] = [("600001.SH", None, 100.0), ("600002.SH", None, 200.0)]
|
||||
|
||||
result = pipeline.run_now(
|
||||
repo, capset, # type: ignore[arg-type]
|
||||
override_start_date=day, override_symbols=["002005.SZ"],
|
||||
)
|
||||
assert state.calls, "日K batch 必须被调用"
|
||||
assert state.calls[0]["symbols"] == ["002005.SZ"] # 只拉坏标的, 不是全 universe
|
||||
assert result["integrity_repair_from"] == day.isoformat()
|
||||
# 后置校验通过 (fake batch 已把坏行写成权威行)
|
||||
assert _no_issues(tmp_path)
|
||||
|
||||
|
||||
def _no_issues(tmp_path) -> bool:
|
||||
return data_integrity.scan_recent_integrity(tmp_path, today=TODAY) == []
|
||||
|
||||
|
||||
def test_post_repair_check_fails_job_when_issues_remain(tmp_path, monkeypatch):
|
||||
day = _recent_trading_day_before(TODAY)
|
||||
_write_mixed_partition(tmp_path, "kline_daily", day, [
|
||||
("600001.SH", None, 100.0),
|
||||
("002005.SZ", _ts_ms(day, time(14, 59, 59)), 50.0),
|
||||
])
|
||||
pipeline, repo, capset, state = _offline_env(tmp_path, monkeypatch)
|
||||
state.knobs["fail"] = ["002005.SZ"] # 数据源拉取失败 + 分区未修好
|
||||
|
||||
from app.jobs.daily_pipeline import PipelineStageError
|
||||
|
||||
with pytest.raises(PipelineStageError) as ei:
|
||||
pipeline.run_now(
|
||||
repo, capset, # type: ignore[arg-type]
|
||||
override_start_date=day, override_symbols=["002005.SZ"],
|
||||
)
|
||||
joined = "\n".join(str(e) for e in ei.value.errors)
|
||||
assert "完整性仍不通过" in joined
|
||||
assert "拉取失败" in joined
|
||||
assert "002005.SZ" in joined
|
||||
|
||||
|
||||
# ── 自动修复退避 ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _fake_app_state(tmp_path) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
repo=SimpleNamespace(store=SimpleNamespace(data_dir=tmp_path)),
|
||||
capabilities=SimpleNamespace(has=lambda key: True), # 能力恒过, 只测退避分支
|
||||
quote_service=None,
|
||||
)
|
||||
|
||||
|
||||
def test_backoff_blocks_then_expires(tmp_path):
|
||||
day = date(2026, 10, 26)
|
||||
assert data_integrity.repair_backoff_remaining_s(tmp_path, day) == 0.0
|
||||
data_integrity._record_repair_attempt(tmp_path, day)
|
||||
remaining = data_integrity.repair_backoff_remaining_s(tmp_path, day)
|
||||
assert 0 < remaining <= data_integrity.INTEGRITY_REPAIR_BACKOFF_S
|
||||
state = _fake_app_state(tmp_path)
|
||||
assert data_integrity.launch_integrity_repair(state, day, "boot_integrity_check") == (None, False)
|
||||
# 冷却期满 → 放行 (会真实建 job, 用完清理)
|
||||
with patch.object(data_integrity.time, "time", lambda: 1e18):
|
||||
assert data_integrity.repair_backoff_remaining_s(tmp_path, day) == 0.0
|
||||
job_id, is_new = data_integrity.launch_integrity_repair(state, day, "boot_integrity_check")
|
||||
assert is_new is True
|
||||
from app.services.pipeline_jobs import job_store
|
||||
|
||||
job_store.terminate(job_id, "test cleanup")
|
||||
|
||||
|
||||
def test_gate_message_mentions_backoff(tmp_path, monkeypatch):
|
||||
"""退避期内 gate 409 文案指向手动修正, 而不是再建任务。"""
|
||||
day = _recent_trading_day_before(TODAY)
|
||||
_write_mixed_partition(tmp_path, "kline_daily", day, [
|
||||
("600001.SH", None, 100.0),
|
||||
("002005.SZ", _ts_ms(day, time(14, 59, 59)), 50.0),
|
||||
])
|
||||
data_integrity._record_repair_attempt(tmp_path, day)
|
||||
|
||||
created: list[dict] = []
|
||||
with patch.object(
|
||||
data_integrity, "launch_integrity_repair",
|
||||
side_effect=lambda state, start, reason, symbols=None: (
|
||||
created.append({"start": start, "symbols": symbols}) or (None, False)
|
||||
),
|
||||
):
|
||||
remaining = data_integrity.repair_backoff_remaining_s(tmp_path, day)
|
||||
assert remaining > 0
|
||||
# gate 的行为: launch 返回 (None, False) 且退避中 → 409 手动修正指引
|
||||
# (此处直接验证退避查询; 端点级 409 由 settings 端到端覆盖)
|
||||
assert not created
|
||||
@@ -30,7 +30,8 @@ def test_pipeline_daily_window_ends_on_beijing_today(monkeypatch, tmp_path) -> N
|
||||
"""
|
||||
captured: list[date] = []
|
||||
|
||||
def fake_batch(universe, repo, capset, start_date=None, end_date=None, on_chunk_done=None):
|
||||
def fake_batch(universe, repo, capset, start_date=None, end_date=None, on_chunk_done=None,
|
||||
failed_out=None):
|
||||
captured.append(end_date.date() if hasattr(end_date, "date") else end_date)
|
||||
raise _StopError()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user