tick-stock-panel/backend/tests/test_mining_lock_admin.py

"""挖掘锁持有者记录 / 陈旧判定 / 强制释放 + 撞锁降级启动测试。"""
from __future__ import annotations

import json
import os
import subprocess
import sys
import time
from pathlib import Path
from types import SimpleNamespace

import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient

from app.api.mining import router as mining_router
from app.services.mining_lock_admin import (
    MiningLockAdminError,
    _classify,
    _count_active_runs,
    _store_idle_seconds,
    force_release,
    lock_status,
)
from app.services.mining_process_lock import (
    MiningProcessLock,
    MiningProcessLockError,
    locate_lock_holder_pid,
    probe_lock_held,
    read_lock_holder,
)

_BACKEND_ROOT = Path(__file__).resolve().parents[1]

_HOLDER_SCRIPT = """
import sys, time
from app.services.mining_process_lock import MiningProcessLock
lock = MiningProcessLock(sys.argv[1])
lock.acquire(role="worker")
print("HELD", flush=True)
time.sleep(600)
"""

# 旧版 (v0.2 时代) 锁写入格式: 只有 b"0", 不含 pid — 升级前启动的残留进程
# 就是这样持锁的, 新版面板无法从内容定位持有者。
_HOLDER_SCRIPT_LEGACY = """
import fcntl, sys, time
path = sys.argv[1] + "/.mining_process.lock"
stream = open(path, "a+b")
stream.write(b"0")
stream.flush()
fcntl.flock(stream.fileno(), fcntl.LOCK_EX)
print("HELD", flush=True)
time.sleep(600)
"""

_POSIX = pytest.mark.skipif(os.name == "nt", reason="外部持锁进程仅在 POSIX 验证")


def _spawn_holder(data_dir: Path) -> subprocess.Popen:
    env = {**os.environ, "PYTHONPATH": str(_BACKEND_ROOT)}
    proc = subprocess.Popen(
        [sys.executable, "-c", _HOLDER_SCRIPT, str(data_dir)],
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        text=True,
        env=env,
        cwd=str(_BACKEND_ROOT),
    )
    line = proc.stdout.readline()
    assert line.strip() == "HELD", f"外部持锁进程未就绪: {line!r} {proc.stderr.read()}"
    return proc


def _spawn_legacy_holder(data_dir: Path) -> subprocess.Popen:
    """旧格式 (b"0") 持锁进程: 模拟升级前启动、升级后仍残留的旧版后端。"""
    env = {**os.environ, "PYTHONPATH": str(_BACKEND_ROOT)}
    proc = subprocess.Popen(
        [sys.executable, "-c", _HOLDER_SCRIPT_LEGACY, str(data_dir)],
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        text=True,
        env=env,
        cwd=str(_BACKEND_ROOT),
    )
    line = proc.stdout.readline()
    assert line.strip() == "HELD", f"旧格式持锁进程未就绪: {line!r} {proc.stderr.read()}"
    return proc


def _kill(proc: subprocess.Popen) -> None:
    if proc.poll() is None:
        proc.kill()
    proc.wait(timeout=10)


# ── 锁内容与探测 ───────────────────────────────────────────────


def test_lock_records_holder_and_release_marker(tmp_path):
    lock = MiningProcessLock(tmp_path)
    lock.acquire()
    holder = read_lock_holder(tmp_path)
    assert holder is not None
    assert holder["pid"] == os.getpid()
    assert holder["role"] == "app"
    assert probe_lock_held(tmp_path) is True

    lock.release()
    assert probe_lock_held(tmp_path) is False
    # 释放标记后不再把内容当作有效持有者
    assert read_lock_holder(tmp_path) is None


def test_second_acquire_conflicts_in_same_process(tmp_path):
    first = MiningProcessLock(tmp_path)
    first.acquire()
    second = MiningProcessLock(tmp_path)
    with pytest.raises(MiningProcessLockError):
        second.acquire()
    first.release()


def test_legacy_lock_content_is_unknown_holder(tmp_path):
    path = tmp_path / ".mining_process.lock"
    path.write_bytes(b"0")
    assert read_lock_holder(tmp_path) is None
    assert probe_lock_held(tmp_path) is False


def test_corrupt_lock_content_is_unknown_holder(tmp_path):
    (tmp_path / ".mining_process.lock").write_text("{not json")
    assert read_lock_holder(tmp_path) is None


def test_status_reports_mine(tmp_path):
    lock = MiningProcessLock(tmp_path)
    lock.acquire()
    try:
        status = lock_status(tmp_path)
        assert status["held"] is True
        assert status["mine"] is True
        assert status["force_releasable"] is False
        assert "当前后端" in status["reason"]
    finally:
        lock.release()


# ── 陈旧判定单元矩阵 ──────────────────────────────────────────


def test_classify_matrix():
    base = dict(
        held=True, mine=False, holder_known=True, holder_alive=True,
        pid_consistent=True, active_runs=0, store_idle=None, stale_after=600.0,
    )
    ok, reason = _classify(**base)
    assert ok and reason is None

    not_held = {**base, "held": False}
    assert _classify(**not_held) == (False, "锁未被持有")

    legacy = {**base, "holder_known": False}
    ok, reason = _classify(**legacy)
    assert not ok and "旧格式" in reason

    reused = {**base, "pid_consistent": False}
    ok, reason = _classify(**reused)
    assert not ok and "复用" in reason

    active = {**base, "active_runs": 2}
    ok, reason = _classify(**active)
    assert not ok and "2 个进行中" in reason

    fresh = {**base, "store_idle": 12.0}
    ok, reason = _classify(**fresh)
    assert not ok and "陈旧阈值" in reason

    dead = {**base, "holder_alive": False}
    ok, reason = _classify(**dead)
    assert not ok and "已不存在" in reason


def test_count_active_runs_and_store_idle(tmp_path):
    runs = tmp_path / "research" / "mining" / "runs"
    assert _count_active_runs(tmp_path) == 0
    assert _store_idle_seconds(tmp_path, now=time.time()) is None

    done = runs / "run-done"
    done.mkdir(parents=True)
    (done / "manifest.json").write_text(json.dumps({"status": "succeeded"}))
    active = runs / "run-live"
    active.mkdir(parents=True)
    (active / "manifest.json").write_text(json.dumps({"status": "running"}))
    (active / "events.jsonl").write_text("{}\n")

    assert _count_active_runs(tmp_path) == 1
    idle = _store_idle_seconds(tmp_path, now=time.time())
    assert idle is not None and idle < 60


def test_count_active_runs_tolerates_corrupt_manifest(tmp_path):
    runs = tmp_path / "research" / "mining" / "runs" / "bad"
    runs.mkdir(parents=True)
    (runs / "manifest.json").write_text("{broken")
    assert _count_active_runs(tmp_path) == 0


# ── 强制释放 (真实外部持有进程, POSIX) ─────────────────────────


@_POSIX
def test_force_release_kills_external_holder_and_takes_over(tmp_path):
    proc = _spawn_holder(tmp_path)
    try:
        status = lock_status(tmp_path)
        assert status["held"] is True
        assert status["mine"] is False
        assert status["holder"]["role"] == "worker"
        assert status["force_releasable"] is True, status["reason"]

        lock = MiningProcessLock(tmp_path)
        report = force_release(tmp_path, lock)
        assert report["released"] is True
        assert report["terminated_pid"] == proc.pid
        assert report["reacquired"] is True
        assert lock.acquired is True
        proc.wait(timeout=10)
        assert lock_status(tmp_path)["mine"] is True
        lock.release()
    finally:
        _kill(proc)


@_POSIX
def test_force_release_refuses_fresh_store(tmp_path):
    proc = _spawn_holder(tmp_path)
    try:
        runs = tmp_path / "research" / "mining" / "runs" / "r1"
        runs.mkdir(parents=True)
        (runs / "manifest.json").write_text(json.dumps({"status": "succeeded"}))
        status = lock_status(tmp_path)
        assert status["force_releasable"] is False
        assert "陈旧阈值" in status["reason"]
        with pytest.raises(MiningLockAdminError, match="陈旧阈值"):
            force_release(tmp_path, MiningProcessLock(tmp_path))
        assert proc.poll() is None, "被拒绝时不得误杀持有者"
    finally:
        _kill(proc)


@_POSIX
def test_force_release_refuses_active_runs(tmp_path):
    proc = _spawn_holder(tmp_path)
    try:
        runs = tmp_path / "research" / "mining" / "runs" / "r2"
        runs.mkdir(parents=True)
        (runs / "manifest.json").write_text(json.dumps({"status": "running"}))
        status = lock_status(tmp_path)
        assert status["force_releasable"] is False
        assert "进行中" in status["reason"]
        assert proc.poll() is None
    finally:
        _kill(proc)


def test_force_release_refuses_when_lock_free(tmp_path):
    with pytest.raises(MiningLockAdminError, match="未被持有"):
        force_release(tmp_path, MiningProcessLock(tmp_path))


# ── API 接线: 撞锁降级 + 锁端点 ────────────────────────────────


def _api_client(tmp_path, *, acquired: bool):
    app = FastAPI()
    app.include_router(mining_router)
    app.state.repo = SimpleNamespace(store=SimpleNamespace(data_dir=tmp_path))
    app.state.mining_lock_acquired = acquired
    app.state.mining_process_lock = MiningProcessLock(tmp_path)
    return TestClient(app)


def test_degraded_mode_rejects_mining_start(tmp_path):
    client = _api_client(tmp_path, acquired=False)
    resp = client.post(
        "/api/backtest/mining/runs",
        json={"factor_names": ["momentum_5d"], "budget_profile": "exploratory"},
    )
    assert resp.status_code == 409
    assert "降级" in resp.json()["detail"]


def test_lock_endpoints_free_lock(tmp_path):
    client = _api_client(tmp_path, acquired=True)
    body = client.get("/api/backtest/mining/lock").json()
    assert body["held"] is False
    assert body["force_releasable"] is False
    resp = client.post("/api/backtest/mining/lock/force-release")
    assert resp.status_code == 409
    assert "未被持有" in resp.json()["detail"]


@_POSIX
def test_lock_endpoints_full_recovery_loop(tmp_path):
    proc = _spawn_holder(tmp_path)
    try:
        client = _api_client(tmp_path, acquired=False)
        body = client.get("/api/backtest/mining/lock").json()
        assert body["held"] is True
        assert body["mine"] is False
        assert body["force_releasable"] is True, body["reason"]

        resp = client.post("/api/backtest/mining/lock/force-release")
        assert resp.status_code == 200
        assert resp.json() == {
            "released": True,
            "terminated_pid": proc.pid,
            "reacquired": True,
        }
        proc.wait(timeout=10)
        after = client.get("/api/backtest/mining/lock").json()
        assert after["mine"] is True

        # 接管后挖掘入口必须立即恢复 — 不再是 409 降级
        # (stub app 无 manager, 返回 503; 真实环境此处应为 200/400)
        resp = client.post(
            "/api/backtest/mining/runs",
            json={"factor_names": ["momentum_5d"], "budget_profile": "exploratory"},
        )
        assert resp.status_code != 409
        assert "降级" not in resp.json().get("detail", "")
    finally:
        _kill(proc)


# ── 旧格式锁的 OS 层面持有者定位 (用户反馈: 升级后挖掘一直不可用) ──────────


@_POSIX
def test_legacy_lock_holder_located_via_os_scan(tmp_path):
    """旧格式锁 (b"0", 无 pid) 也能从系统层面定位持有者。

    用户场景: 升级前启动的旧版后端进程残留 (仍持有 flock, 内容为旧格式),
    新版面板无法从内容定位 → 一直提示"请手动结束后端残留进程"。
    """
    proc = _spawn_legacy_holder(tmp_path)
    try:
        assert read_lock_holder(tmp_path) is None
        assert probe_lock_held(tmp_path) is True

        located = locate_lock_holder_pid(tmp_path)
        assert located == proc.pid

        status = lock_status(tmp_path)
        assert status["held"] is True
        assert status["holder_source"] == "os-scan"
        assert status["holder"] is not None
        assert status["holder"]["pid"] == proc.pid
        assert status["holder_alive"] is True
        assert status["force_releasable"] is True, status["reason"]
    finally:
        _kill(proc)


@_POSIX
def test_legacy_lock_holder_force_release_takes_over(tmp_path):
    """定位到旧格式持有者后, 既有的人工确认强制释放流程可完整恢复挖掘。"""
    proc = _spawn_legacy_holder(tmp_path)
    try:
        lock = MiningProcessLock(tmp_path)
        report = force_release(tmp_path, lock)
        assert report["released"] is True
        assert report["terminated_pid"] == proc.pid
        assert report["reacquired"] is True
        proc.wait(timeout=10)
        assert lock_status(tmp_path)["mine"] is True
        lock.release()
    finally:
        _kill(proc)


@_POSIX
def test_legacy_lock_holder_still_refused_while_mining_active(tmp_path):
    """fail-closed 不放宽: 定位到持有者但仍有进行中挖掘运行/存储刚写入 → 拒绝。"""
    proc = _spawn_legacy_holder(tmp_path)
    try:
        runs = tmp_path / "research" / "mining" / "runs" / "r1"
        runs.mkdir(parents=True)
        (runs / "manifest.json").write_text('{"status": "running"}', encoding="utf-8")

        status = lock_status(tmp_path)
        assert status["holder_source"] == "os-scan"
        assert status["force_releasable"] is False
        assert "进行中" in (status["reason"] or "")
    finally:
        _kill(proc)


def test_locate_returns_none_when_lock_free(tmp_path):
    """锁未被持有 (无进程打开) 时定位返回 None, 不误报持有者。"""
    (tmp_path / ".mining_process.lock").write_bytes(b"0")
    assert locate_lock_holder_pid(tmp_path) is None