mirror of
https://github.com/shy3130/tick-stock-panel.git
synced 2026-11-02 10:26:13 +00:00
shy3130
b573ca43eb
复现的原始问题: 旧实例挂死在 shutdown (热重载撞上挖掘运行) 或 worker
成为孤儿后, `.mining_process.lock` 被无限期占用, 后续所有后端实例
在 lifespan 抢锁时直接启动失败, 用户只能对着笼统报错手动找进程。
- 锁文件记录持有者 {pid, started_at, role, host} (flock 仍是唯一
事实, 内容只是诊断线索; 旧格式 "0" 兼容, 视为持有者未知);
新增非阻塞探测, 无需抢锁即可判断是否被持有。
- 撞锁降级启动: lifespan 抢锁失败不再上抛, 面板照常可用; 挖掘
启动入口 (手动/自动/周度调度) 返回 409 并指引到挖掘页; 桌面版
预检从「秒退失败页」改为留日志继续启动; 守护复验不受影响。
- 挖掘页锁占用横幅: 显示持有者 pid/角色/挖掘存储静默时长; 满足
陈旧判定时可一键「强制停止并接管」(SIGTERM→SIGKILL→当前后端
获取锁, 挖掘入口立即恢复)。
- 陈旧判定全部 fail-closed: 持有者信息可读、进程存活且 pid 未被
复用 (POSIX 比对进程实际启动时间)、无进行中的挖掘运行、挖掘
存储 10 分钟内无写入 — 任一不满足即拒绝并给出原因。不做启动
时自动清理, 终止进程必须人工确认。
测试: 新增 test_mining_lock_admin (含真实外部持锁进程的强停回路与
五类拒绝路径), 重写 lifespan 降级契约与桌面预检继续启动契约;
后端全量 3239 通过, 前端 308 通过, ruff 零新增; 沙盒 E2E 完整
复刻「持锁→降级→强停→接管→恢复」事故场景通过。
Signed-off-by: shy3130 <shy3130@qq.com>
145 lines
4.1 KiB
Python
145 lines
4.1 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
from contextlib import asynccontextmanager
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
import app.main as main_module
|
|
|
|
|
|
def _app_stub() -> SimpleNamespace:
|
|
return SimpleNamespace(state=SimpleNamespace())
|
|
|
|
|
|
def test_lifespan_holds_mining_process_lock_around_application(monkeypatch) -> None:
|
|
events: list[str] = []
|
|
|
|
class LockStub:
|
|
def __init__(self, data_dir) -> None:
|
|
del data_dir
|
|
events.append("lock_created")
|
|
self.acquired = False
|
|
|
|
def acquire(self) -> None:
|
|
self.acquired = True
|
|
events.append("lock_acquired")
|
|
|
|
def release(self) -> None:
|
|
self.acquired = False
|
|
events.append("lock_released")
|
|
|
|
@asynccontextmanager
|
|
async def application_lifespan(_app):
|
|
events.append("application_started")
|
|
try:
|
|
yield
|
|
finally:
|
|
events.append("application_stopped")
|
|
|
|
monkeypatch.setattr(main_module, "MiningProcessLock", LockStub)
|
|
monkeypatch.setattr(main_module, "_application_lifespan", application_lifespan)
|
|
|
|
app = _app_stub()
|
|
|
|
async def exercise() -> None:
|
|
async with main_module.lifespan(app):
|
|
events.append("request_serving")
|
|
|
|
asyncio.run(exercise())
|
|
|
|
assert events == [
|
|
"lock_created",
|
|
"lock_acquired",
|
|
"application_started",
|
|
"request_serving",
|
|
"application_stopped",
|
|
"lock_released",
|
|
]
|
|
assert app.state.mining_lock_acquired is True
|
|
assert app.state.mining_process_lock is not None
|
|
|
|
|
|
def test_lifespan_releases_lock_when_application_shutdown_raises(monkeypatch) -> None:
|
|
events: list[str] = []
|
|
|
|
class LockStub:
|
|
def __init__(self, data_dir) -> None:
|
|
del data_dir
|
|
self.acquired = False
|
|
|
|
def acquire(self) -> None:
|
|
self.acquired = True
|
|
events.append("acquired")
|
|
|
|
def release(self) -> None:
|
|
self.acquired = False
|
|
events.append("released")
|
|
|
|
@asynccontextmanager
|
|
async def application_lifespan(_app):
|
|
try:
|
|
yield
|
|
finally:
|
|
raise RuntimeError("shutdown failed")
|
|
|
|
monkeypatch.setattr(main_module, "MiningProcessLock", LockStub)
|
|
monkeypatch.setattr(main_module, "_application_lifespan", application_lifespan)
|
|
|
|
async def exercise() -> None:
|
|
async with main_module.lifespan(_app_stub()):
|
|
pass
|
|
|
|
with pytest.raises(RuntimeError, match="shutdown failed"):
|
|
asyncio.run(exercise())
|
|
|
|
assert events == ["acquired", "released"]
|
|
|
|
|
|
def test_lifespan_degrades_instead_of_failing_when_lock_held(
|
|
monkeypatch, caplog
|
|
) -> None:
|
|
"""抢锁失败: 不再让 lifespan 上抛 (整个后端起不来), 改为降级启动。
|
|
|
|
面板照常可用, 挖掘启动入口返回 409; WARNING 日志留排查线索;
|
|
未获锁不得在 shutdown 时误调 release。
|
|
"""
|
|
from app.services.mining_process_lock import MiningProcessLockError
|
|
|
|
events: list[str] = []
|
|
|
|
class LockStub:
|
|
def __init__(self, data_dir) -> None:
|
|
del data_dir
|
|
self.acquired = False
|
|
|
|
def acquire(self) -> None:
|
|
raise MiningProcessLockError("another process owns mining")
|
|
|
|
def release(self) -> None:
|
|
raise AssertionError("acquire 失败后不应调用 release")
|
|
|
|
@asynccontextmanager
|
|
async def application_lifespan(_app):
|
|
events.append("application_started")
|
|
yield
|
|
events.append("application_stopped")
|
|
|
|
monkeypatch.setattr(main_module, "MiningProcessLock", LockStub)
|
|
monkeypatch.setattr(main_module, "_application_lifespan", application_lifespan)
|
|
|
|
app = _app_stub()
|
|
|
|
async def exercise() -> None:
|
|
async with main_module.lifespan(app):
|
|
events.append("request_serving")
|
|
|
|
with caplog.at_level("WARNING", logger="app.main"):
|
|
asyncio.run(exercise()) # 不再抛出
|
|
|
|
assert events == ["application_started", "request_serving", "application_stopped"]
|
|
assert "挖掘锁" in caplog.text
|
|
assert "降级" in caplog.text
|
|
assert app.state.mining_lock_acquired is False
|