GaoYiYuan_0626
43cf2a1372
- 跨客户并发 InnoDB 1213 死锁由「抛给上层」改为数据访问层自动重试整个事务 （MySQL 官方口径 Retry the entire transaction）；重试次数/退避基数走 settings（convert_deadlock_retry_max/base_ms），只重试 1213，1205 等照旧上抛 - 真库证据：8 客户并发实测触发 4 次 1213，服务端重试全部第 1 次收敛， 外部退避兜底 0 次触发，8/8 成功、份额守恒 - sqlite 单测 +3（重试后落库/预算耗尽上抛/1205 不重试）+ 2 组突变验证命中后恢复 - 全量 739 passed/10 skipped；CONVERT_STRESS 并发 9/9；真库 apply 24/24、service 35/35 零回归
902 lines
40 KiB
Python
902 lines
40 KiB
Python
"""T-13 真 MySQL 并发测试（架构 §9 测试清单 · §10「50 并发压测口径（评审 Q6）」）。
|
||
|
||
**为什么必须是真 MySQL**：本文件验的是 InnoDB **行锁 + 条件 UPDATE rowcount** 这套
|
||
数据库级并发原语。sqlite 内存库（`conftest.sqlite_engine`）共用单连接，多线程跑不出
|
||
真争抢——在那里"绿"是**假绿**（与本项目「方言语义一律上真库」的既有铁律一致）。
|
||
|
||
**为什么不算 CI 常规门禁**（架构 §10 原话：该用例不进 CI，归入 T-13 一并跑）：
|
||
重载用例要 50 线程 + 真库，耗时长。本文件用环境变量 **`CONVERT_STRESS=1`** 实现该语义：
|
||
未设置时重载用例逐条 skip（给出可操作的原因），常规 `pytest -q` 不受拖累。
|
||
T-13 实测：`CONVERT_STRESS=1 pytest tests/test_convert_concurrency.py -q -s`。
|
||
|
||
**隔离策略**（与 `test_convert_integration.py` 同款，三条同时成立，缺一即污染种子）：
|
||
1. id 前缀：`CNV-CONC-` / `TRD-CONC-`（monkeypatch `convert_service._new_id`）；
|
||
2. 数据自建：客户 `CUST-CONCTEST` / 产品 `PROD-CONCTEST?` 全自建，**绝不碰种子**——
|
||
`risk_demo_env` 的 teardown 不还原 `core_share_lot`/`core_holding`，借种子客户跑转换
|
||
会跨用例污染 `test_integration_risk.py`；
|
||
3. teardown 全清：函数级 fixture 按客户 + 前缀删两库全部自建行（幂等）。
|
||
|
||
**连接池**：并发用例用**专用 Engine**（`pool_size=60`）。默认池 `5+10` 会把 50 个线程
|
||
压成 15 路串行，削弱争抢强度、让"不超卖"变成弱证据；专用池只在压测用，常规用例仍走
|
||
`get_engine` 生产口径。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
import threading
|
||
import time
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
from datetime import date, datetime, time as dtime, timedelta
|
||
from decimal import Decimal
|
||
from statistics import mean
|
||
from uuid import uuid4
|
||
|
||
import pytest
|
||
from sqlalchemy import create_engine, text
|
||
from sqlalchemy.exc import OperationalError
|
||
|
||
from conftest import ensure_risk_demo_ready
|
||
|
||
from app.config.settings import settings # noqa: E402
|
||
from app.gateway.convert_core_repository import ConvertCoreRepository # noqa: E402
|
||
from app.repository.convert_repository import ConvertRepository # noqa: E402
|
||
from app.repository.core_ro import CoreReadOnlyRepository # noqa: E402
|
||
from app.repository.risk_repository import RiskRepository # noqa: E402
|
||
from app.service.convert import convert_service as cs # noqa: E402
|
||
from app.service.convert.convert_service import PROCESSING, convert_fund # noqa: E402
|
||
from app.service.convert.errors import LotConflict # noqa: E402
|
||
from app.service.risk.rules import RiskThresholds # noqa: E402
|
||
from app.utils.db import _resolve_credentials # noqa: E402
|
||
|
||
ensure_risk_demo_ready()
|
||
|
||
# ── 隔离种子常量（全部带 CONCTEST 前缀）──────────────────────────────
|
||
CUSTOMER = "CUST-CONCTEST"
|
||
PROD_OUT = "PROD-CONCTESTO" # 转出：债基，申购费率 0.0030
|
||
PROD_IN = "PROD-CONCTESTI" # 转入：股基，申购费率 0.0080
|
||
COMPANY = "华夏模拟基金"
|
||
TA = "TA-CN-001"
|
||
OUT_RATE = Decimal("0.0030")
|
||
IN_RATE = Decimal("0.0080")
|
||
OUT_NAV = Decimal("1.0300")
|
||
IN_NAV = Decimal("0.9500")
|
||
FEE_TIERS = [
|
||
(0, 7, "0.0150"),
|
||
(7, 30, "0.0100"),
|
||
(30, 180, "0.0050"),
|
||
(180, 365, "0.0025"),
|
||
(365, None, "0.0000"),
|
||
]
|
||
_LOT_DAYS = 100 # 全部批次同一费率档（0.0050）→ 断言只看份额守恒，不被费用档干扰
|
||
|
||
_GROUP_LIKE = "CNV-CONC-%"
|
||
|
||
#: 架构 §8.3 建议的退避间隔（`LOT_CONFLICT` → 调用方重试 ≤3 次）
|
||
BACKOFF_MS = (100, 200, 400)
|
||
#: 可重试异常：409 是明确设计为可重试的；死锁/锁等待超时同属瞬时状态
|
||
_RETRYABLE = (LotConflict, OperationalError)
|
||
|
||
|
||
def _stress_enabled() -> bool:
|
||
return os.environ.get("CONVERT_STRESS") == "1"
|
||
|
||
|
||
requires_stress = pytest.mark.skipif(
|
||
not _stress_enabled(),
|
||
reason="重载并发用例不进 CI 常规门禁（架构 §10）；设 CONVERT_STRESS=1 显式开启",
|
||
)
|
||
|
||
|
||
def _noop_engine(out_trade: dict, in_trade: dict) -> dict:
|
||
"""阶段 1.5 的空引擎：并发用例只验**份额争抢**，不掺引擎/出单噪声。"""
|
||
return {"triggered_rules": [], "alert_ids": [], "aml_hit": False}
|
||
|
||
|
||
def _new_id(prefix: str, now: datetime) -> str:
|
||
"""`_new_id` 替换：id 带 `-CONC-` 段（与 `CNV-CONC-%` 清理前缀一致）。"""
|
||
return f"{prefix}-CONC-{uuid4().hex[:10].upper()}"
|
||
|
||
|
||
# ── 专用 Engine（放大连接池）─────────────────────────────────────────
|
||
def _big_pool_engine(database: str, role: str, size: int = 60):
|
||
"""建一个连接池放大的 Engine（与 `get_engine` 同 URL/凭据，仅池参数不同）。"""
|
||
user, password = _resolve_credentials(database, role)
|
||
auth = f"{user}:{password}" if password else user
|
||
url = (
|
||
f"mysql+pymysql://{auth}@{settings.mysql_host}:{settings.mysql_port}"
|
||
f"/{database}?charset=utf8mb4"
|
||
)
|
||
return create_engine(url, pool_pre_ping=True, pool_size=size, max_overflow=size)
|
||
|
||
|
||
# ── 种子与清理 ──────────────────────────────────────────────────────
|
||
def _seed(core, lots: list[str]) -> Decimal:
|
||
"""自建隔离种子；`lots` = 各批次份额。返回份额池总量。"""
|
||
today = date.today()
|
||
base = datetime.combine(today, dtime(10, 0))
|
||
total = sum(Decimal(q) for q in lots)
|
||
with core.begin() as conn:
|
||
conn.execute(
|
||
text(
|
||
"INSERT INTO core_customer (customer_id, display_name, open_date)"
|
||
" VALUES (:c, 'T13并发测试', :d)"
|
||
),
|
||
{"c": CUSTOMER, "d": today},
|
||
)
|
||
conn.execute(
|
||
text(
|
||
"INSERT INTO core_customer_risk (customer_id, risk_code, evaluated_at, expires_at)"
|
||
" VALUES (:c, 'C5', :t, :exp)"
|
||
),
|
||
{
|
||
"c": CUSTOMER,
|
||
"t": base - timedelta(days=30),
|
||
"exp": base + timedelta(days=300),
|
||
},
|
||
)
|
||
for pid, name, ptype, rate in [
|
||
(PROD_OUT, "T13转出基金", "bond", OUT_RATE),
|
||
(PROD_IN, "T13转入基金", "stock", IN_RATE),
|
||
]:
|
||
conn.execute(
|
||
text(
|
||
"INSERT INTO core_product (product_id, product_name, min_risk_code,"
|
||
" product_type, can_subscribe, can_redeem, subscribe_fee_rate,"
|
||
" min_redeem_qty, min_hold_qty, min_hold_action, fund_company, ta_code)"
|
||
" VALUES (:p, :n, 'R2', :t, 1, 1, :r, 0, 0, 'force_transfer', :co, :ta)"
|
||
),
|
||
{"p": pid, "n": name, "t": ptype, "r": str(rate), "co": COMPANY, "ta": TA},
|
||
)
|
||
for mh, mh_max, rate in FEE_TIERS:
|
||
conn.execute(
|
||
text(
|
||
"INSERT INTO core_fee_rule (product_id, fee_type, min_hold_days,"
|
||
" max_hold_days, rate) VALUES (:p, 'redeem', :mh, :mm, :r)"
|
||
),
|
||
{"p": PROD_OUT, "mh": mh, "mm": mh_max, "r": rate},
|
||
)
|
||
conn.execute(
|
||
text(
|
||
"INSERT INTO core_product_nav (product_id, nav, daily_chg_pct, nav_date)"
|
||
" VALUES (:p, :n, 0, :d)"
|
||
),
|
||
{"p": PROD_IN, "n": str(IN_NAV), "d": today},
|
||
)
|
||
for i, qty in enumerate(lots, start=1):
|
||
conn.execute(
|
||
text(
|
||
"INSERT INTO core_share_lot (lot_id, customer_id, product_id, qty,"
|
||
" remain_qty, nav, confirmed_at)"
|
||
" VALUES (:l, :c, :p, :q, :q, :n, :cat)"
|
||
),
|
||
{
|
||
"l": f"LOT-CONCTEST-{i}",
|
||
"c": CUSTOMER,
|
||
"p": PROD_OUT,
|
||
"q": qty,
|
||
"n": str(OUT_NAV),
|
||
"cat": base - timedelta(days=_LOT_DAYS),
|
||
},
|
||
)
|
||
conn.execute(
|
||
text(
|
||
"INSERT INTO core_holding (customer_id, product_id, qty, cost_amount,"
|
||
" market_value, pnl_pct, as_of) VALUES (:c, :p, :q, :cost, :mv, 0, :d)"
|
||
),
|
||
{
|
||
"c": CUSTOMER,
|
||
"p": PROD_OUT,
|
||
"q": str(total),
|
||
"cost": str(total * OUT_NAV),
|
||
"mv": str(total * OUT_NAV),
|
||
"d": today,
|
||
},
|
||
)
|
||
return total
|
||
|
||
|
||
_CUST_LIKE = "CUST-CONCTEST%"
|
||
_PROD_LIKE = "PROD-CONCTEST%"
|
||
#: 删除顺序按真库 FK 依赖（`information_schema.KEY_COLUMN_USAGE` 实测）：
|
||
#: `core_holding`/`core_share_lot`/`core_trade` 同时引用 `core_customer` 与
|
||
#: `core_product`，故必须先删它们，再删 `core_product`，最后删 `core_customer`。
|
||
_CORE_CLEANUP_BY_CUSTOMER = [
|
||
"DELETE FROM core_trade WHERE customer_id LIKE :cl",
|
||
"DELETE FROM core_share_lot WHERE customer_id LIKE :cl",
|
||
"DELETE FROM core_holding WHERE customer_id LIKE :cl",
|
||
"DELETE FROM core_cash_flow WHERE customer_id LIKE :cl",
|
||
"DELETE FROM core_customer_advisor WHERE customer_id LIKE :cl",
|
||
"DELETE FROM core_customer_risk WHERE customer_id LIKE :cl",
|
||
]
|
||
_CORE_CLEANUP_BY_PRODUCT = [
|
||
f"DELETE FROM core_fee_rule WHERE product_id LIKE '{_PROD_LIKE}'",
|
||
f"DELETE FROM core_product_nav WHERE product_id LIKE '{_PROD_LIKE}'",
|
||
f"DELETE FROM core_product WHERE product_id LIKE '{_PROD_LIKE}'",
|
||
]
|
||
|
||
|
||
def _cleanup(core, agent) -> None:
|
||
"""按客户 + 前缀清两库全部自建行（幂等；seed 失败亦可清）。"""
|
||
params = {"cl": _CUST_LIKE}
|
||
with core.begin() as conn:
|
||
conn.execute(
|
||
text(
|
||
"DELETE FROM core_convert_lot_detail"
|
||
f" WHERE convert_group_id LIKE '{_GROUP_LIKE}'"
|
||
)
|
||
)
|
||
for sql in _CORE_CLEANUP_BY_CUSTOMER:
|
||
conn.execute(text(sql), params)
|
||
for sql in _CORE_CLEANUP_BY_PRODUCT:
|
||
conn.execute(text(sql))
|
||
conn.execute(text("DELETE FROM core_customer WHERE customer_id LIKE :cl"), params)
|
||
with agent.begin() as conn:
|
||
conn.execute(
|
||
text(f"DELETE FROM risk_convert_detail WHERE convert_group_id LIKE '{_GROUP_LIKE}'")
|
||
)
|
||
for table in ("risk_alert", "risk_suitability_log", "audit_log"):
|
||
conn.execute(text(f"DELETE FROM {table} WHERE customer_id LIKE :cl"), params)
|
||
|
||
|
||
@pytest.fixture()
|
||
def conc_env(risk_demo_env, monkeypatch):
|
||
"""真库并发环境：admin 引擎（teardown 需 DELETE）+ 专用大池引擎 + id 前缀注入。
|
||
|
||
种子由用例自己 `_seed`（不同用例份额池不同），fixture 只负责清场与收尾。
|
||
"""
|
||
core_admin = risk_demo_env["core"]
|
||
agent_admin = risk_demo_env["agent"]
|
||
core_ro = _big_pool_engine(settings.mysql_core_database, "ro")
|
||
core_rw = _big_pool_engine(settings.mysql_core_database, "rw")
|
||
agent_rw = _big_pool_engine(settings.mysql_database, "rw")
|
||
monkeypatch.setattr(cs, "_new_id", _new_id)
|
||
_cleanup(core_admin, agent_admin)
|
||
try:
|
||
yield {
|
||
"core": core_admin,
|
||
"agent": agent_admin,
|
||
"core_ro": core_ro,
|
||
"core_rw": core_rw,
|
||
"agent_rw": agent_rw,
|
||
}
|
||
finally:
|
||
_cleanup(core_admin, agent_admin)
|
||
for eng in (core_ro, core_rw, agent_rw):
|
||
eng.dispose()
|
||
|
||
|
||
# ── 调用与度量 ──────────────────────────────────────────────────────
|
||
_NOW = datetime.combine(date.today(), dtime(10, 0))
|
||
|
||
|
||
def _convert(env, qty: str, cid: str | None, hook=_noop_engine):
|
||
"""一次转换（生产账号、生产口径；仅 id 工厂与引擎 hook 被替换）。"""
|
||
return convert_fund(
|
||
{
|
||
"customer_id": CUSTOMER,
|
||
"from_product_id": PROD_OUT,
|
||
"to_product_id": PROD_IN,
|
||
"qty": qty,
|
||
"client_request_id": cid,
|
||
},
|
||
core_ro=CoreReadOnlyRepository(engine=env["core_ro"]),
|
||
risk_repo=RiskRepository(engine=env["agent_rw"]),
|
||
convert_repo=ConvertRepository(engine=env["agent_rw"]),
|
||
core_writer=ConvertCoreRepository(engine=env["core_rw"]),
|
||
thresholds=RiskThresholds.from_settings(),
|
||
now=_NOW,
|
||
engine_hook=hook,
|
||
)
|
||
|
||
|
||
class Outcome:
|
||
"""一个并发请求的终态（是否成功、重试了几次、最终异常）。"""
|
||
|
||
__slots__ = ("ok", "retries", "error", "processing", "group_id")
|
||
|
||
def __init__(self, ok, retries, error=None, processing=False, group_id=None):
|
||
self.ok = ok
|
||
self.retries = retries
|
||
self.error = error
|
||
self.processing = processing
|
||
self.group_id = group_id
|
||
|
||
|
||
def _run_with_backoff(env, qty: str, cid: str | None) -> Outcome:
|
||
"""按架构 §8.3 的建议间隔退避重试（≤3 次）；`client_request_id` 全程不变。"""
|
||
attempt = 0
|
||
while True:
|
||
try:
|
||
resp = _convert(env, qty, cid)
|
||
except _RETRYABLE as exc:
|
||
if attempt >= len(BACKOFF_MS):
|
||
return Outcome(False, attempt, type(exc).__name__)
|
||
time.sleep(BACKOFF_MS[attempt] / 1000.0)
|
||
attempt += 1
|
||
continue
|
||
except Exception as exc: # noqa: BLE001
|
||
return Outcome(False, attempt, type(exc).__name__)
|
||
if resp.get("status") == PROCESSING:
|
||
return Outcome(False, attempt, "PROCESSING", processing=True)
|
||
return Outcome(True, attempt, group_id=resp["convert_group_id"])
|
||
|
||
|
||
def _concurrently(fn, n: int, *, stagger_ms: float = 0.0) -> list:
|
||
"""n 个线程同时起跑；`stagger_ms` 为逐线程错开步长（0 = 全同时）。
|
||
|
||
起跑线用 `threading.Barrier` 对齐，避免"线程创建开销"把并发抹平。
|
||
"""
|
||
barrier = threading.Barrier(n)
|
||
|
||
def _worker(i: int):
|
||
barrier.wait()
|
||
if stagger_ms:
|
||
time.sleep(stagger_ms * i / 1000.0)
|
||
return fn(i)
|
||
|
||
with ThreadPoolExecutor(max_workers=n) as pool:
|
||
return list(pool.map(_worker, range(n)))
|
||
|
||
|
||
# ── 度量（全部直接读库，不看响应自证）────────────────────────────────
|
||
def _remain_total(core) -> Decimal:
|
||
with core.connect() as conn:
|
||
return Decimal(
|
||
str(
|
||
conn.execute(
|
||
text(
|
||
"SELECT COALESCE(SUM(remain_qty), 0) FROM core_share_lot"
|
||
" WHERE customer_id = :c AND product_id = :p"
|
||
),
|
||
{"c": CUSTOMER, "p": PROD_OUT},
|
||
).scalar_one()
|
||
)
|
||
)
|
||
|
||
|
||
def _deducted_total(core) -> Decimal:
|
||
"""实际成交的转出份额 = 全部 `redeem` 流水的 qty 之和（转出端=客户被扣的部分）。"""
|
||
with core.connect() as conn:
|
||
return Decimal(
|
||
str(
|
||
conn.execute(
|
||
text(
|
||
"SELECT COALESCE(SUM(qty), 0) FROM core_trade"
|
||
" WHERE customer_id = :c AND trade_type = 'redeem'"
|
||
),
|
||
{"c": CUSTOMER},
|
||
).scalar_one()
|
||
)
|
||
)
|
||
|
||
|
||
def _trades(core) -> list[dict]:
|
||
with core.connect() as conn:
|
||
return [
|
||
dict(r)
|
||
for r in conn.execute(
|
||
text("SELECT * FROM core_trade WHERE customer_id = :c ORDER BY trade_id"),
|
||
{"c": CUSTOMER},
|
||
).mappings()
|
||
]
|
||
|
||
|
||
def _group_counts(core) -> dict[str, int]:
|
||
"""convert_group_id → 该组流水条数（正常恒为 2）。"""
|
||
counts: dict[str, int] = {}
|
||
for row in _trades(core):
|
||
gid = row["convert_group_id"]
|
||
counts[gid] = counts.get(gid, 0) + 1
|
||
return counts
|
||
|
||
|
||
def _assert_no_oversell(core, pool: Decimal, ok_count: int) -> None:
|
||
"""三条硬不变量：不超卖 / 守恒 / 每组恰好两条流水（无重复组）。"""
|
||
deducted = _deducted_total(core)
|
||
remain = _remain_total(core)
|
||
assert deducted <= pool, f"超卖：扣减 {deducted} > 池 {pool}"
|
||
assert remain == pool - deducted, f"不守恒：余 {remain} ≠ 池 {pool} − 扣 {deducted}"
|
||
counts = _group_counts(core)
|
||
assert set(counts.values()) <= {2}, f"存在流水条数 ≠2 的组：{counts}"
|
||
assert len(counts) == ok_count, f"成功数 {ok_count} 与组数 {len(counts)} 不一致"
|
||
assert len(_trades(core)) == ok_count * 2, "流水总数应为成功的转换数 ×2"
|
||
|
||
|
||
# ── 1. 50 并发争抢同一批份额：不许超卖（评审 Q6 断言 ①③）──────────────
|
||
@requires_stress
|
||
def test_50_concurrent_requests_never_oversell(conc_env):
|
||
"""50 线程 × 各申请 2000、份额池 50000（5 批 × 10000）→ **不许超卖**。
|
||
|
||
典型失效形态：若 `_deduct_lots` 的条件 UPDATE 丢掉 `remain_qty >= :q`
|
||
（并发哨兵），50 笔会**全部**成交 → 扣减 100000 > 池 50000 → 本用例红。
|
||
|
||
⚠️ **为什么成交笔数不是恒等于 25（理论上限）**——实测在 **20~25** 间浮动：
|
||
`plan_lots` 会把一笔需求跨批次拆成多笔扣减（如 `400 + 1600`），而
|
||
`_deduct_lots` 要求**本次全部扣减都成功**，否则整个阶段一事务回滚 →
|
||
只要其中任一批次被别人清零，这一笔就整体 `LotConflict` 并进入重试；
|
||
重试窗口重叠时彼此让路，落点因此不定。**不超卖是硬不变量，成交笔数是观测量。**
|
||
"""
|
||
core = conc_env["core"]
|
||
pool = _seed(core, ["10000"] * 5)
|
||
|
||
outcomes = _concurrently(lambda i: _run_with_backoff(conc_env, "2000", None), 50)
|
||
ok = [o for o in outcomes if o.ok]
|
||
|
||
errs = {o.error for o in outcomes if not o.ok}
|
||
print(
|
||
f"\n[50 并发·不超卖] 成交 {len(ok)}/50（理论上限 25）· 失败类型 {errs} · "
|
||
f"余 {_remain_total(core)} · 扣减 {_deducted_total(core)}"
|
||
)
|
||
# ① 硬不变量：不超卖 / 守恒 / 每组恰好两条流水（无重复组）
|
||
_assert_no_oversell(core, pool, len(ok))
|
||
# ② 失败面：只能是"份额被抢走"这类可重试失败，不得是引擎/映射/锁异常
|
||
assert errs <= {"LotConflict", "InsufficientShares", "OperationalError"}, (
|
||
f"出现了非预期失败类型：{errs}"
|
||
)
|
||
# ③ 反"假绿"地板：成交数塌穿 20 说明并发路径退化（哨兵过严/锁失效），必须红
|
||
assert len(ok) >= 20, f"成交仅 {len(ok)} 笔，并发路径疑似退化：{errs}"
|
||
|
||
|
||
# ── 2. 退避重试的最终成功率（评审 Q6 断言 ②）──────────────────────────
|
||
@requires_stress
|
||
def test_backoff_retry_success_rate_on_exactly_sufficient_pool(conc_env):
|
||
"""50 线程 × 各申请 1000、池恰好 50000 → 需求与供给**恰好相等**（最紧张）。
|
||
|
||
这是「验证 §8.3 建议间隔（100/200/400ms）够不够」的场景：并发读-改-写必然
|
||
产生一批 `LotConflict`，退避重试后能否全部成交，直接量化建议间隔的充分性。
|
||
|
||
**实测结论（2026-09-10 · 本机 MySQL 8.0.46）**：3 次 × 100/200/400ms 只到
|
||
**80%**（40/50，10 笔重试耗尽后仍 `LotConflict`）。根因不是"间隔太短"，而是
|
||
**近空批次的反复争抢**：`plan_lots` 会把一笔需求跨批次拆成
|
||
`400 + 600` 这类碎片，碎片所在批次随时被别人清零 → 该笔重试仍可能抢不到。
|
||
要收敛到 100% 需要**更多次重试**或**冲突后换批次重规划**，而非单纯拉长间隔。
|
||
该结论已写入 `交接文档.md` §B；本用例把数字**钉住**（防日后倒退）。
|
||
"""
|
||
core = conc_env["core"]
|
||
pool = _seed(core, ["10000"] * 5)
|
||
|
||
outcomes = _concurrently(lambda i: _run_with_backoff(conc_env, "1000", None), 50)
|
||
ok = [o for o in outcomes if o.ok]
|
||
retried = [o for o in outcomes if o.retries]
|
||
rate = len(ok) / 50
|
||
|
||
_assert_no_oversell(core, pool, len(ok))
|
||
assert _remain_total(core) == pool - _deducted_total(core)
|
||
# 失败者必须是**可重试类型**（份额被抢走），不得出现引擎/锁/映射异常
|
||
errs = {o.error for o in outcomes if not o.ok}
|
||
assert errs <= {"LotConflict", "InsufficientShares"}, f"非预期失败类型：{errs}"
|
||
print(
|
||
f"\n[退避重试·紧池] 成功率 {len(ok)}/50 = {rate:.0%} · "
|
||
f"发生重试的请求 {len(retried)}/50 · 重试次数 {sorted(o.retries for o in outcomes)} · "
|
||
f"失败 {sorted(o.error for o in outcomes if not o.ok)}"
|
||
)
|
||
# 护栏（**非指标**）：<60% 说明退避链路整体失效（例如锁/哨兵被改坏），必须红
|
||
assert rate >= 0.6, f"退避链路疑似失效：成功率仅 {rate:.0%}"
|
||
|
||
|
||
@requires_stress
|
||
def test_backoff_retry_all_succeed_when_pool_is_not_tight(conc_env):
|
||
"""50 线程 × 各申请 500、池 50000（需求 25000 < 供给）→ **50/50 全成功**。
|
||
|
||
与紧池用例互为对照：证明条件 UPDATE 哨兵 + 退避链路在**非耗尽**场景下
|
||
完全可靠——紧池的 80% 是"供给恰好用尽"的固有争抢，不是链路缺陷。
|
||
"""
|
||
core = conc_env["core"]
|
||
pool = _seed(core, ["10000"] * 5)
|
||
|
||
outcomes = _concurrently(lambda i: _run_with_backoff(conc_env, "500", None), 50)
|
||
ok = [o for o in outcomes if o.ok]
|
||
|
||
assert len(ok) == 50, f"非紧池应全部成交，实际 {len(ok)}：{[o.error for o in outcomes if not o.ok]}"
|
||
_assert_no_oversell(core, pool, len(ok))
|
||
print(
|
||
f"\n[退避重试·松池] 成功率 {len(ok)}/50 · "
|
||
f"发生重试的请求 {sum(1 for o in outcomes if o.retries)}/50 · 余 {_remain_total(core)}"
|
||
)
|
||
|
||
|
||
# ── 3. 同键并发：只允许产生一次转换（架构 §9「同键并发 → 202」）────────
|
||
def test_same_client_request_id_concurrent_produces_single_conversion(conc_env):
|
||
"""同 `client_request_id` 并发提交 → 只允许一组流水、只扣一次份额。
|
||
|
||
期望（架构 §9）：未抢到执行权的那些返回 **202 `processing`**。
|
||
允许的终态：`processing`（202）或**幂等命中**（200，拿到首次结果）；
|
||
绝不允许出现**第二组流水**（双扣）。
|
||
"""
|
||
core = conc_env["core"]
|
||
pool = _seed(core, ["50000"])
|
||
n = 8
|
||
cid = f"CONC-SAME-{uuid4().hex[:8]}"
|
||
|
||
def _one(_i: int) -> Outcome:
|
||
try:
|
||
resp = _convert(conc_env, "120", cid)
|
||
except Exception as exc: # noqa: BLE001
|
||
return Outcome(False, 0, type(exc).__name__)
|
||
if resp.get("status") == PROCESSING:
|
||
return Outcome(False, 0, None, processing=True)
|
||
return Outcome(True, 0, group_id=resp["convert_group_id"])
|
||
|
||
outcomes = _concurrently(_one, n, stagger_ms=1.5)
|
||
kinds: dict[str, int] = {}
|
||
for o in outcomes:
|
||
key = "processing" if o.processing else ("ok" if o.ok else f"error:{o.error}")
|
||
kinds[key] = kinds.get(key, 0) + 1
|
||
print(f"\n[同键并发 {n} 路] 终态 {kinds} · 组数 {len(_group_counts(core))}")
|
||
|
||
# 唯一硬约束：**只扣一次**（不得双组流水）
|
||
assert _deducted_total(core) == Decimal("120.0000"), (
|
||
f"同键并发了 {_deducted_total(core)} 份，应为 120 份（不得双扣）"
|
||
)
|
||
assert len(_group_counts(core)) == 1
|
||
assert len(_trades(core)) == 2
|
||
# 错误面约束：不得出现 5xx（幂等语义应表现为 202 或幂等命中）
|
||
assert not [o for o in outcomes if o.error], (
|
||
f"同键并发出现了 5xx：{[o.error for o in outcomes if o.error]}"
|
||
)
|
||
assert pool == Decimal("50000")
|
||
|
||
|
||
# ── 3b. 同键并发的**竞态窗口**（确定性交错 · 缺陷已修复后的回归闸门）────
|
||
@requires_stress
|
||
def test_same_client_request_id_interleaving_must_not_leak_5xx(conc_env, monkeypatch):
|
||
"""把 T1 **确定性**停在「占位已落、幂等锁已释放、阶段一未提交」那一点，再放 T2 进来。
|
||
|
||
这是并发缺陷的标准证法（确定性交错，不靠碰运气）：`convert_fund` 的
|
||
`with try_lock("convert:idem:{cid}")` 只包住**幂等判定**（出块即释放），
|
||
紧随其后的 ①②③④（校验与折算，多次 DB 往返）+ ⑤（占位）与 T1 的阶段一之间
|
||
形成一个数毫秒的窗口。窗口内进入的第二笔请求读到 `existing.status='pending'`
|
||
且 `has_convert_trades=False` → 判定为「阶段一未成 → 复用 group_id 重跑」，
|
||
于是**两笔带着同一个 `group_id` 同时跑阶段一**。
|
||
|
||
已实测的失效形态（**这个子窗口不是双扣**，是 500）：
|
||
`in_lot_id = f"LOT-{group_id}-IN"` 派生自 group_id，后跑的那笔 INSERT 撞
|
||
`core_share_lot` 主键 → 整个阶段一事务回滚 → **只扣一次**（120 份）。代价是
|
||
**未映射的 `IntegrityError` 直穿到 API = 500**，而架构 §9 的契约是
|
||
「同键并发 → 202」、§7.3 的契约是幂等命中。
|
||
|
||
⚠️ **但"不是双扣"只在本子窗口成立** —— 见 3c：若两笔各自生成了**不同**的
|
||
group_id，`in_lot_id` 不会相撞，那边是**真·双扣**（实测扣 240 份）。
|
||
也就是说 3b 的 120 份是**顺带**被派生主键挡下的，不是被并发设计挡下的。
|
||
|
||
本用例断言正确契约。**修复**（T-13 落地）：幂等判定区分「确定没跑成」与
|
||
「在飞/未知」——`failed`/`expired` 才复用 group_id 重跑，`pending` 一律回 202。
|
||
本用例即修复后的回归闸门：把 `pending` 改回"也重跑"即变红。
|
||
"""
|
||
core = conc_env["core"]
|
||
_seed(core, ["50000"])
|
||
cid = f"CONC-RACE-{uuid4().hex[:8]}"
|
||
|
||
original = ConvertCoreRepository.apply_convert
|
||
in_stage1 = threading.Event()
|
||
release = threading.Event()
|
||
gate_used = {"v": False}
|
||
|
||
def gated(self, req): # noqa: ANN001
|
||
if not gate_used["v"]: # 只拦第一笔（T1）
|
||
gate_used["v"] = True
|
||
in_stage1.set()
|
||
release.wait(10)
|
||
return original(self, req)
|
||
|
||
monkeypatch.setattr(ConvertCoreRepository, "apply_convert", gated)
|
||
|
||
got: dict = {}
|
||
|
||
def _t1() -> None:
|
||
try:
|
||
got["t1"] = _convert(conc_env, "120", cid)
|
||
except Exception as exc: # noqa: BLE001
|
||
got["t1_err"] = type(exc).__name__
|
||
|
||
th = threading.Thread(target=_t1, name="T1")
|
||
th.start()
|
||
assert in_stage1.wait(10), "T1 未进入阶段一，交错构造失败"
|
||
# 此刻 T1：占位 pending 已落库、`convert:idem:` 锁已释放、阶段一未提交
|
||
try:
|
||
got["t2"] = _convert(conc_env, "120", cid)
|
||
except Exception as exc: # noqa: BLE001
|
||
got["t2_err"] = type(exc).__name__
|
||
release.set()
|
||
th.join(20)
|
||
|
||
deducted = _deducted_total(core)
|
||
counts = _group_counts(core)
|
||
print(
|
||
f"\n[同键竞态·确定性交错] T1={got.get('t1_err') or 'ok'} "
|
||
f"T2={got.get('t2_err') or 'ok'} · 扣减 {deducted} · 组→流水数 {counts}"
|
||
)
|
||
# ① 资金安全：只扣一次（当前由 in_lot_id 主键**顺带**保证，非设计保证）
|
||
assert deducted == Decimal("120.0000"), f"同键被扣了 {deducted} 份（应为 120）"
|
||
assert len(counts) == 1 and len(_trades(core)) == 2, f"应一组两条流水：{counts}"
|
||
# ② 错误面：不得有未映射异常直穿（契约是 202 / 幂等命中，不是 5xx）
|
||
leaked = {k: v for k, v in got.items() if k.endswith("_err")}
|
||
assert not leaked, f"同键并发出现未映射异常直穿（应为 202 或幂等命中）：{leaked}"
|
||
|
||
|
||
# ── 3c. 同键并发的**第二个子窗口**：两笔都判定"无占位"（确定性交错）────
|
||
@requires_stress
|
||
def test_same_client_request_id_placeholder_race_must_not_leak_5xx(conc_env, monkeypatch):
|
||
"""把 T1 停在「幂等判定已过、占位尚未插入」那一点，让 T2 先占位成功。
|
||
|
||
这是与 3b **不同的**子窗口：
|
||
- 3b：T2 在 T1 占位**之后**进入 → 看到 `pending` → 旧代码当"未成"重跑 → 双跑阶段一
|
||
（同 group_id，被 `in_lot_id` 主键顺带挡下，表现为 500）；
|
||
- 3c：T2 在 T1 占位**之前**进入 → 也判定"无占位" → **各自生成不同的 group_id**
|
||
→ 后插入的那笔撞 `uk_idem`。
|
||
|
||
**旧代码表现**：撞键直穿未映射的 `IntegrityError`(503)。
|
||
|
||
⚠️ **这条路径比 503 更危险**：若只修仓储（让 `insert_placeholder` 撞键返回 False）
|
||
而**不接住这个让路信号**，两笔会各带**不同 group_id** 一路跑到阶段一 ——
|
||
`in_lot_id` 不再相撞 → **真·双扣**。突变验证实测：扣 **240** 份、**两组流水**
|
||
（T1=ok T2=ok，`{'CNV-…': 2, 'CNV-…': 2}`）。这是本次 T-13 压测最严重的一处发现，
|
||
也是"必须由调用方 `return 202`"而非"仓储静默吞掉"的原因。
|
||
|
||
修复后：`insert_placeholder` 返回"本笔是否持有占位"，撞键即让路 → 调用方回 202。
|
||
断言：① 无未映射异常直穿；② 只扣一次；③ 只有一组流水。
|
||
"""
|
||
core = conc_env["core"]
|
||
_seed(core, ["50000"])
|
||
cid = f"CONC-UKIDEM-{uuid4().hex[:8]}"
|
||
|
||
original = ConvertRepository.insert_placeholder
|
||
in_zero = threading.Event()
|
||
release = threading.Event()
|
||
gate_used = {"v": False}
|
||
|
||
def gated(self, group_id, client_request_id): # noqa: ANN001
|
||
if not gate_used["v"]: # 只拦第一笔（T1），且**拦在插入之前**
|
||
gate_used["v"] = True
|
||
in_zero.set()
|
||
release.wait(10)
|
||
return original(self, group_id, client_request_id)
|
||
|
||
monkeypatch.setattr(ConvertRepository, "insert_placeholder", gated)
|
||
|
||
got: dict = {}
|
||
|
||
def _t1() -> None:
|
||
try:
|
||
got["t1"] = _convert(conc_env, "120", cid)
|
||
except Exception as exc: # noqa: BLE001
|
||
got["t1_err"] = type(exc).__name__
|
||
|
||
th = threading.Thread(target=_t1, name="T1")
|
||
th.start()
|
||
assert in_zero.wait(10), "T1 未到达阶段零，交错构造失败"
|
||
# 此刻 T1：幂等判定已过（无占位）、`convert:idem:` 锁已释放、占位未插
|
||
try:
|
||
got["t2"] = _convert(conc_env, "120", cid)
|
||
except Exception as exc: # noqa: BLE001
|
||
got["t2_err"] = type(exc).__name__
|
||
release.set()
|
||
th.join(20)
|
||
|
||
deducted = _deducted_total(core)
|
||
counts = _group_counts(core)
|
||
print(
|
||
f"\n[占位竞态·确定性交错] T1={got.get('t1_err') or 'ok'} "
|
||
f"T2={got.get('t2_err') or 'ok'} · 扣减 {deducted} · 组→流水数 {counts}"
|
||
)
|
||
leaked = {k: v for k, v in got.items() if k.endswith("_err")}
|
||
assert not leaked, f"uk_idem 竞态直穿未映射异常（应为 202）：{leaked}"
|
||
assert deducted == Decimal("120.0000"), f"同键被扣了 {deducted} 份（应为 120）"
|
||
assert len(counts) == 1 and len(_trades(core)) == 2, f"应一组两条流水：{counts}"
|
||
|
||
|
||
# ── 4. 补跑阶段二加锁：并发补偿只落一次（T-12 的并发面）────────────────
|
||
def test_concurrent_compensation_writes_detail_once(conc_env, monkeypatch):
|
||
"""阶段二失败后并发补偿（`convert:rerun:` 锁）→ 详情只补一次、只有一组流水。"""
|
||
from app.service.convert.convert_service import compensate_convert
|
||
|
||
core = conc_env["core"]
|
||
agent = conc_env["agent"]
|
||
_seed(core, ["50000"])
|
||
|
||
original = ConvertRepository.complete_convert
|
||
|
||
def boom(self, group_id, **kwargs): # noqa: ANN001
|
||
raise RuntimeError("阶段二写失败（制造待补偿态）")
|
||
|
||
monkeypatch.setattr(ConvertRepository, "complete_convert", boom)
|
||
gid = _convert(conc_env, "120", f"CONC-CMP-{uuid4().hex[:8]}")["convert_group_id"]
|
||
monkeypatch.setattr(ConvertRepository, "complete_convert", original)
|
||
|
||
def _comp(_i: int) -> str:
|
||
svc = dict(
|
||
core_ro=CoreReadOnlyRepository(engine=conc_env["core_ro"]),
|
||
risk_repo=RiskRepository(engine=conc_env["agent_rw"]),
|
||
convert_repo=ConvertRepository(engine=conc_env["agent_rw"]),
|
||
)
|
||
out = compensate_convert(gid, thresholds=RiskThresholds.from_settings(), now=_NOW, **svc)
|
||
return out["state"]
|
||
|
||
states = _concurrently(_comp, 6, stagger_ms=1.0)
|
||
print(f"\n[并发补偿 6 路] 状态分布 {states}")
|
||
assert states.count("rebuilt") == 1, f"应恰好 1 路真补写，实际 {states}"
|
||
with agent.connect() as conn:
|
||
rows = [
|
||
dict(r)
|
||
for r in conn.execute(
|
||
text("SELECT status FROM risk_convert_detail WHERE convert_group_id = :g"),
|
||
{"g": gid},
|
||
).mappings()
|
||
]
|
||
assert rows == [{"status": "completed"}], f"详情应只有一行 completed：{rows}"
|
||
assert len(_group_counts(core)) == 1 and len(_trades(core)) == 2
|
||
|
||
|
||
# ── 5. 多客户并发：互不阻塞，但会出 1213 死锁（须靠退避重试收敛）─────────
|
||
@requires_stress
|
||
def test_concurrent_different_customers_need_deadlock_retry(conc_env):
|
||
"""8 个客户并发转换同一转出产品 → 全部成功，但**必须靠重试死锁**。
|
||
|
||
**实测发现（2026-09-10）**：跨客户并发在 `core_holding`/`core_share_lot` 的
|
||
唯一索引上会触发 InnoDB **1213 死锁**（RR 隔离级下 INSERT/UPDATE 的
|
||
插入意向锁与间隙锁互斥）。**T-13 后补（同日用户拍板）**：`apply_convert`
|
||
已在数据访问层对 1213 自动重试整事务（settings 可配）；本用例保留 §8.3
|
||
的**外部退避兜底**，双层重试叠加下 8 客户并发仍须全部成功、份额守恒、
|
||
group_id 两两不同。
|
||
|
||
注意与「50 并发不超卖」的区别：那批是**同一** (客户, 产品)，走的是行锁阻塞；
|
||
这批是**不同**客户，走的是间隙锁 → 死锁而非阻塞。
|
||
"""
|
||
core = conc_env["core"]
|
||
_seed(core, ["5000"] * 2)
|
||
|
||
others = [f"CUST-CONCTEST-{i}" for i in range(1, 8)]
|
||
with core.begin() as conn:
|
||
for c in others:
|
||
conn.execute(
|
||
text(
|
||
"INSERT INTO core_customer (customer_id, display_name, open_date)"
|
||
" VALUES (:c, 'T13并发客户', CURDATE())"
|
||
),
|
||
{"c": c},
|
||
)
|
||
conn.execute(
|
||
text(
|
||
"INSERT INTO core_customer_risk (customer_id, risk_code, evaluated_at,"
|
||
" expires_at) VALUES (:c, 'C5', NOW(), DATE_ADD(NOW(), INTERVAL 300 DAY))"
|
||
),
|
||
{"c": c},
|
||
)
|
||
conn.execute(
|
||
text(
|
||
"INSERT INTO core_share_lot (lot_id, customer_id, product_id, qty,"
|
||
" remain_qty, nav, confirmed_at)"
|
||
" VALUES (:l, :c, :p, 5000, 5000, :n, DATE_SUB(NOW(), INTERVAL 100 DAY))"
|
||
),
|
||
{"l": f"LOT-CONCTEST-OTH-{c}", "c": c, "p": PROD_OUT, "n": str(OUT_NAV)},
|
||
)
|
||
|
||
customers = [CUSTOMER, *others]
|
||
|
||
def _one(i: int) -> Outcome:
|
||
"""走 `_run_with_backoff` 的等价逻辑，但客户可变（死锁重试在此体现）。"""
|
||
attempt = 0
|
||
while True:
|
||
try:
|
||
resp = convert_fund(
|
||
{
|
||
"customer_id": customers[i],
|
||
"from_product_id": PROD_OUT,
|
||
"to_product_id": PROD_IN,
|
||
"qty": "1000",
|
||
},
|
||
core_ro=CoreReadOnlyRepository(engine=conc_env["core_ro"]),
|
||
risk_repo=RiskRepository(engine=conc_env["agent_rw"]),
|
||
convert_repo=ConvertRepository(engine=conc_env["agent_rw"]),
|
||
core_writer=ConvertCoreRepository(engine=conc_env["core_rw"]),
|
||
thresholds=RiskThresholds.from_settings(),
|
||
now=_NOW,
|
||
engine_hook=_noop_engine,
|
||
)
|
||
except _RETRYABLE as exc:
|
||
if attempt >= len(BACKOFF_MS):
|
||
return Outcome(False, attempt, type(exc).__name__)
|
||
time.sleep(BACKOFF_MS[attempt] / 1000.0)
|
||
attempt += 1
|
||
continue
|
||
except Exception as exc: # noqa: BLE001
|
||
return Outcome(False, attempt, type(exc).__name__)
|
||
return Outcome(True, attempt, group_id=resp["convert_group_id"])
|
||
|
||
outcomes = _concurrently(_one, len(customers))
|
||
ok = [o for o in outcomes if o.ok]
|
||
deadlock_retries = sum(o.retries for o in outcomes)
|
||
print(
|
||
f"\n[8 客户并发] 成功 {len(ok)}/8 · 死锁重试累计 {deadlock_retries} 次 · "
|
||
f"失败 {[o.error for o in outcomes if not o.ok]}"
|
||
)
|
||
assert len(ok) == 8, f"退避重试后应全部成功：{[o.error for o in outcomes if not o.ok]}"
|
||
with core.connect() as conn:
|
||
total = Decimal(
|
||
str(
|
||
conn.execute(
|
||
text(
|
||
"SELECT COALESCE(SUM(remain_qty), 0) FROM core_share_lot"
|
||
" WHERE product_id = :p AND lot_id LIKE 'LOT-CONCTEST%'"
|
||
),
|
||
{"p": PROD_OUT},
|
||
).scalar_one()
|
||
)
|
||
)
|
||
# 8 个客户各转出 1000；主客户有 2 批 × 5000 = 10000，其余 7 个各 5000
|
||
# → 初始合计 45000，扣 8000 → 37000
|
||
expected = Decimal("45000") - Decimal("1000") * len(customers)
|
||
assert total == expected, f"剩余合计应为 {expected}，实际 {total}"
|
||
assert len({o.group_id for o in ok}) == 8, "8 个客户的 group_id 必须两两不同"
|
||
|
||
|
||
# ── 6. 性能实测（PRD §9 第 18 条）────────────────────────────────────
|
||
class _TimingWriter:
|
||
"""`ConvertCoreRepository` 代理：只测**阶段一单库事务**耗时，不改任何行为。"""
|
||
|
||
def __init__(self, inner):
|
||
self._inner = inner
|
||
self.samples: list[float] = []
|
||
|
||
def apply_convert(self, req):
|
||
t0 = time.perf_counter()
|
||
try:
|
||
return self._inner.apply_convert(req)
|
||
finally:
|
||
self.samples.append((time.perf_counter() - t0) * 1000.0)
|
||
|
||
|
||
def _pct(values: list[float], p: float) -> float:
|
||
ordered = sorted(values)
|
||
idx = min(len(ordered) - 1, max(0, int(round((p / 100.0) * len(ordered) + 0.5)) - 1))
|
||
return ordered[idx]
|
||
|
||
|
||
@requires_stress
|
||
def test_performance_probe_stage1_and_end_to_end(conc_env):
|
||
"""实测：阶段一单库事务 + 端到端一次转换（PRD §9 第 18 条补录来源）。
|
||
|
||
测量条件：本机 MySQL 8.0.46（127.0.0.1）、模拟库、**单线程顺序**、
|
||
份额池充足的稳态（每笔 1 份，避免 `plan_lots` 跨批波动）。
|
||
端到端含阶段 1.5 空引擎（真实引擎的额外开销在 `test_convert_integration` 另行体现）。
|
||
"""
|
||
core = conc_env["core"]
|
||
n = 60
|
||
_seed(core, [str(n)])
|
||
|
||
writer = _TimingWriter(ConvertCoreRepository(engine=conc_env["core_rw"]))
|
||
totals: list[float] = []
|
||
for i in range(n):
|
||
req = {
|
||
"customer_id": CUSTOMER,
|
||
"from_product_id": PROD_OUT,
|
||
"to_product_id": PROD_IN,
|
||
"qty": "1",
|
||
"client_request_id": f"CONC-PERF-{i}",
|
||
}
|
||
t0 = time.perf_counter()
|
||
convert_fund(
|
||
req,
|
||
core_ro=CoreReadOnlyRepository(engine=conc_env["core_ro"]),
|
||
risk_repo=RiskRepository(engine=conc_env["agent_rw"]),
|
||
convert_repo=ConvertRepository(engine=conc_env["agent_rw"]),
|
||
core_writer=writer,
|
||
thresholds=RiskThresholds.from_settings(),
|
||
now=_NOW,
|
||
engine_hook=_noop_engine,
|
||
)
|
||
totals.append((time.perf_counter() - t0) * 1000.0)
|
||
|
||
# 首个样本含连接池冷启动，剔除（否则把"建连"算进"业务耗时"）
|
||
stage1 = writer.samples[1:]
|
||
e2e = totals[1:]
|
||
print(
|
||
f"\n[性能实测 n={len(e2e)}] 阶段一 ms: P50={_pct(stage1, 50):.1f} "
|
||
f"P95={_pct(stage1, 95):.1f} max={max(stage1):.1f} mean={mean(stage1):.1f}"
|
||
f"\n 端到端 ms: P50={_pct(e2e, 50):.1f} P95={_pct(e2e, 95):.1f} "
|
||
f"max={max(e2e):.1f} mean={mean(e2e):.1f}"
|
||
)
|
||
# PRD §9 第 18 条的验收硬指标是端到端 < 2s（阶段一 <100ms 是预估、非硬指标）
|
||
assert max(e2e) < 2000, f"端到端最大 {max(e2e):.1f}ms 超 PRD §9 第 18 条的 2s"
|