266 lines
8.7 KiB
Python
266 lines
8.7 KiB
Python
"""Evaluate Sprint 1 compliance detection against a labeled Markdown dataset."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from dataclasses import asdict, dataclass
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from app.model.schemas import ComplianceCheckRequest
|
|
from app.service.compliance_check_service import ComplianceCheckService
|
|
from app.service.compliance_semantic_service import ComplianceSemanticService
|
|
|
|
DEFAULT_DATASET = ROOT / "docs" / "开发文档" / "26-Sprint1合规评测集.md"
|
|
REQUIRED_HEADERS = [
|
|
"case_id",
|
|
"text",
|
|
"expected_level",
|
|
"category",
|
|
"scene",
|
|
"rule_layer",
|
|
"expected_copy_allowed",
|
|
"warn_confirm_required",
|
|
"note",
|
|
]
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ComplianceEvalCase:
|
|
case_id: str
|
|
text: str
|
|
expected_level: str
|
|
category: str
|
|
scene: str
|
|
rule_layer: str
|
|
expected_copy_allowed: bool
|
|
warn_confirm_required: bool
|
|
note: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ComplianceEvalReport:
|
|
total_cases: int
|
|
positives: int
|
|
negatives: int
|
|
true_positives: int
|
|
false_negatives: list[str]
|
|
false_positives: list[str]
|
|
true_negatives: int
|
|
recall: float
|
|
false_positive_rate: float
|
|
exact_level_accuracy: float
|
|
copy_gate_cases: int
|
|
copy_gate_mismatches: list[str]
|
|
|
|
def to_json_dict(self) -> dict:
|
|
return asdict(self)
|
|
|
|
|
|
class DeterministicEvalLLMClient:
|
|
"""Local semantic oracle for development regression tests only."""
|
|
|
|
block_cues = (
|
|
"本金安全边际极高",
|
|
"收益区间基本确定",
|
|
"本金安全性可以放心",
|
|
"亏损概率可以忽略",
|
|
)
|
|
warn_cues = (
|
|
"近期上涨把握较大",
|
|
"窗口期比较短",
|
|
"适合尽快配置",
|
|
"强烈建议立即配置",
|
|
)
|
|
|
|
def complete(self, prompt: str, *, timeout_seconds: float) -> str:
|
|
del timeout_seconds
|
|
text = prompt.rsplit("text=", maxsplit=1)[-1]
|
|
if any(cue in text for cue in self.block_cues):
|
|
return json.dumps(
|
|
{
|
|
"risk_level": "BLOCK",
|
|
"reason": "命中开发评测用隐性本金或收益承诺语义。",
|
|
"suggestion": "改为客观风险揭示，不表达本金或收益确定性。",
|
|
},
|
|
ensure_ascii=False,
|
|
)
|
|
if any(cue in text for cue in self.warn_cues):
|
|
return json.dumps(
|
|
{
|
|
"risk_level": "WARN",
|
|
"reason": "命中开发评测用隐性催促或行情预判语义。",
|
|
"suggestion": "改为中性说明，由客户自主判断。",
|
|
},
|
|
ensure_ascii=False,
|
|
)
|
|
return json.dumps(
|
|
{
|
|
"risk_level": "INFO",
|
|
"reason": "未发现开发评测语义风险。",
|
|
"suggestion": None,
|
|
},
|
|
ensure_ascii=False,
|
|
)
|
|
|
|
|
|
def load_eval_cases(path: Path = DEFAULT_DATASET) -> list[ComplianceEvalCase]:
|
|
rows = _read_markdown_table(path)
|
|
cases: list[ComplianceEvalCase] = []
|
|
seen_ids: set[str] = set()
|
|
for index, row in enumerate(rows, start=1):
|
|
case_id = row["case_id"].strip()
|
|
expected_level = row["expected_level"].strip().upper()
|
|
rule_layer = row["rule_layer"].strip()
|
|
if not case_id:
|
|
raise ValueError(f"Missing case_id at row {index}")
|
|
if case_id in seen_ids:
|
|
raise ValueError(f"Duplicate case_id: {case_id}")
|
|
if expected_level not in {"BLOCK", "WARN", "INFO"}:
|
|
raise ValueError(f"Unsupported expected_level at row {index}: {expected_level}")
|
|
if rule_layer not in {"hard_rule", "semantic", "copy_gate"}:
|
|
raise ValueError(f"Unsupported rule_layer at row {index}: {rule_layer}")
|
|
seen_ids.add(case_id)
|
|
cases.append(
|
|
ComplianceEvalCase(
|
|
case_id=case_id,
|
|
text=row["text"].strip(),
|
|
expected_level=expected_level,
|
|
category=row["category"].strip(),
|
|
scene=row["scene"].strip(),
|
|
rule_layer=rule_layer,
|
|
expected_copy_allowed=_parse_bool(row["expected_copy_allowed"]),
|
|
warn_confirm_required=_parse_bool(row["warn_confirm_required"]),
|
|
note=row["note"].strip(),
|
|
)
|
|
)
|
|
return cases
|
|
|
|
|
|
def evaluate_dataset(path: Path = DEFAULT_DATASET) -> ComplianceEvalReport:
|
|
return evaluate_cases(load_eval_cases(path))
|
|
|
|
|
|
def evaluate_cases(cases: list[ComplianceEvalCase]) -> ComplianceEvalReport:
|
|
semantic_service = ComplianceSemanticService(
|
|
llm_client=DeterministicEvalLLMClient(),
|
|
enabled=True,
|
|
)
|
|
service = ComplianceCheckService(semantic_service=semantic_service)
|
|
true_positives = 0
|
|
true_negatives = 0
|
|
false_negatives: list[str] = []
|
|
false_positives: list[str] = []
|
|
exact_matches = 0
|
|
copy_gate_mismatches: list[str] = []
|
|
|
|
for case in cases:
|
|
result = service.check_text(
|
|
ComplianceCheckRequest(
|
|
text=case.text,
|
|
scene=case.scene or None,
|
|
)
|
|
)
|
|
expected_positive = case.expected_level != "INFO"
|
|
predicted_positive = result.risk_level != "INFO"
|
|
|
|
if result.risk_level == case.expected_level:
|
|
exact_matches += 1
|
|
if expected_positive and predicted_positive:
|
|
true_positives += 1
|
|
elif expected_positive and not predicted_positive:
|
|
false_negatives.append(case.case_id)
|
|
elif not expected_positive and predicted_positive:
|
|
false_positives.append(case.case_id)
|
|
else:
|
|
true_negatives += 1
|
|
|
|
if case.rule_layer == "copy_gate":
|
|
_check_copy_gate_expectation(case, result.risk_level, copy_gate_mismatches)
|
|
|
|
positives = true_positives + len(false_negatives)
|
|
negatives = true_negatives + len(false_positives)
|
|
recall = true_positives / positives if positives else 1.0
|
|
false_positive_rate = len(false_positives) / negatives if negatives else 0.0
|
|
return ComplianceEvalReport(
|
|
total_cases=len(cases),
|
|
positives=positives,
|
|
negatives=negatives,
|
|
true_positives=true_positives,
|
|
false_negatives=false_negatives,
|
|
false_positives=false_positives,
|
|
true_negatives=true_negatives,
|
|
recall=round(recall, 4),
|
|
false_positive_rate=round(false_positive_rate, 4),
|
|
exact_level_accuracy=round(exact_matches / len(cases), 4) if cases else 1.0,
|
|
copy_gate_cases=sum(1 for case in cases if case.rule_layer == "copy_gate"),
|
|
copy_gate_mismatches=copy_gate_mismatches,
|
|
)
|
|
|
|
|
|
def _check_copy_gate_expectation(
|
|
case: ComplianceEvalCase,
|
|
predicted_level: str,
|
|
mismatches: list[str],
|
|
) -> None:
|
|
predicted_copy_allowed = predicted_level != "BLOCK"
|
|
predicted_warn_confirm = predicted_level == "WARN"
|
|
if predicted_copy_allowed != case.expected_copy_allowed:
|
|
mismatches.append(f"{case.case_id}: copy_allowed")
|
|
if predicted_warn_confirm != case.warn_confirm_required:
|
|
mismatches.append(f"{case.case_id}: warn_confirm_required")
|
|
|
|
|
|
def _read_markdown_table(path: Path) -> list[dict[str, str]]:
|
|
if not path.exists():
|
|
raise FileNotFoundError(path)
|
|
|
|
table_lines = [
|
|
line.strip()
|
|
for line in path.read_text(encoding="utf-8").splitlines()
|
|
if line.strip().startswith("|") and line.strip().endswith("|")
|
|
]
|
|
if len(table_lines) < 3:
|
|
raise ValueError("Markdown table is missing or empty")
|
|
|
|
headers = _split_markdown_row(table_lines[0])
|
|
if headers != REQUIRED_HEADERS:
|
|
raise ValueError(f"Unexpected table headers: {headers}")
|
|
|
|
rows: list[dict[str, str]] = []
|
|
for line in table_lines[2:]:
|
|
cells = _split_markdown_row(line)
|
|
if len(cells) != len(headers):
|
|
raise ValueError(f"Column count mismatch: {line}")
|
|
rows.append(dict(zip(headers, cells, strict=True)))
|
|
return rows
|
|
|
|
|
|
def _split_markdown_row(line: str) -> list[str]:
|
|
return [cell.strip() for cell in line.strip().strip("|").split("|")]
|
|
|
|
|
|
def _parse_bool(value: str) -> bool:
|
|
normalized = value.strip().lower()
|
|
if normalized == "true":
|
|
return True
|
|
if normalized == "false":
|
|
return False
|
|
raise ValueError(f"Expected true/false, got: {value}")
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description="Evaluate Sprint 1 compliance dataset.")
|
|
parser.add_argument("--dataset", type=Path, default=DEFAULT_DATASET)
|
|
args = parser.parse_args()
|
|
report = evaluate_dataset(args.dataset)
|
|
print(json.dumps(report.to_json_dict(), ensure_ascii=False, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|