"""Evaluate Sprint 1 compliance detection against a labeled Markdown dataset.""" from __future__ import annotations import argparse import json import sys from dataclasses import asdict, dataclass from pathlib import Path ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT)) from app.model.schemas import ComplianceCheckRequest from app.service.compliance_check_service import ComplianceCheckService from app.service.compliance_semantic_service import ComplianceSemanticService DEFAULT_DATASET = ROOT / "docs" / "开发文档" / "26-Sprint1合规评测集.md" REQUIRED_HEADERS = [ "case_id", "text", "expected_level", "category", "scene", "rule_layer", "expected_copy_allowed", "warn_confirm_required", "note", ] @dataclass(frozen=True) class ComplianceEvalCase: case_id: str text: str expected_level: str category: str scene: str rule_layer: str expected_copy_allowed: bool warn_confirm_required: bool note: str @dataclass(frozen=True) class ComplianceEvalReport: total_cases: int positives: int negatives: int true_positives: int false_negatives: list[str] false_positives: list[str] true_negatives: int recall: float false_positive_rate: float exact_level_accuracy: float copy_gate_cases: int copy_gate_mismatches: list[str] def to_json_dict(self) -> dict: return asdict(self) class DeterministicEvalLLMClient: """Local semantic oracle for development regression tests only.""" block_cues = ( "本金安全边际极高", "收益区间基本确定", "本金安全性可以放心", "亏损概率可以忽略", ) warn_cues = ( "近期上涨把握较大", "窗口期比较短", "适合尽快配置", "强烈建议立即配置", ) def complete(self, prompt: str, *, timeout_seconds: float) -> str: del timeout_seconds text = prompt.rsplit("text=", maxsplit=1)[-1] if any(cue in text for cue in self.block_cues): return json.dumps( { "risk_level": "BLOCK", "reason": "命中开发评测用隐性本金或收益承诺语义。", "suggestion": "改为客观风险揭示，不表达本金或收益确定性。", }, ensure_ascii=False, ) if any(cue in text for cue in self.warn_cues): return json.dumps( { "risk_level": "WARN", "reason": "命中开发评测用隐性催促或行情预判语义。", "suggestion": "改为中性说明，由客户自主判断。", }, ensure_ascii=False, ) return json.dumps( { "risk_level": "INFO", "reason": "未发现开发评测语义风险。", "suggestion": None, }, ensure_ascii=False, ) def load_eval_cases(path: Path = DEFAULT_DATASET) -> list[ComplianceEvalCase]: rows = _read_markdown_table(path) cases: list[ComplianceEvalCase] = [] seen_ids: set[str] = set() for index, row in enumerate(rows, start=1): case_id = row["case_id"].strip() expected_level = row["expected_level"].strip().upper() rule_layer = row["rule_layer"].strip() if not case_id: raise ValueError(f"Missing case_id at row {index}") if case_id in seen_ids: raise ValueError(f"Duplicate case_id: {case_id}") if expected_level not in {"BLOCK", "WARN", "INFO"}: raise ValueError(f"Unsupported expected_level at row {index}: {expected_level}") if rule_layer not in {"hard_rule", "semantic", "copy_gate"}: raise ValueError(f"Unsupported rule_layer at row {index}: {rule_layer}") seen_ids.add(case_id) cases.append( ComplianceEvalCase( case_id=case_id, text=row["text"].strip(), expected_level=expected_level, category=row["category"].strip(), scene=row["scene"].strip(), rule_layer=rule_layer, expected_copy_allowed=_parse_bool(row["expected_copy_allowed"]), warn_confirm_required=_parse_bool(row["warn_confirm_required"]), note=row["note"].strip(), ) ) return cases def evaluate_dataset(path: Path = DEFAULT_DATASET) -> ComplianceEvalReport: return evaluate_cases(load_eval_cases(path)) def evaluate_cases(cases: list[ComplianceEvalCase]) -> ComplianceEvalReport: semantic_service = ComplianceSemanticService( llm_client=DeterministicEvalLLMClient(), enabled=True, ) service = ComplianceCheckService(semantic_service=semantic_service) true_positives = 0 true_negatives = 0 false_negatives: list[str] = [] false_positives: list[str] = [] exact_matches = 0 copy_gate_mismatches: list[str] = [] for case in cases: result = service.check_text( ComplianceCheckRequest( text=case.text, scene=case.scene or None, ) ) expected_positive = case.expected_level != "INFO" predicted_positive = result.risk_level != "INFO" if result.risk_level == case.expected_level: exact_matches += 1 if expected_positive and predicted_positive: true_positives += 1 elif expected_positive and not predicted_positive: false_negatives.append(case.case_id) elif not expected_positive and predicted_positive: false_positives.append(case.case_id) else: true_negatives += 1 if case.rule_layer == "copy_gate": _check_copy_gate_expectation(case, result.risk_level, copy_gate_mismatches) positives = true_positives + len(false_negatives) negatives = true_negatives + len(false_positives) recall = true_positives / positives if positives else 1.0 false_positive_rate = len(false_positives) / negatives if negatives else 0.0 return ComplianceEvalReport( total_cases=len(cases), positives=positives, negatives=negatives, true_positives=true_positives, false_negatives=false_negatives, false_positives=false_positives, true_negatives=true_negatives, recall=round(recall, 4), false_positive_rate=round(false_positive_rate, 4), exact_level_accuracy=round(exact_matches / len(cases), 4) if cases else 1.0, copy_gate_cases=sum(1 for case in cases if case.rule_layer == "copy_gate"), copy_gate_mismatches=copy_gate_mismatches, ) def _check_copy_gate_expectation( case: ComplianceEvalCase, predicted_level: str, mismatches: list[str], ) -> None: predicted_copy_allowed = predicted_level != "BLOCK" predicted_warn_confirm = predicted_level == "WARN" if predicted_copy_allowed != case.expected_copy_allowed: mismatches.append(f"{case.case_id}: copy_allowed") if predicted_warn_confirm != case.warn_confirm_required: mismatches.append(f"{case.case_id}: warn_confirm_required") def _read_markdown_table(path: Path) -> list[dict[str, str]]: if not path.exists(): raise FileNotFoundError(path) table_lines = [ line.strip() for line in path.read_text(encoding="utf-8").splitlines() if line.strip().startswith("|") and line.strip().endswith("|") ] if len(table_lines) < 3: raise ValueError("Markdown table is missing or empty") headers = _split_markdown_row(table_lines[0]) if headers != REQUIRED_HEADERS: raise ValueError(f"Unexpected table headers: {headers}") rows: list[dict[str, str]] = [] for line in table_lines[2:]: cells = _split_markdown_row(line) if len(cells) != len(headers): raise ValueError(f"Column count mismatch: {line}") rows.append(dict(zip(headers, cells, strict=True))) return rows def _split_markdown_row(line: str) -> list[str]: return [cell.strip() for cell in line.strip().strip("|").split("|")] def _parse_bool(value: str) -> bool: normalized = value.strip().lower() if normalized == "true": return True if normalized == "false": return False raise ValueError(f"Expected true/false, got: {value}") def main() -> None: parser = argparse.ArgumentParser(description="Evaluate Sprint 1 compliance dataset.") parser.add_argument("--dataset", type=Path, default=DEFAULT_DATASET) args = parser.parse_args() report = evaluate_dataset(args.dataset) print(json.dumps(report.to_json_dict(), ensure_ascii=False, indent=2)) if __name__ == "__main__": main()