60 lines
1.8 KiB
Python
60 lines
1.8 KiB
Python
import json
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
EVAL_DATASET = ROOT / "docs" / "开发文档" / "26-Sprint1合规评测集.md"
|
|
EVAL_SCRIPT = ROOT / "scripts" / "eval" / "evaluate_compliance.py"
|
|
|
|
|
|
def test_eval_dataset_has_prd_minimum_cases_and_risk_coverage():
|
|
assert EVAL_DATASET.exists()
|
|
|
|
from scripts.eval.evaluate_compliance import load_eval_cases
|
|
|
|
cases = load_eval_cases(EVAL_DATASET)
|
|
levels = {case.expected_level for case in cases}
|
|
rule_layers = {case.rule_layer for case in cases}
|
|
|
|
assert len(cases) >= 200
|
|
assert {"BLOCK", "WARN", "INFO"}.issubset(levels)
|
|
assert {"hard_rule", "semantic", "copy_gate"}.issubset(rule_layers)
|
|
|
|
|
|
def test_eval_report_meets_prd_recall_and_false_positive_thresholds():
|
|
assert EVAL_SCRIPT.exists()
|
|
|
|
from scripts.eval.evaluate_compliance import evaluate_dataset
|
|
from scripts.seed.import_compliance_rules import import_rules_from_markdown
|
|
|
|
import_rules_from_markdown()
|
|
report = evaluate_dataset(EVAL_DATASET)
|
|
|
|
assert report.total_cases >= 200
|
|
assert report.recall >= 0.90
|
|
assert report.false_positive_rate <= 0.15
|
|
assert report.false_negatives == []
|
|
assert report.copy_gate_cases >= 3
|
|
|
|
|
|
def test_eval_script_prints_json_report():
|
|
assert EVAL_SCRIPT.exists()
|
|
|
|
from scripts.seed.import_compliance_rules import import_rules_from_markdown
|
|
|
|
import_rules_from_markdown()
|
|
completed = subprocess.run(
|
|
[sys.executable, str(EVAL_SCRIPT), "--dataset", str(EVAL_DATASET)],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
cwd=ROOT,
|
|
)
|
|
report = json.loads(completed.stdout)
|
|
|
|
assert report["total_cases"] >= 200
|
|
assert report["recall"] >= 0.90
|
|
assert report["false_positive_rate"] <= 0.15
|
|
assert report["copy_gate_cases"] >= 3
|