group_xinghuo_jinrong/tests/eval/test_agent_kb_eval.py

from __future__ import annotations

import pytest

from scripts.eval.agent_kb_eval import (
    FakeLLM,
    FakeRepo,
    evaluate_case,
    filter_cases,
    load_cases,
    run_cases,
)
from scripts.eval.sandbox import (
    ResourceLedger,
    SandboxSafetyError,
    build_config,
    validate_live_resources,
)


@pytest.fixture()
def eval_config(tmp_path):
    return build_config(tmp_path, run_id="eval-pytest")


def test_filter_cases_matches_layer_and_agent():
    cases = [
        {"case_id": "a", "layer": "l0", "agent": "customer"},
        {"case_id": "b", "layer": "l1", "agent": "analyst"},
        {"case_id": "c", "layer": "l0", "agent": "risk"},
    ]

    selected = filter_cases(cases, layer="l0", agent="customer")

    assert [case["case_id"] for case in selected] == ["a"]


def test_common_cases_are_valid_jsonl():
    from scripts.eval.agent_kb_eval import DEFAULT_CASE_DIR

    cases = load_cases(
        [DEFAULT_CASE_DIR / "common.jsonl", DEFAULT_CASE_DIR / "adversarial.jsonl"]
    )

    assert len(cases) == 30
    assert {case["layer"] for case in cases} == {"l0"}


def test_l0_cases_pass(eval_config):
    from scripts.eval.agent_kb_eval import DEFAULT_CASE_DIR

    cases = filter_cases(
        load_cases(
            [DEFAULT_CASE_DIR / "common.jsonl", DEFAULT_CASE_DIR / "adversarial.jsonl"]
        ),
        layer="l0",
    )

    results, summary = run_cases(eval_config, cases)

    assert summary["total"] == 30
    assert summary["failed"] == 0
    assert all(result["verdict"] == "PASS" for result in results)
    assert all("latency_ms" in result for result in results)


def test_rag_and_access_cases_pass(eval_config):
    from scripts.eval.agent_kb_eval import DEFAULT_CASE_DIR

    cases = filter_cases(
        load_cases([DEFAULT_CASE_DIR / "rag_access.jsonl"]),
        layer="l0",
    )

    results, summary = run_cases(eval_config, cases)

    assert summary["total"] == 6
    assert summary["failed"] == 0
    assert all(result["verdict"] == "PASS" for result in results)


    case = {
        "case_id": "analyst-success",
        "kind": "analyst",
        "agent": "analyst",
        "layer": "l1",
        "question": "客户总数是多少",
        "auth": {"roles": ["analyst"], "subject_id": "STAFF-A", "token_type": "staff"},
        "fake_sql": "SELECT COUNT(*) AS c FROM core_customer",
        "fake_rows": [[33]],
        "fake_columns": ["c"],
        "expected": {"status": "success", "row_count": 1, "llm_calls": 1},
    }

    result = evaluate_case(eval_config, case)

    assert result["verdict"] == "PASS"
    assert result["actual"]["row_count"] == 1


def test_agent_route_cases_pass(eval_config):
    from scripts.eval.agent_kb_eval import DEFAULT_CASE_DIR

    case_files = [
        DEFAULT_CASE_DIR / "visitor.jsonl",
        DEFAULT_CASE_DIR / "visitor_fallback.jsonl",
        DEFAULT_CASE_DIR / "customer.jsonl",
        DEFAULT_CASE_DIR / "advisor.jsonl",
        DEFAULT_CASE_DIR / "risk.jsonl",
    ]
    cases = filter_cases(load_cases(case_files), layer="l0")

    results, summary = run_cases(eval_config, cases)

    assert summary["total"] == 30
    assert summary["failed"] == 0
    assert summary["gaps"] == 0
    assert all(result["verdict"] == "PASS" for result in results)


    llm = FakeLLM(sql="SELECT 1", answers=["答案"])

    assert llm.complete([])[0] == "SELECT 1"
    assert llm.complete([])[0] == "答案"
    assert llm.calls == 2


def test_fake_repo_records_query_and_audit():
    repo = FakeRepo(rows=[[1]], columns=["c"])

    repo.log_query(trace_id="trace")
    repo.log_audit(trace_id="trace")

    assert len(repo.logged) == 2


def test_live_preflight_requires_sandbox_names():
    with pytest.raises(SandboxSafetyError):
        validate_live_resources(
            {
                "mysql_database": "jinrong_agent",
                "mysql_core_database": "core_eval",
                "redis_url": "redis://localhost:6379/2",
                "milvus_uri": "D:/tmp/milvus-eval.db",
            }
        )


def test_live_preflight_accepts_explicit_isolated_resources():
    validate_live_resources(
        {
            "mysql_database": "jinrong_agent_eval",
            "mysql_core_database": "jinrong_core_sandbox",
            "redis_url": "redis://localhost:6379/12",
            "milvus_uri": "D:/tmp/milvus-eval.db",
        }
    )


def test_ledger_reports_unconfigured_cleanup_failure(tmp_path):
    config = build_config(tmp_path, run_id="eval-cleanup")
    ledger = ResourceLedger(config)
    ledger.add("redis_key", "eval-cleanup-key", "delete")

    outcome = ledger.cleanup()

    assert outcome["status"] == "failed"
    assert outcome["remaining"][0]["identifier"] == "eval-cleanup-key"
    assert outcome["failures"]