袁聪的第一次提交，包含nl2sql,行情数据,场外申购
This commit is contained in:
@@ -0,0 +1,360 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select, text, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.contracts import RequestContext
|
||||
from app.core.offsite_fund_contracts import ReceiveRecognizedMailRequest
|
||||
from app.model.base import Base
|
||||
from app.model.offsite_fund import OffsiteFundMail, OffsiteMailCursor, OffsiteNotification
|
||||
from app.service.offsite_document_recognition_adapter import StructuredRecognitionResult
|
||||
from app.service.offsite_mail_adapter import (
|
||||
RawMailAttachment,
|
||||
RawMailMessage,
|
||||
SavedMailAttachment,
|
||||
SavedMailMessage,
|
||||
)
|
||||
from app.worker.offsite_mail_worker import OffsiteMailWorker
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_worker_advances_uid_only_after_each_mail_succeeds(tmp_path: Path) -> None:
|
||||
maker, engine = await _database()
|
||||
receiver = FakeReceiver(_raw_mail("5"), _raw_mail("6"), _raw_mail("7"))
|
||||
service = FakeService(fail_uid="6")
|
||||
worker = OffsiteMailWorker(
|
||||
_settings(),
|
||||
receiver=receiver,
|
||||
storage=FakeStorage(tmp_path),
|
||||
recognizer=FakeRecognizer(),
|
||||
identity_resolver=_identity,
|
||||
session_factory=maker,
|
||||
service_factory=lambda _session: service,
|
||||
)
|
||||
|
||||
try:
|
||||
assert await worker.run_once()
|
||||
cursor = await _cursor(maker)
|
||||
assert cursor.last_uid == "5"
|
||||
assert cursor.blocked_uid == "6"
|
||||
assert cursor.status == "failed"
|
||||
assert receiver.fetch_last_uid == "0"
|
||||
assert service.calls == ["5", "6"]
|
||||
|
||||
async with maker() as session, session.begin():
|
||||
await session.execute(
|
||||
update(OffsiteMailCursor).values(next_retry_at=None)
|
||||
)
|
||||
service.fail_uid = None
|
||||
|
||||
assert await worker.run_once()
|
||||
cursor = await _cursor(maker)
|
||||
assert cursor.last_uid == "7"
|
||||
assert cursor.status == "idle"
|
||||
assert cursor.blocked_uid is None
|
||||
assert service.calls == ["5", "6", "6", "7"]
|
||||
assert receiver.fetch_last_uid == "5"
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_worker_does_not_write_without_configured_worker_identity(tmp_path: Path) -> None:
|
||||
maker, engine = await _database()
|
||||
receiver = FakeReceiver(_raw_mail("5"))
|
||||
worker = OffsiteMailWorker(
|
||||
_settings(offsite_worker_user_id=""),
|
||||
receiver=receiver,
|
||||
storage=FakeStorage(tmp_path),
|
||||
recognizer=FakeRecognizer(),
|
||||
identity_resolver=_identity,
|
||||
session_factory=maker,
|
||||
service_factory=lambda _session: FakeService(),
|
||||
)
|
||||
|
||||
try:
|
||||
assert not await worker.run_once()
|
||||
assert receiver.health_calls == 0
|
||||
async with maker() as session:
|
||||
assert await session.scalar(select(OffsiteMailCursor.id)) is None
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_worker_wires_saved_attachment_recognition_into_business_service(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
maker, engine = await _database()
|
||||
service = FakeService()
|
||||
worker = OffsiteMailWorker(
|
||||
_settings(),
|
||||
receiver=FakeReceiver(_raw_mail_with_attachment("5")),
|
||||
storage=FakeStorage(tmp_path),
|
||||
recognizer=SuccessfulRecognizer(),
|
||||
identity_resolver=_identity,
|
||||
session_factory=maker,
|
||||
service_factory=lambda _session: service,
|
||||
)
|
||||
|
||||
try:
|
||||
assert await worker.run_once()
|
||||
assert len(service.payloads) == 1
|
||||
attachment = service.payloads[0].attachments[0]
|
||||
assert attachment.document_type == "subscription"
|
||||
assert attachment.original_file_path.endswith("5.eml")
|
||||
assert attachment.extracted_fields["申请编号"] == "SUB-005"
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_worker_recovers_stale_sending_notification_without_marking_success(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
maker, engine = await _database()
|
||||
worker = OffsiteMailWorker(
|
||||
_settings(),
|
||||
receiver=FakeReceiver(),
|
||||
storage=FakeStorage(tmp_path),
|
||||
recognizer=FakeRecognizer(),
|
||||
session_factory=maker,
|
||||
)
|
||||
old = datetime(2026, 9, 1, 0, 0, 0)
|
||||
try:
|
||||
async with maker() as session, session.begin():
|
||||
session.add(
|
||||
OffsiteNotification(
|
||||
id=1,
|
||||
notification_type="mail_return",
|
||||
business_key="20260901-001-A01",
|
||||
receiver_id="15008108550@163.com",
|
||||
operator_id="operator-001",
|
||||
agent_draft="draft",
|
||||
final_content="final",
|
||||
payload={},
|
||||
status="发送中",
|
||||
retry_count=0,
|
||||
created_at=old,
|
||||
updated_at=old,
|
||||
)
|
||||
)
|
||||
|
||||
assert await worker.recover_stale_notifications()
|
||||
async with maker() as session:
|
||||
notification = await session.get(OffsiteNotification, 1)
|
||||
assert notification is not None
|
||||
assert notification.status == "发送失败"
|
||||
assert notification.provider_message_id is None
|
||||
assert notification.failure_reason is not None
|
||||
assert "人工核验" in notification.failure_reason
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
class FakeReceiver:
|
||||
def __init__(self, *messages: RawMailMessage) -> None:
|
||||
self.messages = messages
|
||||
self.last_scanned_uid: str | None = "0"
|
||||
self.fetch_last_uid: str | None = None
|
||||
self.health_calls = 0
|
||||
|
||||
def health_check(self) -> dict[str, object]:
|
||||
self.health_calls += 1
|
||||
return {"status": "ok"}
|
||||
|
||||
def fetch_since(self, last_uid: str | None, *, limit: int) -> tuple[RawMailMessage, ...]:
|
||||
del limit
|
||||
self.fetch_last_uid = last_uid
|
||||
self.last_scanned_uid = self.messages[-1].imap_uid if self.messages else last_uid
|
||||
return self.messages
|
||||
|
||||
def close(self) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class FakeStorage:
|
||||
def __init__(self, root: Path) -> None:
|
||||
self.root = root
|
||||
|
||||
def save(self, mail: RawMailMessage) -> SavedMailMessage:
|
||||
path = self.root / f"{mail.imap_uid}.eml"
|
||||
path.write_bytes(mail.raw_message)
|
||||
return SavedMailMessage(
|
||||
imap_uid=mail.imap_uid,
|
||||
message_id=mail.message_id,
|
||||
sender=mail.sender,
|
||||
return_path=mail.return_path,
|
||||
auth_result=mail.auth_result,
|
||||
eml_path=str(path),
|
||||
attachments=(
|
||||
SavedMailAttachment(
|
||||
filename="empty.txt",
|
||||
file_hash="a" * 64,
|
||||
media_type="text/plain",
|
||||
size_bytes=0,
|
||||
original_file_path=str(path),
|
||||
),
|
||||
) if mail.attachments else (),
|
||||
)
|
||||
|
||||
|
||||
class FakeRecognizer:
|
||||
async def recognize(self, source: Any) -> Any:
|
||||
del source
|
||||
raise AssertionError("本测试邮件没有附件，不应调用识别器")
|
||||
|
||||
|
||||
class SuccessfulRecognizer:
|
||||
async def recognize(self, source: Any) -> StructuredRecognitionResult:
|
||||
del source
|
||||
return StructuredRecognitionResult(
|
||||
document_type="subscription",
|
||||
extracted_fields={
|
||||
"基金代码": "000001",
|
||||
"基金名称": "测试基金",
|
||||
"账户标识": "ACCT-001",
|
||||
"投资者名称": "测试客户",
|
||||
"申请编号": "SUB-005",
|
||||
"申请日期": "2026-09-10",
|
||||
"代销机构": "测试代销",
|
||||
"申购金额": "10000",
|
||||
"金额单位": "元",
|
||||
},
|
||||
field_confidence={"申请编号": Decimal("0.99")},
|
||||
missing_fields=(),
|
||||
low_confidence_fields=(),
|
||||
page_evidence={"申请编号": [{"page": 1}]},
|
||||
ocr_text="申请编号：SUB-005",
|
||||
ocr_status="mock",
|
||||
llm_status="mock",
|
||||
)
|
||||
|
||||
|
||||
class FakeService:
|
||||
def __init__(self, fail_uid: str | None = None) -> None:
|
||||
self.fail_uid = fail_uid
|
||||
self.calls: list[str] = []
|
||||
self.payloads: list[ReceiveRecognizedMailRequest] = []
|
||||
|
||||
async def receive_recognized_mail(
|
||||
self, payload: ReceiveRecognizedMailRequest, context: RequestContext
|
||||
) -> dict[str, object]:
|
||||
del context
|
||||
self.calls.append(payload.imap_uid)
|
||||
self.payloads.append(payload)
|
||||
if payload.imap_uid == self.fail_uid:
|
||||
return {"code": 500, "message": "模拟入库失败", "data": {}}
|
||||
return {"code": 0, "message": "ok", "data": {"business": False}}
|
||||
|
||||
|
||||
async def _database() -> tuple[async_sessionmaker[AsyncSession], Any]:
|
||||
engine = create_async_engine(
|
||||
"sqlite+aiosqlite://",
|
||||
connect_args={"check_same_thread": False},
|
||||
poolclass=StaticPool,
|
||||
)
|
||||
async with engine.begin() as connection:
|
||||
await connection.run_sync(
|
||||
lambda sync: Base.metadata.create_all(
|
||||
sync,
|
||||
tables=[
|
||||
OffsiteFundMail.__table__,
|
||||
OffsiteMailCursor.__table__,
|
||||
OffsiteNotification.__table__,
|
||||
],
|
||||
)
|
||||
)
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE TABLE interaction_audit (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
actor_type VARCHAR(16) NOT NULL,
|
||||
actor_id BIGINT NULL,
|
||||
target_customer_id BIGINT NULL,
|
||||
session_id VARCHAR(64) NULL,
|
||||
portal VARCHAR(32) NULL,
|
||||
action_type VARCHAR(64) NOT NULL,
|
||||
detail JSON NOT NULL,
|
||||
created_at DATETIME NOT NULL
|
||||
)
|
||||
"""
|
||||
)
|
||||
)
|
||||
return async_sessionmaker(engine, expire_on_commit=False), engine
|
||||
|
||||
|
||||
async def _identity(identity: RequestContext) -> RequestContext:
|
||||
return identity.model_copy(
|
||||
update={
|
||||
"roles": ("operator",),
|
||||
"permissions": ("offsite:write",),
|
||||
"data_scope": "all",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _settings(**updates: object) -> Settings:
|
||||
values: dict[str, object] = {
|
||||
"jwt_issuer": "jr-local",
|
||||
"jwt_audience": "jr-agent-platform",
|
||||
"mysql_dsn": "sqlite+aiosqlite:///test.db",
|
||||
"redis_url": "redis://127.0.0.1:6379/0",
|
||||
"milvus_uri": "http://127.0.0.1:19530",
|
||||
"neo4j_uri": "bolt://127.0.0.1:7687",
|
||||
"offsite_mail_worker_enabled": True,
|
||||
"offsite_imap_enabled": True,
|
||||
"offsite_worker_user_id": "1",
|
||||
"offsite_max_retry_count": 3,
|
||||
}
|
||||
values.update(updates)
|
||||
return Settings(**values)
|
||||
|
||||
|
||||
def _raw_mail(uid: str) -> RawMailMessage:
|
||||
return RawMailMessage(
|
||||
imap_uid=uid,
|
||||
message_id=f"<{uid}@worker.test>",
|
||||
sender="15008108550@163.com",
|
||||
return_path="15008108550@163.com",
|
||||
auth_result={"spf": "pass"},
|
||||
raw_message=f"mail-{uid}".encode(),
|
||||
received_at=datetime(2026, 9, 10, 9, 30, 0),
|
||||
attachments=(),
|
||||
)
|
||||
|
||||
|
||||
def _raw_mail_with_attachment(uid: str) -> RawMailMessage:
|
||||
mail = _raw_mail(uid)
|
||||
return RawMailMessage(
|
||||
imap_uid=mail.imap_uid,
|
||||
message_id=mail.message_id,
|
||||
sender=mail.sender,
|
||||
return_path=mail.return_path,
|
||||
auth_result=mail.auth_result,
|
||||
raw_message=mail.raw_message,
|
||||
received_at=mail.received_at,
|
||||
attachments=(
|
||||
RawMailAttachment(
|
||||
filename="申购申请单.pdf",
|
||||
media_type="application/pdf",
|
||||
payload=b"pdf",
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def _cursor(maker: async_sessionmaker[AsyncSession]) -> OffsiteMailCursor:
|
||||
async with maker() as session:
|
||||
cursor = await session.scalar(select(OffsiteMailCursor))
|
||||
assert cursor is not None
|
||||
return cursor
|
||||
Reference in New Issue
Block a user