diff --git a/.env.example b/.env.example index 7e94214..85fe35b 100644 --- a/.env.example +++ b/.env.example @@ -13,6 +13,7 @@ JWT_CLOCK_SKEW_SECONDS=30 MYSQL_DSN=mysql+asyncmy://jr_app:change-me@127.0.0.1:3306/jr_agent MYSQL_POOL_SIZE=5 MYSQL_MAX_OVERFLOW=10 +MYSQL_POOL_PRE_PING=false MYSQL_TX_ISOLATION=READ COMMITTED REDIS_URL=redis://127.0.0.1:6379/0 @@ -42,3 +43,41 @@ WORKER_LEASE_SECONDS=60 WORKER_RETRY_LIMIT=3 WORKER_POLL_SECONDS=1 SSE_CHUNK_CHARACTERS=256 + +OFFSITE_MAILBOX=15273589815@163.com +OFFSITE_ALLOWED_SENDERS=["15008108550@163.com"] +OFFSITE_RISK_RECEIVER_ID= +OFFSITE_SETTLEMENT_RECEIVER_ID= +OFFSITE_MAIL_RETURN_RECEIVER=15008108550@163.com +OFFSITE_MAX_RETRY_COUNT=3 +OFFSITE_IMAP_ENABLED=false +OFFSITE_IMAP_HOST= +OFFSITE_IMAP_PORT=993 +OFFSITE_IMAP_USERNAME= +OFFSITE_IMAP_PASSWORD= +OFFSITE_IMAP_USE_SSL=true +OFFSITE_IMAP_IDLE_ENABLED=false +OFFSITE_MAIL_WORKER_ENABLED=false +OFFSITE_MAIL_WORKER_BATCH_SIZE=20 +OFFSITE_WORKER_USER_ID= +OFFSITE_NOTIFICATION_SENDING_TIMEOUT_SECONDS=900 +OFFSITE_MAIL_STORAGE_DIR=data/offsite_mail +OFFSITE_OCR_ENABLED=false +OFFSITE_OCR_TIMEOUT_SECONDS=30 +OFFSITE_ALIYUN_OCR_ENDPOINT= +OFFSITE_ALIYUN_ACCESS_KEY_ID= +OFFSITE_ALIYUN_ACCESS_KEY_SECRET= +OFFSITE_DEEPSEEK_ENABLED=false +OFFSITE_DEEPSEEK_BASE_URL=https://api.deepseek.com +OFFSITE_DEEPSEEK_API_KEY= +OFFSITE_DEEPSEEK_MODEL=deepseek-v4-flash +OFFSITE_DEEPSEEK_TIMEOUT_SECONDS=30 +OFFSITE_SMTP_ENABLED=false +OFFSITE_SMTP_DRY_RUN=true +OFFSITE_SMTP_HOST= +OFFSITE_SMTP_PORT=465 +OFFSITE_SMTP_USERNAME= +OFFSITE_SMTP_PASSWORD= +OFFSITE_SMTP_SENDER= +OFFSITE_SMTP_USE_SSL=true +OFFSITE_SMTP_TIMEOUT_SECONDS=30 diff --git a/alembic/versions/20260910_offsite_fund.py b/alembic/versions/20260910_offsite_fund.py new file mode 100644 index 0000000..473d89f --- /dev/null +++ b/alembic/versions/20260910_offsite_fund.py @@ -0,0 +1,141 @@ +"""add offsite fund operation tables""" +from alembic import op + +revision = "20260910_offsite_fund" +down_revision = "20260909_api_receipt" +branch_labels = None +depends_on = None + +TABLES = [ + """CREATE TABLE offsite_fund_mail ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + mail_id VARCHAR(16) NOT NULL, + imap_uid VARCHAR(64) NOT NULL, + message_id VARCHAR(255) NOT NULL, + received_date DATE NOT NULL, + sender VARCHAR(255) NOT NULL, + return_path VARCHAR(255) NULL, + auth_result JSON NOT NULL, + original_eml_path VARCHAR(500) NOT NULL, + status VARCHAR(32) NOT NULL DEFAULT 'received', + created_at DATETIME(6) NOT NULL, + updated_at DATETIME(6) NOT NULL, + PRIMARY KEY (id), + UNIQUE KEY uk_offsite_mail_id (mail_id), + UNIQUE KEY uk_offsite_mail_source (imap_uid, message_id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci""", + """CREATE TABLE offsite_fund_attachment ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + attachment_id VARCHAR(24) NOT NULL, + mail_id VARCHAR(16) NOT NULL, + filename VARCHAR(255) NOT NULL, + file_hash VARCHAR(128) NOT NULL, + media_type VARCHAR(128) NOT NULL, + size_bytes BIGINT UNSIGNED NOT NULL DEFAULT 0, + document_type VARCHAR(24) NOT NULL, + original_file_path VARCHAR(500) NOT NULL, + ocr_text MEDIUMTEXT NOT NULL, + extracted_fields JSON NOT NULL, + field_confidence JSON NOT NULL, + page_evidence JSON NOT NULL, + status VARCHAR(32) NOT NULL DEFAULT 'recognized', + created_at DATETIME(6) NOT NULL, + PRIMARY KEY (id), + UNIQUE KEY uk_offsite_attachment_id (attachment_id), + UNIQUE KEY uk_offsite_attachment_hash (file_hash, mail_id), + KEY idx_offsite_attachment_mail (mail_id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci""", + """CREATE TABLE offsite_fund_document ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + task_id VARCHAR(24) NOT NULL, + mail_id VARCHAR(16) NOT NULL, + attachment_id VARCHAR(24) NOT NULL, + document_type VARCHAR(24) NOT NULL, + fund_code VARCHAR(32) NULL, + fund_name VARCHAR(255) NULL, + account_identifier VARCHAR(128) NULL, + investor_name VARCHAR(128) NULL, + application_no VARCHAR(128) NULL, + application_date DATE NULL, + raw_application_date VARCHAR(64) NULL, + agency VARCHAR(128) NULL, + subscription_amount_yuan DECIMAL(20,4) NULL, + redemption_shares DECIMAL(20,4) NULL, + operator_decision VARCHAR(16) NOT NULL DEFAULT '未处理', + status VARCHAR(32) NOT NULL DEFAULT 'planned', + created_at DATETIME(6) NOT NULL, + updated_at DATETIME(6) NOT NULL, + PRIMARY KEY (id), + UNIQUE KEY uk_offsite_document_task (task_id), + KEY idx_offsite_document_summary (fund_code, application_date) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci""", + """CREATE TABLE offsite_execution_plan_task ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + task_id VARCHAR(24) NOT NULL, + stage VARCHAR(16) NOT NULL, + rule_code VARCHAR(64) NOT NULL, + title VARCHAR(128) NOT NULL, + depends_on JSON NOT NULL, + input_json JSON NOT NULL, + output_json JSON NULL, + status VARCHAR(16) NOT NULL DEFAULT '待执行', + error_message VARCHAR(500) NULL, + created_at DATETIME(6) NOT NULL, + updated_at DATETIME(6) NOT NULL, + PRIMARY KEY (id), + KEY idx_offsite_plan_task (task_id, stage, rule_code) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci""", + """CREATE TABLE offsite_rule_result ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + task_id VARCHAR(24) NOT NULL, + rule_code VARCHAR(64) NOT NULL, + rule_name VARCHAR(128) NOT NULL, + result VARCHAR(16) NOT NULL, + document_value JSON NOT NULL, + database_value JSON NOT NULL, + calculation JSON NOT NULL, + created_at DATETIME(6) NOT NULL, + PRIMARY KEY (id), + UNIQUE KEY uk_offsite_rule_task (task_id, rule_code) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci""", + """CREATE TABLE offsite_query_record ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + task_id VARCHAR(24) NOT NULL, + rule_code VARCHAR(64) NOT NULL, + natural_language_request MEDIUMTEXT NOT NULL, + script_path VARCHAR(500) NOT NULL, + result_summary JSON NOT NULL, + status VARCHAR(24) NOT NULL, + error_message VARCHAR(500) NULL, + created_at DATETIME(6) NOT NULL, + PRIMARY KEY (id), + KEY idx_offsite_query_task (task_id, rule_code) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci""", + """CREATE TABLE offsite_notification ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + notification_type VARCHAR(32) NOT NULL, + business_key VARCHAR(64) NOT NULL, + receiver_id VARCHAR(64) NOT NULL, + operator_id VARCHAR(64) NOT NULL, + agent_draft MEDIUMTEXT NOT NULL, + final_content MEDIUMTEXT NOT NULL, + payload JSON NOT NULL, + status VARCHAR(16) NOT NULL DEFAULT '待发送', + retry_count INT UNSIGNED NOT NULL DEFAULT 0, + provider_message_id VARCHAR(128) NULL, + failure_reason VARCHAR(500) NULL, + created_at DATETIME(6) NOT NULL, + sent_at DATETIME(6) NULL, + PRIMARY KEY (id), + KEY idx_offsite_notice_business (business_key, notification_type) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci""", +] + + +def upgrade() -> None: + for statement in TABLES: + op.execute(statement) + + +def downgrade() -> None: + raise RuntimeError("场外运营业务数据禁止自动删除，请使用前向兼容迁移") diff --git a/alembic/versions/20260910_offsite_worker.py b/alembic/versions/20260910_offsite_worker.py new file mode 100644 index 0000000..cdaea37 --- /dev/null +++ b/alembic/versions/20260910_offsite_worker.py @@ -0,0 +1,54 @@ +"""add offsite mail worker state""" + +from alembic import op + +revision = "20260910_offsite_worker" +down_revision = "20260910_offsite_fund" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute( + """ + CREATE TABLE offsite_mail_cursor ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + mailbox VARCHAR(255) NOT NULL, + folder VARCHAR(64) NOT NULL DEFAULT 'INBOX', + last_uid VARCHAR(64) NOT NULL DEFAULT '0', + status VARCHAR(32) NOT NULL DEFAULT 'idle', + retry_count INT UNSIGNED NOT NULL DEFAULT 0, + blocked_uid VARCHAR(64) NULL, + blocked_message_id VARCHAR(255) NULL, + last_error VARCHAR(500) NULL, + next_retry_at DATETIME(6) NULL, + lease_id VARCHAR(64) NULL, + lease_until DATETIME(6) NULL, + created_at DATETIME(6) NOT NULL, + updated_at DATETIME(6) NOT NULL, + PRIMARY KEY (id), + UNIQUE KEY uk_offsite_mail_cursor_scope (mailbox, folder), + KEY idx_offsite_mail_cursor_retry (status, next_retry_at), + KEY idx_offsite_mail_cursor_lease (lease_until) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci + """ + ) + op.execute( + """ + ALTER TABLE offsite_fund_mail + ADD COLUMN retry_count INT UNSIGNED NOT NULL DEFAULT 0, + ADD COLUMN last_error VARCHAR(500) NULL, + ADD COLUMN next_retry_at DATETIME(6) NULL, + ADD COLUMN last_attempt_at DATETIME(6) NULL + """ + ) + op.execute( + """ + ALTER TABLE offsite_notification + ADD COLUMN updated_at DATETIME(6) NULL + """ + ) + + +def downgrade() -> None: + raise RuntimeError("场外运营业务数据禁止自动删除，请使用前向兼容迁移") diff --git a/app/api/controllers/offsite_fund.py b/app/api/controllers/offsite_fund.py new file mode 100644 index 0000000..f379544 --- /dev/null +++ b/app/api/controllers/offsite_fund.py @@ -0,0 +1,100 @@ +"""场外基金运营接口。""" + +from typing import Any + +from fastapi import APIRouter, Depends +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.dependencies.auth import build_request_context +from app.api.dependencies.database import get_session +from app.api.schemas.offsite_fund import ( + OffsiteConfirmRequest, + OffsiteMailRecognizeRequest, + OffsiteNotificationRequest, + OffsiteNotificationSendRequest, + OffsiteRecalculateRequest, + OffsiteTriggerNl2SqlRequest, +) +from app.core.contracts import RequestContext +from app.service.offsite_fund_service import OffsiteFundService + +router = APIRouter(prefix="/api/v1/offsite-fund", tags=["offsite-fund"]) +operation_router = APIRouter(prefix="/api", tags=["offsite-fund"]) + + +@router.post("/recognized-mails") +async def receive_recognized_mail( + payload: OffsiteMailRecognizeRequest, + context: RequestContext = Depends(build_request_context), # noqa: B008 + session: AsyncSession = Depends(get_session), # noqa: B008 +) -> dict[str, Any]: + return await OffsiteFundService(session).receive_recognized_mail(payload, context) + + +@router.post("/documents/{task_id}/confirmations") +async def confirm_document( + task_id: str, + payload: OffsiteConfirmRequest, + context: RequestContext = Depends(build_request_context), # noqa: B008 + session: AsyncSession = Depends(get_session), # noqa: B008 +) -> dict[str, Any]: + return await OffsiteFundService(session).confirm_document( + task_id, payload.decision, payload.operator_id, context) + + +@router.post("/documents/{task_id}/notifications") +async def create_notification( + task_id: str, + payload: OffsiteNotificationRequest, + context: RequestContext = Depends(build_request_context), # noqa: B008 + session: AsyncSession = Depends(get_session), # noqa: B008 +) -> dict[str, Any]: + return await OffsiteFundService(session).create_notification( + task_id, payload.notification_type, payload.operator_id, context) + + +@router.post("/notifications/{notification_id}/send") +async def send_notification( + notification_id: int, + payload: OffsiteNotificationSendRequest, + context: RequestContext = Depends(build_request_context), # noqa: B008 + session: AsyncSession = Depends(get_session), # noqa: B008 +) -> dict[str, Any]: + return await OffsiteFundService(session).send_notification( + notification_id, + payload.operator_id, + payload.operator_confirmed, + payload.final_content, + context, + ) + + +@router.post("/settlement-statistics/recalculate") +async def recalculate_statistics( + payload: OffsiteRecalculateRequest, + context: RequestContext = Depends(build_request_context), # noqa: B008 + session: AsyncSession = Depends(get_session), # noqa: B008 +) -> dict[str, Any]: + return await OffsiteFundService(session).recalculate_statistics( + payload.fund_code, payload.application_date, context) + + +@operation_router.post("/tasks/{task_id}/trigger-agent-nl2sql") +async def trigger_agent_nl2sql( + task_id: str, + payload: OffsiteTriggerNl2SqlRequest, + context: RequestContext = Depends(build_request_context), # noqa: B008 + session: AsyncSession = Depends(get_session), # noqa: B008 +) -> dict[str, Any]: + return await OffsiteFundService(session).trigger_agent_nl2sql( + task_id, payload.operator_id, payload.manual_confirmed, context) + + +@operation_router.post("/settlement-statistics/recalculate") +async def recalculate_statistics_compat( + payload: OffsiteRecalculateRequest, + context: RequestContext = Depends(build_request_context), # noqa: B008 + session: AsyncSession = Depends(get_session), # noqa: B008 +) -> dict[str, Any]: + return await OffsiteFundService(session).recalculate_statistics( + payload.fund_code, payload.application_date, context) diff --git a/app/api/schemas/offsite_fund.py b/app/api/schemas/offsite_fund.py new file mode 100644 index 0000000..6e87148 --- /dev/null +++ b/app/api/schemas/offsite_fund.py @@ -0,0 +1,45 @@ +"""场外基金接口请求结构。""" + +from pydantic import BaseModel, ConfigDict, Field + +from app.core.offsite_fund_contracts import OperationDecision, ReceiveRecognizedMailRequest + + +class OffsiteMailRecognizeRequest(ReceiveRecognizedMailRequest): + pass + + +class OffsiteConfirmRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + decision: OperationDecision + operator_id: str = Field(min_length=1, max_length=64) + + +class OffsiteRecalculateRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + fund_code: str = Field(min_length=1, max_length=32) + application_date: str = Field(min_length=8, max_length=32) + + +class OffsiteNotificationRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + notification_type: str = Field(pattern="^(risk|settlement|mail_return)$") + operator_id: str = Field(min_length=1, max_length=64) + + +class OffsiteNotificationSendRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + operator_id: str = Field(min_length=1, max_length=64) + operator_confirmed: bool + final_content: str | None = Field(default=None, max_length=10000) + + +class OffsiteTriggerNl2SqlRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + operator_id: str = Field(min_length=1, max_length=64) + manual_confirmed: bool diff --git a/app/core/config.py b/app/core/config.py index 5cdd2b5..6d53a25 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -18,6 +18,7 @@ class Settings(BaseSettings): mysql_dsn: str mysql_pool_size: int = Field(default=5, ge=1) mysql_max_overflow: int = Field(default=10, ge=0) + mysql_pool_pre_ping: bool = False mysql_tx_isolation: str = "READ COMMITTED" redis_url: str redis_connect_timeout_seconds: float = Field(default=2, gt=0) @@ -41,6 +42,43 @@ class Settings(BaseSettings): worker_lease_seconds: int = Field(default=60, gt=0) worker_retry_limit: int = Field(default=3, ge=0) worker_poll_seconds: float = Field(default=1, gt=0) + offsite_mailbox: str = "15273589815@163.com" + offsite_allowed_senders: tuple[str, ...] = ("15008108550@163.com",) + offsite_risk_receiver_id: str = "" + offsite_settlement_receiver_id: str = "" + offsite_mail_return_receiver: str = "15008108550@163.com" + offsite_max_retry_count: int = Field(default=3, ge=0) + offsite_imap_enabled: bool = False + offsite_imap_host: str = "" + offsite_imap_port: int = Field(default=993, gt=0) + offsite_imap_username: str = "" + offsite_imap_password: str = "" + offsite_imap_use_ssl: bool = True + offsite_imap_idle_enabled: bool = False + offsite_mail_worker_enabled: bool = False + offsite_mail_worker_batch_size: int = Field(default=20, ge=1, le=100) + offsite_worker_user_id: str = "" + offsite_notification_sending_timeout_seconds: int = Field(default=900, gt=0) + offsite_mail_storage_dir: str = "data/offsite_mail" + offsite_ocr_enabled: bool = False + offsite_ocr_timeout_seconds: float = Field(default=30, gt=0) + offsite_aliyun_ocr_endpoint: str = "" + offsite_aliyun_access_key_id: str = "" + offsite_aliyun_access_key_secret: str = "" + offsite_deepseek_enabled: bool = False + offsite_deepseek_base_url: str = "https://api.deepseek.com" + offsite_deepseek_api_key: str = "" + offsite_deepseek_model: str = "deepseek-v4-flash" + offsite_deepseek_timeout_seconds: float = Field(default=30, gt=0) + offsite_smtp_enabled: bool = False + offsite_smtp_dry_run: bool = True + offsite_smtp_host: str = "" + offsite_smtp_port: int = Field(default=465, gt=0) + offsite_smtp_username: str = "" + offsite_smtp_password: str = "" + offsite_smtp_sender: str = "" + offsite_smtp_use_ssl: bool = True + offsite_smtp_timeout_seconds: float = Field(default=30, gt=0) model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore") diff --git a/app/core/nl2sql_catalog.py b/app/core/nl2sql_catalog.py new file mode 100644 index 0000000..75d42ab --- /dev/null +++ b/app/core/nl2sql_catalog.py @@ -0,0 +1,110 @@ +import re + +DOMAINS: dict[str, set[str]] = { + "market_nav": {"fin_market_price", "fin_nav_history", "fin_product"}, + "customer_risk": { + "sys_customer_assignment", "fin_customer_profile", "fin_risk_assessment", + }, + "product_fee": {"fin_product", "fin_fee_rule"}, + "trading_account": { + "fin_sim_order", "fin_transaction", "fin_sim_account", "fin_cash_ledger", + "fin_holding", "fin_product", + }, + "client_content": {"client_facing_content", "fin_customer_profile"}, +} + +TABLE_COLUMNS: dict[str, set[str]] = { + "sys_customer_assignment": { + "id", "customer_id", "employee_id", "employee_role", "assigned_at", "unassigned_at", + }, + "fin_customer_profile": { + "customer_id", "trade_account", "real_name", "investor_type", "total_asset", + "behavior_score", "risk_tags", "opened_at", "updated_at", + }, + "fin_risk_assessment": { + "id", "customer_id", "total_score", "investor_type", "assessed_at", + "valid_until", "created_at", + }, + "fin_product": { + "id", "product_code", "product_name", "exchange_code", "product_category", + "risk_level", "current_nav", "transaction_fee_rate", "status", + "created_at", "updated_at", + }, + "fin_fee_rule": { + "id", "rule_code", "product_id", "exchange_code", "order_side", "customer_tier", + "fee_rate", "minimum_fee", "fixed_fee", "effective_from", "effective_until", + "status", "created_at", "updated_at", + }, + "fin_market_price": { + "id", "product_id", "trade_date", "open_price", "high_price", "low_price", + "close_price", "volume", "turnover_amount", "created_at", + }, + "fin_nav_history": {"id", "product_id", "nav_date", "nav", "created_at"}, + "fin_holding": { + "id", "customer_id", "trade_account", "product_id", "total_quantity", + "available_quantity", "frozen_quantity", "market_value", "profit_loss", + "profit_loss_ratio", "status", "updated_at", + }, + "fin_transaction": { + "id", "transaction_no", "order_id", "customer_id", "account_id", "product_id", + "order_side", "executed_price", "executed_quantity", "gross_amount", + "fee_amount", "net_amount", "quote_at", "executed_at", "created_at", + }, + "fin_sim_order": { + "id", "order_no", "customer_id", "account_id", "product_id", "order_side", + "price_type", "quantity", "quote_price", "quote_at", "advisor_id", + "filled_quantity", "status", "submitted_at", "cancelled_at", "created_at", + "updated_at", + }, + "fin_sim_account": { + "id", "account_no", "customer_id", "currency", "cash_balance", "available_cash", + "frozen_cash", "initial_balance", "status", "version", "created_at", "updated_at", + }, + "fin_cash_ledger": { + "id", "ledger_no", "account_id", "transaction_id", "entry_type", "amount", + "balance_after", "available_cash_after", "frozen_cash_after", "occurred_at", + "created_at", + }, + "client_facing_content": { + "id", "customer_id", "content_type", "draft_content", "review_status", + "reviewer_user_id", "reviewed_at", "published_at", "created_at", "updated_at", + }, +} + +ALLOWED_TABLES = set(TABLE_COLUMNS) +CURRENT_ONLY_TABLES = {"fin_holding", "fin_sim_account", "sys_customer_assignment"} +BANNED_SQL = re.compile( + r"\b(INSERT|UPDATE|DELETE|DROP|ALTER|TRUNCATE|CREATE|GRANT|REVOKE|CALL|EXEC|MERGE)\b", + re.IGNORECASE, +) + +ALIASES = { + "sys_customer_assignment": "ca", "fin_customer_profile": "cp", + "fin_risk_assessment": "ra", "fin_product": "p", "fin_fee_rule": "f", + "fin_market_price": "m", "fin_nav_history": "n", "fin_holding": "h", + "fin_transaction": "t", "fin_sim_order": "o", "fin_sim_account": "a", + "fin_cash_ledger": "l", "client_facing_content": "c", +} + +JOIN_SQL = { + ("fin_market_price", "fin_product"): "m.product_id = p.id", + ("fin_nav_history", "fin_product"): "n.product_id = p.id", + ("fin_fee_rule", "fin_product"): "f.product_id = p.id", + ("fin_holding", "fin_product"): "h.product_id = p.id", + ("fin_holding", "fin_customer_profile"): "h.customer_id = cp.customer_id", + ("fin_transaction", "fin_product"): "t.product_id = p.id", + ("fin_transaction", "fin_customer_profile"): "t.customer_id = cp.customer_id", + ("fin_transaction", "fin_sim_account"): "t.account_id = a.id", + ("fin_sim_order", "fin_product"): "o.product_id = p.id", + ("fin_sim_order", "fin_customer_profile"): "o.customer_id = cp.customer_id", + ("fin_sim_order", "fin_sim_account"): "o.account_id = a.id", + ("fin_sim_account", "fin_customer_profile"): "a.customer_id = cp.customer_id", + ("fin_cash_ledger", "fin_sim_account"): "l.account_id = a.id", + ("fin_risk_assessment", "fin_customer_profile"): "ra.customer_id = cp.customer_id", + ("sys_customer_assignment", "fin_customer_profile"): "ca.customer_id = cp.customer_id", + ("client_facing_content", "fin_customer_profile"): "c.customer_id = cp.customer_id", +} + + +def alias(table: str) -> str: + return ALIASES[table] diff --git a/app/core/nl2sql_contracts.py b/app/core/nl2sql_contracts.py new file mode 100644 index 0000000..7c92764 --- /dev/null +++ b/app/core/nl2sql_contracts.py @@ -0,0 +1,53 @@ +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field, field_validator + + +class FinancialNL2SQLInput(BaseModel): + """金融 NL2SQL 公共工具入参；身份和权限只信任 RequestContext。""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + question: str = Field(min_length=1, max_length=500) + confirmation: str | None = Field(default=None, max_length=200) + dry_run: bool = False + limit: int = Field(default=50, ge=1, le=200) + + @field_validator("question") + @classmethod + def question_must_not_be_blank(cls, value: str) -> str: + if not value.strip(): + raise ValueError("question must not be blank") + return value.strip() + + +class FinancialQueryPlan(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + intent: str + domains: tuple[str, ...] + tables: tuple[str, ...] + metrics: tuple[str, ...] = () + dimensions: tuple[str, ...] = () + filters: tuple[dict[str, Any], ...] = () + time_mode: str = "none" + time_column: str | None = None + start: str | None = None + end: str | None = None + limit: int = Field(default=50, ge=1, le=200) + confidence: float = Field(default=0.0, ge=0, le=1) + needs_confirmation: bool = False + confirmation_question: str | None = None + unsupported_reason: str | None = None + + +class FinancialNL2SQLResult(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + status: str + message: str + data: dict[str, Any] = Field(default_factory=dict) + query_plan: dict[str, Any] = Field(default_factory=dict) + sql: str | None = None + parameters: dict[str, Any] = Field(default_factory=dict) + audit: dict[str, Any] = Field(default_factory=dict) diff --git a/app/core/offsite_fund_contracts.py b/app/core/offsite_fund_contracts.py new file mode 100644 index 0000000..e5ce02f --- /dev/null +++ b/app/core/offsite_fund_contracts.py @@ -0,0 +1,48 @@ +"""场外基金申购赎回业务契约。""" + +from decimal import Decimal +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + +DocumentType = Literal["summary", "subscription", "redemption", "other"] +PlanTaskStatus = Literal["待执行", "执行中", "已完成", "查询失败", "无法判断", "不适用"] +RuleResultStatus = Literal["正常", "异常", "无法判断"] +OperationDecision = Literal["确认正常", "确认异常", "未处理"] +SendStatus = Literal["待发送", "发送中", "发送成功", "发送失败"] + + +class RecognizedAttachment(BaseModel): + model_config = ConfigDict(extra="forbid") + + filename: str = Field(min_length=1, max_length=255) + file_hash: str = Field(min_length=16, max_length=128) + original_file_path: str = Field(default="", max_length=500) + media_type: str = Field(default="application/octet-stream", max_length=128) + size_bytes: int = Field(default=0, ge=0) + document_type: DocumentType + extracted_fields: dict[str, object] = Field(default_factory=dict) + field_confidence: dict[str, Decimal] = Field(default_factory=dict) + ocr_text: str = "" + page_evidence: dict[str, object] = Field(default_factory=dict) + + +class ReceiveRecognizedMailRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + imap_uid: str = Field(min_length=1, max_length=64) + message_id: str = Field(min_length=1, max_length=255) + sender: str = Field(min_length=3, max_length=255) + return_path: str | None = Field(default=None, max_length=255) + auth_result: dict[str, object] = Field(default_factory=dict) + eml_path: str = Field(default="", max_length=500) + attachments: tuple[RecognizedAttachment, ...] + + +class OffsiteDocumentSummary(BaseModel): + model_config = ConfigDict(extra="forbid") + + task_id: str + document_type: DocumentType + status: str + rule_results: dict[str, RuleResultStatus] diff --git a/app/infrastructure/db.py b/app/infrastructure/db.py index 583d842..bebf78e 100644 --- a/app/infrastructure/db.py +++ b/app/infrastructure/db.py @@ -1,6 +1,12 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine +from sqlalchemy.pool import NullPool from app.core.config import get_settings -engine = create_async_engine(get_settings().mysql_dsn, pool_pre_ping=True) +settings = get_settings() +engine = create_async_engine( + settings.mysql_dsn, + pool_pre_ping=settings.mysql_pool_pre_ping, + poolclass=NullPool, +) SessionFactory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) diff --git a/app/main.py b/app/main.py index 1803639..b0c9842 100644 --- a/app/main.py +++ b/app/main.py @@ -6,6 +6,8 @@ from app.api.controllers.agent_runs import router as agent_runs_router from app.api.controllers.conversations import router as conversations_router from app.api.controllers.health import router as health_router from app.api.controllers.knowledge import router as knowledge_router +from app.api.controllers.offsite_fund import operation_router as offsite_operation_router +from app.api.controllers.offsite_fund import router as offsite_fund_router from app.api.controllers.public_platform import router as public_platform_router from app.core.config import get_settings from app.core.errors import AgentError @@ -25,6 +27,8 @@ def create_app() -> FastAPI: application.include_router(agent_runs_router) application.include_router(conversations_router) application.include_router(public_platform_router) + application.include_router(offsite_fund_router) + application.include_router(offsite_operation_router) application.include_router(knowledge_router) application.include_router(health_router) application.include_router(admin_router) diff --git a/app/model/offsite_fund.py b/app/model/offsite_fund.py new file mode 100644 index 0000000..147f351 --- /dev/null +++ b/app/model/offsite_fund.py @@ -0,0 +1,185 @@ +"""场外基金运营独立持久化模型。""" + +from datetime import date, datetime +from decimal import Decimal +from typing import Any + +from sqlalchemy import ( + JSON, + BigInteger, + Date, + DateTime, + Integer, + Numeric, + String, + Text, + UniqueConstraint, +) +from sqlalchemy.orm import Mapped, mapped_column + +from app.model.base import Base + + +class OffsiteFundMail(Base): + __tablename__ = "offsite_fund_mail" + __table_args__ = ( + UniqueConstraint("imap_uid", "message_id", name="uk_offsite_mail_source"), + ) + + id: Mapped[int] = mapped_column(BigInteger, primary_key=True) + mail_id: Mapped[str] = mapped_column(String(16), unique=True, nullable=False) + imap_uid: Mapped[str] = mapped_column(String(64), nullable=False) + message_id: Mapped[str] = mapped_column(String(255), nullable=False) + received_date: Mapped[date] = mapped_column(Date, nullable=False) + sender: Mapped[str] = mapped_column(String(255), nullable=False) + return_path: Mapped[str | None] = mapped_column(String(255)) + auth_result: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False) + original_eml_path: Mapped[str] = mapped_column(String(500), nullable=False) + status: Mapped[str] = mapped_column(String(32), nullable=False, default="received") + retry_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + last_error: Mapped[str | None] = mapped_column(String(500)) + next_retry_at: Mapped[datetime | None] = mapped_column(DateTime) + last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime) + created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False) + updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False) + + +class OffsiteMailCursor(Base): + """场外收件箱 UID 游标和处理租约。""" + + __tablename__ = "offsite_mail_cursor" + __table_args__ = ( + UniqueConstraint("mailbox", "folder", name="uk_offsite_mail_cursor_scope"), + ) + + id: Mapped[int] = mapped_column( + BigInteger().with_variant(Integer, "sqlite"), primary_key=True + ) + mailbox: Mapped[str] = mapped_column(String(255), nullable=False) + folder: Mapped[str] = mapped_column(String(64), nullable=False, default="INBOX") + last_uid: Mapped[str] = mapped_column(String(64), nullable=False, default="0") + status: Mapped[str] = mapped_column(String(32), nullable=False, default="idle") + retry_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + blocked_uid: Mapped[str | None] = mapped_column(String(64)) + blocked_message_id: Mapped[str | None] = mapped_column(String(255)) + last_error: Mapped[str | None] = mapped_column(String(500)) + next_retry_at: Mapped[datetime | None] = mapped_column(DateTime) + lease_id: Mapped[str | None] = mapped_column(String(64)) + lease_until: Mapped[datetime | None] = mapped_column(DateTime) + created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False) + updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False) + + +class OffsiteFundAttachment(Base): + __tablename__ = "offsite_fund_attachment" + __table_args__ = ( + UniqueConstraint("file_hash", "mail_id", name="uk_offsite_attachment_hash"), + ) + + id: Mapped[int] = mapped_column(BigInteger, primary_key=True) + attachment_id: Mapped[str] = mapped_column(String(24), unique=True, nullable=False) + mail_id: Mapped[str] = mapped_column(String(16), nullable=False) + filename: Mapped[str] = mapped_column(String(255), nullable=False) + file_hash: Mapped[str] = mapped_column(String(128), nullable=False) + media_type: Mapped[str] = mapped_column(String(128), nullable=False) + size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0) + document_type: Mapped[str] = mapped_column(String(24), nullable=False) + original_file_path: Mapped[str] = mapped_column(String(500), nullable=False) + ocr_text: Mapped[str] = mapped_column(Text, nullable=False) + extracted_fields: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False) + field_confidence: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False) + page_evidence: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False) + status: Mapped[str] = mapped_column(String(32), nullable=False, default="recognized") + created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False) + + +class OffsiteFundDocument(Base): + __tablename__ = "offsite_fund_document" + + id: Mapped[int] = mapped_column(BigInteger, primary_key=True) + task_id: Mapped[str] = mapped_column(String(24), unique=True, nullable=False) + mail_id: Mapped[str] = mapped_column(String(16), nullable=False) + attachment_id: Mapped[str] = mapped_column(String(24), nullable=False) + document_type: Mapped[str] = mapped_column(String(24), nullable=False) + fund_code: Mapped[str | None] = mapped_column(String(32)) + fund_name: Mapped[str | None] = mapped_column(String(255)) + account_identifier: Mapped[str | None] = mapped_column(String(128)) + investor_name: Mapped[str | None] = mapped_column(String(128)) + application_no: Mapped[str | None] = mapped_column(String(128)) + application_date: Mapped[date | None] = mapped_column(Date) + raw_application_date: Mapped[str | None] = mapped_column(String(64)) + agency: Mapped[str | None] = mapped_column(String(128)) + subscription_amount_yuan: Mapped[Decimal | None] = mapped_column(Numeric(20, 4)) + redemption_shares: Mapped[Decimal | None] = mapped_column(Numeric(20, 4)) + operator_decision: Mapped[str] = mapped_column(String(16), nullable=False, default="未处理") + status: Mapped[str] = mapped_column(String(32), nullable=False, default="planned") + created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False) + updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False) + + +class OffsiteExecutionPlanTask(Base): + __tablename__ = "offsite_execution_plan_task" + + id: Mapped[int] = mapped_column(BigInteger, primary_key=True) + task_id: Mapped[str] = mapped_column(String(24), nullable=False) + stage: Mapped[str] = mapped_column(String(16), nullable=False) + rule_code: Mapped[str] = mapped_column(String(64), nullable=False) + title: Mapped[str] = mapped_column(String(128), nullable=False) + depends_on: Mapped[list[Any]] = mapped_column(JSON, nullable=False) + input_json: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False) + output_json: Mapped[dict[str, Any] | None] = mapped_column(JSON) + status: Mapped[str] = mapped_column(String(16), nullable=False, default="待执行") + error_message: Mapped[str | None] = mapped_column(String(500)) + created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False) + updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False) + + +class OffsiteRuleResult(Base): + __tablename__ = "offsite_rule_result" + __table_args__ = ( + UniqueConstraint("task_id", "rule_code", name="uk_offsite_rule_task"), + ) + + id: Mapped[int] = mapped_column(BigInteger, primary_key=True) + task_id: Mapped[str] = mapped_column(String(24), nullable=False) + rule_code: Mapped[str] = mapped_column(String(64), nullable=False) + rule_name: Mapped[str] = mapped_column(String(128), nullable=False) + result: Mapped[str] = mapped_column(String(16), nullable=False) + document_value: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False) + database_value: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False) + calculation: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False) + + +class OffsiteQueryRecord(Base): + __tablename__ = "offsite_query_record" + + id: Mapped[int] = mapped_column(BigInteger, primary_key=True) + task_id: Mapped[str] = mapped_column(String(24), nullable=False) + rule_code: Mapped[str] = mapped_column(String(64), nullable=False) + natural_language_request: Mapped[str] = mapped_column(Text, nullable=False) + script_path: Mapped[str] = mapped_column(String(500), nullable=False) + result_summary: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False) + status: Mapped[str] = mapped_column(String(24), nullable=False) + error_message: Mapped[str | None] = mapped_column(String(500)) + created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False) + + +class OffsiteNotification(Base): + __tablename__ = "offsite_notification" + + id: Mapped[int] = mapped_column(BigInteger, primary_key=True) + notification_type: Mapped[str] = mapped_column(String(32), nullable=False) + business_key: Mapped[str] = mapped_column(String(64), nullable=False) + receiver_id: Mapped[str] = mapped_column(String(64), nullable=False) + operator_id: Mapped[str] = mapped_column(String(64), nullable=False) + agent_draft: Mapped[str] = mapped_column(Text, nullable=False) + final_content: Mapped[str] = mapped_column(Text, nullable=False) + payload: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False) + status: Mapped[str] = mapped_column(String(16), nullable=False, default="待发送") + retry_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + provider_message_id: Mapped[str | None] = mapped_column(String(128)) + failure_reason: Mapped[str | None] = mapped_column(String(500)) + created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False) + sent_at: Mapped[datetime | None] = mapped_column(DateTime) + updated_at: Mapped[datetime | None] = mapped_column(DateTime) diff --git a/app/model/session.py b/app/model/session.py index abaab22..c4df5d9 100644 --- a/app/model/session.py +++ b/app/model/session.py @@ -1,7 +1,6 @@ from datetime import datetime -from sqlalchemy import String, text -from sqlalchemy.dialects.mysql import BIGINT, DATETIME, INTEGER, TINYINT +from sqlalchemy import BigInteger, DateTime, Integer, String, text from sqlalchemy.orm import Mapped, mapped_column from app.model.base import Base @@ -9,26 +8,26 @@ from app.model.base import Base class ConversationSession(Base): __tablename__ = "svc_conversation_session" - id: Mapped[int] = mapped_column(BIGINT(unsigned=True), primary_key=True, autoincrement=True) + id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) session_id: Mapped[str] = mapped_column(String(64), unique=True) - user_id: Mapped[int] = mapped_column(BIGINT(unsigned=True)) + user_id: Mapped[int] = mapped_column(BigInteger) portal: Mapped[str] = mapped_column(String(32)) agent_type: Mapped[str | None] = mapped_column(String(32)) status: Mapped[str] = mapped_column(String(16), default="active") - clarification_round: Mapped[int] = mapped_column(TINYINT(unsigned=True), default=0) - message_count: Mapped[int] = mapped_column(INTEGER(unsigned=True), default=0) + clarification_round: Mapped[int] = mapped_column(Integer, default=0) + message_count: Mapped[int] = mapped_column(Integer, default=0) last_intent: Mapped[str | None] = mapped_column(String(32)) config_version: Mapped[str | None] = mapped_column(String(64)) started_at: Mapped[datetime] = mapped_column( - DATETIME(fsp=6), server_default=text("CURRENT_TIMESTAMP(6)") + DateTime, server_default=text("CURRENT_TIMESTAMP(6)") ) last_active_at: Mapped[datetime] = mapped_column( - DATETIME(fsp=6), server_default=text("CURRENT_TIMESTAMP(6)") + DateTime, server_default=text("CURRENT_TIMESTAMP(6)") ) - ended_at: Mapped[datetime | None] = mapped_column(DATETIME(fsp=6)) + ended_at: Mapped[datetime | None] = mapped_column(DateTime) created_at: Mapped[datetime] = mapped_column( - DATETIME(fsp=6), server_default=text("CURRENT_TIMESTAMP(6)") + DateTime, server_default=text("CURRENT_TIMESTAMP(6)") ) updated_at: Mapped[datetime] = mapped_column( - DATETIME(fsp=6), server_default=text("CURRENT_TIMESTAMP(6)") + DateTime, server_default=text("CURRENT_TIMESTAMP(6)") ) diff --git a/app/repository/platform_repository.py b/app/repository/platform_repository.py index ef891d6..e81491b 100644 --- a/app/repository/platform_repository.py +++ b/app/repository/platform_repository.py @@ -1,4 +1,4 @@ -from typing import Any +from typing import Any, cast from sqlalchemy import MetaData, Table, insert, select, update from sqlalchemy.ext.asyncio import AsyncSession @@ -66,7 +66,9 @@ class PlatformRepository: async def create(self, name: str, values: dict[str, Any]) -> dict[str, Any]: table = await self.table(name) result = await self.session.execute(insert(table).values(**values)) - row_id = int(result.inserted_primary_key[0]) # type: ignore[attr-defined] + inserted_primary_key = cast(Any, result).inserted_primary_key + assert inserted_primary_key is not None + row_id = int(inserted_primary_key[0]) row = await self.get(name, row_id) assert row is not None return row diff --git a/app/service/agent/bootstrap.py b/app/service/agent/bootstrap.py index e89e86e..13e2034 100644 --- a/app/service/agent/bootstrap.py +++ b/app/service/agent/bootstrap.py @@ -2,7 +2,10 @@ from functools import lru_cache from typing import Any, cast from app.core.fund_contracts import FundQuoteQuery +from app.core.nl2sql_contracts import FinancialNL2SQLInput from app.service.agent.factory import AgentFactory +from app.service.agent.offsite_fund_agent import OffsiteFundAgent +from app.service.financial_nl2sql_service import query_financial_data_tool from app.service.fund_quote_service import query_fund_quote_tool from app.service.intent_classifier import IntentClassifier from app.service.model_gateway import ( @@ -34,11 +37,24 @@ def get_agent_factory() -> AgentFactory: allowed_roles=("customer", "advisor", "operator", "risk_operator", "admin"), timeout_seconds=5, )) + registry.register(ToolDefinition( + name="query_financial_data", + input_model=FinancialNL2SQLInput, + handler=cast(Any, query_financial_data_tool), + required_permission="financial:nl2sql:read", + allowed_roles=("advisor", "operator", "admin", "super_admin"), + timeout_seconds=10, + )) model_service = ModelGenerationService(ModelDispatchService(DatabaseModelGateway())) endpoint_resolver = DatabaseModelEndpointResolver() - return AgentFactory( + factory = AgentFactory( model_service=model_service, tool_executor=ToolExecutor(registry), intent_classifier=IntentClassifier(model_service), intent_endpoint_resolver=endpoint_resolver, ) + factory.register( + OffsiteFundAgent.definition, + lambda _context: OffsiteFundAgent(OffsiteFundAgent.definition), + ) + return factory diff --git a/app/service/agent/offsite_fund_agent.py b/app/service/agent/offsite_fund_agent.py new file mode 100644 index 0000000..2cfa742 --- /dev/null +++ b/app/service/agent/offsite_fund_agent.py @@ -0,0 +1,24 @@ +"""场外基金运营 Agent。""" + +from app.core.contracts import AgentDefinition, AgentRequest, CoreResult, RequestContext +from app.service.agent.base import BaseAgent + + +class OffsiteFundAgent(BaseAgent): + definition = AgentDefinition( + agent_type="offsite_fund", + version="1.0.0", + allowed_roles=("operator", "risk_operator", "admin"), + allowed_portals=("api",), + supported_intents=("offsite_status", "offsite_reconcile", "general"), + ) + + async def handle(self, request: AgentRequest, context: RequestContext) -> CoreResult: + del context + text = ( + "场外申购赎回后端流程已接入。请通过场外业务接口写入已识别邮件和附件，" + "系统会生成邮件编号、附件编号、执行计划、确定性规则结果和待人工确认状态。" + f"本次请求摘要：{request.message[:120]}" + ) + return CoreResult(text=text) + diff --git a/app/service/financial_nl2sql_service.py b/app/service/financial_nl2sql_service.py new file mode 100644 index 0000000..446801e --- /dev/null +++ b/app/service/financial_nl2sql_service.py @@ -0,0 +1,499 @@ +from __future__ import annotations + +import re +from collections.abc import Callable +from contextlib import AbstractAsyncContextManager +from datetime import UTC, datetime, timedelta +from decimal import Decimal +from typing import Any + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.contracts import RequestContext +from app.core.errors import ForbiddenAgentError, ValidationAgentError +from app.core.nl2sql_catalog import ( + ALLOWED_TABLES, + BANNED_SQL, + CURRENT_ONLY_TABLES, + DOMAINS, + JOIN_SQL, + TABLE_COLUMNS, + alias, +) +from app.core.nl2sql_contracts import ( + FinancialNL2SQLInput, + FinancialNL2SQLResult, + FinancialQueryPlan, +) +from app.infrastructure.db import SessionFactory + +SessionFactoryType = Callable[[], AbstractAsyncContextManager[AsyncSession]] + + +def _range(question: str) -> tuple[str | None, str | None]: + days = 0 + if "近7天" in question or "最近7天" in question: + days = 7 + elif "近30天" in question or "最近30天" in question or "近一个月" in question: + days = 30 + elif "近90天" in question or "最近90天" in question or "近三个月" in question: + days = 90 + if not days: + return None, None + end = datetime.now(UTC).replace(tzinfo=None) + start = end - timedelta(days=days) + return start.strftime("%Y-%m-%d %H:%M:%S"), end.strftime("%Y-%m-%d %H:%M:%S") + + +def _filters(question: str) -> tuple[dict[str, Any], ...]: + return tuple( + {"field": "fin_product.product_code", "operator": "=", "value": code} + for code in sorted(set(re.findall(r"(? FinancialQueryPlan: + question = query.question + start, end = _range(question) + time_mode = "range" if start and end else "none" + if any(word in question for word in ("截至", "历史时点", "当时", "某日")): + time_mode = "as_of" + filters = _filters(question) + for builder in (self._market_plan, self._trade_plan, self._profile_plan): + plan = builder(query, time_mode, start, end, filters) + if plan is not None: + return plan + return FinancialQueryPlan( + intent="unknown", domains=("customer_risk",), tables=("fin_customer_profile",), + limit=query.limit, confidence=0.45, needs_confirmation=True, + confirmation_question="请明确要查询的客户、产品、时间范围或指标口径。", + ) + + @staticmethod + def _market_plan( + query: FinancialNL2SQLInput, + time_mode: str, + start: str | None, + end: str | None, + filters: tuple[dict[str, Any], ...], + ) -> FinancialQueryPlan | None: + question = query.question + if "净值" in question: + return FinancialQueryPlan( + intent="nav_history_query", domains=("market_nav", "product_fee"), + tables=("fin_nav_history", "fin_product"), metrics=("基金净值",), + dimensions=("fin_product.product_code", "fin_product.product_name", + "fin_nav_history.nav_date"), + filters=filters, time_mode=time_mode, time_column="nav_date", + start=start, end=end, limit=query.limit, confidence=0.90, + ) + if any(word in question for word in ("行情", "收盘", "开盘", "最高价", "最低价")): + return FinancialQueryPlan( + intent="market_price_query", domains=("market_nav", "product_fee"), + tables=("fin_market_price", "fin_product"), metrics=("收盘价",), + dimensions=("fin_product.product_code", "fin_product.product_name", + "fin_market_price.trade_date"), + filters=filters, time_mode=time_mode, time_column="trade_date", + start=start, end=end, limit=query.limit, confidence=0.90, + ) + return None + + @staticmethod + def _trade_plan( + query: FinancialNL2SQLInput, + time_mode: str, + start: str | None, + end: str | None, + filters: tuple[dict[str, Any], ...], + ) -> FinancialQueryPlan | None: + question = query.question + if "资金" in question or "现金流水" in question: + cash_metrics = ( + ("历史资金变化",) + if any(w in question for w in ("汇总", "合计", "变化")) + else () + ) + return FinancialQueryPlan( + intent="cash_ledger_query", domains=("trading_account", "customer_risk"), + tables=("fin_cash_ledger", "fin_sim_account", "fin_customer_profile"), + metrics=cash_metrics, + dimensions=("fin_cash_ledger.occurred_at", "fin_cash_ledger.entry_type"), + filters=filters, time_mode=time_mode, time_column="occurred_at", + start=start, end=end, limit=query.limit, confidence=0.88, + ) + if "持仓" in question: + return FinancialQueryPlan( + intent="holding_query", domains=("trading_account", "customer_risk", "product_fee"), + tables=("fin_holding", "fin_customer_profile", "fin_product"), + metrics=("持仓市值", "浮动盈亏") if "盈亏" in question else ("持仓市值",), + dimensions=("fin_customer_profile.customer_id", "fin_product.product_code", + "fin_product.product_name"), + filters=filters, time_mode=time_mode, time_column="updated_at", + start=start, end=end, limit=query.limit, confidence=0.88, + ) + if any(word in question for word in ("成交", "交易")): + transaction_metrics = ( + ("交易金额", "成交数量") + if any(w in question for w in ("统计", "汇总", "合计")) + else () + ) + return FinancialQueryPlan( + intent="transaction_query", + domains=("trading_account", "customer_risk", "product_fee"), + tables=("fin_transaction", "fin_customer_profile", "fin_product"), + metrics=transaction_metrics, + dimensions=("fin_customer_profile.customer_id", "fin_product.product_code", + "fin_transaction.order_side"), + filters=filters, time_mode=time_mode, time_column="executed_at", + start=start, end=end, limit=query.limit, confidence=0.87, + ) + if "委托" in question or "订单" in question: + return FinancialQueryPlan( + intent="order_query", domains=("trading_account", "customer_risk", "product_fee"), + tables=("fin_sim_order", "fin_customer_profile", "fin_product"), + dimensions=("fin_customer_profile.customer_id", "fin_product.product_code", + "fin_sim_order.status"), + filters=filters, time_mode=time_mode, time_column="submitted_at", + start=start, end=end, limit=query.limit, confidence=0.86, + ) + if "账户" in question or "余额" in question or "现金" in question: + return FinancialQueryPlan( + intent="account_query", domains=("trading_account", "customer_risk"), + tables=("fin_sim_account", "fin_customer_profile"), + metrics=("现金余额", "可用现金"), + dimensions=("fin_customer_profile.customer_id",), + time_mode=time_mode, + time_column="updated_at", + start=start, + end=end, + limit=query.limit, + confidence=0.88, + ) + return None + + @staticmethod + def _profile_plan( + query: FinancialNL2SQLInput, + time_mode: str, + start: str | None, + end: str | None, + filters: tuple[dict[str, Any], ...], + ) -> FinancialQueryPlan | None: + question = query.question + if "怎么样" in question: + return FinancialQueryPlan( + intent="unknown", domains=("customer_risk",), + tables=("fin_customer_profile",), limit=query.limit, + confidence=0.45, needs_confirmation=True, + confirmation_question="请明确要查询客户画像、风险测评、交易、持仓还是账户信息。", + ) + if "费率" in question or "费用" in question: + return FinancialQueryPlan( + intent="fee_rule_query", domains=("product_fee",), + tables=("fin_fee_rule", "fin_product"), + dimensions=("fin_product.product_code", "fin_fee_rule.order_side", + "fin_fee_rule.customer_tier"), + filters=filters, limit=query.limit, confidence=0.87, + ) + if "内容" in question or "审核" in question or "发布" in question: + return FinancialQueryPlan( + intent="client_content_query", domains=("client_content", "customer_risk"), + tables=("client_facing_content", "fin_customer_profile"), + dimensions=( + "fin_customer_profile.customer_id", + "client_facing_content.content_type", + "client_facing_content.review_status", + ), + time_mode=time_mode, time_column="created_at", start=start, end=end, + limit=query.limit, confidence=0.84, needs_confirmation=True, + confirmation_question="请确认要查询的是对客内容的审核或发布记录。", + ) + if "风险" in question or "测评" in question or "画像" in question or "客户" in question: + return FinancialQueryPlan( + intent="customer_risk_query", domains=("customer_risk",), + tables=("fin_risk_assessment", "fin_customer_profile"), + metrics=("客户数",) if any(w in question for w in ("多少", "数量", "统计")) else (), + dimensions=( + "fin_customer_profile.customer_id", + "fin_customer_profile.investor_type", + ), + time_mode=time_mode, time_column="assessed_at", start=start, end=end, + limit=query.limit, confidence=0.86, + ) + return None + + +class FinancialNL2SQLService: + def __init__( + self, + planner: RuleBasedFinancialPlanner | None = None, + session_factory: SessionFactoryType = SessionFactory, + ) -> None: + self.planner = planner or RuleBasedFinancialPlanner() + self.session_factory = session_factory + + async def query( + self, arguments: FinancialNL2SQLInput, context: RequestContext + ) -> dict[str, Any]: + self._require_context(context) + plan = self.planner.plan(arguments) + if arguments.confirmation and plan.needs_confirmation: + plan = plan.model_copy(update={ + "needs_confirmation": False, "confidence": max(0.85, plan.confidence), + }) + valid, message = self._validate_plan(plan, context) + if not valid: + return self._result("rejected", message, arguments, context, plan, None, {}, 0) + if plan.needs_confirmation or plan.confidence < 0.85: + question = plan.confirmation_question or "请确认查询范围、指标口径和时间条件。" + return self._result( + "need_confirmation", question, arguments, context, plan, None, {}, 0 + ) + sql, params = self._compile_sql(plan, context) + self._safe_sql_check(sql, params, plan) + if arguments.dry_run: + return self._result( + "ready", "SQL 已生成并通过只读校验", + arguments, context, plan, sql, params, 0, + ) + rows = await self._execute(sql, params) + result = self._result( + "success", "查询成功", arguments, context, plan, sql, params, len(rows) + ) + result["data"] = {"total": len(rows), "rows": rows} + return result + + @staticmethod + def _require_context(context: RequestContext) -> None: + if "financial:nl2sql:read" not in context.permissions: + raise ForbiddenAgentError("缺少金融 NL2SQL 查询权限") + if not {"advisor", "operator", "admin", "super_admin"}.intersection(context.roles): + raise ForbiddenAgentError("当前角色不能使用金融 NL2SQL 查询") + + @staticmethod + def _validate_plan(plan: FinancialQueryPlan, context: RequestContext) -> tuple[bool, str]: + if len(plan.domains) > 3: + return False, "最多支持跨三个业务域查询" + if not set(plan.tables).issubset(ALLOWED_TABLES): + return False, "查询包含未纳入 NL2SQL 范围的数据表" + allowed = set().union(*(DOMAINS[d] for d in plan.domains if d in DOMAINS)) + if not set(plan.tables).issubset(allowed): + return False, "查询计划的数据表和业务域不匹配" + if plan.time_mode == "as_of" and CURRENT_ONLY_TABLES.intersection(plan.tables): + return False, "当前快照表缺少历史版本，暂不支持该历史时点查询" + if context.data_scope not in {"self", "own_customers", "all"}: + return False, "未知的数据权限范围" + return True, "计划校验通过" + + def _compile_sql( + self, plan: FinancialQueryPlan, context: RequestContext + ) -> tuple[str, dict[str, Any]]: + select_parts: list[str] = [] + group_parts: list[str] = [] + for dimension in plan.dimensions: + table, column = self._split_field(dimension, plan.tables[0]) + self._ensure_column(table, column) + select_parts.append(f"{alias(table)}.{column} AS {column}") + group_parts.append(f"{alias(table)}.{column}") + for metric in plan.metrics: + select_parts.append(self._metric_expression(metric)) + if not select_parts: + select_parts = self._default_select(plan.tables[0]) + params: dict[str, Any] = {} + where = ["1=1"] + base_alias = alias(plan.tables[0]) + if plan.time_column and plan.start: + where.append(f"{base_alias}.{plan.time_column} >= :start_time") + params["start_time"] = plan.start + if plan.time_column and plan.end: + where.append(f"{base_alias}.{plan.time_column} <= :end_time") + params["end_time"] = plan.end + for index, item in enumerate(plan.filters): + table, column = self._split_field(str(item.get("field", "")), plan.tables[0]) + self._ensure_column(table, column) + operator = str(item.get("operator", "=")).upper() + if operator not in {"=", "!=", ">", ">=", "<", "<=", "LIKE"}: + raise ValidationAgentError("查询筛选条件不合法") + key = f"filter_{index}" + where.append(f"{alias(table)}.{column} {operator} :{key}") + params[key] = item.get("value") + self._append_customer_scope(plan, context, where, params) + sql = ( + f"SELECT {', '.join(select_parts)} FROM {self._join(plan.tables)} " + f"WHERE {' AND '.join(where)}" + ) + if plan.metrics and group_parts: + sql += f" GROUP BY {', '.join(group_parts)}" + return f"{sql} LIMIT {plan.limit}", params + + @staticmethod + def _metric_expression(metric: str) -> str: + mapping = { + "交易金额": "SUM(t.gross_amount) AS gross_amount", + "成交数量": "SUM(t.executed_quantity) AS executed_quantity", + "持仓市值": "h.market_value AS market_value", + "浮动盈亏": "h.profit_loss AS profit_loss", + "现金余额": "a.cash_balance AS cash_balance", + "可用现金": "a.available_cash AS available_cash", + "历史资金变化": "SUM(l.amount) AS cash_change", + "收盘价": "m.close_price AS close_price", + "基金净值": "n.nav AS nav", + "客户数": "COUNT(DISTINCT cp.customer_id) AS customer_count", + } + if metric not in mapping: + raise ValidationAgentError(f"暂不支持指标：{metric}") + return mapping[metric] + + @staticmethod + def _default_select(primary: str) -> list[str]: + defaults = { + "fin_fee_rule": [ + "p.product_code AS product_code", + "f.order_side AS order_side", + "f.fee_rate AS fee_rate", + ], + "fin_sim_order": [ + "o.order_no AS order_no", + "o.status AS status", + "o.submitted_at AS submitted_at", + ], + "client_facing_content": [ + "c.content_type AS content_type", + "c.review_status AS review_status", + ], + "fin_transaction": [ + "t.transaction_no AS transaction_no", + "t.gross_amount AS gross_amount", + ], + } + return defaults.get(primary, [f"{alias(primary)}.id AS id"]) + + @staticmethod + def _split_field(field: str, default_table: str) -> tuple[str, str]: + if "." not in field: + return default_table, field + table, column = field.split(".", 1) + return table, column + + @staticmethod + def _ensure_column(table: str, column: str) -> None: + if column not in TABLE_COLUMNS.get(table, set()): + raise ValidationAgentError("查询字段不在白名单内") + + @staticmethod + def _join(tables: tuple[str, ...]) -> str: + primary = tables[0] + joined = {primary} + pending = set(tables) - joined + sql = f"{primary} {alias(primary)}" + while pending: + for table in sorted(pending): + condition = next( + ( + JOIN_SQL.get((existing, table)) or JOIN_SQL.get((table, existing)) + for existing in joined + if JOIN_SQL.get((existing, table)) or JOIN_SQL.get((table, existing)) + ), + None, + ) + if condition: + sql += f" JOIN {table} {alias(table)} ON {condition}" + joined.add(table) + pending.remove(table) + break + else: + raise ValidationAgentError("查询涉及的表缺少合法关联路径") + return sql + + @staticmethod + def _append_customer_scope( + plan: FinancialQueryPlan, context: RequestContext, where: list[str], params: dict[str, Any] + ) -> None: + aliases = [ + alias(table) + for table in plan.tables + if "customer_id" in TABLE_COLUMNS.get(table, set()) + ] + if not aliases or context.data_scope == "all": + return + customer_ids = tuple(context.customer_ids) or (context.user_id,) + keys = [] + for index, customer_id in enumerate(customer_ids): + key = f"scope_customer_{index}" + keys.append(f":{key}") + params[key] = int(customer_id) + where.append(f"{aliases[0]}.customer_id IN ({', '.join(keys)})") + + @staticmethod + def _safe_sql_check(sql: str, params: dict[str, Any], plan: FinancialQueryPlan) -> None: + normalized = re.sub(r"\s+", " ", sql.strip()) + if not normalized.upper().startswith("SELECT "): + raise ForbiddenAgentError("仅支持 SELECT 查询") + if ";" in normalized.rstrip(";") or BANNED_SQL.search(normalized): + raise ForbiddenAgentError("检测到非只读数据库操作") + if "*" in normalized: + raise ForbiddenAgentError("禁止使用 SELECT *") + refs = set(re.findall(r"\b(?:FROM|JOIN)\s+([A-Za-z_][A-Za-z0-9_]*)", normalized, re.I)) + if refs != set(plan.tables): + raise ForbiddenAgentError("SQL 使用的数据表与查询计划不一致") + if len(params) > 30: + raise ForbiddenAgentError("查询参数过多") + + async def _execute(self, sql: str, params: dict[str, Any]) -> list[dict[str, Any]]: + async with self.session_factory() as session: + result = await session.execute(text(sql), params) + return [self._jsonable(dict(row)) for row in result.mappings().all()] + + @staticmethod + def _jsonable(row: dict[str, Any]) -> dict[str, Any]: + converted: dict[str, Any] = {} + for key, value in row.items(): + if isinstance(value, Decimal): + converted[key] = str(value) + elif isinstance(value, datetime): + converted[key] = value.isoformat(sep=" ", timespec="seconds") + else: + converted[key] = value + return converted + + @staticmethod + def _result( + status: str, + message: str, + arguments: FinancialNL2SQLInput, + context: RequestContext, + plan: FinancialQueryPlan, + sql: str | None, + params: dict[str, Any], + row_count: int, + ) -> dict[str, Any]: + audit = { + "tool": "query_financial_data", + "engine_version": "mvp-1", + "trace_id": context.trace_id, + "question": arguments.question, + "query_plan": plan.model_dump(mode="json"), + "generated_sql": sql, + "parameters": params, + "permission_check": { + "status": "passed" if status in {"ready", "success"} else status, + "roles": list(context.roles), + "data_scope": context.data_scope, + "permission": "financial:nl2sql:read", + "allowed_tables": list(plan.tables), + }, + "execution": {"status": status, "row_count": row_count}, + "created_at": datetime.now(UTC).replace(tzinfo=None).isoformat(timespec="seconds"), + } + return FinancialNL2SQLResult( + status=status, message=message, query_plan=plan.model_dump(mode="json"), + sql=sql, parameters=params, audit=audit, + ).model_dump(mode="json") + + +async def query_financial_data_tool( + arguments: FinancialNL2SQLInput, context: RequestContext +) -> dict[str, Any]: + return await FinancialNL2SQLService().query(arguments, context) diff --git a/app/service/health_service.py b/app/service/health_service.py index 8707da0..3cb4d19 100644 --- a/app/service/health_service.py +++ b/app/service/health_service.py @@ -38,6 +38,6 @@ class HealthService: finally: if client is not None: try: - await client.aclose() + await client.close() except Exception: pass diff --git a/app/service/model_gateway.py b/app/service/model_gateway.py index 0eb362a..29d1bae 100644 --- a/app/service/model_gateway.py +++ b/app/service/model_gateway.py @@ -1,7 +1,7 @@ import os from collections.abc import Mapping from dataclasses import dataclass -from typing import Any, Protocol +from typing import Any, Protocol, cast import httpx from sqlalchemy import select @@ -91,7 +91,9 @@ class DatabaseModelGateway: )) if endpoint is None: raise RecoverableAgentError("模型端点未注册或未激活") - adapter = OpenAICompatibleGateway({endpoint.endpoint_code: endpoint}) + adapter = OpenAICompatibleGateway({ + endpoint.endpoint_code: cast(EndpointSettings, endpoint) + }) return await adapter.generate( endpoint_code=endpoint.endpoint_code, prompt=prompt, timeout_ms=timeout_ms ) diff --git a/app/service/offsite_document_recognition_adapter.py b/app/service/offsite_document_recognition_adapter.py new file mode 100644 index 0000000..dac80a0 --- /dev/null +++ b/app/service/offsite_document_recognition_adapter.py @@ -0,0 +1,440 @@ +"""场外基金附件 OCR 与字段识别适配层。""" + +from __future__ import annotations + +import base64 +import json +import re +from collections.abc import Mapping +from dataclasses import dataclass +from decimal import Decimal, InvalidOperation +from typing import Any, Literal, cast + +import httpx + +from app.core.config import Settings +from app.core.offsite_fund_contracts import DocumentType + +RecognitionStatus = Literal["mock", "success", "disabled", "misconfigured", "error"] + +REQUIRED_FIELDS: dict[str, tuple[str, ...]] = { + "subscription": ( + "基金代码", + "基金名称", + "账户标识", + "申请编号", + "申请日期", + "代销机构", + "申购金额", + "金额单位", + ), + "redemption": ( + "基金代码", + "基金名称", + "账户标识", + "申请编号", + "申请日期", + "代销机构", + "赎回份额", + ), +} + + +@dataclass(frozen=True) +class RecognitionSourceFile: + filename: str + media_type: str + payload: bytes + file_hash: str + original_file_path: str = "" + + +@dataclass(frozen=True) +class OcrRecognitionResult: + status: RecognitionStatus + ocr_text: str + tables: list[object] + page_evidence: dict[str, object] + raw_summary: dict[str, object] + error_message: str | None = None + + +@dataclass(frozen=True) +class StructuredRecognitionResult: + document_type: DocumentType + extracted_fields: dict[str, object] + field_confidence: dict[str, Decimal] + missing_fields: tuple[str, ...] + low_confidence_fields: tuple[str, ...] + page_evidence: dict[str, object] + ocr_text: str + ocr_status: RecognitionStatus + llm_status: RecognitionStatus + error_message: str | None = None + + +class OffsiteDocumentRecognitionAdapter: + """先 OCR，再用 DeepSeek 或本地 Mock 生成统一字段。""" + + def __init__(self, settings: Settings, *, client: httpx.AsyncClient | None = None) -> None: + self.settings = settings + self.client = client + self._owns_client = client is None + + def health_check(self) -> dict[str, object]: + ocr_missing = self._ocr_missing_config() if self.settings.offsite_ocr_enabled else [] + llm_missing = ( + self._deepseek_missing_config() if self.settings.offsite_deepseek_enabled else [] + ) + return { + "ocr": ( + {"status": "misconfigured", "missing": ocr_missing} + if ocr_missing + else {"status": "ok" if self.settings.offsite_ocr_enabled else "disabled"} + ), + "deepseek": ( + {"status": "misconfigured", "missing": llm_missing} + if llm_missing + else { + "status": "ok" if self.settings.offsite_deepseek_enabled else "disabled", + "model": self.settings.offsite_deepseek_model, + } + ), + } + + async def recognize(self, source: RecognitionSourceFile) -> StructuredRecognitionResult: + ocr = await self._recognize_ocr(source) + structured = await self._extract_fields(source, ocr) + if self._owns_client and self.client is not None: + await self.client.aclose() + return structured + + async def _recognize_ocr(self, source: RecognitionSourceFile) -> OcrRecognitionResult: + if not self.settings.offsite_ocr_enabled: + text = _mock_text(source) + return OcrRecognitionResult( + status="mock", + ocr_text=text, + tables=[], + page_evidence={}, + raw_summary={"provider": "mock", "filename": source.filename}, + ) + missing = self._ocr_missing_config() + if missing: + return OcrRecognitionResult( + status="misconfigured", + ocr_text="", + tables=[], + page_evidence={}, + raw_summary={}, + error_message=f"OCR配置缺失：{', '.join(missing)}", + ) + client = self._client() + payload = { + "filename": source.filename, + "media_type": source.media_type, + "file_hash": source.file_hash, + "content_base64": base64.b64encode(source.payload).decode("ascii"), + } + try: + response = await client.post( + self.settings.offsite_aliyun_ocr_endpoint, + headers={ + "X-Acs-AccessKey-Id": self.settings.offsite_aliyun_access_key_id, + "X-Acs-Signed-Gateway": "offsite-fund", + }, + json=payload, + timeout=httpx.Timeout(self.settings.offsite_ocr_timeout_seconds), + ) + response.raise_for_status() + body = _json_mapping(response) + data = _mapping_value(body, "data") + text = _text_from_mapping(data) or _text_from_mapping(body) + tables = _list_value(data, "tables") or _list_value(body, "tables") + page_evidence = _dict_value(data, "page_evidence") or _dict_value(body, "page_evidence") + return OcrRecognitionResult( + status="success", + ocr_text=text, + tables=tables, + page_evidence=page_evidence, + raw_summary={ + "provider": "aliyun_document_ai", + "response_keys": sorted(str(key) for key in body.keys()), + }, + ) + except (httpx.HTTPError, ValueError) as exc: + return OcrRecognitionResult( + status="error", + ocr_text="", + tables=[], + page_evidence={}, + raw_summary={"provider": "aliyun_document_ai"}, + error_message=type(exc).__name__, + ) + + async def _extract_fields( + self, source: RecognitionSourceFile, ocr: OcrRecognitionResult + ) -> StructuredRecognitionResult: + if not self.settings.offsite_deepseek_enabled: + return _mock_structured_result(source, ocr, "mock") + missing = self._deepseek_missing_config() + if missing: + result = _mock_structured_result(source, ocr, "misconfigured") + return _with_error(result, f"DeepSeek配置缺失：{', '.join(missing)}") + try: + body = await self._call_deepseek(source, ocr) + document_type = _document_type(body.get("document_type")) + fields = _dict_value(body, "extracted_fields") + confidence = _confidence_map(_dict_value(body, "field_confidence")) + page_evidence = _dict_value(body, "page_evidence") or ocr.page_evidence + missing_fields = tuple(str(item) for item in _list_value(body, "missing_fields")) + low_fields = tuple(str(item) for item in _list_value(body, "low_confidence_fields")) + if document_type in ("subscription", "redemption") and not missing_fields: + missing_fields = _missing_fields(document_type, fields) + if not low_fields: + low_fields = _low_confidence_fields(confidence) + return StructuredRecognitionResult( + document_type=document_type, + extracted_fields=fields, + field_confidence=confidence, + missing_fields=missing_fields, + low_confidence_fields=low_fields, + page_evidence=page_evidence, + ocr_text=ocr.ocr_text, + ocr_status=ocr.status, + llm_status="success", + error_message=ocr.error_message, + ) + except (httpx.HTTPError, ValueError, KeyError, TypeError) as exc: + result = _mock_structured_result(source, ocr, "error") + return _with_error(result, type(exc).__name__) + + async def _call_deepseek( + self, source: RecognitionSourceFile, ocr: OcrRecognitionResult + ) -> Mapping[str, object]: + client = self._client() + response = await client.post( + self.settings.offsite_deepseek_base_url.rstrip("/") + "/chat/completions", + headers={ + "Authorization": f"Bearer {self.settings.offsite_deepseek_api_key}", + "Content-Type": "application/json", + }, + json={ + "model": self.settings.offsite_deepseek_model, + "messages": [ + { + "role": "system", + "content": ( + "你只输出JSON。字段名必须使用申请日期和申请编号，" + "document_type只能是summary、subscription、redemption、other。" + ), + }, + { + "role": "user", + "content": json.dumps( + { + "filename": source.filename, + "media_type": source.media_type, + "ocr_text": ocr.ocr_text[:12000], + "tables": ocr.tables, + "page_evidence": ocr.page_evidence, + }, + ensure_ascii=False, + ), + }, + ], + "temperature": 0, + "response_format": {"type": "json_object"}, + }, + timeout=httpx.Timeout(self.settings.offsite_deepseek_timeout_seconds), + ) + response.raise_for_status() + content = _deepseek_content(_json_mapping(response)) + parsed = json.loads(content) + if not isinstance(parsed, dict): + raise ValueError("DeepSeek响应不是JSON对象") + return cast(Mapping[str, object], parsed) + + def _client(self) -> httpx.AsyncClient: + if self.client is None: + self.client = httpx.AsyncClient() + return self.client + + def _ocr_missing_config(self) -> list[str]: + missing = [] + if not self.settings.offsite_aliyun_ocr_endpoint: + missing.append("OFFSITE_ALIYUN_OCR_ENDPOINT") + if not self.settings.offsite_aliyun_access_key_id: + missing.append("OFFSITE_ALIYUN_ACCESS_KEY_ID") + if not self.settings.offsite_aliyun_access_key_secret: + missing.append("OFFSITE_ALIYUN_ACCESS_KEY_SECRET") + return missing + + def _deepseek_missing_config(self) -> list[str]: + missing = [] + if not self.settings.offsite_deepseek_base_url: + missing.append("OFFSITE_DEEPSEEK_BASE_URL") + if not self.settings.offsite_deepseek_api_key: + missing.append("OFFSITE_DEEPSEEK_API_KEY") + if not self.settings.offsite_deepseek_model: + missing.append("OFFSITE_DEEPSEEK_MODEL") + return missing + + +def _mock_text(source: RecognitionSourceFile) -> str: + text = source.payload.decode("utf-8", errors="ignore").strip() + return text or f"文件名：{source.filename}" + + +def _mock_structured_result( + source: RecognitionSourceFile, ocr: OcrRecognitionResult, llm_status: RecognitionStatus +) -> StructuredRecognitionResult: + document_type = _classify(source.filename + "\n" + ocr.ocr_text) + fields = _extract_simple_fields(ocr.ocr_text) + confidence = {key: Decimal("1") for key in fields} + missing_fields: tuple[str, ...] = () + if document_type in ("subscription", "redemption"): + missing_fields = _missing_fields(document_type, fields) + return StructuredRecognitionResult( + document_type=document_type, + extracted_fields=fields, + field_confidence=confidence, + missing_fields=missing_fields, + low_confidence_fields=(), + page_evidence=ocr.page_evidence, + ocr_text=ocr.ocr_text, + ocr_status=ocr.status, + llm_status=llm_status, + error_message=ocr.error_message, + ) + + +def _with_error(result: StructuredRecognitionResult, message: str) -> StructuredRecognitionResult: + return StructuredRecognitionResult( + document_type=result.document_type, + extracted_fields=result.extracted_fields, + field_confidence=result.field_confidence, + missing_fields=result.missing_fields, + low_confidence_fields=result.low_confidence_fields, + page_evidence=result.page_evidence, + ocr_text=result.ocr_text, + ocr_status=result.ocr_status, + llm_status=result.llm_status, + error_message=message, + ) + + +def _classify(text: str) -> DocumentType: + if "汇总" in text: + return "summary" + if "赎回" in text: + return "redemption" + if "申购" in text: + return "subscription" + return "other" + + +def _extract_simple_fields(text: str) -> dict[str, object]: + fields: dict[str, object] = {} + for name in ( + "基金代码", + "基金名称", + "账户标识", + "投资者名称", + "客户标识", + "申请编号", + "申请日期", + "代销机构", + "申购金额", + "金额单位", + "赎回份额", + ): + value = _find_label_value(text, name) + if value: + fields[name] = value + return fields + + +def _find_label_value(text: str, label: str) -> str | None: + pattern = rf"{re.escape(label)}\s*[:：]?\s*([^\s,，;；]+)" + matched = re.search(pattern, text) + if not matched: + return None + return matched.group(1).strip() + + +def _missing_fields(document_type: str, fields: Mapping[str, object]) -> tuple[str, ...]: + required = list(REQUIRED_FIELDS.get(document_type, ())) + if not (fields.get("投资者名称") or fields.get("客户标识")): + required.append("投资者名称/客户标识") + return tuple(name for name in required if not str(fields.get(name) or "").strip()) + + +def _low_confidence_fields(confidence: Mapping[str, Decimal]) -> tuple[str, ...]: + return tuple(name for name, value in confidence.items() if value < Decimal("0.95")) + + +def _confidence_map(value: Mapping[str, object]) -> dict[str, Decimal]: + result: dict[str, Decimal] = {} + for key, raw in value.items(): + try: + result[str(key)] = Decimal(str(raw)) + except (InvalidOperation, ValueError): + continue + return result + + +def _document_type(value: object) -> DocumentType: + if value in ("summary", "subscription", "redemption", "other"): + return cast(DocumentType, value) + return "other" + + +def _json_mapping(response: httpx.Response) -> Mapping[str, object]: + body: Any = response.json() + if not isinstance(body, dict): + raise ValueError("响应不是JSON对象") + return cast(Mapping[str, object], body) + + +def _mapping_value(value: Mapping[str, object], key: str) -> Mapping[str, object]: + nested = value.get(key) + if isinstance(nested, dict): + return cast(Mapping[str, object], nested) + return {} + + +def _dict_value(value: Mapping[str, object], key: str) -> dict[str, object]: + nested = value.get(key) + if isinstance(nested, dict): + return dict(cast(Mapping[str, object], nested)) + return {} + + +def _list_value(value: Mapping[str, object], key: str) -> list[object]: + nested = value.get(key) + return list(nested) if isinstance(nested, list) else [] + + +def _text_from_mapping(value: Mapping[str, object]) -> str: + for key in ("ocr_text", "text", "content", "markdown"): + candidate = value.get(key) + if isinstance(candidate, str): + return candidate + return "" + + +def _deepseek_content(body: Mapping[str, object]) -> str: + choices = body.get("choices") + if not isinstance(choices, list) or not choices: + raise ValueError("DeepSeek响应缺少choices") + first = choices[0] + if not isinstance(first, dict): + raise ValueError("DeepSeek响应格式错误") + message = first.get("message") + if not isinstance(message, dict): + raise ValueError("DeepSeek响应缺少message") + content = message.get("content") + if not isinstance(content, str) or not content.strip(): + raise ValueError("DeepSeek响应缺少content") + return content diff --git a/app/service/offsite_fund_rules.py b/app/service/offsite_fund_rules.py new file mode 100644 index 0000000..3ba264a --- /dev/null +++ b/app/service/offsite_fund_rules.py @@ -0,0 +1,152 @@ +"""场外基金申购赎回确定性规则。""" + +from dataclasses import dataclass +from datetime import date, datetime +from decimal import Decimal, InvalidOperation + +from app.core.offsite_fund_contracts import RuleResultStatus + +HUNDRED = Decimal("100") +TEN_PERCENT = Decimal("0.10") +TWENTY_PERCENT = Decimal("0.20") +ONE_YUAN = Decimal("1") + + +@dataclass(frozen=True) +class RuleDecision: + rule_code: str + rule_name: str + result: RuleResultStatus + document_value: dict[str, object] + database_value: dict[str, object] + calculation: dict[str, object] + + +def decimal_from(value: object) -> Decimal | None: + if value is None or value == "": + return None + try: + return Decimal(str(value).replace(",", "").strip()) + except (InvalidOperation, ValueError): + return None + + +def normalize_amount_yuan(raw_value: object, unit: object) -> Decimal | None: + amount = decimal_from(raw_value) + if amount is None: + return None + raw_unit = str(unit or "元").strip() + if raw_unit == "万元": + return amount * Decimal("10000") + if raw_unit in {"元", "人民币", "CNY"}: + return amount + return None + + +def parse_application_date(raw_value: object) -> date | None: + text = str(raw_value or "").strip() + for fmt in ("%Y-%m-%d", "%Y/%m/%d", "%Y年%m月%d日"): + try: + return datetime.strptime(text, fmt).date() + except ValueError: + continue + return None + + +class OffsiteFundRuleEngine: + """只做程序化计算，不读取数据库，也不调用模型。""" + + def check_subscription( + self, + *, + amount_yuan: Decimal | None, + nav: Decimal | None, + total_fund_shares: Decimal | None, + before_holding_shares: Decimal | None, + ) -> list[RuleDecision]: + decisions = [self._minimum_subscription(amount_yuan)] + if (amount_yuan is None or nav is None or nav <= 0 + or total_fund_shares is None or total_fund_shares == 0): + return decisions + [ + self._unknown("subscription_holding_ratio", "申购后单一投资者持有比例"), + self._unknown("subscription_single_share_limit", "申购单笔份额上限"), + ] + current_shares = amount_yuan / nav + before = before_holding_shares or Decimal("0") + ratio = (before + current_shares) / total_fund_shares + decisions.append(RuleDecision( + rule_code="subscription_holding_ratio", + rule_name="申购后单一投资者持有比例", + result="异常" if ratio > TWENTY_PERCENT else "正常", + document_value={"申购金额元": str(amount_yuan)}, + database_value={"最新净值": str(nav), "基金最新总份额": str(total_fund_shares), + "申请前持有份额": str(before)}, + calculation={"本次申购份额": str(current_shares), "申购后持有比例": str(ratio)}, + )) + limit = total_fund_shares * TEN_PERCENT + decisions.append(RuleDecision( + rule_code="subscription_single_share_limit", + rule_name="申购单笔份额上限", + result="异常" if current_shares > limit else "正常", + document_value={"申购金额元": str(amount_yuan)}, + database_value={"最新净值": str(nav), "基金最新总份额": str(total_fund_shares)}, + calculation={"本次申购份额": str(current_shares), "份额上限": str(limit)}, + )) + return decisions + + def check_redemption( + self, + *, + redemption_shares: Decimal | None, + total_fund_shares: Decimal | None, + available_quantity: Decimal | None, + ) -> list[RuleDecision]: + if redemption_shares is None or total_fund_shares is None or total_fund_shares == 0: + ratio = self._unknown("redemption_large_ratio", "赎回巨额比例") + else: + value = redemption_shares / total_fund_shares + ratio = RuleDecision( + rule_code="redemption_large_ratio", + rule_name="赎回巨额比例", + result="异常" if value > TWENTY_PERCENT else "正常", + document_value={"赎回份额": str(redemption_shares)}, + database_value={"产品最新总份额": str(total_fund_shares)}, + calculation={"赎回比例": str(value)}, + ) + if redemption_shares is None or available_quantity is None: + available = self._unknown("redemption_available_quantity", "账户可用份额") + else: + available = RuleDecision( + rule_code="redemption_available_quantity", + rule_name="账户可用份额", + result="异常" if redemption_shares > available_quantity else "正常", + document_value={"赎回份额": str(redemption_shares)}, + database_value={"当前最新可用份额": str(available_quantity)}, + calculation={"是否超出可用份额": redemption_shares > available_quantity}, + ) + return [ratio, available] + + @staticmethod + def _minimum_subscription(amount_yuan: Decimal | None) -> RuleDecision: + result: RuleResultStatus = "无法判断" + if amount_yuan is not None: + result = "异常" if amount_yuan <= ONE_YUAN else "正常" + return RuleDecision( + rule_code="subscription_minimum_amount", + rule_name="申购最低金额", + result=result, + document_value={"申购金额元": str(amount_yuan) if amount_yuan is not None else None}, + database_value={}, + calculation={"判断口径": "标准化申购金额 <= 1 元为异常"}, + ) + + @staticmethod + def _unknown(rule_code: str, rule_name: str) -> RuleDecision: + return RuleDecision( + rule_code=rule_code, + rule_name=rule_name, + result="无法判断", + document_value={}, + database_value={}, + calculation={"原因": "必要识别字段或查询结果缺失"}, + ) diff --git a/app/service/offsite_fund_service.py b/app/service/offsite_fund_service.py new file mode 100644 index 0000000..952f4ff --- /dev/null +++ b/app/service/offsite_fund_service.py @@ -0,0 +1,710 @@ +"""场外基金申购赎回业务编排服务。""" + +import asyncio +from collections.abc import Sequence +from datetime import UTC, datetime +from decimal import Decimal +from pathlib import Path +from typing import Literal, cast +from zoneinfo import ZoneInfo + +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.config import get_settings +from app.core.contracts import RequestContext +from app.core.offsite_fund_contracts import ( + DocumentType, + OffsiteDocumentSummary, + OperationDecision, + ReceiveRecognizedMailRequest, + RecognizedAttachment, +) +from app.model.audit import InteractionAudit +from app.model.offsite_fund import ( + OffsiteExecutionPlanTask, + OffsiteFundAttachment, + OffsiteFundDocument, + OffsiteFundMail, + OffsiteNotification, + OffsiteQueryRecord, + OffsiteRuleResult, +) +from app.service.offsite_fund_rules import ( + OffsiteFundRuleEngine, + decimal_from, + normalize_amount_yuan, + parse_application_date, +) +from app.service.offsite_nl2sql_adapter import OffsiteNl2SqlAdapter +from app.service.offsite_smtp_adapter import ( + OffsiteMailReplyRequest, + OffsiteSmtpSender, + SmtpAttachment, + SmtpSendResult, +) + +SHANGHAI = ZoneInfo("Asia/Shanghai") + + +class OffsiteFundService: + def __init__( + self, session: AsyncSession, smtp_sender: OffsiteSmtpSender | None = None + ) -> None: + self.session = session + self.rules = OffsiteFundRuleEngine() + self.nl2sql = OffsiteNl2SqlAdapter() + self.smtp_sender = smtp_sender or OffsiteSmtpSender(get_settings()) + + async def receive_recognized_mail( + self, payload: ReceiveRecognizedMailRequest, context: RequestContext + ) -> dict[str, object]: + denied = self._permission_error(context, ("offsite:write",)) + if denied is not None: + return denied + settings = get_settings() + if payload.sender not in settings.offsite_allowed_senders: + return {"code": 403, "message": "发件人不在场外业务白名单", "data": {}} + business = [item for item in payload.attachments if item.document_type != "other"] + if not business: + return {"code": 0, "message": "ok", "data": {"business": False}} + async with self.session.begin(): + existing = await self.session.scalar(select(OffsiteFundMail).where( + OffsiteFundMail.imap_uid == payload.imap_uid, + OffsiteFundMail.message_id == payload.message_id, + )) + if existing is not None: + return {"code": 0, "message": "ok", "data": {"mail_id": existing.mail_id}} + mail_id = await self._next_mail_id() + now = datetime.now(UTC).replace(tzinfo=None) + mail = OffsiteFundMail( + mail_id=mail_id, imap_uid=payload.imap_uid, message_id=payload.message_id, + received_date=datetime.now(SHANGHAI).date(), sender=payload.sender, + return_path=payload.return_path, auth_result=payload.auth_result, + original_eml_path=payload.eml_path, status="recognized", + created_at=now, updated_at=now, + ) + self.session.add(mail) + summaries: list[OffsiteDocumentSummary] = [] + for index, item in enumerate(payload.attachments, start=1): + attachment_id = f"{mail_id}-A{index:02d}" + self.session.add(OffsiteFundAttachment( + attachment_id=attachment_id, mail_id=mail_id, filename=item.filename, + file_hash=item.file_hash, media_type=item.media_type, + size_bytes=item.size_bytes, document_type=item.document_type, + original_file_path=item.original_file_path, ocr_text=item.ocr_text, + extracted_fields=item.extracted_fields, + field_confidence={ + key: str(value) for key, value in item.field_confidence.items() + }, + page_evidence=item.page_evidence, status="recognized", created_at=now, + )) + if item.document_type == "other": + continue + if item.document_type == "summary": + summaries.append(await self._create_document( + mail_id, f"{attachment_id}-S", attachment_id, item, "subscription", now)) + summaries.append(await self._create_document( + mail_id, f"{attachment_id}-R", attachment_id, item, "redemption", now)) + else: + summaries.append(await self._create_document( + mail_id, attachment_id, attachment_id, item, item.document_type, now)) + self._add_audit(context, "offsite.mail_recognized", { + "mail_id": mail_id, + "imap_uid": payload.imap_uid, + "message_id": payload.message_id, + "business_attachment_count": len(business), + }) + return {"code": 0, "message": "ok", "data": { + "mail_id": mail_id, + "documents": [item.model_dump(mode="json") for item in summaries], + }} + + async def confirm_document( + self, task_id: str, decision: OperationDecision, operator_id: str, + context: RequestContext, + ) -> dict[str, object]: + denied = self._permission_error(context, ("offsite:confirm", "offsite:write")) + if denied is not None: + return denied + async with self.session.begin(): + document = await self.session.scalar(select(OffsiteFundDocument).where( + OffsiteFundDocument.task_id == task_id).with_for_update()) + if document is None: + return {"code": 404, "message": "单据不存在", "data": {}} + document.operator_decision = decision + document.status = "operator_confirmed" + document.updated_at = datetime.now(UTC).replace(tzinfo=None) + self._add_audit(context, "offsite.document_confirmed", { + "task_id": task_id, + "operator_id": operator_id, + "decision": decision, + }) + return {"code": 0, "message": "ok", "data": {"task_id": task_id, "decision": decision}} + + async def recalculate_statistics( + self, fund_code: str, application_date: str, context: RequestContext + ) -> dict[str, object]: + denied = self._permission_error(context, ("offsite:read", "offsite:write")) + if denied is not None: + return denied + target_date = parse_application_date(application_date) + if target_date is None: + return {"code": 422, "message": "申请日期格式不正确", "data": {}} + rows = (await self.session.execute(select(OffsiteFundDocument).where( + OffsiteFundDocument.fund_code == fund_code, + OffsiteFundDocument.application_date == target_date, + OffsiteFundDocument.operator_decision == "确认正常", + ))).scalars().all() + subscription_total = sum( + ((row.subscription_amount_yuan or Decimal("0")) for row in rows), + Decimal("0"), + ) + redemption_total = sum( + ((row.redemption_shares or Decimal("0")) for row in rows), + Decimal("0"), + ) + agency_breakdown = self._agency_breakdown(rows) + redemption_amount_yuan: Decimal | None = None + net_flow_amount_yuan: Decimal | None = subscription_total if redemption_total == 0 else None + return {"code": 0, "message": "ok", "data": { + "fund_code": fund_code, "application_date": target_date.isoformat(), + "fund_name": next((row.fund_name for row in rows if row.fund_name), None), + "subscription_amount_yuan": str(subscription_total), + "subscription_count": sum(1 for row in rows if row.document_type == "subscription"), + "redemption_shares": str(redemption_total), + "redemption_count": sum(1 for row in rows if row.document_type == "redemption"), + "latest_nav": None, + "redemption_amount_yuan": ( + str(redemption_amount_yuan) if redemption_amount_yuan is not None else None + ), + "net_flow_amount_yuan": ( + str(net_flow_amount_yuan) if net_flow_amount_yuan is not None else None + ), + "agency_breakdown": agency_breakdown, + }} + + async def trigger_agent_nl2sql( + self, task_id: str, operator_id: str, manual_confirmed: bool, + context: RequestContext, + ) -> dict[str, object]: + denied = self._permission_error( + context, ("offsite:nl2sql", "offsite:write", "financial:nl2sql:read") + ) + if denied is not None: + return denied + if not manual_confirmed: + return {"code": 422, "message": "必须传递人工已确认原始文件内容状态", "data": {}} + now = datetime.now(UTC).replace(tzinfo=None) + records: list[dict[str, object]] = [] + async with self.session.begin(): + document = await self.session.scalar(select(OffsiteFundDocument).where( + OffsiteFundDocument.task_id == task_id)) + if document is None: + return {"code": 404, "message": "单据不存在", "data": {}} + questions = self._nl2sql_questions(document) + if not questions: + return {"code": 422, "message": "查询条件不足", "data": {}} + for rule_code, question in questions: + result = self.nl2sql.query(question, context) + self.session.add(OffsiteQueryRecord( + task_id=task_id, rule_code=rule_code, + natural_language_request=question, script_path=self.nl2sql.script_path, + result_summary=result, status=str(result.get("status", "error")), + error_message=( + result.get("message") if isinstance(result.get("message"), str) else None + ), + created_at=now, + )) + await self._update_query_plan_status(task_id, rule_code, result, now) + records.append({"rule_code": rule_code, "status": result.get("status")}) + self._add_audit(context, "offsite.nl2sql_triggered", { + "task_id": task_id, + "operator_id": operator_id, + "query_count": len(records), + }) + return {"code": 0, "message": "ok", "data": {"task_id": task_id, "queries": records}} + + async def create_notification( + self, task_id: str, notification_type: str, operator_id: str, + context: RequestContext, + ) -> dict[str, object]: + denied = self._permission_error(context, ("offsite:notify", "offsite:write")) + if denied is not None: + return denied + settings = get_settings() + receiver = { + "risk": settings.offsite_risk_receiver_id, + "settlement": settings.offsite_settlement_receiver_id, + "mail_return": settings.offsite_mail_return_receiver, + }[notification_type] + draft = f"{task_id} 待发送{notification_type}通知" + now = datetime.now(UTC).replace(tzinfo=None) + async with self.session.begin(): + document = await self.session.scalar(select(OffsiteFundDocument).where( + OffsiteFundDocument.task_id == task_id)) + if document is None: + return {"code": 404, "message": "单据不存在", "data": {}} + validation = self._validate_notification(document, notification_type) + if validation is not None: + return validation + rule_results = (await self.session.execute(select(OffsiteRuleResult).where( + OffsiteRuleResult.task_id == task_id + ))).scalars().all() + payload = self._notification_payload(document, rule_results, operator_id) + notice = OffsiteNotification( + notification_type=notification_type, business_key=task_id, + receiver_id=receiver, operator_id=operator_id, agent_draft=draft, + final_content=draft, payload=payload, status="待发送", + created_at=now, updated_at=now, + ) + self.session.add(notice) + await self.session.flush() + notice_id = notice.id + self._add_audit(context, "offsite.notification_created", { + "task_id": task_id, + "notification_type": notification_type, + "operator_id": operator_id, + "receiver_id": receiver, + }) + return {"code": 0, "message": "ok", "data": {"notification_id": str(notice_id)}} + + async def send_notification( + self, + notification_id: int, + operator_id: str, + operator_confirmed: bool, + final_content: str | None, + context: RequestContext, + ) -> dict[str, object]: + denied = self._permission_error(context, ("offsite:notify", "offsite:write")) + if denied is not None: + return denied + if not operator_confirmed: + return {"code": 422, "message": "邮件发送前必须完成运营确认", "data": {}} + prepared = await self._prepare_notification_send( + notification_id, operator_id, final_content, context + ) + if isinstance(prepared, dict): + return prepared + notice, mail, attachment, receiver = prepared + try: + request = self._build_mail_reply_request( + notice, mail, attachment, receiver, operator_id, final_content + ) + result = await asyncio.to_thread(self.smtp_sender.send_reply, request) + except OSError as exc: + result = SmtpSendResult( + status="发送失败", + dry_run=False, + provider_message_id=None, + failure_reason=type(exc).__name__, + retry_count=notice.retry_count + 1, + request_summary={"notification_id": str(notification_id)}, + ) + await self._finish_notification(notification_id, result, context) + return { + "code": 0, + "message": "ok", + "data": { + "notification_id": str(notification_id), + "status": result.status, + "dry_run": result.dry_run, + "provider_message_id": result.provider_message_id, + "failure_reason": result.failure_reason, + "retry_count": result.retry_count, + }, + } + + async def _prepare_notification_send( + self, + notification_id: int, + operator_id: str, + final_content: str | None, + context: RequestContext, + ) -> tuple[ + OffsiteNotification, + OffsiteFundMail, + OffsiteFundAttachment, + str, + ] | dict[str, object]: + settings = get_settings() + async with self.session.begin(): + notice = await self.session.scalar(select(OffsiteNotification).where( + OffsiteNotification.id == notification_id + ).with_for_update()) + if notice is None: + return {"code": 404, "message": "通知不存在", "data": {}} + if notice.status == "发送成功": + return { + "code": 0, + "message": "ok", + "data": { + "notification_id": str(notification_id), + "status": notice.status, + "provider_message_id": notice.provider_message_id, + }, + } + if notice.status == "发送中": + return {"code": 409, "message": "通知正在发送中", "data": {}} + if notice.retry_count > 0 and notice.retry_count >= settings.offsite_max_retry_count: + return {"code": 422, "message": "通知已达到最大重试次数", "data": {}} + document = await self.session.scalar(select(OffsiteFundDocument).where( + OffsiteFundDocument.task_id == notice.business_key)) + if document is None: + return {"code": 404, "message": "通知关联的单据或邮件不存在", "data": {}} + mail = await self.session.scalar(select(OffsiteFundMail).where( + OffsiteFundMail.mail_id == document.mail_id)) + if mail is None: + return {"code": 404, "message": "通知关联的单据或邮件不存在", "data": {}} + if notice.notification_type == "mail_return": + receiver = settings.offsite_mail_return_receiver + else: + receiver = notice.receiver_id if "@" in notice.receiver_id else "" + if not receiver: + return {"code": 422, "message": "通知对象未配置可发送的邮箱地址", "data": {}} + attachment = await self.session.scalar(select(OffsiteFundAttachment).where( + OffsiteFundAttachment.attachment_id == document.attachment_id)) + if attachment is None: + return {"code": 404, "message": "通知关联的原始附件不存在", "data": {}} + if final_content is not None and final_content != notice.final_content: + self._add_audit(context, "offsite.notification_content_changed", { + "notification_id": str(notification_id), + "operator_id": operator_id, + "changed": True, + }) + notice.final_content = final_content + notice.operator_id = operator_id + notice.status = "发送中" + return notice, mail, attachment, receiver + + def _build_mail_reply_request( + self, + notice: OffsiteNotification, + mail: OffsiteFundMail, + attachment: OffsiteFundAttachment, + receiver: str, + operator_id: str, + final_content: str | None, + ) -> OffsiteMailReplyRequest: + path = Path(attachment.original_file_path) + payload = path.read_bytes() + return OffsiteMailReplyRequest( + to_address=receiver, + subject="场外基金申购赎回处理结果", + body=final_content if final_content is not None else notice.final_content, + operator_id=operator_id, + operator_confirmed=True, + reply_to_message_id=mail.message_id, + attachments=( + SmtpAttachment( + filename=attachment.filename, + media_type=attachment.media_type, + payload=payload, + ), + ), + retry_count=notice.retry_count, + ) + + async def _finish_notification( + self, notification_id: int, result: SmtpSendResult, context: RequestContext + ) -> None: + now = datetime.now(UTC).replace(tzinfo=None) + async with self.session.begin(): + notice = await self.session.scalar(select(OffsiteNotification).where( + OffsiteNotification.id == notification_id + ).with_for_update()) + if notice is None: + return + notice.status = result.status + notice.provider_message_id = result.provider_message_id + notice.failure_reason = result.failure_reason + notice.retry_count = result.retry_count + notice.sent_at = now if result.status == "发送成功" else None + notice.updated_at = now + if result.status == "发送成功" and notice.notification_type == "mail_return": + document = await self.session.scalar(select(OffsiteFundDocument).where( + OffsiteFundDocument.task_id == notice.business_key)) + if document is not None: + mail = await self.session.scalar(select(OffsiteFundMail).where( + OffsiteFundMail.mail_id == document.mail_id).with_for_update()) + if mail is not None: + mail.status = "normal_return_sent" + self._add_audit(context, "offsite.notification_send_finished", { + "notification_id": str(notification_id), + "status": result.status, + "dry_run": result.dry_run, + "retry_count": result.retry_count, + }) + + async def _next_mail_id(self) -> str: + today = datetime.now(SHANGHAI).date() + prefix = today.strftime("%Y%m%d") + current = await self.session.scalar(select(func.count()).select_from(OffsiteFundMail).where( + OffsiteFundMail.received_date == today)) + return f"{prefix}-{int(current or 0) + 1:03d}" + + async def _create_document( + self, + mail_id: str, + task_id: str, + attachment_id: str, + item: RecognizedAttachment, + document_type: Literal["subscription", "redemption"], + now: datetime, + ) -> OffsiteDocumentSummary: + fields = item.extracted_fields + raw_date = fields.get("申请日期") + amount_yuan = normalize_amount_yuan(fields.get("申购金额"), fields.get("金额单位")) + redemption_shares = decimal_from(fields.get("赎回份额")) + status = self._recognition_status(item, document_type) + document = OffsiteFundDocument( + task_id=task_id, mail_id=mail_id, attachment_id=attachment_id, + document_type=document_type, fund_code=self._text(fields.get("基金代码")), + fund_name=self._text(fields.get("基金名称")), + account_identifier=self._text(fields.get("账户标识")), + investor_name=self._text(fields.get("投资者名称") or fields.get("客户标识")), + application_no=self._text(fields.get("申请编号")), + application_date=parse_application_date(raw_date), + raw_application_date=self._text(raw_date), agency=self._text(fields.get("代销机构")), + subscription_amount_yuan=amount_yuan, redemption_shares=redemption_shares, + status=status, created_at=now, updated_at=now, + ) + self.session.add(document) + self._append_plan_tasks(task_id, document_type, fields, now) + decisions = self.rules.check_subscription( + amount_yuan=amount_yuan, nav=decimal_from(fields.get("最新净值")), + total_fund_shares=decimal_from(fields.get("基金最新总份额")), + before_holding_shares=decimal_from(fields.get("申请前持有份额")), + ) if document_type == "subscription" else self.rules.check_redemption( + redemption_shares=redemption_shares, + total_fund_shares=decimal_from(fields.get("基金最新总份额")), + available_quantity=decimal_from(fields.get("当前最新可用份额")), + ) + results: dict[str, Literal["正常", "异常", "无法判断"]] = {} + for decision in decisions: + results[decision.rule_code] = decision.result + self.session.add(OffsiteRuleResult( + task_id=task_id, rule_code=decision.rule_code, rule_name=decision.rule_name, + result=decision.result, document_value=decision.document_value, + database_value=decision.database_value, calculation=decision.calculation, + created_at=now, + )) + return OffsiteDocumentSummary( + task_id=task_id, document_type=cast(DocumentType, document_type), status=status, + rule_results=results) + + def _append_plan_tasks( + self, task_id: str, document_type: str, fields: dict[str, object], now: datetime + ) -> None: + rules: tuple[tuple[str, str], ...] = ( + ("subscription_minimum_amount", "申购最低金额"), + ("subscription_holding_ratio", "申购后单一投资者持有比例"), + ("subscription_single_share_limit", "申购单笔份额上限"), + ) + if document_type == "redemption": + rules = (("redemption_large_ratio", "赎回巨额比例"), + ("redemption_available_quantity", "账户可用份额")) + for rule_code, title in rules: + for stage in ("查询", "计算", "核对"): + if rule_code == "subscription_minimum_amount" and stage == "查询": + status = "不适用" + else: + status = "已完成" if stage != "查询" else "待执行" + self.session.add(OffsiteExecutionPlanTask( + task_id=task_id, stage=stage, rule_code=rule_code, title=title, + depends_on=[], input_json=fields, output_json=None, + status=status, + error_message=None, created_at=now, updated_at=now, + )) + + @staticmethod + def _nl2sql_questions(document: OffsiteFundDocument) -> list[tuple[str, str]]: + if not document.fund_code: + return [] + base = f"基金代码为{document.fund_code}" + if document.account_identifier: + base += f"，账户标识为{document.account_identifier}" + if document.document_type == "subscription": + return [ + ( + "subscription_holding_ratio", + base + "，查询基金最新总份额、最新净值和申请前持有份额", + ), + ("subscription_single_share_limit", base + "，查询基金最新总份额和最新净值"), + ] + return [ + ("redemption_large_ratio", base + "，查询产品最新总份额"), + ("redemption_available_quantity", base + "，查询账户当前最新可用份额"), + ] + + @staticmethod + def _text(value: object) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None + + async def _update_query_plan_status( + self, task_id: str, rule_code: str, result: dict[str, object], now: datetime + ) -> None: + task = await self.session.scalar(select(OffsiteExecutionPlanTask).where( + OffsiteExecutionPlanTask.task_id == task_id, + OffsiteExecutionPlanTask.rule_code == rule_code, + OffsiteExecutionPlanTask.stage == "查询", + ).with_for_update()) + if task is None: + return + status = str(result.get("status", "error")) + task.status = "已完成" if status in {"ready", "success"} else "查询失败" + if status == "need_confirmation": + task.status = "无法判断" + task.output_json = result + message = result.get("message") + task.error_message = message if isinstance(message, str) else None + task.updated_at = now + + @staticmethod + def _permission_error( + context: RequestContext, permissions: tuple[str, ...] + ) -> dict[str, object] | None: + roles = {"operator", "risk_operator", "admin", "super_admin"} + if not roles.intersection(context.roles): + return {"code": 403, "message": "当前角色不能操作场外基金流程", "data": {}} + if not set(permissions).intersection(context.permissions): + return {"code": 403, "message": "缺少场外基金操作权限", "data": {}} + return None + + def _add_audit( + self, context: RequestContext, action_type: str, detail: dict[str, object] + ) -> None: + self.session.add(InteractionAudit( + actor_type="user", + actor_id=int(context.user_id) if context.user_id.isdigit() else None, + target_customer_id=None, + session_id=None, + portal=context.portal, + action_type=action_type, + detail={**detail, "trace_id": context.trace_id}, + created_at=datetime.now(UTC).replace(tzinfo=None), + )) + + @staticmethod + def _recognition_status( + item: RecognizedAttachment, document_type: Literal["subscription", "redemption"] + ) -> str: + fields = item.extracted_fields + required = ["基金代码", "基金名称", "账户标识", "申请编号", "申请日期", "代销机构"] + if not (fields.get("投资者名称") or fields.get("客户标识")): + return "recognition_exception" + if document_type == "subscription": + required.extend(["申购金额", "金额单位"]) + else: + required.append("赎回份额") + if any(not str(fields.get(name) or "").strip() for name in required): + return "recognition_exception" + if not item.field_confidence: + return "recognition_review" + confidence_values = [ + value for value in (decimal_from(value) for value in item.field_confidence.values()) + if value is not None + ] + if not confidence_values: + return "recognition_review" + min_confidence = min(confidence_values) + if min_confidence < Decimal("0.80"): + return "recognition_exception" + if min_confidence < Decimal("0.95"): + return "recognition_review" + return "planned" + + @staticmethod + def _agency_breakdown(rows: Sequence[OffsiteFundDocument]) -> list[dict[str, object]]: + grouped: dict[str, dict[str, object]] = {} + for row in rows: + agency = row.agency or "未识别代销机构" + item = grouped.setdefault(agency, { + "agency": agency, + "subscription_amount_yuan": Decimal("0"), + "subscription_count": 0, + "redemption_shares": Decimal("0"), + "redemption_count": 0, + }) + if row.document_type == "subscription": + item["subscription_amount_yuan"] = cast( + Decimal, item["subscription_amount_yuan"] + ) + (row.subscription_amount_yuan or Decimal("0")) + subscription_count = item["subscription_count"] + item["subscription_count"] = ( + subscription_count + 1 if isinstance(subscription_count, int) else 1 + ) + elif row.document_type == "redemption": + item["redemption_shares"] = cast( + Decimal, item["redemption_shares"] + ) + (row.redemption_shares or Decimal("0")) + redemption_count = item["redemption_count"] + item["redemption_count"] = ( + redemption_count + 1 if isinstance(redemption_count, int) else 1 + ) + return [ + { + **item, + "subscription_amount_yuan": str(item["subscription_amount_yuan"]), + "redemption_shares": str(item["redemption_shares"]), + } + for item in grouped.values() + ] + + @staticmethod + def _validate_notification( + document: OffsiteFundDocument, notification_type: str + ) -> dict[str, object] | None: + if notification_type == "risk" and document.operator_decision != "确认异常": + return {"code": 422, "message": "风控通知只允许发送已确认异常单据", "data": {}} + if notification_type == "settlement" and document.operator_decision != "确认正常": + return {"code": 422, "message": "资金清算通知只允许发送已确认正常单据", "data": {}} + if notification_type == "mail_return" and document.operator_decision == "未处理": + return {"code": 422, "message": "邮件返回前必须先完成人工确认", "data": {}} + return None + + @staticmethod + def _notification_payload( + document: OffsiteFundDocument, + rule_results: Sequence[OffsiteRuleResult], + operator_id: str, + ) -> dict[str, object]: + anomalies = [ + { + "rule_code": result.rule_code, + "rule_name": result.rule_name, + "result": result.result, + "document_value": result.document_value, + "database_value": result.database_value, + "calculation": result.calculation, + } + for result in rule_results if result.result == "异常" + ] + return { + "task_id": document.task_id, + "mail_id": document.mail_id, + "attachment_id": document.attachment_id, + "document_type": document.document_type, + "fund_code": document.fund_code, + "fund_name": document.fund_name, + "account_identifier": OffsiteFundService._mask_account(document.account_identifier), + "application_no": document.application_no, + "application_date": ( + document.application_date.isoformat() if document.application_date else None + ), + "agency": document.agency, + "operator_id": operator_id, + "operator_decision": document.operator_decision, + "confirmed_at": datetime.now(UTC).replace(tzinfo=None).isoformat(timespec="seconds"), + "anomalies": anomalies, + } + + @staticmethod + def _mask_account(value: str | None) -> str | None: + if value is None or len(value) <= 4: + return value + return f"{value[:2]}***{value[-2:]}" diff --git a/app/service/offsite_mail_adapter.py b/app/service/offsite_mail_adapter.py new file mode 100644 index 0000000..32a3052 --- /dev/null +++ b/app/service/offsite_mail_adapter.py @@ -0,0 +1,269 @@ +"""场外基金真实邮件接收和原始文件保存适配层。""" + +from __future__ import annotations + +import hashlib +import imaplib +import os +import re +from contextlib import suppress +from dataclasses import dataclass +from datetime import UTC, datetime +from email import message_from_bytes +from email.message import EmailMessage, Message +from email.utils import parseaddr +from pathlib import Path +from typing import Protocol, cast + +from app.core.config import Settings + + +@dataclass(frozen=True) +class RawMailAttachment: + filename: str + media_type: str + payload: bytes + + +@dataclass(frozen=True) +class RawMailMessage: + imap_uid: str + message_id: str + sender: str + return_path: str | None + auth_result: dict[str, object] + raw_message: bytes + received_at: datetime + attachments: tuple[RawMailAttachment, ...] + + +@dataclass(frozen=True) +class SavedMailAttachment: + filename: str + file_hash: str + media_type: str + size_bytes: int + original_file_path: str + + +@dataclass(frozen=True) +class SavedMailMessage: + imap_uid: str + message_id: str + sender: str + return_path: str | None + auth_result: dict[str, object] + eml_path: str + attachments: tuple[SavedMailAttachment, ...] + + +class ImapConnection(Protocol): + def login(self, user: str, password: str) -> tuple[str, list[bytes]]: ... + + def select(self, mailbox: str, readonly: bool = False) -> tuple[str, list[bytes]]: ... + + def uid(self, command: str, *args: object) -> tuple[str, list[object]]: ... + + def noop(self) -> tuple[str, list[bytes]]: ... + + def logout(self) -> tuple[str, list[bytes]]: ... + + +class OffsiteMailStorage: + """保存原始 eml 和附件，避免覆盖已有原始文件。""" + + def __init__(self, root_dir: str | Path) -> None: + self.root_dir = Path(root_dir) + + def save(self, mail: RawMailMessage) -> SavedMailMessage: + folder = self._folder_for(mail) + folder.mkdir(parents=True, exist_ok=True) + eml_path = folder / "message.eml" + self._write_once(eml_path, mail.raw_message) + saved = [] + for index, attachment in enumerate(mail.attachments, start=1): + file_hash = hashlib.sha256(attachment.payload).hexdigest() + filename = f"A{index:02d}_{file_hash[:12]}_{self._safe_name(attachment.filename)}" + path = folder / filename + self._write_once(path, attachment.payload) + saved.append(SavedMailAttachment( + filename=attachment.filename, + file_hash=file_hash, + media_type=attachment.media_type, + size_bytes=len(attachment.payload), + original_file_path=str(path), + )) + return SavedMailMessage( + imap_uid=mail.imap_uid, + message_id=mail.message_id, + sender=mail.sender, + return_path=mail.return_path, + auth_result=mail.auth_result, + eml_path=str(eml_path), + attachments=tuple(saved), + ) + + def _folder_for(self, mail: RawMailMessage) -> Path: + day = mail.received_at.strftime("%Y%m%d") + source = mail.message_id or mail.imap_uid + return self.root_dir / day / self._safe_name(source.strip("<>") or mail.imap_uid) + + @staticmethod + def _safe_name(value: str) -> str: + safe = re.sub(r"[^0-9A-Za-z._-]+", "_", value.strip()) + return safe[:120] or "unknown" + + @staticmethod + def _write_once(path: Path, payload: bytes) -> None: + if path.exists(): + return + path.write_bytes(payload) + with suppress(OSError): + os.chmod(path, 0o444) + + +class OffsiteImapReceiver: + """只监听收件箱，按 UID 增量拉取原始邮件。""" + + inbox_name = "INBOX" + + def __init__(self, settings: Settings, connection: ImapConnection | None = None) -> None: + self.settings = settings + self._connection = connection + self.last_scanned_uid: str | None = None + + def health_check(self) -> dict[str, object]: + if not self.settings.offsite_imap_enabled: + return {"status": "disabled", "message": "场外 IMAP 未启用"} + missing = self._missing_config() + if missing: + return {"status": "misconfigured", "missing": missing} + try: + connection = self._ensure_connection() + connection.noop() + return {"status": "ok", "mailbox": self.settings.offsite_mailbox} + except Exception as exc: + self.close() + return {"status": "error", "message": type(exc).__name__} + + def fetch_since(self, last_uid: str | None, *, limit: int = 20) -> tuple[RawMailMessage, ...]: + if not self.settings.offsite_imap_enabled: + return () + self.last_scanned_uid = None + missing = self._missing_config() + if missing: + raise RuntimeError(f"场外IMAP配置缺失：{', '.join(missing)}") + connection = self._ensure_connection() + connection.select(self.inbox_name, readonly=True) + start_uid = int(last_uid) + 1 if last_uid and last_uid.isdigit() else 1 + status, data = connection.uid("search", None, f"UID {start_uid}:*") + search_payload = self._first_bytes(data) + if status != "OK" or not search_payload: + return () + uid_values = search_payload.split()[:limit] + if uid_values: + self.last_scanned_uid = uid_values[-1].decode("ascii") + messages: list[RawMailMessage] = [] + for uid in uid_values: + item = self._fetch_one(connection, uid.decode("ascii")) + if item and item.sender in self.settings.offsite_allowed_senders: + messages.append(item) + return tuple(messages) + + def close(self) -> None: + if self._connection is None: + return + try: + self._connection.logout() + finally: + self._connection = None + + def _ensure_connection(self) -> ImapConnection: + if self._connection is not None: + return self._connection + client_cls = imaplib.IMAP4_SSL if self.settings.offsite_imap_use_ssl else imaplib.IMAP4 + connection = cast( + ImapConnection, + client_cls(self.settings.offsite_imap_host, self.settings.offsite_imap_port), + ) + connection.login(self.settings.offsite_imap_username, self.settings.offsite_imap_password) + self._connection = connection + return connection + + def _fetch_one(self, connection: ImapConnection, uid: str) -> RawMailMessage | None: + status, data = connection.uid("fetch", uid, "(RFC822)") + if status != "OK" or not data: + return None + raw = self._extract_rfc822(data) + if raw is None: + return None + parsed = message_from_bytes(raw) + message = cast(EmailMessage, parsed) + sender = parseaddr(message.get("From", ""))[1] + return_path = parseaddr(message.get("Return-Path", ""))[1] or None + message_id = message.get("Message-ID", f"") + return RawMailMessage( + imap_uid=uid, + message_id=message_id, + sender=sender, + return_path=return_path, + auth_result=self._auth_result(message), + raw_message=raw, + received_at=datetime.now(UTC).replace(tzinfo=None), + attachments=self._attachments(message), + ) + + @staticmethod + def _first_bytes(data: list[object]) -> bytes | None: + for item in data: + if isinstance(item, bytes): + return item + return None + + @staticmethod + def _extract_rfc822(data: list[object]) -> bytes | None: + for item in data: + if isinstance(item, tuple): + payload = item[1] + if isinstance(payload, bytes): + return payload + if isinstance(item, bytes) and item.startswith(b"From:"): + return item + return None + + @staticmethod + def _attachments(message: Message) -> tuple[RawMailAttachment, ...]: + attachments: list[RawMailAttachment] = [] + for part in message.walk(): + if part.is_multipart(): + continue + filename = part.get_filename() + disposition = part.get_content_disposition() + if not filename and disposition != "attachment": + continue + raw_payload = part.get_payload(decode=True) + payload = raw_payload if isinstance(raw_payload, bytes) else b"" + attachments.append(RawMailAttachment( + filename=filename or "attachment.bin", + media_type=part.get_content_type(), + payload=payload, + )) + return tuple(attachments) + + @staticmethod + def _auth_result(message: Message) -> dict[str, object]: + return { + "authentication_results": message.get_all("Authentication-Results", []), + "received_spf": message.get_all("Received-SPF", []), + "dkim_signature_present": bool(message.get("DKIM-Signature")), + } + + def _missing_config(self) -> list[str]: + missing = [] + if not self.settings.offsite_imap_host: + missing.append("OFFSITE_IMAP_HOST") + if not self.settings.offsite_imap_username: + missing.append("OFFSITE_IMAP_USERNAME") + if not self.settings.offsite_imap_password: + missing.append("OFFSITE_IMAP_PASSWORD") + return missing diff --git a/app/service/offsite_nl2sql_adapter.py b/app/service/offsite_nl2sql_adapter.py new file mode 100644 index 0000000..c177209 --- /dev/null +++ b/app/service/offsite_nl2sql_adapter.py @@ -0,0 +1,45 @@ +"""场外基金专用 NL2SQL 适配器。 + +该适配器只隔离调用边界，不改写 `nl2sql_yc.query_dict` 的业务能力。 +""" + +from importlib import import_module +from typing import cast + +from app.core.contracts import RequestContext + + +class OffsiteNl2SqlAdapter: + """把场外基金上下文转换为 `nl2sql_yc.query_dict` 可接受的稳定参数。""" + + script_path = "nl2sql_yc.py" + + def query(self, question: str, context: RequestContext) -> dict[str, object]: + try: + query_dict = getattr(import_module("nl2sql_yc"), "query_dict", None) + if not callable(query_dict): + return {"status": "error", "message": "NL2SQL入口query_dict不存在"} + result = cast(dict[str, object], query_dict( + question, + self._auth_context(context), + use_llm=False, + persist_audit=False, + )) + if isinstance(result, dict): + return result + return {"status": "error", "message": "NL2SQL返回格式不正确"} + except Exception as exc: + return {"status": "error", "message": f"NL2SQL调用失败：{type(exc).__name__}"} + + @staticmethod + def _auth_context(context: RequestContext) -> dict[str, object]: + return { + "user_id": int(context.user_id) if context.user_id.isdigit() else None, + "roles": list(context.roles), + "allowed_domains": ["market_nav", "trading_account", "product_fee"], + "customer_scope": context.data_scope if context.data_scope in { + "self", "own_customers", "all", + } else "all", + "max_rows": 100, + "max_query_seconds": 10, + } diff --git a/app/service/offsite_smtp_adapter.py b/app/service/offsite_smtp_adapter.py new file mode 100644 index 0000000..324dcda --- /dev/null +++ b/app/service/offsite_smtp_adapter.py @@ -0,0 +1,205 @@ +"""场外基金邮件回复 SMTP 适配层。""" + +from __future__ import annotations + +import mimetypes +import smtplib +from dataclasses import dataclass +from email.message import EmailMessage +from email.utils import formatdate, make_msgid +from pathlib import Path +from typing import Protocol, cast + +from app.core.config import Settings +from app.core.offsite_fund_contracts import SendStatus + + +@dataclass(frozen=True) +class SmtpAttachment: + filename: str + media_type: str + payload: bytes + + +@dataclass(frozen=True) +class OffsiteMailReplyRequest: + to_address: str + subject: str + body: str + operator_id: str + operator_confirmed: bool + reply_to_message_id: str | None = None + attachments: tuple[SmtpAttachment, ...] = () + retry_count: int = 0 + + +@dataclass(frozen=True) +class SmtpSendResult: + status: SendStatus + dry_run: bool + provider_message_id: str | None + failure_reason: str | None + retry_count: int + request_summary: dict[str, object] + + +class SmtpConnection(Protocol): + def login(self, user: str, password: str) -> object: ... + + def send_message(self, msg: EmailMessage) -> object: ... + + def quit(self) -> object: ... + + +class OffsiteSmtpSender: + """默认 dry-run，真实发送必须显式启用并经过运营确认。""" + + def __init__(self, settings: Settings, connection: SmtpConnection | None = None) -> None: + self.settings = settings + self._connection = connection + + def health_check(self) -> dict[str, object]: + if not self.settings.offsite_smtp_enabled: + return {"status": "disabled", "message": "场外 SMTP 未启用"} + if self.settings.offsite_smtp_dry_run: + return {"status": "dry_run", "message": "场外 SMTP 处于 dry-run 模式"} + missing = self._missing_config() + if missing: + return {"status": "misconfigured", "missing": missing} + return { + "status": "ready", + "host": self.settings.offsite_smtp_host, + "sender": self.settings.offsite_smtp_sender, + } + + def send_reply(self, request: OffsiteMailReplyRequest) -> SmtpSendResult: + message_id = make_msgid(domain="offsite-fund.local") + summary = self._summary(request) + if not request.operator_confirmed: + return SmtpSendResult( + status="发送失败", + dry_run=self.settings.offsite_smtp_dry_run, + provider_message_id=None, + failure_reason="邮件发送前必须完成运营确认", + retry_count=request.retry_count, + request_summary=summary, + ) + if not self.settings.offsite_smtp_enabled or self.settings.offsite_smtp_dry_run: + return SmtpSendResult( + status="待发送", + dry_run=True, + provider_message_id=None, + failure_reason=None, + retry_count=request.retry_count, + request_summary=summary, + ) + missing = self._missing_config() + if missing: + return SmtpSendResult( + status="发送失败", + dry_run=False, + provider_message_id=None, + failure_reason=f"SMTP配置缺失：{', '.join(missing)}", + retry_count=request.retry_count + 1, + request_summary=summary, + ) + try: + message = self._build_message(request, message_id) + connection = self._ensure_connection() + connection.send_message(message) + return SmtpSendResult( + status="发送成功", + dry_run=False, + provider_message_id=message_id, + failure_reason=None, + retry_count=request.retry_count, + request_summary=summary, + ) + except (OSError, smtplib.SMTPException) as exc: + self.close() + return SmtpSendResult( + status="发送失败", + dry_run=False, + provider_message_id=None, + failure_reason=type(exc).__name__, + retry_count=request.retry_count + 1, + request_summary=summary, + ) + + def close(self) -> None: + if self._connection is None: + return + try: + self._connection.quit() + finally: + self._connection = None + + def _ensure_connection(self) -> SmtpConnection: + if self._connection is not None: + return self._connection + smtp_cls = smtplib.SMTP_SSL if self.settings.offsite_smtp_use_ssl else smtplib.SMTP + connection = cast( + SmtpConnection, + smtp_cls( + self.settings.offsite_smtp_host, + self.settings.offsite_smtp_port, + timeout=self.settings.offsite_smtp_timeout_seconds, + ), + ) + connection.login(self.settings.offsite_smtp_username, self.settings.offsite_smtp_password) + self._connection = connection + return connection + + def _build_message(self, request: OffsiteMailReplyRequest, message_id: str) -> EmailMessage: + message = EmailMessage() + message["From"] = self.settings.offsite_smtp_sender + message["To"] = request.to_address + message["Subject"] = request.subject + message["Date"] = formatdate(localtime=True) + message["Message-ID"] = message_id + if request.reply_to_message_id: + message["In-Reply-To"] = request.reply_to_message_id + message["References"] = request.reply_to_message_id + message.set_content(request.body) + for attachment in request.attachments: + maintype, subtype = self._media_type(attachment) + message.add_attachment( + attachment.payload, + maintype=maintype, + subtype=subtype, + filename=attachment.filename, + ) + return message + + def _missing_config(self) -> list[str]: + missing = [] + if not self.settings.offsite_smtp_host: + missing.append("OFFSITE_SMTP_HOST") + if not self.settings.offsite_smtp_username: + missing.append("OFFSITE_SMTP_USERNAME") + if not self.settings.offsite_smtp_password: + missing.append("OFFSITE_SMTP_PASSWORD") + if not self.settings.offsite_smtp_sender: + missing.append("OFFSITE_SMTP_SENDER") + return missing + + @staticmethod + def _media_type(attachment: SmtpAttachment) -> tuple[str, str]: + media_type = attachment.media_type + if not media_type or "/" not in media_type: + guessed, _ = mimetypes.guess_type(attachment.filename) + media_type = guessed or "application/octet-stream" + maintype, subtype = media_type.split("/", 1) + return maintype, subtype + + @staticmethod + def _summary(request: OffsiteMailReplyRequest) -> dict[str, object]: + return { + "to_address": request.to_address, + "subject": request.subject, + "operator_id": request.operator_id, + "operator_confirmed": request.operator_confirmed, + "reply_to_message_id": request.reply_to_message_id, + "attachment_count": len(request.attachments), + "attachment_names": [Path(item.filename).name for item in request.attachments], + } diff --git a/app/service/tool_executor.py b/app/service/tool_executor.py index 847b5e9..2b827f5 100644 --- a/app/service/tool_executor.py +++ b/app/service/tool_executor.py @@ -88,13 +88,27 @@ class ToolExecutor: record = ToolCallRecord( tool_name=name, status="succeeded", input_summary={key: "[redacted]" for key in arguments}, - output_summary={"result_type": type(output).__name__}, + output_summary=self._output_summary(output), ) await self._audit(name, intent, context, "succeeded", "ok") reference = SourceReference(source_type="tool", source_id=f"{context.trace_id}:{name}", title=name) return ToolExecution(output=output, record=record, references=(reference,)) + @staticmethod + def _output_summary(output: Any) -> dict[str, Any]: + summary: dict[str, Any] = {"result_type": type(output).__name__} + if not isinstance(output, dict): + return summary + summary["status"] = output.get("status") + audit = output.get("audit") + if isinstance(audit, dict): + summary["query_plan"] = audit.get("query_plan") + summary["generated_sql"] = audit.get("generated_sql") + summary["permission_check"] = audit.get("permission_check") + summary["execution"] = audit.get("execution") + return summary + async def _audit( self, name: str, intent: str, context: RequestContext, status: str, reason: str ) -> None: diff --git a/app/worker/__main__.py b/app/worker/__main__.py index ec8efd7..56c001a 100644 --- a/app/worker/__main__.py +++ b/app/worker/__main__.py @@ -4,20 +4,24 @@ import logging from app.core.config import get_settings from app.infrastructure.db import engine +from app.worker.offsite_mail_worker import OffsiteMailWorker from app.worker.runtime import WorkerRuntime async def serve(*, once: bool = False) -> None: settings = get_settings() runtime = WorkerRuntime(settings=settings) + offsite_worker = OffsiteMailWorker(settings) try: while True: worked = await runtime.run_once() + worked = await offsite_worker.run_once() or worked if once: return if not worked: await asyncio.sleep(settings.worker_poll_seconds) finally: + await offsite_worker.close() await engine.dispose() diff --git a/app/worker/offsite_mail_worker.py b/app/worker/offsite_mail_worker.py new file mode 100644 index 0000000..7828d18 --- /dev/null +++ b/app/worker/offsite_mail_worker.py @@ -0,0 +1,458 @@ +"""场外基金收件箱独立 Worker。""" + +from __future__ import annotations + +import asyncio +import logging +from collections.abc import Awaitable, Callable +from contextlib import suppress +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Protocol +from uuid import uuid4 + +from sqlalchemy import func, select, update +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.config import Settings +from app.core.contracts import RequestContext +from app.core.offsite_fund_contracts import ( + ReceiveRecognizedMailRequest, + RecognizedAttachment, +) +from app.infrastructure.db import SessionFactory +from app.model.audit import InteractionAudit +from app.model.offsite_fund import OffsiteFundMail, OffsiteMailCursor, OffsiteNotification +from app.service.offsite_document_recognition_adapter import ( + OffsiteDocumentRecognitionAdapter, + RecognitionSourceFile, + StructuredRecognitionResult, +) +from app.service.offsite_fund_service import OffsiteFundService +from app.service.offsite_mail_adapter import ( + OffsiteImapReceiver, + OffsiteMailStorage, + RawMailMessage, + SavedMailAttachment, +) + +logger = logging.getLogger(__name__) + + +class MailReceiver(Protocol): + last_scanned_uid: str | None + + def health_check(self) -> dict[str, object]: ... + + def fetch_since(self, last_uid: str | None, *, limit: int) -> tuple[RawMailMessage, ...]: ... + + def close(self) -> None: ... + + +class MailRecognizer(Protocol): + async def recognize(self, source: RecognitionSourceFile) -> StructuredRecognitionResult: ... + + +class MailService(Protocol): + async def receive_recognized_mail( + self, payload: ReceiveRecognizedMailRequest, context: RequestContext + ) -> dict[str, object]: ... + + +class CursorLease: + def __init__(self, lease_id: str, last_uid: str) -> None: + self.lease_id = lease_id + self.last_uid = last_uid + + +class OffsiteMailWorker: + """按 UID 顺序处理邮件，失败时停在当前 UID 并等待补偿。""" + + def __init__( + self, + settings: Settings, + *, + receiver: MailReceiver | None = None, + storage: OffsiteMailStorage | None = None, + recognizer: MailRecognizer | None = None, + identity_resolver: Callable[[RequestContext], Awaitable[RequestContext]] | None = None, + session_factory: Callable[[], AsyncSession] = SessionFactory, + service_factory: Callable[[AsyncSession], MailService] | None = None, + ) -> None: + self.settings = settings + self.receiver = receiver or OffsiteImapReceiver(settings) + self.storage = storage or OffsiteMailStorage(settings.offsite_mail_storage_dir) + self.recognizer = recognizer or OffsiteDocumentRecognitionAdapter(settings) + self.identity_resolver = identity_resolver + self.session_factory = session_factory + self.service_factory = service_factory or OffsiteFundService + + async def run_once(self) -> bool: + if not self.settings.offsite_mail_worker_enabled: + return False + recovered = await self.recover_stale_notifications() + if not self.settings.offsite_imap_enabled: + return recovered + if not self.settings.offsite_worker_user_id: + logger.error("场外 Worker 未配置合法操作用户 ID，拒绝自动写入业务数据") + return recovered + try: + context = await self._resolve_worker_context() + lease = await self._claim_cursor() + if lease is None: + return recovered + return await self._process_batch(lease, context) or recovered + except asyncio.CancelledError: + raise + except Exception: + logger.exception("场外邮件 Worker 执行失败") + return True + + async def close(self) -> None: + self.receiver.close() + close = getattr(self.recognizer, "close", None) + if callable(close): + result = close() + if asyncio.iscoroutine(result): + await result + + async def recover_stale_notifications(self) -> bool: + now = datetime.now(UTC).replace(tzinfo=None) + cutoff = now - timedelta( + seconds=self.settings.offsite_notification_sending_timeout_seconds + ) + async with self.session_factory() as session, session.begin(): + activity = func.coalesce( + OffsiteNotification.updated_at, OffsiteNotification.created_at + ) + result = await session.execute( + update(OffsiteNotification) + .where( + OffsiteNotification.status == "发送中", + activity < cutoff, + ) + .values( + status="发送失败", + failure_reason="发送状态不确定，需人工核验外部邮箱后再决定是否重试", + updated_at=now, + ) + ) + count = int(getattr(result, "rowcount", 0) or 0) + if count: + session.add( + InteractionAudit( + actor_type="system", + actor_id=None, + target_customer_id=None, + session_id=None, + portal="worker", + action_type="offsite.notification_timeout_recovered", + detail={"count": count}, + created_at=now, + ) + ) + return count > 0 + + async def _resolve_worker_context(self) -> RequestContext: + identity = RequestContext( + user_id=self.settings.offsite_worker_user_id, + trace_id=f"offsite-worker-{uuid4()}", + portal="worker", + ) + resolver = self.identity_resolver + if resolver is None: + from app.service.identity_service import IdentityService + + resolver = IdentityService().resolve + context = await resolver(identity) + if not context.roles or not context.permissions: + raise RuntimeError("场外 Worker 操作用户未通过角色和权限校验") + return context + + async def _claim_cursor(self) -> CursorLease | None: + for attempt in range(2): + lease_id = str(uuid4()) + now = datetime.now(UTC).replace(tzinfo=None) + try: + async with self.session_factory() as session, session.begin(): + cursor = await session.scalar( + select(OffsiteMailCursor) + .where( + OffsiteMailCursor.mailbox == self.settings.offsite_mailbox, + OffsiteMailCursor.folder == OffsiteImapReceiver.inbox_name, + ) + .with_for_update() + ) + if cursor is None: + cursor = OffsiteMailCursor( + mailbox=self.settings.offsite_mailbox, + folder=OffsiteImapReceiver.inbox_name, + last_uid="0", + status="idle", + retry_count=0, + created_at=now, + updated_at=now, + ) + session.add(cursor) + await session.flush() + if cursor.status == "blocked": + return None + if cursor.lease_until is not None and cursor.lease_until > now: + return None + if cursor.next_retry_at is not None and cursor.next_retry_at > now: + return None + cursor.status = "processing" + cursor.lease_id = lease_id + cursor.lease_until = now + timedelta( + seconds=self.settings.worker_lease_seconds + ) + cursor.updated_at = now + return CursorLease(lease_id, cursor.last_uid) + except IntegrityError: + if attempt == 1: + raise + return None + + async def _process_batch(self, lease: CursorLease, context: RequestContext) -> bool: + task = asyncio.current_task() + if task is None: + raise RuntimeError("场外 Worker 无法建立当前任务租约") + heartbeat = asyncio.create_task(self._cursor_heartbeat(lease.lease_id, task)) + try: + return await self._process_batch_work(lease, context) + finally: + heartbeat.cancel() + with suppress(asyncio.CancelledError): + await heartbeat + + async def _process_batch_work(self, lease: CursorLease, context: RequestContext) -> bool: + try: + health = await asyncio.to_thread(self.receiver.health_check) + if health.get("status") != "ok": + message = str(health.get("message") or health.get("status") or "IMAP不可用") + raise RuntimeError(f"场外 IMAP 健康检查失败：{message}") + messages = await asyncio.to_thread( + self.receiver.fetch_since, + lease.last_uid, + limit=self.settings.offsite_mail_worker_batch_size, + ) + current_uid = lease.last_uid + for mail in sorted(messages, key=self._uid_sort_key): + if not self._uid_after(mail.imap_uid, current_uid): + continue + try: + await self._process_message(mail, context) + except asyncio.CancelledError: + raise + except Exception as exc: + await self._record_failure(lease, mail, exc) + return True + current_uid = mail.imap_uid + await self._advance_cursor(lease.lease_id, current_uid, release=False) + scanned_uid = self.receiver.last_scanned_uid + if scanned_uid is not None and self._uid_after(scanned_uid, current_uid): + current_uid = scanned_uid + await self._advance_cursor(lease.lease_id, current_uid, release=True) + return bool(messages) or scanned_uid is not None + except asyncio.CancelledError: + raise + except Exception as exc: + await self._record_cursor_failure(lease, exc) + self.receiver.close() + return True + + async def _cursor_heartbeat(self, lease_id: str, task: asyncio.Task[bool]) -> None: + try: + while True: + await asyncio.sleep(self.settings.worker_lease_seconds / 3) + now = datetime.now(UTC).replace(tzinfo=None) + async with self.session_factory() as session, session.begin(): + result = await session.execute( + update(OffsiteMailCursor) + .where(OffsiteMailCursor.lease_id == lease_id) + .values( + lease_until=now + timedelta( + seconds=self.settings.worker_lease_seconds + ), + updated_at=now, + ) + ) + if int(getattr(result, "rowcount", 0) or 0) != 1: + task.cancel() + return + except asyncio.CancelledError: + raise + except Exception: + logger.exception("场外 Worker 游标租约续期失败") + task.cancel() + + async def _process_message(self, mail: RawMailMessage, context: RequestContext) -> None: + saved = await asyncio.to_thread(self.storage.save, mail) + attachments: list[RecognizedAttachment] = [] + for saved_attachment in saved.attachments: + payload = await asyncio.to_thread(Path(saved_attachment.original_file_path).read_bytes) + recognition = await self.recognizer.recognize( + RecognitionSourceFile( + filename=saved_attachment.filename, + media_type=saved_attachment.media_type, + payload=payload, + file_hash=saved_attachment.file_hash, + original_file_path=saved_attachment.original_file_path, + ) + ) + if self._recognition_failed(recognition): + raise RuntimeError("附件识别失败，已保留原始文件并等待补偿") + attachments.append( + self._recognized_attachment(saved_attachment, recognition) + ) + request = ReceiveRecognizedMailRequest( + imap_uid=saved.imap_uid, + message_id=saved.message_id, + sender=saved.sender, + return_path=saved.return_path, + auth_result=saved.auth_result, + eml_path=saved.eml_path, + attachments=tuple(attachments), + ) + async with self.session_factory() as session: + service = self.service_factory(session) + result = await service.receive_recognized_mail(request, context) + if result.get("code") != 0: + raise RuntimeError(f"场外业务入库失败：{result.get('message', '未知错误')}") + + async def _advance_cursor( + self, lease_id: str, last_uid: str, *, release: bool + ) -> None: + now = datetime.now(UTC).replace(tzinfo=None) + async with self.session_factory() as session, session.begin(): + cursor = await session.scalar( + select(OffsiteMailCursor) + .where(OffsiteMailCursor.lease_id == lease_id) + .with_for_update() + ) + if cursor is None: + return + cursor.last_uid = last_uid + cursor.status = "idle" if release else "processing" + if release: + cursor.retry_count = 0 + cursor.last_error = None + cursor.blocked_uid = None + cursor.blocked_message_id = None + cursor.next_retry_at = None + cursor.lease_id = None + cursor.lease_until = None + cursor.updated_at = now + + async def _record_failure( + self, lease: CursorLease, mail: RawMailMessage, exc: Exception + ) -> None: + now = datetime.now(UTC).replace(tzinfo=None) + message = self._error_message(exc) + async with self.session_factory() as session, session.begin(): + cursor = await session.scalar( + select(OffsiteMailCursor) + .where(OffsiteMailCursor.lease_id == lease.lease_id) + .with_for_update() + ) + if cursor is None: + return + cursor.retry_count += 1 + cursor.status = ( + "blocked" + if cursor.retry_count >= self.settings.offsite_max_retry_count + else "failed" + ) + cursor.blocked_uid = mail.imap_uid + cursor.blocked_message_id = mail.message_id + cursor.last_error = message + cursor.next_retry_at = ( + None + if cursor.status == "blocked" + else now + timedelta(seconds=min(300, 2**cursor.retry_count)) + ) + cursor.lease_id = None + cursor.lease_until = None + cursor.updated_at = now + await session.execute( + update(OffsiteFundMail) + .where( + OffsiteFundMail.imap_uid == mail.imap_uid, + OffsiteFundMail.message_id == mail.message_id, + ) + .values( + retry_count=OffsiteFundMail.retry_count + 1, + last_error=message, + next_retry_at=cursor.next_retry_at, + last_attempt_at=now, + updated_at=now, + ) + ) + logger.error( + "场外邮件处理失败 uid=%s message_id=%s error=%s", + mail.imap_uid, + mail.message_id, + message, + ) + + async def _record_cursor_failure(self, lease: CursorLease, exc: Exception) -> None: + message = self._error_message(exc) + now = datetime.now(UTC).replace(tzinfo=None) + async with self.session_factory() as session, session.begin(): + cursor = await session.scalar( + select(OffsiteMailCursor) + .where(OffsiteMailCursor.lease_id == lease.lease_id) + .with_for_update() + ) + if cursor is None: + return + cursor.retry_count += 1 + cursor.status = "failed" + cursor.last_error = message + cursor.next_retry_at = now + timedelta( + seconds=min(300, 2**cursor.retry_count) + ) + cursor.lease_id = None + cursor.lease_until = None + cursor.updated_at = now + logger.error("场外邮件批处理失败 error=%s", message) + + @staticmethod + def _uid_sort_key(mail: RawMailMessage) -> tuple[int, str]: + return (int(mail.imap_uid) if mail.imap_uid.isdigit() else 2**63 - 1, mail.imap_uid) + + @staticmethod + def _uid_after(candidate: str, current: str) -> bool: + if candidate.isdigit() and current.isdigit(): + return int(candidate) > int(current) + return candidate > current + + @staticmethod + def _recognition_failed(result: object) -> bool: + error = getattr(result, "error_message", None) + ocr_status = getattr(result, "ocr_status", None) + llm_status = getattr(result, "llm_status", None) + failed = {"error", "misconfigured"} + return bool(error and (ocr_status in failed or llm_status in failed)) + + @staticmethod + def _recognized_attachment( + saved: SavedMailAttachment, result: StructuredRecognitionResult + ) -> RecognizedAttachment: + return RecognizedAttachment( + filename=saved.filename, + file_hash=saved.file_hash, + original_file_path=saved.original_file_path, + media_type=saved.media_type, + size_bytes=saved.size_bytes, + document_type=result.document_type, + extracted_fields=result.extracted_fields, + field_confidence=result.field_confidence, + ocr_text=result.ocr_text, + page_evidence=result.page_evidence, + ) + + @staticmethod + def _error_message(exc: Exception) -> str: + return f"{type(exc).__name__}: {str(exc)[:450]}" diff --git a/app/worker/runtime.py b/app/worker/runtime.py index 2636002..0ee99f4 100644 --- a/app/worker/runtime.py +++ b/app/worker/runtime.py @@ -9,7 +9,7 @@ from uuid import uuid4 from sqlalchemy import select, update from app.core.config import Settings, get_settings -from app.core.contracts import AgentRequest, AgentResult, RequestContext +from app.core.contracts import AgentRequest, AgentRequestMetadata, AgentResult, RequestContext from app.core.errors import AgentError, RecoverableAgentError, RunLeaseLostError from app.infrastructure.db import SessionFactory from app.model.audit import InteractionAudit @@ -127,10 +127,11 @@ class WorkerRuntime: DomainEventOutbox.event_type == "agent.run_requested").limit(1)) if message is None or idem is None: raise ValueError("run input missing") + metadata = event.payload.get("metadata", {}) if event else {} request = AgentRequest( agent_type=run.agent_type, message=message.content, session_id=run.session_id, idempotency_key=idem.idempotency_key, - metadata=event.payload.get("metadata", {}) if event else {}, + metadata=AgentRequestMetadata.model_validate(metadata), ) identity = RequestContext(user_id=str(run.user_id), trace_id=run.trace_id) # Re-check account and permissions at execution time, including delayed jobs. diff --git a/docs/15-金融NL2SQL工具接入说明.md b/docs/15-金融NL2SQL工具接入说明.md new file mode 100644 index 0000000..4b839a0 --- /dev/null +++ b/docs/15-金融NL2SQL工具接入说明.md @@ -0,0 +1,105 @@ +# 金融 NL2SQL 工具接入说明 + +## 1. 工具定位 + +`query_financial_data` 是运营和投顾共用的金融只读 NL2SQL 工具。业务 Agent 只负责判断是否调用工具，不直接连库、不读取密钥、不自行写审计。 + +调用链固定为： + +```text +业务 Agent -> BaseAgent.call_tool -> ToolExecutor -> query_financial_data -> 金融只读表 +``` + +## 2. 工具声明 + +业务 Agent 的 `AgentDefinition.allowed_tools` 需要包含： + +```python +allowed_tools=("query_financial_data",) +supported_intents=("financial_query", "general") +``` + +发布配置需要按意图开启工具白名单： + +```json +{ + "namespace": "agent_tools", + "config_key": ":financial_query", + "value_json": {"allowed_tools": ["query_financial_data"]} +} +``` + +工具权限码： + +```text +financial:nl2sql:read +``` + +允许角色： + +```text +advisor、operator、admin、super_admin +``` + +## 3. 调用示例 + +```python +result = await self.call_tool( + "query_financial_data", + {"question": request.message, "dry_run": False, "limit": 50}, + intent="financial_query", + context=context, +) +``` + +低置信度或模糊问题会返回： + +```json +{"status": "need_confirmation", "message": "请确认查询范围、指标口径和时间条件。"} +``` + +Agent 应把 `message` 原样返回给用户，等待用户补充或确认后再次调用，并传入 `confirmation`。 + +## 4. MVP 范围 + +当前纳入 NL2SQL 的表： + +```text +sys_customer_assignment +fin_customer_profile +fin_risk_assessment +fin_product +fin_fee_rule +fin_market_price +fin_nav_history +fin_holding +fin_transaction +fin_sim_order +fin_sim_account +fin_cash_ledger +client_facing_content +``` + +第一期只支持只读查询；支持跨两个或三个业务域查询；支持历史范围查询。历史时点查询如果依赖当前快照表，例如 `fin_holding`、`fin_sim_account`、`sys_customer_assignment`，会拒绝执行。 + +## 5. 审计与持久化 + +工具输出包含 `audit`，其中有： + +- `query_plan`：查询计划； +- `generated_sql`：生成 SQL； +- `permission_check`：权限校验结果； +- `execution`：执行状态和返回行数。 + +`ToolExecutor` 会把这些摘要写入 `ToolCallRecord.output_summary`，公共运行持久化会最终写入 `conversation_message.tool_calls`。 + +## 6. 接入验收 + +运营或投顾 Agent 接入时至少验证： + +- AgentDefinition 声明了 `query_financial_data`； +- 发布配置按目标意图放行该工具； +- 调用方角色具备 `financial:nl2sql:read`； +- 模糊问题会先要求确认； +- 生成 SQL 只包含 `SELECT`，且表名均在白名单内； +- `conversation_message.tool_calls` 能看到查询计划、SQL 和权限校验摘要。 diff --git a/docs/场外外部服务灰度与回滚方案.md b/docs/场外外部服务灰度与回滚方案.md new file mode 100644 index 0000000..af4d558 --- /dev/null +++ b/docs/场外外部服务灰度与回滚方案.md @@ -0,0 +1,82 @@ +# 场外基金外部服务灰度与回滚方案 + +## 1. 当前实现边界 + +当前代码已经提供 IMAP、OCR/DeepSeek 和 SMTP 适配层，但所有真实外部调用均由配置开关控制： + +- `OFFSITE_IMAP_ENABLED=false`：不连接收件邮箱。 +- `OFFSITE_MAIL_WORKER_ENABLED=false`：不启动场外邮件自动编排。 +- `OFFSITE_OCR_ENABLED=false`：OCR 使用本地 Mock。 +- `OFFSITE_DEEPSEEK_ENABLED=false`：字段识别使用本地 Mock。 +- `OFFSITE_SMTP_ENABLED=false`：不创建真实 SMTP 发送连接。 +- `OFFSITE_SMTP_DRY_RUN=true`：即使发送接口被调用，也只生成待发送结果，不标记发送成功。 + +通知创建接口只创建待发送记录。真正发送必须调用独立发送接口，并传递运营确认状态。 + +场外 Worker 另受以下安全条件约束： + +- `OFFSITE_MAIL_WORKER_ENABLED=true` 只是允许编排，不代表可以绕过身份校验。 +- `OFFSITE_WORKER_USER_ID` 必须是数据库中已存在、经过实时角色和权限解析的操作用户；为空或无权限时自动写入失败关闭。 +- Worker 使用 `offsite_mail_cursor` 持久化 INBOX UID、失败 UID、重试时间和租约；失败邮件不推进后续 UID。 +- 当前实现使用健康检查加轮询增量补偿；IMAP IDLE 仍需单独联调，不能把轮询版本描述为已完成 IDLE。 + +## 2. 上线前检查 + +上线前必须由技术和运营共同确认： + +1. 收件邮箱、授权码、发件人白名单和 SMTP 发件地址均使用密钥管理系统注入，不写入代码、日志或 Git。 +2. IMAP 账号只能访问指定收件箱，服务账号不使用登录密码，使用邮箱授权码。 +3. 原始 `.eml` 和附件目录有容量、权限、备份和保留期限方案。 +4. OCR 网关已确认请求格式、签名方式、超时和费用上限。 +5. DeepSeek 模型、配额、超时、结构化 JSON 格式和敏感数据处理策略已确认。 +6. SMTP 收件人、回复原邮件规则、附件重新附加规则已经过运营验收。 +7. 监控至少能发现 IMAP 连接失败、邮件解析失败、OCR/模型失败、SMTP 失败、重试耗尽和磁盘写入失败。 + +## 3. 灰度顺序 + +### 阶段一：离线验证 + +保持所有真实开关关闭，使用 Mock 邮件、OCR、DeepSeek 和 SMTP。验证重复邮件、白名单、附件哈希、识别异常、规则核对和审计。 + +### 阶段二：真实 IMAP、业务发送关闭 + +仅打开 `OFFSITE_IMAP_ENABLED=true` 和 `OFFSITE_MAIL_WORKER_ENABLED=true`，保持 OCR、DeepSeek 和 SMTP 关闭，并配置已核验的 `OFFSITE_WORKER_USER_ID`。先只做收件箱健康检查和增量扫描，确认 UID 不回退、断线后能补偿、非白名单邮件不进入业务队列；IMAP IDLE 未联调前按轮询模式验收。 + +### 阶段三：真实识别、发送保持 dry-run + +打开 `OFFSITE_OCR_ENABLED=true` 和 `OFFSITE_DEEPSEEK_ENABLED=true`，保持 `OFFSITE_SMTP_ENABLED=false` 或 `OFFSITE_SMTP_DRY_RUN=true`。使用已脱敏测试附件验证字段、置信度、页面证据、缺失字段和失败降级。 + +### 阶段四：单收件人真实 SMTP + +打开 `OFFSITE_SMTP_ENABLED=true`，保持 `OFFSITE_SMTP_DRY_RUN=false`，只允许运营选择的一封测试邮件和一个确认收件人。核对 `Message-ID`、`In-Reply-To`、附件内容、发送状态和 provider message ID。 + +### 阶段五：小流量业务灰度 + +先限定单个业务时段或有限业务邮件，持续观察一个完整业务周期。未完成发送、失败重试和人工确认记录不得直接计入资金清算统计。 + +### 阶段六：正式运行 + +只有阶段五无未解释失败、重复发送、错发收件人、原始文件覆盖或统计污染后，才扩大到正式业务范围。 + +## 4. 回滚操作 + +发生错发、重复发送、认证失败、外部费用异常、识别结果大面积异常或存储故障时： + +1. 立即将 `OFFSITE_SMTP_ENABLED=false`，停止新的真实发送。 +2. 如收件范围或解析异常，将 `OFFSITE_IMAP_ENABLED=false`，暂停收件扫描。 +3. 如识别服务异常，只关闭 `OFFSITE_OCR_ENABLED` 或 `OFFSITE_DEEPSEEK_ENABLED`，保留原始邮件和附件，禁止用不完整识别结果自动核对。 +4. 保留 `offsite_notification`、原始 `.eml`、附件和审计记录，不执行删除、覆盖或数据库降级迁移。 +5. 将发送中的通知交由运营核对邮箱实际投递结果；不得仅依据本地请求超时再次发送。 +6. 修复后先回到 dry-run 和单收件人灰度，再恢复正式发送。 + +## 5. 回滚后的数据处理 + +- `发送成功` 不自动重发，重复调用只返回已有成功状态。 +- `发送失败` 只按剩余重试次数重试，超过上限转人工处理。 +- `待发送` 记录保留，恢复服务后由运营重新确认。 +- 原始识别值、原始查询值、程序计算值和运营最终发送正文不得互相覆盖。 +- 统计必须重新执行并核对范围，未成功正常返回的邮件不纳入资金清算统计。 + +## 6. 验收记录 + +每次灰度至少记录：部署版本、配置开关、测试邮件标识、测试附件哈希、操作人、开始结束时间、实际收件人、发送状态、provider message ID、失败原因和回滚结论。 diff --git a/docs/场外申购赎回工作流程图.md b/docs/场外申购赎回工作流程图.md new file mode 100644 index 0000000..70ab653 --- /dev/null +++ b/docs/场外申购赎回工作流程图.md @@ -0,0 +1,38 @@ +# 场外基金申购赎回工作流程图 + +```mermaid +flowchart TD + A[收件箱收到邮件] --> B[IMAP IDLE 事件] + B --> C[按 UID 拉取完整 MIME 和附件] + C --> D{发件人和邮件认证通过} + D -- 否 --> D1[记录告警并停止进入业务队列] + D -- 是 --> E[保存原始 EML 和附件哈希] + E --> F[OCR/Document AI 一次识别] + F --> G[DeepSeek 分类与字段抽取] + G --> H{附件类型} + H -- other --> H1[只归档] + H -- summary/subscription/redemption --> I[生成 mail_id 和 attachment_id] + I --> J[SSE 推送新业务邮件事件] + J --> K[生成查询/计算/核对三阶段执行计划] + K --> L[调用 nl2sql_yc 只读查询] + L --> M[确定性金额、份额、比例计算] + M --> N[固定申购/赎回规则核对] + N --> O{识别或查询是否异常} + O -- 是 --> P[运营人工确认异常或未处理] + O -- 否 --> Q[运营确认正常] + P --> R[独立邮件返回或风控通知候选] + Q --> S[正常邮件返回] + S --> T[资金清算统计重算] + R --> U[发送状态、重试和审计] + T --> U + U --> V{邮件内单据均有最终人工状态} + V -- 否 --> W[保持待处理] + V -- 是 --> X[邮件完成并保留全链路审计] +``` + +## 边界 + +- 本流程只覆盖后端、Agent、数据、接口、异步事件和审计。 +- 反洗钱和开放期规则不纳入本次实现。 +- 邮件返回、风控通知和资金清算通知互相独立，不自动联动。 + diff --git a/hq.py b/hq.py new file mode 100644 index 0000000..d7a4ccb --- /dev/null +++ b/hq.py @@ -0,0 +1,224 @@ +# -*- coding: utf-8 -*- +"""南方基金指定产品行情模块。""" +from __future__ import annotations +import re +import time +import logging +from datetime import date, datetime, time as clock_time +from typing import Any +import httpx + +logger = logging.getLogger(__name__) + +NAV_API = "https://api.fund.eastmoney.com/f10/lsjz" +RETURN_API = "https://api.fund.eastmoney.com/pinzhong/LJSYLZS" +QUOTE_API = "https://push2.eastmoney.com/api/qt/ulist.np/get" +DETAIL_API = "https://fund.eastmoney.com/pingzhongdata/{code}.js" +REQUEST_TIMEOUT = 12.0 +MAX_FUNDS_PER_CALL = 1000 +SOUTHERN_FUND_CODES = tuple(dict.fromkeys("202308 020480 007161 003776 020281 018019 014189 018020 020553 016449 008854 008264 008736 010592 160127 588890 020839 589700 159382 159511 002900 021958 159948 009059 001421".split())) +FUND_TYPE_GROUPS = { + "货币型": ("202308", "020480"), + "债券型": ("007161", "003776", "020281"), + "混合型": ("018019", "014189", "018020"), + "股票型": ("020553", "016449", "008854", "008264", "008736", "010592", "160127", "588890", "020839", "589700", "159382", "159511", "002900", "021958", "159948", "009059", "001421"), +} +FUND_TYPE_BY_CODE = {code: fund_type for fund_type, codes in FUND_TYPE_GROUPS.items() for code in codes} +HEADERS = {"User-Agent": "Mozilla/5.0", "Referer": "https://fund.eastmoney.com/"} +_history_date: str | None = None +_history_cache: dict[str, dict[str, str | None]] = {} +_name_cache: dict[str, str] = {} + + +def is_market_trading_time(now: datetime | None = None) -> bool: + """判断中国大陆工作日盘中时段。""" + current = now or datetime.now() + if current.weekday() >= 5: + return False + current_time = current.time() + return (clock_time(9, 30) <= current_time <= clock_time(11, 30) + or clock_time(13, 0) <= current_time <= clock_time(15, 0)) + + +def get_southern_fund_market(target_date: str | None = None, limit: int | None = None, fund_type: str | None = None) -> list[dict[str, Any]]: + """获取指定南方基金的完整行情表。 + + 每次调用刷新整张表的实时行情；历史收益同一日期只请求一次并保存在进程缓存。 + limit 不传时返回 SOUTHERN_FUND_CODES 中的全部产品。 + """ + query_date = target_date or date.today().isoformat() + date.fromisoformat(query_date) + if fund_type and fund_type not in FUND_TYPE_GROUPS: + raise ValueError("基金类型必须是货币型、债券型、混合型或股票型") + available_codes = FUND_TYPE_GROUPS[fund_type] if fund_type else SOUTHERN_FUND_CODES + count = len(available_codes) if limit is None else min(max(limit, 1), MAX_FUNDS_PER_CALL) + codes = list(available_codes[:count]) + names = _get_names(codes) + history = _get_history(codes, query_date) + quotes = _get_quotes(codes) if is_market_trading_time() else {} + now = time.strftime("%Y-%m-%d") + rows = [] + for code in codes: + old = history.get(code, {}) + live = quotes.get(code, {}) + rows.append({ + "基金代码": code, "基金名称": names.get(code, f"南方基金 {code}"), + "基金类型": FUND_TYPE_BY_CODE.get(code, "未分类"), + "基金净值": live.get("基金净值") or old.get("基金净值"), + "日期": live.get("日期") or old.get("日期"), + "日涨幅": live.get("日涨幅") or old.get("日涨幅"), + "最近半年": old.get("最近半年"), "最近一年": old.get("最近一年"), + "今年以来": old.get("今年以来"), "成立以来": old.get("成立以来"), + "行情时间": now, + "行情来源": "盘中实时行情" if live.get("基金净值") else "收盘后最新净值", + "是否盘中": is_market_trading_time(), + }) + return rows + + +def _get_names(codes: list[str]) -> dict[str, str]: + for code in codes: + if code in _name_cache: + continue + try: + response = httpx.get(DETAIL_API.format(code=code), headers=HEADERS, timeout=REQUEST_TIMEOUT) + response.raise_for_status() + match = re.search(r"var\s+fS_name\s*=\s*[\"']([^\"']+)", response.text) + _name_cache[code] = match.group(1).strip() if match else f"南方基金 {code}" + except (httpx.HTTPError, UnicodeError) as exc: + logger.warning("基金名称接口失败 code=%s error=%s", code, type(exc).__name__) + _name_cache[code] = f"南方基金 {code}" + return {code: _name_cache.get(code, f"南方基金 {code}") for code in codes} + + +def _get_quotes(codes: list[str]) -> dict[str, dict[str, str | None]]: + secids = ",".join(("1." if code.startswith(("5", "6", "9")) else "0.") + code for code in codes) + try: + response = httpx.get(QUOTE_API, params={"fltt": 2, "invt": 2, "fields": "f12,f2,f3", "secids": secids}, headers=HEADERS, timeout=REQUEST_TIMEOUT) + response.raise_for_status() + items = ((response.json().get("data") or {}).get("diff") or []) + except (httpx.HTTPError, ValueError, TypeError) as exc: + logger.warning("实时行情接口失败 count=%s error=%s", len(codes), type(exc).__name__) + return {} + return {str(item["f12"]): {"基金净值": str(item["f2"]) if item.get("f2") not in (None, "-") else None, "日期": None, "日涨幅": f"{item.get('f3')}%" if item.get("f3") not in (None, "-") else None} for item in items if item.get("f12")} + + +def _get_history(codes: list[str], query_date: str) -> dict[str, dict[str, str | None]]: + global _history_date, _history_cache + if _history_date == query_date and all(code in _history_cache for code in codes): + return _history_cache + result = {} + for code in codes: + try: + result[code] = _get_history_snapshot(code, query_date) + except (httpx.HTTPError, ValueError, TypeError) as exc: + logger.warning("历史净值计算失败 code=%s error=%s", code, type(exc).__name__) + result[code] = _empty_returns() + _history_date, _history_cache = query_date, result + return result + + +def _get_history_snapshot(fund_code: str, query_date: str) -> dict[str, str | None]: + """读取净值与累计收益率快照。""" + latest_records = _fetch_nav_records(fund_code, all_pages=False) + valid = _valid_records(latest_records) + if not valid: + return _empty_returns() + target = date.fromisoformat(query_date) + latest_date, _, latest = next((item for item in valid if item[0] == target), valid[0]) + return { + "基金净值": latest.get("DWJZ"), + "日期": latest.get("FSRQ"), + "日涨幅": _format_percent(latest.get("JZZZL")), + "最近半年": _fetch_return_rate(fund_code, "6月"), + "最近一年": _fetch_return_rate(fund_code, "1年"), + "今年以来": _fetch_return_rate(fund_code, "今年来"), + "成立以来": _fetch_return_rate(fund_code, "成立来"), + } + + +def _fetch_nav_page(fund_code: str, page_index: int = 1, start_date: str | None = None, end_date: str | None = None) -> tuple[list[dict[str, Any]], int | None]: + """读取一页历史净值，并返回接口提供的总条数。""" + response = httpx.get(NAV_API, params={"fundCode": fund_code, "pageIndex": page_index, "pageSize": 30, "startDate": start_date or "", "endDate": end_date or ""}, headers=HEADERS, timeout=REQUEST_TIMEOUT) + response.raise_for_status() + data = response.json().get("Data") or {} + records = data.get("LSJZList") or [] + total = data.get("TotalCount") or data.get("totalCount") + try: + total = int(total) if total is not None else None + except (TypeError, ValueError): + total = None + return records, total + + +def _fetch_nav_records(fund_code: str, start_date: str | None = None, end_date: str | None = None, all_pages: bool = False) -> list[dict[str, Any]]: + """调用历史净值接口；区间收益需要时读取完整分页，避免重复使用同一基准日。""" + records, total = _fetch_nav_page(fund_code, 1, start_date, end_date) + if not all_pages or not total or total <= len(records): + return records + page_count = min((total + 29) // 30, 40) + for page_index in range(2, page_count + 1): + page_records, _ = _fetch_nav_page(fund_code, page_index, start_date, end_date) + if not page_records: + break + records.extend(page_records) + return records + + +def _fetch_return_rate(fund_code: str, period: str) -> str | None: + """通过东方财富累计收益率接口读取指定区间的最新收益率。""" + period_map = { + "1月": "m", + "3月": "q", + "6月": "hy", + "1年": "y", + "3年": "try", + "5年": "fiy", + "今年来": "sy", + "成立来": "se", + } + try: + response = httpx.get( + RETURN_API, + params={"fundCode": fund_code, "indexcode": "000300", "type": period_map[period]}, + headers={"Referer": "https://fund.eastmoney.com/"}, + timeout=REQUEST_TIMEOUT, + ) + response.raise_for_status() + payload = response.json() + series = (((payload.get("Data") or [{}])[0]).get("data") or []) + if not series: + return None + latest = series[-1] + if isinstance(latest, dict): + value = latest.get("y") + elif isinstance(latest, (list, tuple)) and len(latest) >= 2: + value = latest[1] + else: + value = None + if value in (None, ""): + return None + return _format_percent(value) + except (httpx.HTTPError, ValueError, TypeError, KeyError, IndexError) as exc: + logger.warning("累计收益率接口失败 code=%s period=%s error=%s", fund_code, period, type(exc).__name__) + return None + +def _empty_returns() -> dict[str, str | None]: + return {key: None for key in ("基金净值", "日期", "日涨幅", "最近半年", "最近一年", "今年以来", "成立以来")} + + +def _valid_records(records: list[dict[str, Any]]) -> list[tuple[date, float, dict[str, Any]]]: + valid = [] + for item in records: + try: + valid.append((date.fromisoformat(item["FSRQ"]), float(item["DWJZ"]), item)) + except (KeyError, TypeError, ValueError): + continue + return sorted(valid, key=lambda item: item[0], reverse=True) + + +def _format_percent(value: Any) -> str | None: + if value in (None, ""): + return None + text = str(value).strip() + return text if text.endswith("%") else f"{text}%" diff --git a/nl2sql_yc.py b/nl2sql_yc.py new file mode 100644 index 0000000..e437d1c --- /dev/null +++ b/nl2sql_yc.py @@ -0,0 +1,648 @@ +"""金融 NL2SQL MVP。 + +查询数据只允许使用参数化 SELECT；审计记录可通过回调写入 +conversation_message.tool_calls。该文件可被运营和投顾 Agent 直接导入调用。 +""" +from __future__ import annotations + +import json +import logging +import os +import re +import uuid +from dataclasses import asdict, dataclass, field +from datetime import datetime, time, timedelta +from pathlib import Path +from typing import Any, Callable + +logger = logging.getLogger(__name__) + +ALLOWED_TABLES = { + "sys_customer_assignment", "fin_customer_profile", "fin_risk_assessment", + "fin_product", "fin_fee_rule", "fin_market_price", "fin_nav_history", + "fin_holding", "fin_transaction", "fin_sim_order", "fin_sim_account", + "fin_cash_ledger", "client_facing_content", +} + +DOMAINS = { + "market_nav": {"fin_market_price", "fin_nav_history", "fin_product"}, + "customer_risk": { + "sys_customer_assignment", "fin_customer_profile", "fin_risk_assessment", + }, + "product_fee": {"fin_product", "fin_fee_rule"}, + "trading_account": { + "fin_sim_order", "fin_transaction", "fin_sim_account", + "fin_cash_ledger", "fin_holding", "fin_product", + }, + "client_content": {"client_facing_content"}, +} + +TABLE_COLUMNS = { + "sys_customer_assignment": { + "id", "customer_id", "employee_id", "employee_role", "assigned_at", "unassigned_at", + }, + "fin_customer_profile": { + "customer_id", "trade_account", "real_name", "birth_date", "occupation", + "mobile_masked", "investor_type", "investment_horizon", "preferred_asset_class", + "trading_frequency", "last_active_at", "total_asset", "behavior_score", + "risk_tags", "opened_at", "updated_at", + }, + "fin_risk_assessment": { + "id", "customer_id", "questionnaire_version", "answers", "total_score", + "investor_type", "assessed_at", "valid_until", "created_at", + }, + "fin_product": { + "id", "product_code", "product_name", "exchange_code", "product_category", + "risk_level", "fund_manager", "currency", "lot_size", "price_tick", + "current_nav", "current_nav_at", "min_amount", "open_start_at", "open_end_at", + "open_period_start", "open_period_end", "transaction_fee_rate", + "single_investor_max_holding_ratio", "management_fee_rate", "custodian_fee_rate", + "risk_disclosure_required", "second_confirmation_required", "recording_required", + "status", "created_at", "updated_at", + }, + "fin_fee_rule": { + "id", "rule_code", "product_id", "exchange_code", "order_side", "customer_tier", + "min_trade_amount", "max_trade_amount", "fee_rate", "minimum_fee", "fixed_fee", + "priority", "effective_from", "effective_until", "status", "created_at", "updated_at", + }, + "fin_market_price": { + "id", "product_id", "trade_date", "open_price", "high_price", "low_price", + "close_price", "volume", "turnover_amount", "total_fund_shares", "source", + "source_updated_at", "created_at", + }, + "fin_nav_history": {"id", "product_id", "nav_date", "nav", "created_at"}, + "fin_holding": { + "id", "customer_id", "trade_account", "product_id", "total_quantity", + "shares", "available_quantity", "frozen_quantity", "average_cost", "cost_amount", + "market_value", "current_value", "profit_loss", "profit_loss_ratio", "status", + "first_acquired_at", "version", "updated_at", + }, + "fin_transaction": { + "id", "transaction_no", "order_id", "work_order_id", "customer_id", "account_id", + "product_id", "order_side", "transaction_type", "executed_price", "nav", "executed_quantity", + "shares", "gross_amount", "amount", "fee_rule_id", "fee_rate_snapshot", "fee_amount", + "fee", "net_amount", "quote_at", "quote_source", "executed_at", "confirmed_at", + "confirmed_by", "auto_confirmed", "created_at", + }, + "fin_sim_order": { + "id", "order_no", "customer_id", "account_id", "product_id", "order_side", + "price_type", "quantity", "limit_price", "quote_price", "quote_at", "quote_source", + "channel", "advisor_id", "filled_quantity", "average_executed_price", "status", + "risk_rule_hits", "risk_disclosure_ack_at", "second_confirmation_at", + "recording_reference", "ops_handler_id", "ops_handled_at", "compliance_handler_id", + "compliance_handled_at", "reject_reason", "submitted_at", "cancelled_at", + "created_at", "updated_at", + }, + "fin_sim_account": { + "id", "account_no", "customer_id", "currency", "cash_balance", "available_cash", + "frozen_cash", "initial_balance", "status", "version", "created_at", "updated_at", + }, + "fin_cash_ledger": { + "id", "ledger_no", "account_id", "transaction_id", "entry_type", "amount", + "balance_after", "available_cash_after", "frozen_cash_after", "idempotency_key", + "occurred_at", "created_at", + }, + "client_facing_content": { + "id", "customer_id", "content_type", "draft_content", "generated_by_portal", + "review_status", "reviewer_user_id", "reviewed_at", "published_at", + "created_at", "updated_at", + }, +} + +JOIN_SQL = { + ("fin_transaction", "fin_product"): "t.product_id = p.id", + ("fin_holding", "fin_product"): "h.product_id = p.id", + ("fin_market_price", "fin_product"): "m.product_id = p.id", + ("fin_nav_history", "fin_product"): "n.product_id = p.id", + ("fin_sim_order", "fin_product"): "o.product_id = p.id", + ("fin_fee_rule", "fin_product"): "f.product_id = p.id", + ("fin_transaction", "fin_customer_profile"): "t.customer_id = cp.customer_id", + ("fin_holding", "fin_customer_profile"): "h.customer_id = cp.customer_id", + ("fin_sim_account", "fin_customer_profile"): "a.customer_id = cp.customer_id", + ("fin_cash_ledger", "fin_sim_account"): "l.account_id = a.id", + ("fin_sim_account", "fin_customer_profile"): "a.customer_id = cp.customer_id", + ("fin_cash_ledger", "fin_sim_account"): "l.account_id = a.id", + ("fin_transaction", "fin_sim_account"): "t.account_id = a.id", + ("fin_transaction", "sys_customer_assignment"): "t.customer_id = ca.customer_id", + ("fin_holding", "sys_customer_assignment"): "h.customer_id = ca.customer_id", + ("fin_customer_profile", "sys_customer_assignment"): "cp.customer_id = ca.customer_id", +} + + +def _load_local_config() -> dict[str, str]: + """加载标准环境变量，并兼容 0901/.evn 的中文标签密钥格式。""" + values = dict(os.environ) + path = Path(__file__).with_name(".evn") + if not path.exists(): + return values + raw_lines = [line.strip() for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + for line in raw_lines: + if "=" in line: + key, value = line.split("=", 1) + values.setdefault(key.strip(), value.strip().strip('"').strip("'")) + if raw_lines: + first_value = raw_lines[0].split("：", 1)[-1].strip() if "：" in raw_lines[0] else raw_lines[0] + values.setdefault("DEEPSEEK_API_KEY_NL2SQL", first_value) + if len(raw_lines) > 1: + second_value = raw_lines[1].split("：", 1)[-1].strip() if "：" in raw_lines[1] else raw_lines[1] + values.setdefault("ALIYUN_API_KEY", second_value) + return values + + +CONFIG = _load_local_config() +DATABASE_URL = CONFIG.get("DATABASE_URL", "") +LLM_BASE_URL = CONFIG.get("DEEPSEEK_BASE_URL") or "https://api.deepseek.com" +LLM_MODEL = CONFIG.get("DEEPSEEK_NL2SQL_MODEL") or CONFIG.get("DEEPSEEK_MODEL") or "deepseek-chat" +LLM_KEY = CONFIG.get("DEEPSEEK_API_KEY_NL2SQL") or CONFIG.get("DEEPSEEK_API_KEY", "") + + +@dataclass +class AuthContext: + user_id: int | None = None + roles: list[str] = field(default_factory=list) + allowed_domains: list[str] = field(default_factory=lambda: list(DOMAINS)) + customer_scope: str = "all" + allowed_fields: list[str] | None = None + masked_fields: list[str] = field(default_factory=list) + max_rows: int = 500 + max_query_seconds: int = 10 + + +@dataclass +class QueryRequest: + question: str + auth_context: AuthContext + conversation_id: str | None = None + request_id: str | None = None + timezone: str = "Asia/Shanghai" + confirmation: str | None = None + + +@dataclass +class QueryPlan: + intent: str + domains: list[str] + tables: list[str] + time_mode: str = "none" + time_column: str | None = None + start: str | None = None + end: str | None = None + metrics: list[str] = field(default_factory=list) + dimensions: list[str] = field(default_factory=list) + filters: list[dict[str, Any]] = field(default_factory=list) + sort: list[dict[str, str]] = field(default_factory=list) + limit: int = 50 + confidence: float = 0.0 + needs_confirmation: bool = False + confirmation_question: str | None = None + unsupported_reason: str | None = None + + +def _json_object(text_value: str) -> dict[str, Any]: + cleaned = re.sub(r"```(?:json)?|```", "", text_value or "").strip() + match = re.search(r"\{.*\}", cleaned, flags=re.S) + if not match: + raise ValueError("模型未返回结构化查询计划") + value = json.loads(match.group(0)) + if not isinstance(value, dict): + raise ValueError("查询计划必须是 JSON 对象") + return value + + +def _days_range(days: int) -> tuple[str, str]: + end = datetime.now() + start = end - timedelta(days=days) + return start.strftime("%Y-%m-%d %H:%M:%S"), end.strftime("%Y-%m-%d %H:%M:%S") + + +def _catalog_text(tables: set[str]) -> str: + lines = ["只允许使用以下表和字段："] + for table in sorted(tables): + lines.append(f"{table}: {', '.join(sorted(TABLE_COLUMNS[table]))}") + lines.append("指标口径：交易金额=gross_amount；净交易金额=net_amount；收盘价=close_price；基金净值=nav；") + lines.append("成交价格=executed_price；当前持仓市值=market_value；历史资金变化=fin_cash_ledger.amount。") + return "\n".join(lines) + + +def _route_keywords(question: str) -> list[str]: + routes = [] + if any(word in question for word in ("行情", "收盘", "开盘", "最高价", "最低价")): + routes.append("market_nav") + if "净值" in question: + routes.append("market_nav") + if any(word in question for word in ("客户", "风险", "测评", "画像", "投顾", "运营")): + routes.append("customer_risk") + if any(word in question for word in ("产品", "基金", "费率", "适配", "风险等级")): + routes.append("product_fee") + if any(word in question for word in ("委托", "成交", "交易", "持仓", "账户", "资金", "现金", "盈亏")): + routes.append("trading_account") + if any(word in question for word in ("内容", "审核", "发布")): + routes.append("client_content") + return list(dict.fromkeys(routes)) or ["trading_account"] + + +def _llm_plan(request: QueryRequest, domains: list[str]) -> dict[str, Any]: + try: + from openai import OpenAI + except ImportError as exc: + raise RuntimeError("缺少 openai 依赖，无法调用模型") from exc + tables = set().union(*(DOMAINS[d] for d in domains)) + system = f"""你是金融查询计划生成器。只输出 JSON，不输出解释。 +用户身份已由后端确认，customer_scope={request.auth_context.customer_scope}。 +{_catalog_text(tables)} +返回字段：intent, domains, tables, time_mode, time_column, start, end, metrics, +dimensions, filters, sort, limit, confidence, needs_confirmation, confirmation_question。 +只从给定表和字段中选择；无法准确判断时降低 confidence。历史时点查询不能使用当前 +fin_holding、fin_sim_account 或当前归属表冒充历史数据。""" + client = OpenAI(api_key=LLM_KEY, base_url=LLM_BASE_URL) + response = client.chat.completions.create( + model=LLM_MODEL, + messages=[{"role": "system", "content": system}, {"role": "user", "content": request.question}], + temperature=0, + response_format={"type": "json_object"}, + timeout=request.auth_context.max_query_seconds, + ) + return _json_object(response.choices[0].message.content) + + +def _normalize_plan(raw: dict[str, Any], request: QueryRequest, domains: list[str]) -> QueryPlan: + tables = [name for name in raw.get("tables", []) if name in ALLOWED_TABLES] + if not tables: + tables = sorted(set().union(*(DOMAINS[d] for d in domains))) + confidence = float(raw.get("confidence", 0.0) or 0.0) + confidence = max(0.0, min(confidence, 1.0)) + plan = QueryPlan( + intent=str(raw.get("intent") or "unknown"), + domains=[d for d in raw.get("domains", domains) if d in DOMAINS] or domains, + tables=tables, + time_mode=str(raw.get("time_mode") or raw.get("temporal", {}).get("mode") or "none"), + time_column=raw.get("time_column") or raw.get("temporal", {}).get("time_column"), + start=raw.get("start") or raw.get("temporal", {}).get("start"), + end=raw.get("end") or raw.get("temporal", {}).get("end"), + metrics=list(raw.get("metrics") or []), + dimensions=list(raw.get("dimensions") or []), + filters=list(raw.get("filters") or []), + sort=list(raw.get("sort") or []), + limit=min(max(int(raw.get("limit", 50) or 50), 1), request.auth_context.max_rows), + confidence=confidence, + needs_confirmation=bool(raw.get("needs_confirmation", False)), + confirmation_question=raw.get("confirmation_question"), + unsupported_reason=raw.get("unsupported_reason"), + ) + if plan.confidence < 0.60: + plan.needs_confirmation = True + plan.confirmation_question = plan.confirmation_question or "请补充客户、产品、时间范围或指标口径。" + elif plan.confidence < 0.85: + plan.needs_confirmation = True + plan.confirmation_question = plan.confirmation_question or "请确认我对查询范围和指标口径的理解是否正确。" + return plan + + +def _validate_plan(plan: QueryPlan, auth: AuthContext) -> tuple[bool, str]: + allowed_tables = set().union(*(DOMAINS[d] for d in auth.allowed_domains if d in DOMAINS)) + if not set(plan.tables).issubset(allowed_tables): + return False, "查询包含当前角色未授权的数据表" + if len(plan.tables) > 8: + return False, "查询涉及的数据表过多" + if plan.time_mode in {"range", "as_of"} and not plan.time_column: + return False, "历史查询缺少时间字段" + temporal_current = {"fin_holding", "fin_sim_account", "sys_customer_assignment"} + if plan.time_mode == "as_of" and temporal_current.intersection(plan.tables): + return False, "当前表缺少历史时点来源，暂不支持该历史时点查询" + if plan.start and plan.end and plan.start > plan.end: + return False, "查询开始时间不能晚于结束时间" + return True, "计划校验通过" + + +def _field_allowed(table: str, field_name: str, auth: AuthContext) -> bool: + if field_name not in TABLE_COLUMNS.get(table, set()): + return False + if auth.allowed_fields is None: + return True + return f"{table}.{field_name}" in auth.allowed_fields or field_name in auth.allowed_fields + + +def _safe_sql_check(sql: str, params: dict[str, Any], plan: QueryPlan, auth: AuthContext) -> tuple[bool, str]: + normalized = re.sub(r"\s+", " ", sql.strip()) + upper = normalized.upper() + if not upper.startswith("SELECT "): + return False, "仅支持 SELECT 查询" + if ";" in normalized.rstrip(";"): + return False, "禁止执行多语句" + if re.search(r"\b(INSERT|UPDATE|DELETE|DROP|ALTER|TRUNCATE|CREATE|GRANT|REVOKE|CALL|INTO\s+OUTFILE)\b", upper): + return False, "检测到禁止的数据库操作" + if "*" in normalized: + return False, "禁止使用 SELECT *" + table_refs = set(re.findall(r"\b(?:FROM|JOIN)\s+([A-Za-z_][A-Za-z0-9_]*)", normalized, flags=re.I)) + if not table_refs.issubset(set(plan.tables)): + return False, "SQL 使用了计划外的数据表" + for table in table_refs: + aliases = re.findall(rf"\b{re.escape(table)}\s+(?:AS\s+)?([A-Za-z_][A-Za-z0-9_]*)", normalized, flags=re.I) + del aliases + if plan.time_mode in {"range", "as_of"} and plan.time_column and plan.time_column not in normalized: + return False, "历史查询缺少计划中的时间条件" + if len(params) > 30: + return False, "查询参数过多" + return True, "SQL 安全校验通过" + + +def _alias(table: str) -> str: + return { + "fin_transaction": "t", "fin_product": "p", "fin_holding": "h", + "fin_market_price": "m", "fin_nav_history": "n", "fin_sim_order": "o", + "fin_sim_account": "a", "fin_cash_ledger": "l", + "fin_customer_profile": "cp", "sys_customer_assignment": "ca", + "fin_risk_assessment": "ra", "fin_fee_rule": "f", + "client_facing_content": "c", + }.get(table, table[:1]) + + +def _compile_sql(plan: QueryPlan, auth: AuthContext) -> tuple[str, dict[str, Any]]: + if plan.unsupported_reason: + raise ValueError(plan.unsupported_reason) + primary = plan.tables[0] + alias = _alias(primary) + select_parts = [] + group_parts = [] + for dimension in plan.dimensions: + if "." not in dimension: + dimension = f"{primary}.{dimension}" + table, column = dimension.split(".", 1) + if not _field_allowed(table, column, auth): + raise PermissionError(f"字段未授权：{dimension}") + select_parts.append(f"{_alias(table)}.{column} AS {column}") + group_parts.append(f"{_alias(table)}.{column}") + metric_map = { + "交易金额": "SUM(t.gross_amount) AS gross_amount", + "gross_amount": "SUM(t.gross_amount) AS gross_amount", + "净交易金额": "SUM(t.net_amount) AS net_amount", + "net_amount": "SUM(t.net_amount) AS net_amount", + "成交数量": "SUM(t.executed_quantity) AS executed_quantity", + "持仓数量": "h.total_quantity AS total_quantity", + "持仓市值": "h.market_value AS market_value", + "浮动盈亏": "h.profit_loss AS profit_loss", + "可用现金": "a.available_cash AS available_cash", + "现金余额": "a.cash_balance AS cash_balance", + "历史资金变化": "SUM(l.amount) AS cash_change", + "收盘价": "m.close_price AS close_price", + "基金净值": "n.nav AS nav", + "成交价格": "t.executed_price AS executed_price", + "客户数": "COUNT(DISTINCT cp.customer_id) AS customer_count", + } + for metric in plan.metrics or ["客户数"]: + expression = metric_map.get(str(metric)) + if not expression: + raise ValueError(f"暂不支持指标：{metric}") + select_parts.append(expression) + if not select_parts: + raise ValueError("查询计划没有可返回的字段") + params: dict[str, Any] = {} + where = ["1=1"] + if primary == "fin_transaction": + where.append("t.order_side IN ('买入', '卖出')") + if plan.time_column and plan.start: + params["start_time"] = plan.start + where.append(f"{alias}.{plan.time_column} >= :start_time") + if plan.time_column and plan.end: + params["end_time"] = plan.end + where.append(f"{alias}.{plan.time_column} <= :end_time") + for index, item in enumerate(plan.filters): + field_name = item.get("field") + operator = str(item.get("operator", "=")).upper() + if not isinstance(field_name, str) or "." not in field_name or operator not in {"=", "!=", ">", ">=", "<", "<=", "LIKE"}: + raise ValueError("查询筛选条件不合法") + table, column = field_name.split(".", 1) + if not _field_allowed(table, column, auth): + raise PermissionError(f"字段未授权：{field_name}") + key = f"filter_{index}" + params[key] = item.get("value") + where.append(f"{_alias(table)}.{column} {operator} :{key}") + from_sql = f"{primary} {alias}" + pending = set(plan.tables) - {primary} + joined = {primary} + while pending: + progress = False + for table in sorted(pending): + join = None + for existing in joined: + join = JOIN_SQL.get((existing, table)) or JOIN_SQL.get((table, existing)) + if join: + break + if not join: + continue + from_sql += f" JOIN {table} {_alias(table)} ON {join}" + joined.add(table) + pending.remove(table) + progress = True + if not progress: + break + if pending: + raise ValueError(f"缺少合法 Join 路径：{', '.join(sorted(pending))}") + sql = f"SELECT {', '.join(select_parts)} FROM {from_sql} WHERE {' AND '.join(where)}" + if group_parts: + sql += f" GROUP BY {', '.join(group_parts)}" + if plan.sort: + sort_items = [] + for item in plan.sort: + field = str(item.get("field", "")).split(".")[-1] + direction = "DESC" if str(item.get("direction", "desc")).lower() == "desc" else "ASC" + if field not in {part.split(" AS ")[-1] for part in select_parts}: + continue + sort_items.append(f"{field} {direction}") + if sort_items: + sql += " ORDER BY " + ", ".join(sort_items) + sql += f" LIMIT {min(plan.limit, auth.max_rows)}" + return sql, params + + +def _mock_plan(request: QueryRequest, domains: list[str]) -> dict[str, Any]: + """无模型或测试时的保守规则，便于单元测试和离线联调。""" + question = request.question + if "近30天" in question or "最近30天" in question: + start, end = _days_range(30) + else: + start = end = None + if "收盘" in question or "行情" in question: + return {"intent": "market_price", "domains": ["market_nav"], "tables": ["fin_market_price", "fin_product"], + "time_mode": "range", "time_column": "trade_date", "start": start, "end": end, + "metrics": ["收盘价"], "dimensions": ["fin_product.product_name", "fin_market_price.trade_date"], + "confidence": 0.88} + if "净值" in question: + return {"intent": "nav_history", "domains": ["market_nav"], "tables": ["fin_nav_history", "fin_product"], + "time_mode": "range", "time_column": "nav_date", "start": start, "end": end, + "metrics": ["基金净值"], "dimensions": ["fin_product.product_name", "fin_nav_history.nav_date"], + "confidence": 0.88} + if "当前持仓" in question or "持仓市值" in question: + return {"intent": "current_holding", "domains": ["customer_risk", "trading_account"], + "tables": ["fin_holding", "fin_product", "fin_customer_profile"], + "metrics": ["持仓市值"], "dimensions": ["fin_customer_profile.real_name", "fin_product.product_name"], + "confidence": 0.87} + if "账户余额" in question or "可用现金" in question: + return {"intent": "current_account", "domains": ["trading_account"], + "tables": ["fin_sim_account", "fin_customer_profile"], "metrics": ["可用现金"], + "dimensions": ["fin_customer_profile.real_name"], "confidence": 0.87} + if "资金变动" in question or "资金流水" in question: + return {"intent": "cash_ledger", "domains": ["trading_account"], + "tables": ["fin_cash_ledger", "fin_sim_account"], "time_mode": "range", + "time_column": "occurred_at", "start": start, "end": end, + "metrics": ["历史资金变化"], "dimensions": ["fin_cash_ledger.occurred_at"], "confidence": 0.87} + return {"intent": "unknown", "domains": domains, "tables": sorted(set().union(*(DOMAINS[d] for d in domains))), + "confidence": 0.45, "needs_confirmation": True, + "confirmation_question": "请明确要查询的业务对象、指标和时间范围。"} + + +def _audit_payload(request: QueryRequest, plan: QueryPlan, sql: str | None, params: dict[str, Any], + validation: dict[str, Any], execution: dict[str, Any]) -> dict[str, Any]: + return { + "tool": "nl2sql", "engine_version": "mvp-1", "request_id": request.request_id, + "conversation_id": request.conversation_id, "intent": plan.intent, + "confidence": plan.confidence, "domains": plan.domains, + "query_plan": asdict(plan), "generated_sql": sql, + "parameters": {key: "" if "password" in key.lower() else value for key, value in params.items()}, + "authorized_context": {"user_id": request.auth_context.user_id, "roles": request.auth_context.roles, + "customer_scope": request.auth_context.customer_scope}, + "validation": validation, "execution": execution, + "created_at": datetime.now().isoformat(timespec="seconds"), + } + + +def write_conversation_tool_calls(connection: Any, request: QueryRequest, audit: dict[str, Any]) -> None: + """将审计对象写入已有 conversation_message；宿主也可使用 audit_writer 自行落库。""" + from sqlalchemy import text + connection.execute( + text("""INSERT INTO conversation_message + (session_id, customer_id, portal, role, content, tool_calls, intent, confidence, created_at) + VALUES (:session_id, :customer_id, :portal, 'assistant', :content, :tool_calls, + :intent, :confidence, :created_at)"""), + { + "session_id": request.conversation_id or request.request_id, + "customer_id": request.auth_context.user_id, + "portal": "nl2sql", + "content": audit.get("execution", {}).get("status", "nl2sql"), + "tool_calls": json.dumps(audit, ensure_ascii=False, default=str), + "intent": audit.get("intent"), + "confidence": audit.get("confidence"), + "created_at": datetime.now(), + }, + ) + + +def query(request: QueryRequest, *, db_engine: Any = None, + audit_writer: Callable[[dict[str, Any]], None] | None = None, + use_llm: bool = True, persist_audit: bool = False) -> dict[str, Any]: + """统一入口：返回结构化结果，供运营和投顾 Agent 直接调用。""" + if not request.question or not request.question.strip(): + return {"status": "rejected", "message": "查询问题不能为空"} + request.request_id = request.request_id or str(uuid.uuid4()) + domains = _route_keywords(request.question) + try: + raw = _llm_plan(request, domains) if use_llm and LLM_KEY else _mock_plan(request, domains) + plan = _normalize_plan(raw, request, domains) + if request.confirmation and plan.needs_confirmation: + plan.needs_confirmation = False + plan.confidence = max(plan.confidence, 0.85) + valid, message = _validate_plan(plan, request.auth_context) + if not valid: + audit = _audit_payload(request, plan, None, {}, {"valid": False, "message": message}, {"status": "rejected"}) + if audit_writer: + audit_writer(audit) + if persist_audit and db_engine is not None: + with db_engine.begin() as connection: + write_conversation_tool_calls(connection, request, audit) + return {"status": "rejected", "message": message, "audit": audit} + if plan.needs_confirmation: + audit = _audit_payload(request, plan, None, {}, {"valid": True, "message": "等待确认"}, {"status": "waiting"}) + if audit_writer: + audit_writer(audit) + if persist_audit and db_engine is not None: + with db_engine.begin() as connection: + write_conversation_tool_calls(connection, request, audit) + return {"status": "need_confirmation", "message": plan.confirmation_question, "query_plan": asdict(plan), "audit": audit} + sql, params = _compile_sql(plan, request.auth_context) + safe, safety_message = _safe_sql_check(sql, params, plan, request.auth_context) + if not safe: + raise PermissionError(safety_message) + if db_engine is None: + execution = {"status": "not_executed", "reason": "未提供数据库连接，仅返回已校验 SQL"} + audit = _audit_payload(request, plan, sql, params, {"valid": True, "message": safety_message}, execution) + if audit_writer: + audit_writer(audit) + if persist_audit and db_engine is not None: + with db_engine.begin() as connection: + write_conversation_tool_calls(connection, request, audit) + return {"status": "ready", "sql": sql, "parameters": params, "query_plan": asdict(plan), "audit": audit} + from sqlalchemy import text + with db_engine.connect() as connection: + result = connection.execute(text(sql), params) + columns = list(result.keys()) + rows = [dict(zip(columns, row)) for row in result.fetchmany(request.auth_context.max_rows)] + execution = {"status": "success", "row_count": len(rows), "truncated": len(rows) >= request.auth_context.max_rows} + audit = _audit_payload(request, plan, sql, params, {"valid": True, "message": safety_message}, execution) + if audit_writer: + audit_writer(audit) + if persist_audit: + with db_engine.begin() as connection: + write_conversation_tool_calls(connection, request, audit) + return {"status": "success", "data": {"total": len(rows), "rows": rows}, + "query_plan": asdict(plan), "sql": sql, "audit": audit} + except (ValueError, PermissionError, RuntimeError) as exc: + logger.warning("NL2SQL业务失败：%s", exc) + return {"status": "error", "message": str(exc), "request_id": request.request_id} + except Exception: + logger.exception("NL2SQL执行失败") + return {"status": "error", "message": "查询执行失败，请稍后重试", "request_id": request.request_id} + + +def build_request(question: str, auth_context: dict[str, Any], **kwargs: Any) -> QueryRequest: + """将 Agent 的字典请求转换为统一请求对象。""" + return QueryRequest(question=question, auth_context=AuthContext(**auth_context), **kwargs) + + +def query_dict(question: str, auth_context: dict[str, Any], *, + db_engine: Any = None, + audit_writer: Callable[[dict[str, Any]], None] | None = None, + use_llm: bool = True, persist_audit: bool = False, + **request_kwargs: Any) -> dict[str, Any]: + """给 Agent 使用的字典式快捷入口。""" + return query( + build_request(question, auth_context, **request_kwargs), + db_engine=db_engine, + audit_writer=audit_writer, + use_llm=use_llm, + persist_audit=persist_audit, + ) + + +def get_tool_definition() -> dict[str, Any]: + """返回可注册到运营或投顾 Agent 的统一工具定义。""" + return { + "name": "financial_nl2sql", + "description": "对金融业务数据执行只读自然语言查询；低置信度时先确认。", + "input_schema": { + "type": "object", + "required": ["question", "auth_context"], + "properties": { + "question": {"type": "string"}, + "auth_context": { + "type": "object", + "required": ["roles", "customer_scope"], + "properties": { + "user_id": {"type": ["integer", "null"]}, + "roles": {"type": "array", "items": {"type": "string"}}, + "customer_scope": {"type": "string", "enum": ["self", "own_customers", "all"]}, + "allowed_domains": {"type": "array", "items": {"type": "string"}}, + }, + }, + "conversation_id": {"type": ["string", "null"]}, + "request_id": {"type": ["string", "null"]}, + "timezone": {"type": "string"}, + "confirmation": {"type": ["string", "null"]}, + }, + }, + } + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + demo = build_request("查询最近30天的行情收盘价", {"roles": ["advisor"], "customer_scope": "all"}) + print(json.dumps(query(demo, use_llm=False), ensure_ascii=False, indent=2, default=str)) diff --git a/pyproject.toml b/pyproject.toml index fe73c5a..e62170a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ include = ["app*"] [tool.pytest.ini_options] testpaths = ["tests"] asyncio_mode = "auto" +asyncio_default_fixture_loop_scope = "function" markers = ["integration: requires local database services"] [tool.ruff] diff --git a/tests/conftest.py b/tests/conftest.py index 860ce92..e2c9a45 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,11 @@ +import asyncio +import sys + import pytest +if sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + from app.core.contracts import AgentDefinition, CoreResult, ResolvedAgentConfig from app.service.agent.base import BaseAgent from app.service.agent.factory import AgentFactory diff --git a/tests/contract/test_financial_nl2sql_tool_contract.py b/tests/contract/test_financial_nl2sql_tool_contract.py new file mode 100644 index 0000000..929ed3a --- /dev/null +++ b/tests/contract/test_financial_nl2sql_tool_contract.py @@ -0,0 +1,10 @@ +from app.service.agent.bootstrap import get_agent_factory + + +def test_financial_nl2sql_tool_is_registered_as_read_only() -> None: + factory = get_agent_factory() + tool = factory._tool_executor.registry.get("query_financial_data") + assert tool.read_only is True + assert tool.required_permission == "financial:nl2sql:read" + assert "operator" in tool.allowed_roles + assert "advisor" in tool.allowed_roles diff --git a/tests/integration/test_offsite_fund_api.py b/tests/integration/test_offsite_fund_api.py new file mode 100644 index 0000000..8ec75c9 --- /dev/null +++ b/tests/integration/test_offsite_fund_api.py @@ -0,0 +1,424 @@ +import asyncio +from collections.abc import AsyncIterator +from pathlib import Path +from uuid import uuid4 + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import delete, func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.dependencies.auth import build_request_context +from app.api.dependencies.database import get_session +from app.core.contracts import RequestContext +from app.infrastructure.db import SessionFactory +from app.main import app +from app.model.audit import InteractionAudit +from app.model.offsite_fund import ( + OffsiteExecutionPlanTask, + OffsiteFundAttachment, + OffsiteFundDocument, + OffsiteFundMail, + OffsiteNotification, + OffsiteQueryRecord, + OffsiteRuleResult, +) +from app.service.offsite_smtp_adapter import SmtpSendResult + +TEST_TRACE_ID = "" + + +async def override_context() -> RequestContext: + return RequestContext( + user_id="1", + trace_id=TEST_TRACE_ID or str(uuid4()), + roles=("operator",), + permissions=("offsite:write",), + data_scope="all", + ) + + +async def override_session() -> AsyncIterator[AsyncSession]: + async with SessionFactory() as session: + yield session + + +@pytest.mark.integration +def test_offsite_recognized_mail_persists_workflow_and_notification() -> None: + global TEST_TRACE_ID + TEST_TRACE_ID = f"trace-offsite-{uuid4()}" + uid = f"offsite-{uuid4()}" + message_id = f"<{uuid4()}@integration.local>" + payload = { + "imap_uid": uid, + "message_id": message_id, + "sender": "15008108550@163.com", + "return_path": "15008108550@163.com", + "auth_result": {"spf": "pass", "dkim": "pass"}, + "eml_path": "mock/offsite.eml", + "attachments": [ + { + "filename": "申购申请单.pdf", + "file_hash": f"hash-{uuid4().hex}", + "media_type": "application/pdf", + "size_bytes": 2048, + "document_type": "subscription", + "ocr_text": "基金代码 000001 申购金额 10000", + "extracted_fields": { + "基金代码": "000001", + "基金名称": "测试基金", + "账户标识": "ACCT-001", + "投资者名称": "测试客户", + "申请编号": f"SUB-{uuid4().hex[:8]}", + "申请日期": "2026-09-10", + "代销机构": "测试代销", + "申购金额": "10000", + "金额单位": "元", + "最新净值": "1.0000", + "基金最新总份额": "1000000", + "申请前持有份额": "1000", + }, + "field_confidence": {"基金代码": "0.99", "申购金额": "0.98"}, + "page_evidence": {"基金代码": [1], "申购金额": [1]}, + } + ], + } + + app.dependency_overrides[build_request_context] = override_context + app.dependency_overrides[get_session] = override_session + mail_id = "" + task_id = "" + try: + with TestClient(app) as client: + response = client.post("/api/v1/offsite-fund/recognized-mails", json=payload) + assert response.status_code == 200 + body = response.json() + assert body["code"] == 0 + mail_id = body["data"]["mail_id"] + task_id = body["data"]["documents"][0]["task_id"] + assert body["data"]["documents"][0]["rule_results"] == { + "subscription_minimum_amount": "正常", + "subscription_holding_ratio": "正常", + "subscription_single_share_limit": "正常", + } + + duplicate = client.post("/api/v1/offsite-fund/recognized-mails", json=payload) + assert duplicate.status_code == 200 + assert duplicate.json()["data"] == {"mail_id": mail_id} + + confirm = client.post( + f"/api/v1/offsite-fund/documents/{task_id}/confirmations", + json={"decision": "确认正常", "operator_id": "operator-001"}, + ) + assert confirm.status_code == 200 + assert confirm.json()["code"] == 0 + + recalc = client.post( + "/api/v1/offsite-fund/settlement-statistics/recalculate", + json={"fund_code": "000001", "application_date": "2026-09-10"}, + ) + assert recalc.status_code == 200 + assert recalc.json()["data"]["subscription_amount_yuan"] == "10000.0000" + + notice = client.post( + f"/api/v1/offsite-fund/documents/{task_id}/notifications", + json={"notification_type": "settlement", "operator_id": "operator-001"}, + ) + assert notice.status_code == 200 + assert notice.json()["data"]["notification_id"] + + asyncio.run(_assert_offsite_rows(mail_id, task_id, TEST_TRACE_ID)) + finally: + asyncio.run(_cleanup_offsite_rows(mail_id, task_id, TEST_TRACE_ID)) + app.dependency_overrides.clear() + TEST_TRACE_ID = "" + + +@pytest.mark.integration +def test_offsite_trigger_nl2sql_uses_query_dict_adapter() -> None: + global TEST_TRACE_ID + TEST_TRACE_ID = f"trace-offsite-nl2sql-{uuid4()}" + uid = f"offsite-{uuid4()}" + message_id = f"<{uuid4()}@integration.local>" + payload = _subscription_payload(uid, message_id) + app.dependency_overrides[build_request_context] = override_context + app.dependency_overrides[get_session] = override_session + mail_id = "" + task_id = "" + try: + with TestClient(app) as client: + created = client.post("/api/v1/offsite-fund/recognized-mails", json=payload) + assert created.status_code == 200 + body = created.json() + mail_id = body["data"]["mail_id"] + task_id = body["data"]["documents"][0]["task_id"] + + response = client.post( + f"/api/tasks/{task_id}/trigger-agent-nl2sql", + json={"operator_id": "1", "manual_confirmed": True}, + ) + assert response.status_code == 200 + data = response.json()["data"] + assert data["task_id"] == task_id + assert len(data["queries"]) == 2 + assert {item["status"] for item in data["queries"]} <= { + "ready", "success", "need_confirmation", "rejected", "error", + } + + asyncio.run(_assert_nl2sql_rows(task_id)) + finally: + asyncio.run(_cleanup_offsite_rows(mail_id, task_id, TEST_TRACE_ID)) + app.dependency_overrides.clear() + TEST_TRACE_ID = "" + + +@pytest.mark.integration +def test_offsite_low_confidence_document_goes_to_recognition_exception() -> None: + global TEST_TRACE_ID + TEST_TRACE_ID = f"trace-offsite-low-confidence-{uuid4()}" + uid = f"offsite-{uuid4()}" + message_id = f"<{uuid4()}@integration.local>" + payload = _subscription_payload(uid, message_id) + payload["attachments"][0]["field_confidence"] = {"基金代码": "0.79", "申购金额": "0.99"} + app.dependency_overrides[build_request_context] = override_context + app.dependency_overrides[get_session] = override_session + mail_id = "" + task_id = "" + try: + with TestClient(app) as client: + response = client.post("/api/v1/offsite-fund/recognized-mails", json=payload) + assert response.status_code == 200 + body = response.json() + mail_id = body["data"]["mail_id"] + document = body["data"]["documents"][0] + task_id = document["task_id"] + assert document["status"] == "recognition_exception" + finally: + asyncio.run(_cleanup_offsite_rows(mail_id, task_id, TEST_TRACE_ID)) + app.dependency_overrides.clear() + TEST_TRACE_ID = "" + + +@pytest.mark.integration +def test_offsite_service_rejects_missing_permission() -> None: + async def no_permission_context() -> RequestContext: + return RequestContext( + user_id="1", + trace_id=f"trace-offsite-denied-{uuid4()}", + roles=("operator",), + permissions=(), + ) + + payload = _subscription_payload(f"offsite-{uuid4()}", f"<{uuid4()}@integration.local>") + app.dependency_overrides[build_request_context] = no_permission_context + app.dependency_overrides[get_session] = override_session + try: + with TestClient(app) as client: + response = client.post("/api/v1/offsite-fund/recognized-mails", json=payload) + assert response.status_code == 200 + assert response.json()["code"] == 403 + assert "缺少场外基金操作权限" in response.json()["message"] + finally: + app.dependency_overrides.clear() + + +@pytest.mark.integration +def test_offsite_mail_return_send_updates_notification_without_external_call( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + global TEST_TRACE_ID + TEST_TRACE_ID = f"trace-offsite-send-{uuid4()}" + uid = f"offsite-{uuid4()}" + message_id = f"<{uuid4()}@integration.local>" + payload = _subscription_payload(uid, message_id) + attachment_path = tmp_path / "申购申请单.pdf" + attachment_path.write_bytes(b"pdf-bytes") + payload["attachments"][0]["original_file_path"] = str(attachment_path) + + class FakeDryRunSender: + def __init__(self, _settings: object) -> None: + pass + + def send_reply(self, request: object) -> SmtpSendResult: + del request + return SmtpSendResult( + status="待发送", + dry_run=True, + provider_message_id=None, + failure_reason=None, + retry_count=0, + request_summary={"provider": "test"}, + ) + + monkeypatch.setattr("app.service.offsite_fund_service.OffsiteSmtpSender", FakeDryRunSender) + app.dependency_overrides[build_request_context] = override_context + app.dependency_overrides[get_session] = override_session + mail_id = "" + task_id = "" + notification_id = 0 + try: + with TestClient(app) as client: + created = client.post("/api/v1/offsite-fund/recognized-mails", json=payload) + assert created.status_code == 200 + data = created.json()["data"] + mail_id = data["mail_id"] + task_id = data["documents"][0]["task_id"] + + confirmed = client.post( + f"/api/v1/offsite-fund/documents/{task_id}/confirmations", + json={"decision": "确认正常", "operator_id": "operator-001"}, + ) + assert confirmed.status_code == 200 + + notice = client.post( + f"/api/v1/offsite-fund/documents/{task_id}/notifications", + json={"notification_type": "mail_return", "operator_id": "operator-001"}, + ) + assert notice.status_code == 200 + notification_id = int(notice.json()["data"]["notification_id"]) + + sent = client.post( + f"/api/v1/offsite-fund/notifications/{notification_id}/send", + json={ + "operator_id": "operator-001", + "operator_confirmed": True, + "final_content": "运营确认后的回复正文", + }, + ) + assert sent.status_code == 200 + assert sent.json()["data"] == { + "notification_id": str(notification_id), + "status": "待发送", + "dry_run": True, + "provider_message_id": None, + "failure_reason": None, + "retry_count": 0, + } + + asyncio.run(_assert_notification_pending(notification_id)) + finally: + asyncio.run(_cleanup_offsite_rows(mail_id, task_id, TEST_TRACE_ID)) + app.dependency_overrides.clear() + TEST_TRACE_ID = "" + + +def _subscription_payload(uid: str, message_id: str) -> dict[str, object]: + return { + "imap_uid": uid, + "message_id": message_id, + "sender": "15008108550@163.com", + "return_path": "15008108550@163.com", + "auth_result": {"spf": "pass", "dkim": "pass"}, + "eml_path": "mock/offsite.eml", + "attachments": [ + { + "filename": "申购申请单.pdf", + "file_hash": f"hash-{uuid4().hex}", + "original_file_path": "mock/申购申请单.pdf", + "media_type": "application/pdf", + "size_bytes": 2048, + "document_type": "subscription", + "ocr_text": "基金代码 000001 申购金额 10000", + "extracted_fields": { + "基金代码": "000001", + "基金名称": "测试基金", + "账户标识": "ACCT-001", + "投资者名称": "测试客户", + "申请编号": f"SUB-{uuid4().hex[:8]}", + "申请日期": "2026-09-10", + "代销机构": "测试代销", + "申购金额": "10000", + "金额单位": "元", + "最新净值": "1.0000", + "基金最新总份额": "1000000", + "申请前持有份额": "1000", + }, + "field_confidence": {"基金代码": "0.99", "申购金额": "0.98"}, + "page_evidence": {"基金代码": [1], "申购金额": [1]}, + } + ], + } + + +async def _assert_offsite_rows(mail_id: str, task_id: str, trace_id: str) -> None: + async with SessionFactory() as session: + assert await _count(session, OffsiteFundMail, OffsiteFundMail.mail_id == mail_id) == 1 + assert await _count( + session, OffsiteFundAttachment, OffsiteFundAttachment.mail_id == mail_id + ) == 1 + assert await _count( + session, OffsiteFundDocument, OffsiteFundDocument.task_id == task_id + ) == 1 + assert await _count(session, OffsiteRuleResult, OffsiteRuleResult.task_id == task_id) == 3 + assert await _count( + session, OffsiteExecutionPlanTask, OffsiteExecutionPlanTask.task_id == task_id + ) == 9 + assert await _count( + session, OffsiteNotification, OffsiteNotification.business_key == task_id + ) == 1 + assert await _count( + session, InteractionAudit, InteractionAudit.action_type.like("offsite.%") + ) >= 1 + + +async def _assert_nl2sql_rows(task_id: str) -> None: + async with SessionFactory() as session: + assert await _count(session, OffsiteQueryRecord, OffsiteQueryRecord.task_id == task_id) == 2 + tasks = (await session.execute(select(OffsiteExecutionPlanTask).where( + OffsiteExecutionPlanTask.task_id == task_id, + OffsiteExecutionPlanTask.stage == "查询", + OffsiteExecutionPlanTask.rule_code != "subscription_minimum_amount", + ))).scalars().all() + assert {task.status for task in tasks} <= {"已完成", "查询失败", "无法判断"} + assert all(task.output_json is not None for task in tasks) + + +async def _assert_notification_pending(notification_id: int) -> None: + async with SessionFactory() as session: + notice = await session.scalar(select(OffsiteNotification).where( + OffsiteNotification.id == notification_id + )) + assert notice is not None + assert notice.status == "待发送" + assert notice.provider_message_id is None + assert notice.failure_reason is None + assert notice.sent_at is None + + +async def _count(session: AsyncSession, model: type, criterion: object) -> int: + return int(await session.scalar(select(func.count()).select_from(model).where(criterion)) or 0) + + +async def _cleanup_offsite_rows(mail_id: str, task_id: str, trace_id: str) -> None: + if not mail_id and not task_id: + return + async with SessionFactory() as session, session.begin(): + if trace_id: + await session.execute( + delete(InteractionAudit).where( + InteractionAudit.detail["trace_id"].as_string() == trace_id + ) + ) + if task_id: + await session.execute( + delete(OffsiteNotification).where(OffsiteNotification.business_key == task_id) + ) + await session.execute( + delete(OffsiteQueryRecord).where(OffsiteQueryRecord.task_id == task_id) + ) + await session.execute( + delete(OffsiteRuleResult).where(OffsiteRuleResult.task_id == task_id) + ) + await session.execute( + delete(OffsiteExecutionPlanTask).where( + OffsiteExecutionPlanTask.task_id == task_id + ) + ) + await session.execute( + delete(OffsiteFundDocument).where(OffsiteFundDocument.task_id == task_id) + ) + if mail_id: + await session.execute( + delete(OffsiteFundAttachment).where(OffsiteFundAttachment.mail_id == mail_id) + ) + await session.execute(delete(OffsiteFundMail).where(OffsiteFundMail.mail_id == mail_id)) diff --git a/tests/integration/test_offsite_notification_send.py b/tests/integration/test_offsite_notification_send.py new file mode 100644 index 0000000..210a7fb --- /dev/null +++ b/tests/integration/test_offsite_notification_send.py @@ -0,0 +1,275 @@ +from __future__ import annotations + +import asyncio +from collections.abc import AsyncIterator +from pathlib import Path +from uuid import uuid4 + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import delete, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.dependencies.auth import build_request_context +from app.api.dependencies.database import get_session +from app.core.contracts import RequestContext +from app.infrastructure.db import SessionFactory +from app.main import app +from app.model.audit import InteractionAudit +from app.model.offsite_fund import ( + OffsiteExecutionPlanTask, + OffsiteFundAttachment, + OffsiteFundDocument, + OffsiteFundMail, + OffsiteNotification, + OffsiteQueryRecord, + OffsiteRuleResult, +) +from app.service.offsite_smtp_adapter import SmtpSendResult + +TRACE_ID = "" + + +async def override_context() -> RequestContext: + return RequestContext( + user_id="1", + trace_id=TRACE_ID or str(uuid4()), + roles=("operator",), + permissions=("offsite:write",), + data_scope="all", + ) + + +async def override_session() -> AsyncIterator[AsyncSession]: + async with SessionFactory() as session: + yield session + + +@pytest.mark.integration +def test_successful_notification_send_is_idempotent( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + global TRACE_ID + TRACE_ID = f"trace-send-success-{uuid4()}" + payload = _subscription_payload() + attachment_path = tmp_path / "申购申请单.pdf" + attachment_path.write_bytes(b"original-pdf") + payload["attachments"][0]["original_file_path"] = str(attachment_path) + sent_requests: list[object] = [] + + class FakeSuccessSender: + def __init__(self, _settings: object) -> None: + pass + + def send_reply(self, request: object) -> SmtpSendResult: + sent_requests.append(request) + return SmtpSendResult( + status="发送成功", + dry_run=False, + provider_message_id="provider-test-001", + failure_reason=None, + retry_count=0, + request_summary={"provider": "test"}, + ) + + monkeypatch.setattr("app.service.offsite_fund_service.OffsiteSmtpSender", FakeSuccessSender) + app.dependency_overrides[build_request_context] = override_context + app.dependency_overrides[get_session] = override_session + mail_id = "" + task_id = "" + notification_id = 0 + try: + with TestClient(app) as client: + created = client.post("/api/v1/offsite-fund/recognized-mails", json=payload) + data = created.json()["data"] + mail_id = data["mail_id"] + task_id = data["documents"][0]["task_id"] + _confirm_and_create_notice(client, task_id) + notice = client.post( + f"/api/v1/offsite-fund/documents/{task_id}/notifications", + json={"notification_type": "mail_return", "operator_id": "operator-001"}, + ) + notification_id = int(notice.json()["data"]["notification_id"]) + + first = client.post( + f"/api/v1/offsite-fund/notifications/{notification_id}/send", + json={"operator_id": "operator-001", "operator_confirmed": True}, + ) + second = client.post( + f"/api/v1/offsite-fund/notifications/{notification_id}/send", + json={"operator_id": "operator-001", "operator_confirmed": True}, + ) + + assert first.json()["data"]["status"] == "发送成功" + assert first.json()["data"]["provider_message_id"] == "provider-test-001" + assert second.json()["data"]["status"] == "发送成功" + assert len(sent_requests) == 1 + asyncio.run(_assert_notification(notification_id, "发送成功", "provider-test-001")) + asyncio.run(_assert_mail_status(mail_id, "normal_return_sent")) + finally: + asyncio.run(_cleanup(mail_id, task_id, notification_id, TRACE_ID)) + app.dependency_overrides.clear() + TRACE_ID = "" + + +@pytest.mark.integration +def test_failed_notification_send_persists_failure_and_retry_count( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + global TRACE_ID + TRACE_ID = f"trace-send-failure-{uuid4()}" + payload = _subscription_payload() + attachment_path = tmp_path / "申购申请单.pdf" + attachment_path.write_bytes(b"original-pdf") + payload["attachments"][0]["original_file_path"] = str(attachment_path) + + class FakeFailureSender: + def __init__(self, _settings: object) -> None: + pass + + def send_reply(self, request: object) -> SmtpSendResult: + del request + return SmtpSendResult( + status="发送失败", + dry_run=False, + provider_message_id=None, + failure_reason="SMTPException", + retry_count=1, + request_summary={"provider": "test"}, + ) + + monkeypatch.setattr("app.service.offsite_fund_service.OffsiteSmtpSender", FakeFailureSender) + app.dependency_overrides[build_request_context] = override_context + app.dependency_overrides[get_session] = override_session + mail_id = "" + task_id = "" + notification_id = 0 + try: + with TestClient(app) as client: + created = client.post("/api/v1/offsite-fund/recognized-mails", json=payload) + data = created.json()["data"] + mail_id = data["mail_id"] + task_id = data["documents"][0]["task_id"] + _confirm_and_create_notice(client, task_id) + notice = client.post( + f"/api/v1/offsite-fund/documents/{task_id}/notifications", + json={"notification_type": "mail_return", "operator_id": "operator-001"}, + ) + notification_id = int(notice.json()["data"]["notification_id"]) + sent = client.post( + f"/api/v1/offsite-fund/notifications/{notification_id}/send", + json={"operator_id": "operator-001", "operator_confirmed": True}, + ) + + assert sent.json()["data"]["status"] == "发送失败" + assert sent.json()["data"]["failure_reason"] == "SMTPException" + assert sent.json()["data"]["retry_count"] == 1 + asyncio.run(_assert_notification(notification_id, "发送失败", None)) + finally: + asyncio.run(_cleanup(mail_id, task_id, notification_id, TRACE_ID)) + app.dependency_overrides.clear() + TRACE_ID = "" + + +def _confirm_and_create_notice(client: TestClient, task_id: str) -> None: + confirmed = client.post( + f"/api/v1/offsite-fund/documents/{task_id}/confirmations", + json={"decision": "确认正常", "operator_id": "operator-001"}, + ) + assert confirmed.status_code == 200 + + +def _subscription_payload() -> dict[str, object]: + suffix = uuid4().hex[:8] + return { + "imap_uid": f"offsite-{suffix}", + "message_id": f"<{suffix}@integration.local>", + "sender": "15008108550@163.com", + "return_path": "15008108550@163.com", + "auth_result": {"spf": "pass", "dkim": "pass"}, + "eml_path": "mock/offsite.eml", + "attachments": [ + { + "filename": "申购申请单.pdf", + "file_hash": f"hash-{uuid4().hex}", + "original_file_path": "mock/申购申请单.pdf", + "media_type": "application/pdf", + "size_bytes": 2048, + "document_type": "subscription", + "ocr_text": "基金代码 000001 申购金额 10000", + "extracted_fields": { + "基金代码": "000001", + "基金名称": "测试基金", + "账户标识": "ACCT-001", + "投资者名称": "测试客户", + "申请编号": f"SUB-{suffix}", + "申请日期": "2026-09-10", + "代销机构": "测试代销", + "申购金额": "10000", + "金额单位": "元", + "最新净值": "1.0000", + "基金最新总份额": "1000000", + "申请前持有份额": "1000", + }, + "field_confidence": {"基金代码": "0.99", "申购金额": "0.98"}, + "page_evidence": {"基金代码": [1], "申购金额": [1]}, + } + ], + } + + +async def _assert_notification( + notification_id: int, status: str, provider_message_id: str | None +) -> None: + async with SessionFactory() as session: + notice = await session.scalar(select(OffsiteNotification).where( + OffsiteNotification.id == notification_id + )) + assert notice is not None + assert notice.status == status + assert notice.provider_message_id == provider_message_id + if status == "发送成功": + assert notice.sent_at is not None + else: + assert notice.sent_at is None + + +async def _assert_mail_status(mail_id: str, status: str) -> None: + async with SessionFactory() as session: + mail = await session.scalar(select(OffsiteFundMail).where( + OffsiteFundMail.mail_id == mail_id + )) + assert mail is not None + assert mail.status == status + + +async def _cleanup(mail_id: str, task_id: str, notification_id: int, trace_id: str) -> None: + async with SessionFactory() as session, session.begin(): + if trace_id: + await session.execute(delete(InteractionAudit).where( + InteractionAudit.detail["trace_id"].as_string() == trace_id + )) + if notification_id: + await session.execute(delete(OffsiteNotification).where( + OffsiteNotification.id == notification_id + )) + if task_id: + await session.execute(delete(OffsiteQueryRecord).where( + OffsiteQueryRecord.task_id == task_id + )) + await session.execute(delete(OffsiteRuleResult).where( + OffsiteRuleResult.task_id == task_id + )) + await session.execute(delete(OffsiteExecutionPlanTask).where( + OffsiteExecutionPlanTask.task_id == task_id + )) + await session.execute(delete(OffsiteFundDocument).where( + OffsiteFundDocument.task_id == task_id + )) + if mail_id: + await session.execute(delete(OffsiteFundAttachment).where( + OffsiteFundAttachment.mail_id == mail_id + )) + await session.execute(delete(OffsiteFundMail).where( + OffsiteFundMail.mail_id == mail_id + )) diff --git a/tests/unit/service/test_financial_nl2sql_agent_integration.py b/tests/unit/service/test_financial_nl2sql_agent_integration.py new file mode 100644 index 0000000..1437f14 --- /dev/null +++ b/tests/unit/service/test_financial_nl2sql_agent_integration.py @@ -0,0 +1,104 @@ +from typing import Any, cast + +import pytest +from _pytest.monkeypatch import MonkeyPatch + +from app.core.contracts import ( + AgentDefinition, + AgentRequest, + AgentResult, + CoreResult, + RecalledMemory, + RequestContext, + ResolvedAgentConfig, +) +from app.core.nl2sql_contracts import FinancialNL2SQLInput +from app.service.agent.base import BaseAgent +from app.service.agent.factory import AgentFactory +from app.service.financial_nl2sql_service import query_financial_data_tool +from app.service.tool_executor import ToolDefinition, ToolExecutor, ToolRegistry + + +class ExampleBusinessAgent(BaseAgent): + definition = AgentDefinition( + agent_type="example_business", + version="1.0.0", + allowed_tools=("query_financial_data",), + allowed_roles=("advisor", "operator"), + allowed_portals=("api",), + supported_intents=("financial_query",), + ) + + async def handle(self, request: AgentRequest, context: RequestContext) -> CoreResult: + result = cast(dict[str, Any], await self.call_tool( + "query_financial_data", + {"question": request.message, "dry_run": True}, + intent="financial_query", + context=context, + )) + return CoreResult(text=result["message"]) + + +@pytest.mark.asyncio +async def test_external_agent_can_call_financial_tool(monkeypatch: MonkeyPatch) -> None: + registry = ToolRegistry() + registry.register(ToolDefinition( + name="query_financial_data", + input_model=FinancialNL2SQLInput, + handler=cast(Any, query_financial_data_tool), + required_permission="financial:nl2sql:read", + allowed_roles=("advisor", "operator"), + )) + executor = ToolExecutor(registry) + monkeypatch.setattr(executor, "_audit", lambda *args: __import__("asyncio").sleep(0)) + + class TestGovernance: + async def resolve( + self, definition: AgentDefinition, context: RequestContext + ) -> ResolvedAgentConfig: + del definition, context + return ResolvedAgentConfig( + config_version="test", + prompt_version="test", + model_endpoint="test", + allowed_tools_by_intent={ + "financial_query": ("query_financial_data",) + }, + ) + + async def recall(self, context: RequestContext) -> tuple[RecalledMemory, ...]: + del context + return () + + async def review( + self, + result: AgentResult, + context: RequestContext, + config: ResolvedAgentConfig, + memories: tuple[RecalledMemory, ...], + ) -> AgentResult: + del context, config, memories + return result + + factory = AgentFactory(TestGovernance(), tool_executor=executor) + factory.register( + ExampleBusinessAgent.definition, + lambda _context: ExampleBusinessAgent(ExampleBusinessAgent.definition), + ) + context = RequestContext( + user_id="1", + trace_id="trace-agent-integration", + roles=("advisor",), + permissions=("agent:run", "financial:nl2sql:read"), + data_scope="all", + ) + agent = factory.create("example_business", context) + await agent.resolve_config(context) + request = AgentRequest( + agent_type="example_business", + message="查询159511近30天行情收盘价", + session_id="s", + idempotency_key="integration-key-123456", + ) + result = await agent.handle(request, context) + assert result.text == "SQL 已生成并通过只读校验" diff --git a/tests/unit/service/test_financial_nl2sql_golden_cases.py b/tests/unit/service/test_financial_nl2sql_golden_cases.py new file mode 100644 index 0000000..2d16bed --- /dev/null +++ b/tests/unit/service/test_financial_nl2sql_golden_cases.py @@ -0,0 +1,100 @@ +import pytest + +from app.core.contracts import RequestContext +from app.core.nl2sql_contracts import FinancialNL2SQLInput +from app.service.financial_nl2sql_service import FinancialNL2SQLService + + +def case( + question: str, status: str, intent: str, tables: set[str] +) -> tuple[str, str, str, set[str]]: + return question, status, intent, tables + + +GOLDEN_CASES = [ + case("查询159511近30天行情收盘价", "ready", "market_price_query", + {"fin_market_price", "fin_product"}), + case("查询588890最近7天行情", "ready", "market_price_query", + {"fin_market_price", "fin_product"}), + case("查看159948近90天收盘价", "ready", "market_price_query", + {"fin_market_price", "fin_product"}), + case("查询159511近30天净值", "ready", "nav_history_query", + {"fin_nav_history", "fin_product"}), + case("查看588890最近7天基金净值", "ready", "nav_history_query", + {"fin_nav_history", "fin_product"}), + case("查询客户当前持仓市值", "ready", "holding_query", + {"fin_holding", "fin_customer_profile", "fin_product"}), + case("查询159511客户持仓盈亏", "ready", "holding_query", + {"fin_holding", "fin_customer_profile", "fin_product"}), + case("统计客户成交交易金额", "ready", "transaction_query", + {"fin_transaction", "fin_customer_profile"}), + case("查询159511成交明细", "ready", "transaction_query", + {"fin_transaction", "fin_product"}), + case("汇总最近30天交易金额和成交数量", "ready", "transaction_query", + {"fin_transaction"}), + case("查询客户委托订单状态", "ready", "order_query", + {"fin_sim_order", "fin_customer_profile"}), + case("查看159511委托状态", "ready", "order_query", + {"fin_sim_order", "fin_product"}), + case("查询客户账户余额", "ready", "account_query", + {"fin_sim_account", "fin_customer_profile"}), + case("查看客户可用现金", "ready", "account_query", + {"fin_sim_account", "fin_customer_profile"}), + case("查询近7天资金流水变化", "ready", "cash_ledger_query", + {"fin_cash_ledger", "fin_sim_account"}), + case("汇总最近30天资金变动", "ready", "cash_ledger_query", + {"fin_cash_ledger", "fin_sim_account"}), + case("查询客户现金流水", "ready", "cash_ledger_query", + {"fin_cash_ledger", "fin_sim_account"}), + case("查询159511费率规则", "ready", "fee_rule_query", + {"fin_fee_rule", "fin_product"}), + case("查看基金费用规则", "ready", "fee_rule_query", + {"fin_fee_rule", "fin_product"}), + case("查询买入卖出费率", "ready", "fee_rule_query", {"fin_fee_rule"}), + case("查询客户风险测评", "ready", "customer_risk_query", + {"fin_risk_assessment"}), + case("统计客户风险等级数量", "ready", "customer_risk_query", + {"fin_customer_profile"}), + case("查看客户画像", "ready", "customer_risk_query", {"fin_customer_profile"}), + case("查询近30天客户风险测评", "ready", "customer_risk_query", + {"fin_risk_assessment"}), + case("查询对客内容审核状态", "need_confirmation", "client_content_query", + {"client_facing_content"}), + case("查看客户内容发布记录", "need_confirmation", "client_content_query", + {"client_facing_content"}), + case("截至某日客户当前持仓市值", "rejected", "holding_query", {"fin_holding"}), + case("截至历史时点账户余额", "rejected", "account_query", {"fin_sim_account"}), + case("帮我看看这个情况", "need_confirmation", "unknown", {"fin_customer_profile"}), + case("这个客户怎么样", "need_confirmation", "unknown", {"fin_customer_profile"}), +] + + +def context() -> RequestContext: + return RequestContext( + user_id="1", + trace_id="trace-golden", + roles=("operator",), + permissions=("financial:nl2sql:read",), + data_scope="all", + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("question", "status", "intent", "tables"), + GOLDEN_CASES, +) +async def test_financial_nl2sql_golden_cases( + question: str, status: str, intent: str, tables: set[str] +) -> None: + result = await FinancialNL2SQLService().query( + FinancialNL2SQLInput(question=question, dry_run=True), + context(), + ) + assert result["status"] == status + assert result["query_plan"]["intent"] == intent + assert tables.issubset(set(result["query_plan"]["tables"])) + + +def test_golden_case_count_reaches_mvp_threshold() -> None: + assert len(GOLDEN_CASES) >= 30 diff --git a/tests/unit/service/test_financial_nl2sql_service.py b/tests/unit/service/test_financial_nl2sql_service.py new file mode 100644 index 0000000..9c57b13 --- /dev/null +++ b/tests/unit/service/test_financial_nl2sql_service.py @@ -0,0 +1,72 @@ +import pytest + +from app.core.contracts import RequestContext +from app.core.nl2sql_contracts import FinancialNL2SQLInput +from app.service.financial_nl2sql_service import FinancialNL2SQLService + + +def context(**updates): + base = RequestContext( + user_id="1", + trace_id="trace-nl2sql", + roles=("advisor",), + permissions=("financial:nl2sql:read",), + data_scope="all", + ) + return base.model_copy(update=updates) + + +@pytest.mark.asyncio +async def test_generates_read_only_market_sql_with_product_filter() -> None: + result = await FinancialNL2SQLService().query( + FinancialNL2SQLInput(question="查询159511近30天行情收盘价", dry_run=True), + context(), + ) + assert result["status"] == "ready" + assert result["sql"].startswith("SELECT ") + assert "fin_market_price" in result["sql"] + assert "DROP" not in result["sql"] + assert result["parameters"]["filter_0"] == "159511" + assert result["audit"]["permission_check"]["status"] == "passed" + + +@pytest.mark.asyncio +async def test_low_confidence_question_requires_confirmation() -> None: + result = await FinancialNL2SQLService().query( + FinancialNL2SQLInput(question="帮我看看这个情况", dry_run=True), + context(), + ) + assert result["status"] == "need_confirmation" + assert result["query_plan"]["confidence"] < 0.85 + + +@pytest.mark.asyncio +async def test_as_of_query_rejects_current_snapshot_tables() -> None: + result = await FinancialNL2SQLService().query( + FinancialNL2SQLInput(question="截至某日客户当前持仓市值", dry_run=True), + context(), + ) + assert result["status"] == "rejected" + assert "历史版本" in result["message"] + + +@pytest.mark.asyncio +async def test_customer_scope_is_injected_for_non_all_scope() -> None: + result = await FinancialNL2SQLService().query( + FinancialNL2SQLInput(question="查询客户账户余额", dry_run=True), + context(data_scope="own_customers", customer_ids=("7", "8")), + ) + assert "customer_id IN" in result["sql"] + assert result["parameters"]["scope_customer_0"] == 7 + assert result["parameters"]["scope_customer_1"] == 8 + + +@pytest.mark.asyncio +async def test_cash_ledger_is_in_nl2sql_scope() -> None: + result = await FinancialNL2SQLService().query( + FinancialNL2SQLInput(question="查询近7天资金流水变化", dry_run=True), + context(), + ) + assert result["status"] == "ready" + assert "fin_cash_ledger" in result["sql"] + assert result["query_plan"]["intent"] == "cash_ledger_query" diff --git a/tests/unit/service/test_offsite_document_recognition_adapter.py b/tests/unit/service/test_offsite_document_recognition_adapter.py new file mode 100644 index 0000000..78bd548 --- /dev/null +++ b/tests/unit/service/test_offsite_document_recognition_adapter.py @@ -0,0 +1,190 @@ +from __future__ import annotations + +import json + +import httpx +import pytest + +from app.core.config import Settings +from app.service.offsite_document_recognition_adapter import ( + OffsiteDocumentRecognitionAdapter, + RecognitionSourceFile, +) + + +@pytest.mark.asyncio +async def test_default_recognition_uses_mock_without_external_call() -> None: + source = RecognitionSourceFile( + filename="申购申请单.txt", + media_type="text/plain", + file_hash="hash-subscription", + payload=( + "基金代码：000001 基金名称：测试基金 账户标识：ACCT-001 " + "投资者名称：测试客户 申请编号：SUB-001 申请日期：2026-09-10 " + "代销机构：测试代销 申购金额：10000 金额单位：元" + ).encode(), + ) + + result = await OffsiteDocumentRecognitionAdapter(_settings()).recognize(source) + + assert result.document_type == "subscription" + assert result.ocr_status == "mock" + assert result.llm_status == "mock" + assert result.extracted_fields["申请编号"] == "SUB-001" + assert result.missing_fields == () + + +def test_health_check_reports_missing_external_config_only_when_enabled() -> None: + disabled = OffsiteDocumentRecognitionAdapter(_settings()).health_check() + assert disabled["ocr"]["status"] == "disabled" + assert disabled["deepseek"]["status"] == "disabled" + + enabled = OffsiteDocumentRecognitionAdapter( + _settings( + offsite_ocr_enabled=True, + offsite_deepseek_enabled=True, + offsite_aliyun_ocr_endpoint="", + offsite_aliyun_access_key_id="", + offsite_aliyun_access_key_secret="", + offsite_deepseek_api_key="", + ) + ).health_check() + assert enabled["ocr"]["status"] == "misconfigured" + assert enabled["deepseek"]["status"] == "misconfigured" + assert "OFFSITE_ALIYUN_OCR_ENDPOINT" in enabled["ocr"]["missing"] + assert "OFFSITE_DEEPSEEK_API_KEY" in enabled["deepseek"]["missing"] + + +@pytest.mark.asyncio +async def test_enabled_recognition_calls_ocr_and_deepseek_with_masked_request_body() -> None: + requests: list[httpx.Request] = [] + + async def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + if str(request.url) == "https://ocr.example.local/recognize": + return httpx.Response( + 200, + json={ + "data": { + "ocr_text": "基金代码：000001 申请编号：SUB-001 申购金额：10000", + "tables": [{"rows": 1}], + "page_evidence": {"基金代码": [{"page": 1}]}, + } + }, + ) + if str(request.url) == "https://api.deepseek.test/chat/completions": + return httpx.Response( + 200, + json={ + "choices": [ + { + "message": { + "content": json.dumps( + { + "document_type": "subscription", + "extracted_fields": { + "基金代码": "000001", + "基金名称": "测试基金", + "账户标识": "ACCT-001", + "投资者名称": "测试客户", + "申请编号": "SUB-001", + "申请日期": "2026-09-10", + "代销机构": "测试代销", + "申购金额": "10000", + "金额单位": "元", + }, + "field_confidence": {"基金代码": "0.99"}, + "missing_fields": [], + "low_confidence_fields": [], + "page_evidence": {"基金代码": [{"page": 1}]}, + }, + ensure_ascii=False, + ) + } + } + ] + }, + ) + return httpx.Response(404) + + client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + adapter = OffsiteDocumentRecognitionAdapter( + _settings( + offsite_ocr_enabled=True, + offsite_aliyun_ocr_endpoint="https://ocr.example.local/recognize", + offsite_aliyun_access_key_id="aliyun-id", + offsite_aliyun_access_key_secret="aliyun-secret", + offsite_deepseek_enabled=True, + offsite_deepseek_base_url="https://api.deepseek.test", + offsite_deepseek_api_key="deepseek-secret", + ), + client=client, + ) + + try: + result = await adapter.recognize( + RecognitionSourceFile( + filename="申购申请单.pdf", + media_type="application/pdf", + file_hash="hash-subscription", + payload=b"pdf bytes", + ) + ) + finally: + await client.aclose() + + assert result.document_type == "subscription" + assert result.ocr_status == "success" + assert result.llm_status == "success" + assert result.extracted_fields["基金代码"] == "000001" + assert result.field_confidence["基金代码"].to_eng_string() == "0.99" + assert result.missing_fields == () + assert requests[0].headers["x-acs-accesskey-id"] == "aliyun-id" + assert requests[1].headers["authorization"] == "Bearer deepseek-secret" + assert b"aliyun-secret" not in requests[0].content + assert b"deepseek-secret" not in requests[1].content + + +@pytest.mark.asyncio +async def test_deepseek_error_falls_back_to_mock_with_error_status() -> None: + async def handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response(503) + + client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + adapter = OffsiteDocumentRecognitionAdapter( + _settings( + offsite_deepseek_enabled=True, + offsite_deepseek_base_url="https://api.deepseek.test", + offsite_deepseek_api_key="deepseek-secret", + ), + client=client, + ) + + try: + result = await adapter.recognize( + RecognitionSourceFile( + filename="赎回申请单.txt", + media_type="text/plain", + file_hash="hash-redemption", + payload="基金代码：000001 赎回份额：200".encode(), + ) + ) + finally: + await client.aclose() + + assert result.document_type == "redemption" + assert result.llm_status == "error" + assert result.error_message == "HTTPStatusError" + + +def _settings(**updates: object) -> Settings: + values: dict[str, object] = { + "jwt_issuer": "jr-local", + "jwt_audience": "jr-agent-platform", + "mysql_dsn": "sqlite+aiosqlite:///test.db", + "redis_url": "redis://127.0.0.1:6379/0", + "milvus_uri": "http://127.0.0.1:19530", + "neo4j_uri": "bolt://127.0.0.1:7687", + } + values.update(updates) + return Settings(**values) diff --git a/tests/unit/service/test_offsite_fund_rules.py b/tests/unit/service/test_offsite_fund_rules.py new file mode 100644 index 0000000..379b1a0 --- /dev/null +++ b/tests/unit/service/test_offsite_fund_rules.py @@ -0,0 +1,56 @@ +from decimal import Decimal + +from app.service.offsite_fund_rules import OffsiteFundRuleEngine, normalize_amount_yuan + + +def test_subscription_minimum_amount_boundary() -> None: + engine = OffsiteFundRuleEngine() + below = engine.check_subscription( + amount_yuan=Decimal("0.99"), nav=None, total_fund_shares=None, + before_holding_shares=None) + equal = engine.check_subscription( + amount_yuan=Decimal("1.00"), nav=None, total_fund_shares=None, + before_holding_shares=None) + above = engine.check_subscription( + amount_yuan=Decimal("1.01"), nav=None, total_fund_shares=None, + before_holding_shares=None) + assert below[0].result == "异常" + assert equal[0].result == "异常" + assert above[0].result == "正常" + + +def test_subscription_ratio_and_share_limit_boundary() -> None: + engine = OffsiteFundRuleEngine() + normal = engine.check_subscription( + amount_yuan=Decimal("100"), nav=Decimal("1"), total_fund_shares=Decimal("1000"), + before_holding_shares=Decimal("100")) + abnormal = engine.check_subscription( + amount_yuan=Decimal("101"), nav=Decimal("1"), total_fund_shares=Decimal("1000"), + before_holding_shares=Decimal("100")) + assert {item.rule_code: item.result for item in normal}["subscription_holding_ratio"] == "正常" + assert {item.rule_code: item.result for item in abnormal}[ + "subscription_holding_ratio"] == "异常" + assert {item.rule_code: item.result for item in abnormal}[ + "subscription_single_share_limit"] == "异常" + + +def test_redemption_ratio_and_available_quantity_boundary() -> None: + engine = OffsiteFundRuleEngine() + normal = engine.check_redemption( + redemption_shares=Decimal("200"), total_fund_shares=Decimal("1000"), + available_quantity=Decimal("200")) + abnormal = engine.check_redemption( + redemption_shares=Decimal("201"), total_fund_shares=Decimal("1000"), + available_quantity=Decimal("200")) + assert {item.rule_code: item.result for item in normal}["redemption_large_ratio"] == "正常" + assert {item.rule_code: item.result for item in normal}[ + "redemption_available_quantity"] == "正常" + assert {item.rule_code: item.result for item in abnormal}["redemption_large_ratio"] == "异常" + assert {item.rule_code: item.result for item in abnormal}[ + "redemption_available_quantity"] == "异常" + + +def test_normalize_amount_keeps_original_unit_semantics() -> None: + assert normalize_amount_yuan("2.50", "万元") == Decimal("25000.00") + assert normalize_amount_yuan("2.50", "元") == Decimal("2.50") + assert normalize_amount_yuan("2.50", "美元") is None diff --git a/tests/unit/service/test_offsite_mail_adapter.py b/tests/unit/service/test_offsite_mail_adapter.py new file mode 100644 index 0000000..bb6f66c --- /dev/null +++ b/tests/unit/service/test_offsite_mail_adapter.py @@ -0,0 +1,182 @@ +from __future__ import annotations + +import os +from datetime import datetime +from email.message import EmailMessage +from pathlib import Path + +from app.core.config import Settings +from app.service.offsite_mail_adapter import ( + OffsiteImapReceiver, + OffsiteMailStorage, + RawMailAttachment, + RawMailMessage, +) + + +class FakeImapConnection: + def __init__(self, messages: dict[str, bytes] | None = None) -> None: + self.messages = messages or {} + self.noop_count = 0 + self.logout_count = 0 + self.selected: list[tuple[str, bool]] = [] + self.uid_calls: list[tuple[str, tuple[object, ...]]] = [] + + def login(self, user: str, password: str) -> tuple[str, list[bytes]]: + return "OK", [b"LOGIN completed"] + + def select(self, mailbox: str, readonly: bool = False) -> tuple[str, list[bytes]]: + self.selected.append((mailbox, readonly)) + return "OK", [b"2"] + + def uid(self, command: str, *args: object) -> tuple[str, list[object]]: + self.uid_calls.append((command, args)) + if command == "search": + return "OK", [b" ".join(uid.encode("ascii") for uid in self.messages)] + if command == "fetch": + uid = str(args[0]) + return "OK", [(b"RFC822", self.messages[uid]), b")"] + return "NO", [] + + def noop(self) -> tuple[str, list[bytes]]: + self.noop_count += 1 + return "OK", [b"NOOP completed"] + + def logout(self) -> tuple[str, list[bytes]]: + self.logout_count += 1 + return "BYE", [b"LOGOUT completed"] + + +def test_health_check_returns_disabled_when_imap_switch_is_off() -> None: + connection = FakeImapConnection() + receiver = OffsiteImapReceiver(_settings(offsite_imap_enabled=False), connection) + + assert receiver.health_check() == {"status": "disabled", "message": "场外 IMAP 未启用"} + assert connection.noop_count == 0 + + +def test_health_check_reports_missing_required_imap_config() -> None: + connection = FakeImapConnection() + receiver = OffsiteImapReceiver( + _settings( + offsite_imap_enabled=True, + offsite_imap_host="", + offsite_imap_username="", + offsite_imap_password="", + ), + connection, + ) + + result = receiver.health_check() + + assert result["status"] == "misconfigured" + assert result["missing"] == [ + "OFFSITE_IMAP_HOST", + "OFFSITE_IMAP_USERNAME", + "OFFSITE_IMAP_PASSWORD", + ] + assert connection.noop_count == 0 + + +def test_fetch_since_selects_inbox_and_filters_sender_whitelist() -> None: + accepted = _raw_email("15008108550@163.com", "") + rejected = _raw_email("blocked@example.com", "") + connection = FakeImapConnection({"5": accepted, "6": rejected}) + receiver = OffsiteImapReceiver(_settings(offsite_imap_enabled=True), connection) + + messages = receiver.fetch_since("4", limit=20) + + assert connection.selected == [("INBOX", True)] + assert ("search", (None, "UID 5:*")) in connection.uid_calls + assert ("fetch", ("5", "(RFC822)")) in connection.uid_calls + assert ("fetch", ("6", "(RFC822)")) in connection.uid_calls + assert len(messages) == 1 + assert messages[0].imap_uid == "5" + assert messages[0].message_id == "" + assert messages[0].sender == "15008108550@163.com" + assert messages[0].return_path == "15008108550@163.com" + assert messages[0].attachments[0].filename == "申购申请单.pdf" + assert messages[0].auth_result["dkim_signature_present"] is True + + +def test_mail_storage_saves_eml_and_attachments_without_overwriting(tmp_path: Path) -> None: + storage = OffsiteMailStorage(tmp_path) + mail = _raw_mail_message( + raw_message=b"From: first@example.com\r\n\r\nfirst", + attachment_payload=b"first attachment", + ) + + first = storage.save(mail) + second = storage.save( + _raw_mail_message( + raw_message=b"From: second@example.com\r\n\r\nsecond", + attachment_payload=b"second attachment", + ) + ) + + eml_path = Path(first.eml_path) + first_attachment_path = Path(first.attachments[0].original_file_path) + second_attachment_path = Path(second.attachments[0].original_file_path) + assert eml_path.read_bytes() == b"From: first@example.com\r\n\r\nfirst" + assert first_attachment_path.read_bytes() == b"first attachment" + assert second_attachment_path.read_bytes() == b"second attachment" + assert first_attachment_path != second_attachment_path + assert first.attachments[0].file_hash != second.attachments[0].file_hash + + for path in (eml_path, first_attachment_path, second_attachment_path): + os.chmod(path, 0o666) + + +def _settings(**updates: object) -> Settings: + values: dict[str, object] = { + "jwt_issuer": "jr-local", + "jwt_audience": "jr-agent-platform", + "mysql_dsn": "sqlite+aiosqlite:///test.db", + "redis_url": "redis://127.0.0.1:6379/0", + "milvus_uri": "http://127.0.0.1:19530", + "neo4j_uri": "bolt://127.0.0.1:7687", + "offsite_imap_host": "imap.example.local", + "offsite_imap_username": "15273589815@163.com", + "offsite_imap_password": "test-auth-code", + "offsite_allowed_senders": ("15008108550@163.com",), + } + values.update(updates) + return Settings(**values) + + +def _raw_email(sender: str, message_id: str) -> bytes: + message = EmailMessage() + message["From"] = sender + message["To"] = "15273589815@163.com" + message["Message-ID"] = message_id + message["Return-Path"] = sender + message["Authentication-Results"] = "mx.example.local; spf=pass" + message["Received-SPF"] = "pass" + message["DKIM-Signature"] = "v=1; a=rsa-sha256; b=test" + message.set_content("场外基金业务邮件") + message.add_attachment( + b"pdf-bytes", + maintype="application", + subtype="pdf", + filename="申购申请单.pdf", + ) + return message.as_bytes() + + +def _raw_mail_message(raw_message: bytes, attachment_payload: bytes) -> RawMailMessage: + return RawMailMessage( + imap_uid="8", + message_id="", + sender="15008108550@163.com", + return_path="15008108550@163.com", + auth_result={"spf": "pass"}, + raw_message=raw_message, + received_at=datetime(2026, 9, 10, 9, 30, 0), + attachments=( + RawMailAttachment( + filename="申购申请单.pdf", + media_type="application/pdf", + payload=attachment_payload, + ), + ), + ) diff --git a/tests/unit/service/test_offsite_smtp_adapter.py b/tests/unit/service/test_offsite_smtp_adapter.py new file mode 100644 index 0000000..7dd07ee --- /dev/null +++ b/tests/unit/service/test_offsite_smtp_adapter.py @@ -0,0 +1,172 @@ +from __future__ import annotations + +import smtplib +from email.message import EmailMessage + +from app.core.config import Settings +from app.service.offsite_smtp_adapter import ( + OffsiteMailReplyRequest, + OffsiteSmtpSender, + SmtpAttachment, +) + + +class FakeSmtpConnection: + def __init__(self, *, fail_send: bool = False) -> None: + self.fail_send = fail_send + self.login_calls: list[tuple[str, str]] = [] + self.sent_messages: list[EmailMessage] = [] + self.quit_count = 0 + + def login(self, user: str, password: str) -> object: + self.login_calls.append((user, password)) + return {} + + def send_message(self, msg: EmailMessage) -> object: + if self.fail_send: + raise smtplib.SMTPException("send failed") + self.sent_messages.append(msg) + return {} + + def quit(self) -> object: + self.quit_count += 1 + return {} + + +def test_health_check_reports_disabled_and_dry_run_modes() -> None: + disabled = OffsiteSmtpSender(_settings()).health_check() + assert disabled == {"status": "disabled", "message": "场外 SMTP 未启用"} + + dry_run = OffsiteSmtpSender( + _settings(offsite_smtp_enabled=True, offsite_smtp_dry_run=True) + ).health_check() + assert dry_run == {"status": "dry_run", "message": "场外 SMTP 处于 dry-run 模式"} + + +def test_send_reply_requires_operator_confirmation_before_any_send() -> None: + connection = FakeSmtpConnection() + sender = OffsiteSmtpSender( + _settings(offsite_smtp_enabled=True, offsite_smtp_dry_run=False), + connection, + ) + + result = sender.send_reply(_request(operator_confirmed=False)) + + assert result.status == "发送失败" + assert result.failure_reason == "邮件发送前必须完成运营确认" + assert connection.sent_messages == [] + + +def test_send_reply_defaults_to_dry_run_without_real_smtp_call() -> None: + connection = FakeSmtpConnection() + sender = OffsiteSmtpSender(_settings(), connection) + + result = sender.send_reply(_request()) + + assert result.status == "待发送" + assert result.dry_run is True + assert result.provider_message_id is None + assert result.failure_reason is None + assert connection.sent_messages == [] + assert result.request_summary["attachment_count"] == 1 + + +def test_enabled_real_send_builds_reply_message_and_attaches_original_file() -> None: + connection = FakeSmtpConnection() + sender = OffsiteSmtpSender( + _settings(offsite_smtp_enabled=True, offsite_smtp_dry_run=False), + connection, + ) + + result = sender.send_reply(_request(reply_to_message_id="")) + + assert result.status == "发送成功" + assert result.dry_run is False + assert result.retry_count == 0 + assert connection.login_calls == [] + assert len(connection.sent_messages) == 1 + message = connection.sent_messages[0] + assert message["From"] == "15273589815@163.com" + assert message["To"] == "15008108550@163.com" + assert message["In-Reply-To"] == "" + assert message["References"] == "" + assert message["Message-ID"] == result.provider_message_id + attachments = list(message.iter_attachments()) + assert len(attachments) == 1 + assert attachments[0].get_filename() == "申购申请单.pdf" + assert attachments[0].get_payload(decode=True) == b"pdf-bytes" + + +def test_enabled_real_send_reports_failure_and_increments_retry_count() -> None: + connection = FakeSmtpConnection(fail_send=True) + sender = OffsiteSmtpSender( + _settings(offsite_smtp_enabled=True, offsite_smtp_dry_run=False), + connection, + ) + + result = sender.send_reply(_request(retry_count=2)) + + assert result.status == "发送失败" + assert result.failure_reason == "SMTPException" + assert result.retry_count == 3 + assert connection.quit_count == 1 + + +def test_enabled_real_send_fails_closed_when_config_is_missing() -> None: + sender = OffsiteSmtpSender( + _settings( + offsite_smtp_enabled=True, + offsite_smtp_dry_run=False, + offsite_smtp_host="", + offsite_smtp_username="", + offsite_smtp_password="", + offsite_smtp_sender="", + ) + ) + + result = sender.send_reply(_request(retry_count=1)) + + assert result.status == "发送失败" + assert result.retry_count == 2 + assert "OFFSITE_SMTP_HOST" in str(result.failure_reason) + + +def _request( + *, + operator_confirmed: bool = True, + retry_count: int = 0, + reply_to_message_id: str | None = None, +) -> OffsiteMailReplyRequest: + return OffsiteMailReplyRequest( + to_address="15008108550@163.com", + subject="场外基金申购赎回处理结果", + body="已处理。", + operator_id="operator-001", + operator_confirmed=operator_confirmed, + reply_to_message_id=reply_to_message_id, + retry_count=retry_count, + attachments=( + SmtpAttachment( + filename="申购申请单.pdf", + media_type="application/pdf", + payload=b"pdf-bytes", + ), + ), + ) + + +def _settings(**updates: object) -> Settings: + values: dict[str, object] = { + "jwt_issuer": "jr-local", + "jwt_audience": "jr-agent-platform", + "mysql_dsn": "sqlite+aiosqlite:///test.db", + "redis_url": "redis://127.0.0.1:6379/0", + "milvus_uri": "http://127.0.0.1:19530", + "neo4j_uri": "bolt://127.0.0.1:7687", + "offsite_smtp_host": "smtp.example.local", + "offsite_smtp_username": "15273589815@163.com", + "offsite_smtp_password": "test-auth-code", + "offsite_smtp_sender": "15273589815@163.com", + } + values.update(updates) + return Settings(**values) diff --git a/tests/unit/worker/test_offsite_mail_worker.py b/tests/unit/worker/test_offsite_mail_worker.py new file mode 100644 index 0000000..cfd1054 --- /dev/null +++ b/tests/unit/worker/test_offsite_mail_worker.py @@ -0,0 +1,360 @@ +from __future__ import annotations + +from datetime import datetime +from decimal import Decimal +from pathlib import Path +from typing import Any + +import pytest +from sqlalchemy import select, text, update +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine +from sqlalchemy.pool import StaticPool + +from app.core.config import Settings +from app.core.contracts import RequestContext +from app.core.offsite_fund_contracts import ReceiveRecognizedMailRequest +from app.model.base import Base +from app.model.offsite_fund import OffsiteFundMail, OffsiteMailCursor, OffsiteNotification +from app.service.offsite_document_recognition_adapter import StructuredRecognitionResult +from app.service.offsite_mail_adapter import ( + RawMailAttachment, + RawMailMessage, + SavedMailAttachment, + SavedMailMessage, +) +from app.worker.offsite_mail_worker import OffsiteMailWorker + + +@pytest.mark.asyncio +async def test_worker_advances_uid_only_after_each_mail_succeeds(tmp_path: Path) -> None: + maker, engine = await _database() + receiver = FakeReceiver(_raw_mail("5"), _raw_mail("6"), _raw_mail("7")) + service = FakeService(fail_uid="6") + worker = OffsiteMailWorker( + _settings(), + receiver=receiver, + storage=FakeStorage(tmp_path), + recognizer=FakeRecognizer(), + identity_resolver=_identity, + session_factory=maker, + service_factory=lambda _session: service, + ) + + try: + assert await worker.run_once() + cursor = await _cursor(maker) + assert cursor.last_uid == "5" + assert cursor.blocked_uid == "6" + assert cursor.status == "failed" + assert receiver.fetch_last_uid == "0" + assert service.calls == ["5", "6"] + + async with maker() as session, session.begin(): + await session.execute( + update(OffsiteMailCursor).values(next_retry_at=None) + ) + service.fail_uid = None + + assert await worker.run_once() + cursor = await _cursor(maker) + assert cursor.last_uid == "7" + assert cursor.status == "idle" + assert cursor.blocked_uid is None + assert service.calls == ["5", "6", "6", "7"] + assert receiver.fetch_last_uid == "5" + finally: + await engine.dispose() + + +@pytest.mark.asyncio +async def test_worker_does_not_write_without_configured_worker_identity(tmp_path: Path) -> None: + maker, engine = await _database() + receiver = FakeReceiver(_raw_mail("5")) + worker = OffsiteMailWorker( + _settings(offsite_worker_user_id=""), + receiver=receiver, + storage=FakeStorage(tmp_path), + recognizer=FakeRecognizer(), + identity_resolver=_identity, + session_factory=maker, + service_factory=lambda _session: FakeService(), + ) + + try: + assert not await worker.run_once() + assert receiver.health_calls == 0 + async with maker() as session: + assert await session.scalar(select(OffsiteMailCursor.id)) is None + finally: + await engine.dispose() + + +@pytest.mark.asyncio +async def test_worker_wires_saved_attachment_recognition_into_business_service( + tmp_path: Path, +) -> None: + maker, engine = await _database() + service = FakeService() + worker = OffsiteMailWorker( + _settings(), + receiver=FakeReceiver(_raw_mail_with_attachment("5")), + storage=FakeStorage(tmp_path), + recognizer=SuccessfulRecognizer(), + identity_resolver=_identity, + session_factory=maker, + service_factory=lambda _session: service, + ) + + try: + assert await worker.run_once() + assert len(service.payloads) == 1 + attachment = service.payloads[0].attachments[0] + assert attachment.document_type == "subscription" + assert attachment.original_file_path.endswith("5.eml") + assert attachment.extracted_fields["申请编号"] == "SUB-005" + finally: + await engine.dispose() + + +@pytest.mark.asyncio +async def test_worker_recovers_stale_sending_notification_without_marking_success( + tmp_path: Path, +) -> None: + maker, engine = await _database() + worker = OffsiteMailWorker( + _settings(), + receiver=FakeReceiver(), + storage=FakeStorage(tmp_path), + recognizer=FakeRecognizer(), + session_factory=maker, + ) + old = datetime(2026, 9, 1, 0, 0, 0) + try: + async with maker() as session, session.begin(): + session.add( + OffsiteNotification( + id=1, + notification_type="mail_return", + business_key="20260901-001-A01", + receiver_id="15008108550@163.com", + operator_id="operator-001", + agent_draft="draft", + final_content="final", + payload={}, + status="发送中", + retry_count=0, + created_at=old, + updated_at=old, + ) + ) + + assert await worker.recover_stale_notifications() + async with maker() as session: + notification = await session.get(OffsiteNotification, 1) + assert notification is not None + assert notification.status == "发送失败" + assert notification.provider_message_id is None + assert notification.failure_reason is not None + assert "人工核验" in notification.failure_reason + finally: + await engine.dispose() + + +class FakeReceiver: + def __init__(self, *messages: RawMailMessage) -> None: + self.messages = messages + self.last_scanned_uid: str | None = "0" + self.fetch_last_uid: str | None = None + self.health_calls = 0 + + def health_check(self) -> dict[str, object]: + self.health_calls += 1 + return {"status": "ok"} + + def fetch_since(self, last_uid: str | None, *, limit: int) -> tuple[RawMailMessage, ...]: + del limit + self.fetch_last_uid = last_uid + self.last_scanned_uid = self.messages[-1].imap_uid if self.messages else last_uid + return self.messages + + def close(self) -> None: + return None + + +class FakeStorage: + def __init__(self, root: Path) -> None: + self.root = root + + def save(self, mail: RawMailMessage) -> SavedMailMessage: + path = self.root / f"{mail.imap_uid}.eml" + path.write_bytes(mail.raw_message) + return SavedMailMessage( + imap_uid=mail.imap_uid, + message_id=mail.message_id, + sender=mail.sender, + return_path=mail.return_path, + auth_result=mail.auth_result, + eml_path=str(path), + attachments=( + SavedMailAttachment( + filename="empty.txt", + file_hash="a" * 64, + media_type="text/plain", + size_bytes=0, + original_file_path=str(path), + ), + ) if mail.attachments else (), + ) + + +class FakeRecognizer: + async def recognize(self, source: Any) -> Any: + del source + raise AssertionError("本测试邮件没有附件，不应调用识别器") + + +class SuccessfulRecognizer: + async def recognize(self, source: Any) -> StructuredRecognitionResult: + del source + return StructuredRecognitionResult( + document_type="subscription", + extracted_fields={ + "基金代码": "000001", + "基金名称": "测试基金", + "账户标识": "ACCT-001", + "投资者名称": "测试客户", + "申请编号": "SUB-005", + "申请日期": "2026-09-10", + "代销机构": "测试代销", + "申购金额": "10000", + "金额单位": "元", + }, + field_confidence={"申请编号": Decimal("0.99")}, + missing_fields=(), + low_confidence_fields=(), + page_evidence={"申请编号": [{"page": 1}]}, + ocr_text="申请编号：SUB-005", + ocr_status="mock", + llm_status="mock", + ) + + +class FakeService: + def __init__(self, fail_uid: str | None = None) -> None: + self.fail_uid = fail_uid + self.calls: list[str] = [] + self.payloads: list[ReceiveRecognizedMailRequest] = [] + + async def receive_recognized_mail( + self, payload: ReceiveRecognizedMailRequest, context: RequestContext + ) -> dict[str, object]: + del context + self.calls.append(payload.imap_uid) + self.payloads.append(payload) + if payload.imap_uid == self.fail_uid: + return {"code": 500, "message": "模拟入库失败", "data": {}} + return {"code": 0, "message": "ok", "data": {"business": False}} + + +async def _database() -> tuple[async_sessionmaker[AsyncSession], Any]: + engine = create_async_engine( + "sqlite+aiosqlite://", + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + async with engine.begin() as connection: + await connection.run_sync( + lambda sync: Base.metadata.create_all( + sync, + tables=[ + OffsiteFundMail.__table__, + OffsiteMailCursor.__table__, + OffsiteNotification.__table__, + ], + ) + ) + await connection.execute( + text( + """ + CREATE TABLE interaction_audit ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + actor_type VARCHAR(16) NOT NULL, + actor_id BIGINT NULL, + target_customer_id BIGINT NULL, + session_id VARCHAR(64) NULL, + portal VARCHAR(32) NULL, + action_type VARCHAR(64) NOT NULL, + detail JSON NOT NULL, + created_at DATETIME NOT NULL + ) + """ + ) + ) + return async_sessionmaker(engine, expire_on_commit=False), engine + + +async def _identity(identity: RequestContext) -> RequestContext: + return identity.model_copy( + update={ + "roles": ("operator",), + "permissions": ("offsite:write",), + "data_scope": "all", + } + ) + + +def _settings(**updates: object) -> Settings: + values: dict[str, object] = { + "jwt_issuer": "jr-local", + "jwt_audience": "jr-agent-platform", + "mysql_dsn": "sqlite+aiosqlite:///test.db", + "redis_url": "redis://127.0.0.1:6379/0", + "milvus_uri": "http://127.0.0.1:19530", + "neo4j_uri": "bolt://127.0.0.1:7687", + "offsite_mail_worker_enabled": True, + "offsite_imap_enabled": True, + "offsite_worker_user_id": "1", + "offsite_max_retry_count": 3, + } + values.update(updates) + return Settings(**values) + + +def _raw_mail(uid: str) -> RawMailMessage: + return RawMailMessage( + imap_uid=uid, + message_id=f"<{uid}@worker.test>", + sender="15008108550@163.com", + return_path="15008108550@163.com", + auth_result={"spf": "pass"}, + raw_message=f"mail-{uid}".encode(), + received_at=datetime(2026, 9, 10, 9, 30, 0), + attachments=(), + ) + + +def _raw_mail_with_attachment(uid: str) -> RawMailMessage: + mail = _raw_mail(uid) + return RawMailMessage( + imap_uid=mail.imap_uid, + message_id=mail.message_id, + sender=mail.sender, + return_path=mail.return_path, + auth_result=mail.auth_result, + raw_message=mail.raw_message, + received_at=mail.received_at, + attachments=( + RawMailAttachment( + filename="申购申请单.pdf", + media_type="application/pdf", + payload=b"pdf", + ), + ), + ) + + +async def _cursor(maker: async_sessionmaker[AsyncSession]) -> OffsiteMailCursor: + async with maker() as session: + cursor = await session.scalar(select(OffsiteMailCursor)) + assert cursor is not None + return cursor diff --git a/to_do_list.md b/to_do_list.md new file mode 100644 index 0000000..e4083c9 --- /dev/null +++ b/to_do_list.md @@ -0,0 +1,37 @@ +# 场外基金申购赎回后续任务清单 + +> 更新方式：完成一项实时打勾，并补充验证结果。 +> 范围：场外基金申购赎回独立接口、独立表、真实能力配置入口和可回归测试。 + +- [x] 1. 阅读项目规范、现有代码、调用链、数据流、权限和影响面。 + 备注：已阅读根目录与子项目 AGENTS、CLAUDE、AI 阅读/执行/测试/输出规范、MVC 规范、Agent 接入说明、数据库基线、建表设计、接口文档、NL2SQL 接入说明、场外流程图和本 SKILL。 +- [x] 2. 明确本次需求并拆分邮件、识别、核对、统计、通知、审计任务。 + 备注：本轮按用户确认分两阶段推进：先做当前业务闭环兼容完善，再按上线方案接入真实 IMAP/OCR/SMTP；不改变现有成功链路和接口路径。 +- [x] 3. 完善邮件接收、补偿、幂等、编号和原始文件保存。 + 备注：新增独立 `OffsiteMailWorker`，按 INBOX UID 增量扫描，持久化游标、失败 UID、重试时间和租约；成功邮件才推进游标，失败邮件停留并可重启补偿。当前采用健康检查 + 轮询补偿，IMAP IDLE 事件循环仍未实现，不能据此直接进入正式生产。 +- [x] 4. 完善附件分类、OCR/LLM 一次识别和识别异常处理。 + 备注：保持 summary 拆分、other 归档；新增字段缺失、低置信识别状态分流，不覆盖原始识别字段。 +- [x] 5. 完善 NL2SQL、Plan and Execute 和查询失败隔离。 + 备注：恢复并复用 `nl2sql_yc.query_dict`；新增场外专用适配器隔离调用参数和副作用；触发查询后独立保存每条规则查询记录并回填查询阶段计划状态。 +- [x] 6. 完善确定性计算、申购规则和赎回规则。 + 备注：继续使用确定性规则引擎；已覆盖申购金额 0.99/1.00/1.01 元、20% 比例边界、赎回可用份额等于和超出边界。 +- [ ] 7. 完善人工确认、汇总统计、邮件返回和独立通知路由。 + 备注：已补充人工确认审计、统计扩展字段、通知发送前状态约束、脱敏 payload 和发送中超时接管；邮件整体完成判定、资金清算触发及完整混合邮件路由仍未闭环。 +- [x] 8. 完善配置、权限、日志、重试、审计和数据安全。 + 备注：场外写接口在 Service 层增加角色和权限二次校验；人工确认、NL2SQL 触发、通知创建写入 `interaction_audit`；通知 payload 对账户标识脱敏。 +- [x] 9. 新增真实 IMAP 收信适配层，默认关闭并支持健康检查、自动重连和增量补偿扫描。 + 备注：新增 `OffsiteImapReceiver`，默认由 `OFFSITE_IMAP_ENABLED=false` 关闭；支持健康检查、缺配置识别、只选 `INBOX`、按 UID 增量扫描、白名单发件人过滤、扫描边界记录和连接关闭后下一轮重连。IMAP IDLE 仍未实现，当前由 Worker 轮询补偿。 +- [x] 10. 新增原始 MIME 邮件和附件受控保存流程，保留 UID、Message-ID、附件哈希、路径、失败原因和重试记录。 + 备注：新增 `OffsiteMailStorage` 保存原始 `.eml` 与附件；附件路径包含 SHA256 前缀，避免同名不同内容覆盖；Worker 失败时持久化游标和邮件级错误/重试字段，并保留原始文件。真实对象存储、容量治理、备份和保留期限仍需上线前配置。 +- [x] 11. 新增阿里云 OCR/Document AI 与 DeepSeek 识别适配层，默认 Mock，可配置真实调用、超时、降级和脱敏日志。 + 备注：新增 `OffsiteDocumentRecognitionAdapter`，默认 OCR/DeepSeek 均关闭并走本地 Mock；开启后通过配置化 HTTP endpoint 调用 OCR 网关，通过 DeepSeek OpenAI-compatible `/chat/completions` 进行分类和字段 JSON 映射；支持超时、缺配置识别、失败关闭、Mock 降级、结构化字段、字段置信度、缺失字段和低置信字段输出。阿里云官方直连签名/SDK 接入仍需在真实联调阶段确认，不在本轮新增依赖。 +- [x] 12. 新增 SMTP 邮件发送适配层，默认 dry-run，发送前必须运营确认，记录发送状态、provider message ID、失败原因和重试次数。 + 备注：新增 `OffsiteSmtpSender`，默认 `OFFSITE_SMTP_ENABLED=false` 且 `OFFSITE_SMTP_DRY_RUN=true`；发送前强制检查 `operator_confirmed`，支持回复原邮件、重新附加原附件、返回发送状态、失败原因和重试次数。新增发送中超时接管为“发送失败 + 人工核验”，不自动判断外部是否已投递；真实 SMTP 灰度尚未执行。 +- [x] 13. 为真实外部服务补齐 Mock 单元测试、集成测试、灰度验证步骤和上线回滚方案。 + 备注：新增 IMAP/原始文件保存、OCR/DeepSeek 识别、SMTP dry-run/真实发送 Mock 单元测试；新增通知成功幂等、失败状态回写和正常回复完成状态集成测试；新增 `docs/场外外部服务灰度与回滚方案.md`。真实生产灰度尚未执行，需上线前按文档人工执行。 +- [x] 14. 执行代码检查和自动化验证。 + 备注：本轮迁移后 `pytest` 全量 183 passed；`ruff`、`mypy`、权威文档检查和数据库结构审计均通过，当前数据库为 59 张业务表。 +- [x] 15. 再次阅读本 SKILL，核查是否有遗漏的内容和任务未生成到清单执行。 + 备注：已再次阅读桌面 SKILL；真实 IMAP/OCR/SMTP、真实发送状态机和邮件完成判定保留在后续未完成任务中。 +- [x] 16. 对功能和 BUG 进行测试。 + 备注：新增并通过场外专项测试，覆盖 `nl2sql_yc.query_dict` 适配、计划查询状态回填、低置信识别异常、缺权限拦截、IMAP 健康检查、只读收件箱增量扫描、白名单过滤、原始文件防覆盖保存、Worker UID 断点/失败恢复/身份拒绝/附件接线、SMTP dry-run、通知成功幂等、失败状态回写、发送中超时接管和正常回复完成状态。 diff --git a/tools/audit_schema.py b/tools/audit_schema.py index c36de55..971b54f 100644 --- a/tools/audit_schema.py +++ b/tools/audit_schema.py @@ -1,38 +1,57 @@ from __future__ import annotations import re +import sys from pathlib import Path +from urllib.parse import unquote, urlparse import pymysql ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + SQL_FILE = ROOT / "alembic" / "baseline_generated.sql" +VERSIONS_DIR = ROOT / "alembic" / "versions" def expected_tables() -> set[str]: text = SQL_FILE.read_text(encoding="utf-8") - return set(re.findall(r"CREATE TABLE `?([A-Za-z0-9_]+)`?", text)) + tables = set(re.findall(r"CREATE TABLE `?([A-Za-z0-9_]+)`?", text)) + for path in VERSIONS_DIR.glob("*.py"): + tables.update(re.findall(r"CREATE TABLE `?([A-Za-z0-9_]+)`?", path.read_text(encoding="utf-8"))) + return tables + + +def mysql_connection() -> pymysql.Connection: + from app.core.config import get_settings + + parsed = urlparse(get_settings().mysql_dsn.replace("mysql+asyncmy://", "mysql+pymysql://")) + return pymysql.connect( + host=parsed.hostname or "127.0.0.1", + port=parsed.port or 3306, + user=unquote(parsed.username or ""), + password=unquote(parsed.password or ""), + database=(parsed.path or "/").lstrip("/"), + charset="utf8mb4", + ) def main() -> None: - connection = pymysql.connect(host="127.0.0.1", port=3306, user="root", password="123456", database="jr") + connection = mysql_connection() + database = connection.db.decode() if isinstance(connection.db, bytes) else connection.db with connection.cursor() as cursor: - cursor.execute("SELECT table_name FROM information_schema.tables WHERE table_schema=%s", ("jr",)) + cursor.execute("SELECT table_name FROM information_schema.tables WHERE table_schema=%s", (database,)) actual = {row[0] for row in cursor.fetchall()} - {"alembic_version"} - expected = expected_tables() | { - "agent_run", "domain_event_outbox", "request_idempotency", "config_release", - "platform_config_item", "model_endpoint_config", "model_routing_rule", - "model_routing_fallback", "prompt_template_version", - "outbox_delivery", - "svc_conversation_session", - "api_request_receipt", - } + expected = expected_tables() missing = expected - actual unexpected = actual - expected if missing or unexpected: raise SystemExit(f"table mismatch missing={sorted(missing)} unexpected={sorted(unexpected)}") - cursor.execute("SELECT table_name, column_name FROM information_schema.columns WHERE table_schema=%s", ("jr",)) + cursor.execute( + "SELECT table_name, column_name FROM information_schema.columns WHERE table_schema=%s", + (database,), + ) actual_columns: dict[str, set[str]] = {} for table, column in cursor.fetchall(): actual_columns.setdefault(table, set()).add(column)