"""场外基金附件 OCR 与字段识别适配层。""" from __future__ import annotations import base64 import json import re from collections.abc import Mapping from dataclasses import dataclass from decimal import Decimal, InvalidOperation from typing import Any, Literal, cast import httpx from app.core.config import Settings from app.core.offsite_fund_contracts import DocumentType RecognitionStatus = Literal["mock", "success", "disabled", "misconfigured", "error"] REQUIRED_FIELDS: dict[str, tuple[str, ...]] = { "subscription": ( "基金代码", "基金名称", "账户标识", "申请编号", "申请日期", "代销机构", "申购金额", "金额单位", ), "redemption": ( "基金代码", "基金名称", "账户标识", "申请编号", "申请日期", "代销机构", "赎回份额", ), } @dataclass(frozen=True) class RecognitionSourceFile: filename: str media_type: str payload: bytes file_hash: str original_file_path: str = "" @dataclass(frozen=True) class OcrRecognitionResult: status: RecognitionStatus ocr_text: str tables: list[object] page_evidence: dict[str, object] raw_summary: dict[str, object] error_message: str | None = None @dataclass(frozen=True) class StructuredRecognitionResult: document_type: DocumentType extracted_fields: dict[str, object] field_confidence: dict[str, Decimal] missing_fields: tuple[str, ...] low_confidence_fields: tuple[str, ...] page_evidence: dict[str, object] ocr_text: str ocr_status: RecognitionStatus llm_status: RecognitionStatus error_message: str | None = None class OffsiteDocumentRecognitionAdapter: """先 OCR,再用 DeepSeek 或本地 Mock 生成统一字段。""" def __init__(self, settings: Settings, *, client: httpx.AsyncClient | None = None) -> None: self.settings = settings self.client = client self._owns_client = client is None def health_check(self) -> dict[str, object]: ocr_missing = self._ocr_missing_config() if self.settings.offsite_ocr_enabled else [] llm_missing = ( self._deepseek_missing_config() if self.settings.offsite_deepseek_enabled else [] ) return { "ocr": ( {"status": "misconfigured", "missing": ocr_missing} if ocr_missing else {"status": "ok" if self.settings.offsite_ocr_enabled else "disabled"} ), "deepseek": ( {"status": "misconfigured", "missing": llm_missing} if llm_missing else { "status": "ok" if self.settings.offsite_deepseek_enabled else "disabled", "model": self.settings.offsite_deepseek_model, } ), } async def recognize(self, source: RecognitionSourceFile) -> StructuredRecognitionResult: ocr = await self._recognize_ocr(source) structured = await self._extract_fields(source, ocr) if self._owns_client and self.client is not None: await self.client.aclose() return structured async def _recognize_ocr(self, source: RecognitionSourceFile) -> OcrRecognitionResult: if not self.settings.offsite_ocr_enabled: text = _mock_text(source) return OcrRecognitionResult( status="mock", ocr_text=text, tables=[], page_evidence={}, raw_summary={"provider": "mock", "filename": source.filename}, ) missing = self._ocr_missing_config() if missing: return OcrRecognitionResult( status="misconfigured", ocr_text="", tables=[], page_evidence={}, raw_summary={}, error_message=f"OCR配置缺失:{', '.join(missing)}", ) client = self._client() payload = { "filename": source.filename, "media_type": source.media_type, "file_hash": source.file_hash, "content_base64": base64.b64encode(source.payload).decode("ascii"), } try: response = await client.post( self.settings.offsite_aliyun_ocr_endpoint, headers={ "X-Acs-AccessKey-Id": self.settings.offsite_aliyun_access_key_id, "X-Acs-Signed-Gateway": "offsite-fund", }, json=payload, timeout=httpx.Timeout(self.settings.offsite_ocr_timeout_seconds), ) response.raise_for_status() body = _json_mapping(response) data = _mapping_value(body, "data") text = _text_from_mapping(data) or _text_from_mapping(body) tables = _list_value(data, "tables") or _list_value(body, "tables") page_evidence = _dict_value(data, "page_evidence") or _dict_value(body, "page_evidence") return OcrRecognitionResult( status="success", ocr_text=text, tables=tables, page_evidence=page_evidence, raw_summary={ "provider": "aliyun_document_ai", "response_keys": sorted(str(key) for key in body.keys()), }, ) except (httpx.HTTPError, ValueError) as exc: return OcrRecognitionResult( status="error", ocr_text="", tables=[], page_evidence={}, raw_summary={"provider": "aliyun_document_ai"}, error_message=type(exc).__name__, ) async def _extract_fields( self, source: RecognitionSourceFile, ocr: OcrRecognitionResult ) -> StructuredRecognitionResult: if not self.settings.offsite_deepseek_enabled: return _mock_structured_result(source, ocr, "mock") missing = self._deepseek_missing_config() if missing: result = _mock_structured_result(source, ocr, "misconfigured") return _with_error(result, f"DeepSeek配置缺失:{', '.join(missing)}") try: body = await self._call_deepseek(source, ocr) document_type = _document_type(body.get("document_type")) fields = _dict_value(body, "extracted_fields") confidence = _confidence_map(_dict_value(body, "field_confidence")) page_evidence = _dict_value(body, "page_evidence") or ocr.page_evidence missing_fields = tuple(str(item) for item in _list_value(body, "missing_fields")) low_fields = tuple(str(item) for item in _list_value(body, "low_confidence_fields")) if document_type in ("subscription", "redemption") and not missing_fields: missing_fields = _missing_fields(document_type, fields) if not low_fields: low_fields = _low_confidence_fields(confidence) return StructuredRecognitionResult( document_type=document_type, extracted_fields=fields, field_confidence=confidence, missing_fields=missing_fields, low_confidence_fields=low_fields, page_evidence=page_evidence, ocr_text=ocr.ocr_text, ocr_status=ocr.status, llm_status="success", error_message=ocr.error_message, ) except (httpx.HTTPError, ValueError, KeyError, TypeError) as exc: result = _mock_structured_result(source, ocr, "error") return _with_error(result, type(exc).__name__) async def _call_deepseek( self, source: RecognitionSourceFile, ocr: OcrRecognitionResult ) -> Mapping[str, object]: client = self._client() response = await client.post( self.settings.offsite_deepseek_base_url.rstrip("/") + "/chat/completions", headers={ "Authorization": f"Bearer {self.settings.offsite_deepseek_api_key}", "Content-Type": "application/json", }, json={ "model": self.settings.offsite_deepseek_model, "messages": [ { "role": "system", "content": ( "你只输出JSON。字段名必须使用申请日期和申请编号," "document_type只能是summary、subscription、redemption、other。" ), }, { "role": "user", "content": json.dumps( { "filename": source.filename, "media_type": source.media_type, "ocr_text": ocr.ocr_text[:12000], "tables": ocr.tables, "page_evidence": ocr.page_evidence, }, ensure_ascii=False, ), }, ], "temperature": 0, "response_format": {"type": "json_object"}, }, timeout=httpx.Timeout(self.settings.offsite_deepseek_timeout_seconds), ) response.raise_for_status() content = _deepseek_content(_json_mapping(response)) parsed = json.loads(content) if not isinstance(parsed, dict): raise ValueError("DeepSeek响应不是JSON对象") return cast(Mapping[str, object], parsed) def _client(self) -> httpx.AsyncClient: if self.client is None: self.client = httpx.AsyncClient() return self.client def _ocr_missing_config(self) -> list[str]: missing = [] if not self.settings.offsite_aliyun_ocr_endpoint: missing.append("OFFSITE_ALIYUN_OCR_ENDPOINT") if not self.settings.offsite_aliyun_access_key_id: missing.append("OFFSITE_ALIYUN_ACCESS_KEY_ID") if not self.settings.offsite_aliyun_access_key_secret: missing.append("OFFSITE_ALIYUN_ACCESS_KEY_SECRET") return missing def _deepseek_missing_config(self) -> list[str]: missing = [] if not self.settings.offsite_deepseek_base_url: missing.append("OFFSITE_DEEPSEEK_BASE_URL") if not self.settings.offsite_deepseek_api_key: missing.append("OFFSITE_DEEPSEEK_API_KEY") if not self.settings.offsite_deepseek_model: missing.append("OFFSITE_DEEPSEEK_MODEL") return missing def _mock_text(source: RecognitionSourceFile) -> str: text = source.payload.decode("utf-8", errors="ignore").strip() return text or f"文件名:{source.filename}" def _mock_structured_result( source: RecognitionSourceFile, ocr: OcrRecognitionResult, llm_status: RecognitionStatus ) -> StructuredRecognitionResult: document_type = _classify(source.filename + "\n" + ocr.ocr_text) fields = _extract_simple_fields(ocr.ocr_text) confidence = {key: Decimal("1") for key in fields} missing_fields: tuple[str, ...] = () if document_type in ("subscription", "redemption"): missing_fields = _missing_fields(document_type, fields) return StructuredRecognitionResult( document_type=document_type, extracted_fields=fields, field_confidence=confidence, missing_fields=missing_fields, low_confidence_fields=(), page_evidence=ocr.page_evidence, ocr_text=ocr.ocr_text, ocr_status=ocr.status, llm_status=llm_status, error_message=ocr.error_message, ) def _with_error(result: StructuredRecognitionResult, message: str) -> StructuredRecognitionResult: return StructuredRecognitionResult( document_type=result.document_type, extracted_fields=result.extracted_fields, field_confidence=result.field_confidence, missing_fields=result.missing_fields, low_confidence_fields=result.low_confidence_fields, page_evidence=result.page_evidence, ocr_text=result.ocr_text, ocr_status=result.ocr_status, llm_status=result.llm_status, error_message=message, ) def _classify(text: str) -> DocumentType: if "汇总" in text: return "summary" if "赎回" in text: return "redemption" if "申购" in text: return "subscription" return "other" def _extract_simple_fields(text: str) -> dict[str, object]: fields: dict[str, object] = {} for name in ( "基金代码", "基金名称", "账户标识", "投资者名称", "客户标识", "申请编号", "申请日期", "代销机构", "申购金额", "金额单位", "赎回份额", ): value = _find_label_value(text, name) if value: fields[name] = value return fields def _find_label_value(text: str, label: str) -> str | None: pattern = rf"{re.escape(label)}\s*[::]?\s*([^\s,,;;]+)" matched = re.search(pattern, text) if not matched: return None return matched.group(1).strip() def _missing_fields(document_type: str, fields: Mapping[str, object]) -> tuple[str, ...]: required = list(REQUIRED_FIELDS.get(document_type, ())) if not (fields.get("投资者名称") or fields.get("客户标识")): required.append("投资者名称/客户标识") return tuple(name for name in required if not str(fields.get(name) or "").strip()) def _low_confidence_fields(confidence: Mapping[str, Decimal]) -> tuple[str, ...]: return tuple(name for name, value in confidence.items() if value < Decimal("0.95")) def _confidence_map(value: Mapping[str, object]) -> dict[str, Decimal]: result: dict[str, Decimal] = {} for key, raw in value.items(): try: result[str(key)] = Decimal(str(raw)) except (InvalidOperation, ValueError): continue return result def _document_type(value: object) -> DocumentType: if value in ("summary", "subscription", "redemption", "other"): return cast(DocumentType, value) return "other" def _json_mapping(response: httpx.Response) -> Mapping[str, object]: body: Any = response.json() if not isinstance(body, dict): raise ValueError("响应不是JSON对象") return cast(Mapping[str, object], body) def _mapping_value(value: Mapping[str, object], key: str) -> Mapping[str, object]: nested = value.get(key) if isinstance(nested, dict): return cast(Mapping[str, object], nested) return {} def _dict_value(value: Mapping[str, object], key: str) -> dict[str, object]: nested = value.get(key) if isinstance(nested, dict): return dict(cast(Mapping[str, object], nested)) return {} def _list_value(value: Mapping[str, object], key: str) -> list[object]: nested = value.get(key) return list(nested) if isinstance(nested, list) else [] def _text_from_mapping(value: Mapping[str, object]) -> str: for key in ("ocr_text", "text", "content", "markdown"): candidate = value.get(key) if isinstance(candidate, str): return candidate return "" def _deepseek_content(body: Mapping[str, object]) -> str: choices = body.get("choices") if not isinstance(choices, list) or not choices: raise ValueError("DeepSeek响应缺少choices") first = choices[0] if not isinstance(first, dict): raise ValueError("DeepSeek响应格式错误") message = first.get("message") if not isinstance(message, dict): raise ValueError("DeepSeek响应缺少message") content = message.get("content") if not isinstance(content, str) or not content.strip(): raise ValueError("DeepSeek响应缺少content") return content