lzf_0626
87a753f7b3
## 解决的问题 全流程验收时发现：**20 只产品里只有 2 只有行情，其余下单直接 503**；而且行情一旦过期 就**没有任何机制刷新**。根因是这张表此前**只有演示种子脚本写**，而它是下单的硬前置 （TradeService 要求 close_price>0、total_fund_shares>0、source_updated_at 在 MAX_QUOTE_AGE 内）。 ## 数据源（踩了两个坑才选对） 1) **东财 push2 / push2his 两个行情域名在本环境一律连不上** （RemoteProtocolError: Server disconnected），而它的 api.fund（净值）与 fundf10（概况/费率）**正常** —— 即东财只有"行情类"接口不可达。 一开始按 K 线方案写完，20 只全失败；排查时还被我自己的 except 吞掉过异常。 2) 一度怀疑被限流，等 90 秒仍失败；做源可用性对照才确认是**域名级不可达**。 最终选**腾讯行情**（qt.gtimg.cn）：一次请求可带多只，给今开/最高/最低/现价/ 成交量(手)/成交额(万元)/**总市值(亿元)**，正好够写一行日行情。 K 线能力仍保留在适配器里（别的网络环境可能可达），注释写明本环境不可用。 ## 实现 - EastmoneyFundAdapter.fetch_tencent_quotes：解析腾讯的位置约定格式，只取语义明确的 字段并对价格做合理性校验；成交额万元→元、成交量按"手"（与东财 K 线口径一致，实测同为 9666652）。 - MarketPriceSyncService（新）：编排 + 落库。总份额按 「已有值 → 腾讯总市值推算 → 季度规模兜底」的优先级确定，source 标明是否含推算成分； 三者都拿不到就**跳过该产品**，不编份额。 - ools/sync_market_prices.py（新）：CLI，演示/验收前跑一次。 ## 另一个坑：非交易日的假行情行 种子脚本用 rade_date = date.today() 写行情，而 2026-09-13 是**周六**、真实行情时间是 09-11 16:14。TradeService 取 order_by(trade_date.desc()).limit(1)，于是那两行假的"今天" 永远排在真实行情前面——表现是"同步成功了、下单还说行情过期"。已删除那两行。 ## 验证 - 同步：请求 20 只、落库 19 行（160129 腾讯源没有该代码） - 下单：510300 @4.579、510500 @7.611、159948 @3.693、511810 @100.012 全部 201 已成交 —— 用的是**真实行情价**，不再是种子编的 4.5/6.2 - 风控处置闭环：新建 ALDEMO0003 后确认接收→进入调查→到达终态， ack_status=已确认、handler_id=9002 ## 顺带发现的疑点（未改，留给风控线确认） ALDEMO0003 到达终态时 status='已关闭' 但 **closed_at 与 handle_result 都是 NULL**； 对照 ALDEMO0001（已排除）则 closed_at 有值。结案时间是否应该写入，需业务侧确认。 门禁：ruff 通过 / mypy 250 文件 0 错 / 单元+契约 1380 passed 2 skipped。
398 lines
17 KiB
Python
398 lines
17 KiB
Python
"""东方财富基金行情适配器。
|
||
|
||
该模块只负责外部 HTTP 调用和供应商响应解析，不依赖 Agent、Controller 或数据库。
|
||
"""
|
||
|
||
import asyncio
|
||
import json
|
||
import re
|
||
from datetime import date
|
||
from decimal import Decimal, InvalidOperation
|
||
from typing import Any, Protocol
|
||
|
||
import httpx
|
||
|
||
from app.core.errors import RecoverableAgentError
|
||
|
||
NAV_API = "https://api.fund.eastmoney.com/f10/lsjz"
|
||
RETURN_API = "https://api.fund.eastmoney.com/pinzhong/LJSYLZS"
|
||
QUOTE_API = "https://push2.eastmoney.com/api/qt/ulist.np/get"
|
||
DETAIL_API = "https://fund.eastmoney.com/pingzhongdata/{code}.js"
|
||
#: 费率只能从 f10 基金概况页抓。**没有可用的 JSON 接口** ——
|
||
#: `FundArchivesDatas.aspx?type=jjfl` 实测返回空（正文就一句 `var apidata=`），
|
||
#: 所以这里按 HTML 去标签后匹配字段名。
|
||
FEE_API = "https://fundf10.eastmoney.com/jbgk_{code}.html"
|
||
#: 日 K 线。**这是本环境唯一可用的行情源**：`QUOTE_API`（push2 实时快照）实测
|
||
#: `Server disconnected`（连接被拒），而 push2his 稳定返回。它给的
|
||
#: 开/收/高/低/成交量/成交额正好是 `fin_market_price` 这张**日行情**表需要的字段。
|
||
KLINE_API = "https://push2his.eastmoney.com/api/qt/stock/kline/get"
|
||
#: 基金规模（亿份的"亿元"口径）与净值走势所在的 JS 文件，用于推算总份额。
|
||
PINGZHONG_API = "https://fund.eastmoney.com/pingzhongdata/{code}.js"
|
||
#: 腾讯行情。**本环境唯一可用的行情源**：东财的两个行情域名（push2 / push2his）
|
||
#: 实测一律 `Server disconnected`，而它的净值域名（api.fund）与概况域名（fundf10）正常 ——
|
||
#: 即东财只有"行情类"接口不可达。腾讯一次请求可带多只，给
|
||
#: 今开/最高/最低/现价/成交量(手)/成交额(万元)/总市值(亿元)。
|
||
TENCENT_QUOTE_API = "https://qt.gtimg.cn/q="
|
||
TENCENT_HEADERS = {"User-Agent": "Mozilla/5.0", "Referer": "https://gu.qq.com/"}
|
||
HEADERS = {"User-Agent": "Mozilla/5.0", "Referer": "https://fund.eastmoney.com/"}
|
||
|
||
#: 费率字段 → 概况页里的标签与取值模式。值形如「0.15」（百分号已剥离），
|
||
#: 页面写 `---` 表示该项不适用（如 ETF 没有申购费），统一映射成 `None`。
|
||
FEE_FIELD_PATTERNS: dict[str, str] = {
|
||
"management_fee_rate": r"管理费率\s*([\d.]+)\s*%",
|
||
"custodian_fee_rate": r"托管费率\s*([\d.]+)\s*%",
|
||
"service_fee_rate": r"销售服务费率\s*([\d.]+)\s*%",
|
||
"subscribe_fee_rate": r"最高申购费率\s*(--|[\d.]+)",
|
||
"redeem_fee_rate": r"最高赎回费率\s*(--|[\d.]+)",
|
||
}
|
||
|
||
|
||
class FundMarketAdapter(Protocol):
|
||
async def fetch_names(self, codes: list[str]) -> dict[str, str]: ...
|
||
|
||
async def fetch_quotes(self, codes: list[str]) -> dict[str, dict[str, Any]]: ...
|
||
|
||
async def fetch_history(self, code: str, target_date: date) -> dict[str, Any]: ...
|
||
|
||
async def fetch_fees(self, codes: list[str]) -> dict[str, dict[str, Any]]: ...
|
||
|
||
async def fetch_kline(
|
||
self, codes: list[str], *, limit: int = 5
|
||
) -> dict[str, list[dict[str, Any]]]: ...
|
||
|
||
async def fetch_fund_scale(self, codes: list[str]) -> dict[str, Any]: ...
|
||
|
||
|
||
class EastmoneyFundAdapter:
|
||
def __init__(
|
||
self,
|
||
client: httpx.AsyncClient | None = None,
|
||
*,
|
||
timeout_seconds: float = 12.0,
|
||
retries: int = 2,
|
||
) -> None:
|
||
self._client = client
|
||
self._owns_client = client is None
|
||
self.timeout_seconds = timeout_seconds
|
||
self.retries = max(0, retries)
|
||
|
||
async def fetch_names(self, codes: list[str]) -> dict[str, str]:
|
||
result: dict[str, str] = {}
|
||
for code in codes:
|
||
try:
|
||
response = await self._request("GET", DETAIL_API.format(code=code))
|
||
match = re.search(r"var\s+fS_name\s*=\s*[\"']([^\"']+)", response.text)
|
||
result[code] = match.group(1).strip() if match else f"基金 {code}"
|
||
except RecoverableAgentError:
|
||
result[code] = f"基金 {code}"
|
||
return result
|
||
|
||
async def fetch_quotes(self, codes: list[str]) -> dict[str, dict[str, Any]]:
|
||
if not codes:
|
||
return {}
|
||
secids = ",".join(
|
||
("1." if code.startswith(("5", "6", "9")) else "0.") + code
|
||
for code in codes
|
||
)
|
||
try:
|
||
response = await self._request(
|
||
"GET", QUOTE_API,
|
||
params={"fltt": 2, "invt": 2, "fields": "f12,f2,f3", "secids": secids},
|
||
)
|
||
items = ((response.json().get("data") or {}).get("diff") or [])
|
||
except (RecoverableAgentError, ValueError, TypeError):
|
||
return {}
|
||
result: dict[str, dict[str, Any]] = {}
|
||
for item in items:
|
||
code = str(item.get("f12")) if item.get("f12") else ""
|
||
if code:
|
||
result[code] = {
|
||
"nav": self._decimal(item.get("f2")),
|
||
"daily_change": self._decimal(item.get("f3")),
|
||
}
|
||
return result
|
||
|
||
async def fetch_history(self, code: str, target_date: date) -> dict[str, Any]:
|
||
try:
|
||
response = await self._request(
|
||
"GET", NAV_API,
|
||
params={"fundCode": code, "pageIndex": 1, "pageSize": 30,
|
||
"startDate": "", "endDate": ""},
|
||
)
|
||
records = ((response.json().get("Data") or {}).get("LSJZList") or [])
|
||
except (RecoverableAgentError, ValueError, TypeError):
|
||
return {"degraded": True}
|
||
selected = next(
|
||
(item for item in records if item.get("FSRQ") == target_date.isoformat()), None
|
||
)
|
||
if selected is None and records:
|
||
selected = records[0]
|
||
if not selected:
|
||
return {"degraded": True}
|
||
return {
|
||
"nav": self._decimal(selected.get("DWJZ")),
|
||
"nav_date": self._date(selected.get("FSRQ")),
|
||
"daily_change": self._decimal(selected.get("JZZZL")),
|
||
"degraded": False,
|
||
}
|
||
|
||
async def fetch_fees(self, codes: list[str]) -> dict[str, dict[str, Any]]:
|
||
"""取每只基金的费率（管理费/托管费/销售服务费/申购费/赎回费）与基金全称。
|
||
|
||
为什么单独留一个入口：费率是**长期稳定**字段，而概况页约 45KB/只，
|
||
所以调用方应当按需刷新、不要每次问答都抓 —— `tools/fetch_live_quotes.py`
|
||
就是按"需要时才拉"来用它的。
|
||
|
||
取数失败的基金**不会出现在返回里**（而不是返回一个全 None 的 dict），
|
||
免得调用方把"没抓到"误当成"该基金确实没有费率"。
|
||
"""
|
||
results: dict[str, dict[str, Any]] = {}
|
||
for code in codes:
|
||
try:
|
||
response = await self._request("GET", FEE_API.format(code=code))
|
||
except RecoverableAgentError:
|
||
continue
|
||
parsed = self._parse_fees(response.text)
|
||
if parsed is not None:
|
||
results[code] = parsed
|
||
return results
|
||
|
||
@staticmethod
|
||
def _parse_fees(html: str) -> dict[str, Any] | None:
|
||
text = re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", html))
|
||
fees: dict[str, Any] = {}
|
||
for field, pattern in FEE_FIELD_PATTERNS.items():
|
||
match = re.search(pattern, text)
|
||
value = match.group(1) if match else None
|
||
fees[field] = (
|
||
None if value in (None, "", "--") else EastmoneyFundAdapter._decimal(value)
|
||
)
|
||
# 一个字段都没解析出来，说明页面结构变了或该代码不存在 —— 返回 None 让调用方
|
||
# 与"该基金确实没有费率"区分开。
|
||
if all(value is None for value in fees.values()):
|
||
return None
|
||
name = re.search(r"基金全称\s*(\S{4,60}?(?:基金|集合资产管理计划))", text)
|
||
fees["full_name"] = name.group(1) if name else None
|
||
return fees
|
||
|
||
@staticmethod
|
||
def _secid(code: str) -> str:
|
||
"""东财的 secid：沪市前缀 `1.`、深市 `0.`（与 `hq.py` 同一口径）。"""
|
||
return ("1." if code.startswith(("5", "6", "9")) else "0.") + code
|
||
|
||
async def fetch_kline(
|
||
self, codes: list[str], *, limit: int = 5
|
||
) -> dict[str, list[dict[str, Any]]]:
|
||
"""取每只基金最近若干交易日的日 K 线。
|
||
|
||
返回 `{code: [{trade_date, open, close, high, low, volume, turnover}, ...]}`
|
||
（按日期升序，最后一条是最新交易日）。取不到的代码**不出现**在返回里。
|
||
|
||
为什么用它而不是 `fetch_quotes`：`push2` 实时快照在本环境实测连接被拒，
|
||
而 `push2his` 的日 K 线稳定可用；且这张表本来就是**日行情**，
|
||
开/收/高/低/量/额比"实时快照"更贴合语义。
|
||
"""
|
||
results: dict[str, list[dict[str, Any]]] = {}
|
||
for code in codes:
|
||
# ⚠️ 这条请求**刻意走同步 httpx**（`asyncio.to_thread` 包装）：实测
|
||
# `AsyncClient` 请求 push2his 会被对端直接断连
|
||
# （`RemoteProtocolError: Server disconnected without sending a response`），
|
||
# 而同参数、同 header 的同步客户端稳定成功 —— 排查时一度以为是解析问题。
|
||
try:
|
||
klines = await asyncio.to_thread(self._fetch_kline_sync, code, limit)
|
||
except Exception: # noqa: BLE001 - 单只失败不影响其余
|
||
continue
|
||
rows = [self._parse_kline(line) for line in klines]
|
||
usable = [row for row in rows if row is not None]
|
||
if usable:
|
||
results[code] = usable
|
||
return results
|
||
|
||
@staticmethod
|
||
def _fetch_kline_sync(code: str, limit: int) -> list[Any]:
|
||
"""同步版 K 线请求（为什么不用 `AsyncClient` 见 `fetch_kline` 的注释）。"""
|
||
response = httpx.get(
|
||
KLINE_API,
|
||
params={
|
||
"secid": EastmoneyFundAdapter._secid(code),
|
||
"fields1": "f1,f2,f3,f4,f5,f6",
|
||
# f51 日期 f52 开 f53 收 f54 高 f55 低 f56 成交量 f57 成交额
|
||
"fields2": "f51,f52,f53,f54,f55,f56,f57",
|
||
"klt": "101", "fqt": "0", "end": "20500101", "lmt": str(limit),
|
||
},
|
||
headers=HEADERS,
|
||
timeout=20.0,
|
||
)
|
||
response.raise_for_status()
|
||
return list((response.json().get("data") or {}).get("klines") or [])
|
||
|
||
@staticmethod
|
||
def _parse_kline(line: Any) -> dict[str, Any] | None:
|
||
"""`"2026-09-11,4.592,4.579,4.592,4.532,9666652,4409407029.000"` → dict。"""
|
||
if not isinstance(line, str):
|
||
return None
|
||
parts = line.split(",")
|
||
if len(parts) < 7:
|
||
return None
|
||
try:
|
||
trade_date = date.fromisoformat(parts[0])
|
||
except ValueError:
|
||
return None
|
||
return {
|
||
"trade_date": trade_date,
|
||
"open": EastmoneyFundAdapter._decimal(parts[1]),
|
||
"close": EastmoneyFundAdapter._decimal(parts[2]),
|
||
"high": EastmoneyFundAdapter._decimal(parts[3]),
|
||
"low": EastmoneyFundAdapter._decimal(parts[4]),
|
||
"volume": EastmoneyFundAdapter._decimal(parts[5]),
|
||
"turnover": EastmoneyFundAdapter._decimal(parts[6]),
|
||
}
|
||
|
||
async def fetch_fund_scale(self, codes: list[str]) -> dict[str, Any]:
|
||
"""取基金规模（**亿元**）与规模所属报告期，用于推算总份额。
|
||
|
||
⚠️ 规模是**季度披露值**（`Data_fluctuationScale` 的最后一期），不是实时值，
|
||
所以推算出的总份额只当量级用 —— 它服务的是"单一投资者持仓占比不超过 X%"
|
||
这类校验，不是对外披露数据。
|
||
"""
|
||
results: dict[str, Any] = {}
|
||
for code in codes:
|
||
try:
|
||
response = await self._request("GET", PINGZHONG_API.format(code=code))
|
||
except RecoverableAgentError:
|
||
continue
|
||
scale = self._parse_scale(response.text)
|
||
if scale is not None:
|
||
results[code] = scale
|
||
return results
|
||
|
||
@staticmethod
|
||
def _parse_scale(text: str) -> dict[str, Any] | None:
|
||
"""从 pingzhongdata 取最后一期规模（亿元）。"""
|
||
match = re.search(r"var\s+Data_fluctuationScale\s*=\s*(\{.*?\});", text, re.S)
|
||
if match is None:
|
||
return None
|
||
try:
|
||
payload = json.loads(match.group(1))
|
||
except (ValueError, TypeError):
|
||
return None
|
||
series = payload.get("series") or []
|
||
categories = payload.get("categories") or []
|
||
if not series:
|
||
return None
|
||
last = series[-1]
|
||
value = last.get("y") if isinstance(last, dict) else None
|
||
try:
|
||
scale_yi = Decimal(str(value))
|
||
except (InvalidOperation, TypeError):
|
||
return None
|
||
return {
|
||
"scale_yi": scale_yi,
|
||
"scale_date": categories[-1] if categories else None,
|
||
}
|
||
|
||
@staticmethod
|
||
def _tencent_symbol(code: str) -> str:
|
||
"""腾讯的代码前缀：沪市 `sh`、深市 `sz`。"""
|
||
return ("sh" if code.startswith(("5", "6", "9")) else "sz") + code
|
||
|
||
async def fetch_tencent_quotes(self, codes: list[str]) -> dict[str, dict[str, Any]]:
|
||
"""腾讯当日行情快照（一次请求可带多只）。
|
||
|
||
**本环境唯一可用的行情源**：东财的 `push2`/`push2his` 实测连接被拒，
|
||
而腾讯这个源稳定返回。它给的字段正好够写一行 `fin_market_price`：
|
||
今开 / 最高 / 最低 / 现价 / 成交量(手) / 成交额(万元) / 总市值(亿元)。
|
||
取不到的代码不出现在返回里。
|
||
"""
|
||
if not codes:
|
||
return {}
|
||
symbols = ",".join(self._tencent_symbol(code) for code in codes)
|
||
try:
|
||
text = await asyncio.to_thread(self._fetch_tencent_sync, symbols)
|
||
except Exception: # noqa: BLE001 - 整批失败就返回空，由调用方按"未同步"处理
|
||
return {}
|
||
return self._parse_tencent(text)
|
||
|
||
@staticmethod
|
||
def _fetch_tencent_sync(symbols: str) -> str:
|
||
response = httpx.get(
|
||
TENCENT_QUOTE_API + symbols, headers=TENCENT_HEADERS, timeout=20.0
|
||
)
|
||
response.raise_for_status()
|
||
return response.text
|
||
|
||
@staticmethod
|
||
def _parse_tencent(text: str) -> dict[str, dict[str, Any]]:
|
||
"""解析 `v_sh510300="1~名称~代码~现价~昨收~今开~成交量~…"` 形式的返回。
|
||
|
||
字段是**位置约定**（腾讯没有自描述），所以这里只取前 46 个位置里语义明确的那些，
|
||
并对结果做合理性校验（价格必须为正），拿不到就丢弃该只而不是给出错值。
|
||
"""
|
||
results: dict[str, dict[str, Any]] = {}
|
||
for line in text.splitlines():
|
||
if "=" not in line or "~" not in line:
|
||
continue
|
||
_, _, body = line.partition("=")
|
||
parts = body.strip().strip('";').split("~")
|
||
if len(parts) < 46:
|
||
continue
|
||
code = parts[2].strip()
|
||
if not code:
|
||
continue
|
||
close = EastmoneyFundAdapter._decimal(parts[3])
|
||
if close is None or close <= 0:
|
||
continue
|
||
turnover_wan = EastmoneyFundAdapter._decimal(parts[37])
|
||
results[code] = {
|
||
"code": code,
|
||
"name": parts[1] or None,
|
||
"open": EastmoneyFundAdapter._decimal(parts[5]),
|
||
"close": close,
|
||
"high": EastmoneyFundAdapter._decimal(parts[33]),
|
||
"low": EastmoneyFundAdapter._decimal(parts[34]),
|
||
# 成交量单位是**手**（与东财 K 线口径一致，实测同为 9666652）。
|
||
"volume": EastmoneyFundAdapter._decimal(parts[6]),
|
||
# 成交额单位是**万元**，统一换算成元。
|
||
"turnover": None if turnover_wan is None else turnover_wan * 10000,
|
||
# 总市值（亿元），用于推算 `total_fund_shares`。
|
||
"total_market_value_yi": EastmoneyFundAdapter._decimal(parts[45]),
|
||
"quoted_at": parts[30] or None,
|
||
}
|
||
return results
|
||
|
||
async def _request(self, method: str, url: str, **kwargs: Any) -> httpx.Response:
|
||
client = self._client or httpx.AsyncClient(headers=HEADERS)
|
||
try:
|
||
last_error: Exception | None = None
|
||
for attempt in range(self.retries + 1):
|
||
try:
|
||
response = await client.request(
|
||
method, url, timeout=self.timeout_seconds, **kwargs
|
||
)
|
||
response.raise_for_status()
|
||
return response
|
||
except (httpx.HTTPError, TimeoutError) as exc:
|
||
last_error = exc
|
||
if attempt < self.retries:
|
||
await asyncio.sleep(0.2 * (2**attempt))
|
||
raise RecoverableAgentError("基金行情数据源不可用") from last_error
|
||
finally:
|
||
if self._owns_client:
|
||
await client.aclose()
|
||
|
||
@staticmethod
|
||
def _decimal(value: Any) -> Decimal | None:
|
||
if value in (None, "", "-"):
|
||
return None
|
||
try:
|
||
return Decimal(str(value).replace("%", ""))
|
||
except (InvalidOperation, ValueError):
|
||
return None
|
||
|
||
@staticmethod
|
||
def _date(value: Any) -> date | None:
|
||
try:
|
||
return date.fromisoformat(str(value))
|
||
except (TypeError, ValueError):
|
||
return None
|