#!/usr/bin/env python3
"""无损性深挖: 偏差分布 + 是否由 adj_factor 6位舍入解释 + qfq 离群定位"""
import glob, os
import pandas as pd
import numpy as np

BASE = os.path.expanduser('~/zt_app/backtest_zt_full')
BAK = os.path.expanduser('~/zt_app/_hfq_ohlc_backup_20260912')
POOLS = ['hs300','zz500','zz1000','zz2000']
cur = {}
for p in POOLS:
    for f in glob.glob(f'{BASE}/daily_{p}/*.csv'):
        cur[os.path.basename(f)[:-4]] = (p, f)

buckets = {'<=1e-9':0,'<=1e-6':0,'<=1e-4':0,'<=1e-2':0,'>1e-2':0}
rel_buckets = {'<=1e-9':0,'<=1e-7':0,'<=1e-5':0,'>1e-5':0}
worst = []
rows = 0
# 用 round(fac,6) 假设复算
max_resid_rounded = 0.0; worst_rounded = None
for bf in sorted(glob.glob(f'{BAK}/*.csv')):
    code = os.path.basename(bf)[:-4]
    if code not in cur: continue
    d = pd.read_csv(bf, dtype={'trade_date': str})
    if 'hfq_close' not in d.columns: continue
    c = pd.read_csv(cur[code][1], dtype={'trade_date': str})
    m = d.merge(c, on='trade_date', suffixes=('_old','_cur'))
    if not len(m): continue
    rows += len(m)
    rc = m['raw_close_cur'].astype(float).values
    fac = m['adj_factor'].astype(float).values
    old = m['hfq_close'].astype(float).values
    dev = np.abs(old - rc*fac)
    rel = dev / np.clip(np.abs(old), 1e-9, None)
    for k, thr in zip(buckets, [1e-9,1e-6,1e-4,1e-2,1e18]):
        pass
    buckets['<=1e-9'] += int((dev<=1e-9).sum())
    buckets['<=1e-6'] += int(((dev>1e-9)&(dev<=1e-6)).sum())
    buckets['<=1e-4'] += int(((dev>1e-6)&(dev<=1e-4)).sum())
    buckets['<=1e-2'] += int(((dev>1e-4)&(dev<=1e-2)).sum())
    buckets['>1e-2'] += int((dev>1e-2).sum())
    rel_buckets['<=1e-9'] += int((rel<=1e-9).sum())
    rel_buckets['<=1e-7'] += int(((rel>1e-9)&(rel<=1e-7)).sum())
    rel_buckets['<=1e-5'] += int(((rel>1e-7)&(rel<=1e-5)).sum())
    rel_buckets['>1e-5'] += int((rel>1e-5).sum())
    i = int(dev.argmax())
    if len(worst) < 6 or dev[i] > worst[0][0]:
        worst.append((float(dev[i]), code, m.iloc[i]['trade_date'], float(old[i]), float(rc[i]*fac[i]), float(fac[i])))
        worst.sort(reverse=True); worst = worst[:6]
    # 假设: 旧 hfq 用的因子 = 未舍入? 试 round(fac,6)
    fac6 = np.round(fac, 6)
    dev6 = np.abs(old - rc*fac6)
    j = int(dev6.argmax())
    if dev6[j] > max_resid_rounded:
        max_resid_rounded = float(dev6[j]); worst_rounded = (code, m.iloc[j]['trade_date'], fac[j])

print('逐行合计:', rows)
print('hfq 绝对偏差分档:', buckets)
print('hfq 相对偏差分档:', rel_buckets)
print('绝对偏差 top:', worst)
print('若因子取 round(6): 残差 max =', f'{max_resid_rounded:.3e}', worst_rounded)
print()
# qfq 离群: 600837.SH
code = '600837.SH'
d = pd.read_csv(f'{BAK}/{code}.csv', dtype={'trade_date': str})
c = pd.read_csv(cur[code][1], dtype={'trade_date': str})
m = d.merge(c, on='trade_date', suffixes=('_old','_cur'))
m['qfq_derived'] = m['raw_close_cur']*m['adj_factor']/m['adj_factor'].iloc[-1]
m['dev'] = (m['qfq_close']-m['qfq_derived']).abs()
print(f'{code} 行数 {len(m)} 末行日 {m.iloc[-1]["trade_date"]} 末因子 {m["adj_factor"].iloc[-1]}')
print(m.nlargest(3, 'dev')[['trade_date','raw_close_old','raw_close_cur','qfq_close','adj_factor','qfq_derived','dev']].to_string())
print('原始 factor(旧文件末行隐含) =', m['hfq_close'].iloc[-1]/m['raw_close_old'].iloc[-1], ' 新列 factor =', m['adj_factor'].iloc[-1])
print('旧 qfq 末行 =', m['qfq_close'].iloc[-1], ' raw 末行 =', m['raw_close_cur'].iloc[-1])