# Detector evaluation (train <2026-01-01, test 2026-01-01..2026-10-26) + option-buyer trades with costs
import pandas as pd,numpy as np
from scipy.stats import fisher_exact,binomtest
R='/workspace/research/flow-forensics-2026-10/recov'
C=pd.read_csv(f'{R}/results/candidates.csv',parse_dates=['date'])
C['test']=C.date>='2026-01-01'
tr=C[~C.test]
F={'heavy_put_near':C.heavy_put_dist.abs()<=40,'put_writing':C.put_oi_chg_pct>0,'call_cover':C.call_oi_chg_pct<0,
   'basis_widen':C.basis_chg>0,'futoi_down':C.futoi_chg_pct<0,'bn_first':C.bn_lead_min>=1,'vix_falling':C.vix_chg<0,
   'straddle_crush':C.straddle_chg_pct<0,'expiry_day':C.dte==0,'expiry_le1':C.dte<=1,'gap_down':C.gap_down==True,
   'below_pdl':C.below_pdl==True,'early_low_le30m':C.min_since_open<=30,'deep_fall_ge200':C.depth_pc>=200,'round_low_le20':C.round_dist<=20,
   'fast_bounce_le5m':C.bounce_min<=5,'fast_fall':C.fall_speed>=tr.fall_speed.median(),'vix_high':C.vix_lvl>=tr.vix_lvl.median(),
   'basis_widen_big':C.basis_chg>=5,'put_writing_big':C.put_oi_chg_pct>=5,'thursday':C.weekday==3}
avail={'heavy_put_near':C.heavy_put_dist.notna(),'put_writing':C.put_oi_chg_pct.notna(),'call_cover':C.call_oi_chg_pct.notna(),'straddle_crush':C.straddle_chg_pct.notna(),
       'bn_first':C.bn_lead_min.notna(),'vix_falling':C.vix_chg.notna(),'vix_high':C.vix_lvl.notna(),'futoi_down':C.futoi_chg_pct.notna(),'put_writing_big':C.put_oi_chg_pct.notna()}
for k,v in F.items(): C[k]=v.where(avail.get(k,pd.Series(True,index=C.index)),np.nan)
def bh(p):
    p=np.asarray(p,float); n=len(p); o=np.argsort(p); q=np.empty(n); m=1
    for r,i in enumerate(o[::-1]): m=min(m,p[i]*n/(n-r)); q[i]=m
    return q
Y='y200'
def uni(df,tag):
    out=[]
    for k in F:
        s=df[df[k].notna()]; a=s[s[k]==1]; b=s[s[k]==0]
        if len(a)<3 or len(b)<3: continue
        t=[[a[Y].sum(),len(a)-a[Y].sum()],[b[Y].sum(),len(b)-b[Y].sum()]]
        out.append(dict(feature=k,split=tag,n_true=len(a),hit_true=a[Y].mean()*100,n_false=len(b),hit_false=b[Y].mean()*100,lift=a[Y].mean()-b[Y].mean(),p=fisher_exact(t)[1]))
    o=pd.DataFrame(out); o['q_bh']=bh(o.p); return o
U=pd.concat([uni(C[~C.test],'train'),uni(C[C.test],'test'),uni(C,'all')]); U.round(4).to_csv(f'{R}/results/feature_univariate.csv',index=False)
print('base y200: train %.1f%% (n=%d)  test %.1f%% (n=%d)'%(tr[Y].mean()*100,len(tr),C[C.test][Y].mean()*100,C.test.sum()))
print(U.pivot(index='feature',columns='split',values=['hit_true','hit_false','n_true','p','q_bh']).round(3).to_string())
# rules
def ev(mask,tag):
    rows=[]
    for sp,s in [('train',~C.test),('test',C.test),('all',C.index==C.index)]:
        m=mask&s; base=C[s][Y].mean(); n=int(m.sum()); h=int(C[m][Y].sum())
        p=binomtest(h,n,base,alternative='greater').pvalue if n else np.nan
        rng=np.random.default_rng(1); pool=C[s][Y].values
        rnd=np.array([rng.choice(pool,n,replace=False).mean() for _ in range(5000)]) if 0<n<=len(pool) else np.array([np.nan])
        rows.append(dict(rule=tag,split=sp,n_fire=n,days=C[m].date.nunique(),hits=h,hit_rate=h/n*100 if n else np.nan,base=base*100,p_binom=p,random_pctile=(rnd<h/n).mean()*100 if n else np.nan,random_p95=np.nanpercentile(rnd,95)*100))
    return rows
f=lambda k:(C[k]==1)
rules={'A_hypothesis(heavyput&putwrite&basis&BNfirst)':f('heavy_put_near')&f('put_writing')&f('basis_widen')&f('bn_first'),
       'A2(putwrite&basis&BNfirst)':f('put_writing')&f('basis_widen')&f('bn_first'),
       'A3(putwrite&callcover&basis)':f('put_writing')&f('call_cover')&f('basis_widen')}
# data-selected on TRAIN only: top-3 positive-lift features by train p
trU=U[(U.split=='train')&(U.lift>0)].sort_values('p')
top=list(trU.feature[:3]); print('train-selected top3:',top, trU.p[:3].round(4).tolist())
rules['B_train_top2('+'&'.join(top[:2])+')']=f(top[0])&f(top[1])
rules['B_train_top3_all']=f(top[0])&f(top[1])&f(top[2])
sc=sum(f(k).astype(int) for k in ['put_writing','call_cover','basis_widen','bn_first','vix_falling','straddle_crush'])
rules['S_flowscore>=4of6']=sc>=4
rules['S_flowscore>=5of6']=sc>=5
RR=[]
for k,m in rules.items(): RR+=ev(m.fillna(False).astype(bool),k)
RR=pd.DataFrame(RR); RR['q_bh_test']=np.nan
t=RR.split=='test'; RR.loc[t,'q_bh_test']=bh(RR.loc[t,'p_binom'].fillna(1)); RR.round(4).to_csv(f'{R}/results/rules_oos.csv',index=False)
print(RR.round(3).to_string())
C['flowscore']=sc
for k,m in rules.items(): C['rule_'+k.split('(')[0]]=m.fillna(False).astype(bool)
C.to_csv(f'{R}/results/candidates_scored.csv',index=False)
# other outcomes for A / B
for y in ['y150','y250','final_low']:
    print(y,'base all %.1f'%(C[y].mean()*100), {k.split('(')[0]:round(C[rules[k].fillna(False).astype(bool)][y].mean()*100,1) for k in rules})