# TWENTY-NO-TV stats (PREREG 20:31 IST 27-Oct-2026)
import pandas as pd,numpy as np
from scipy import stats
L='/workspace/research/popular-strategies'; R=f'{L}/results'; rng=np.random.default_rng(2910)
E=pd.read_csv(f'{R}/ntv_events.csv',parse_dates=['date']); RN=pd.read_csv(f'{R}/ntv_random.csv',parse_dates=['date']); NEG=pd.read_csv(f'{R}/ntv_negtv.csv',parse_dates=['date'])
LAB=pd.read_csv(f'{R}/regime_labels.csv',parse_dates=['date']).set_index('date')
E=E[E.status=='ok'].copy(); E['grp']=E.ix+'_'+E.set
def freq(d):
    return pd.Series(dict(n=len(d),days=d.date.nunique(),x2=100*(d.max_ratio>=2).mean(),x3=100*(d.max_ratio>=3).mean(),zero=100*d.near_zero.astype(float).mean(),peak_min_med=d.peak_min.median(),sig_time_med=d.t.median(),tv_med=d.tv.median() if 'tv' in d else np.nan))
F=E.groupby(['grp','tvdef']).apply(freq).reset_index(); F.insert(2,'split_by','all')
FM=E[E.tvdef=='tv2'].groupby(['grp','mode']).apply(freq).reset_index()
FW=E[E.tvdef=='tv2'].groupby(['grp','win']).apply(freq).reset_index()
# OTM-20 comparison from w20_scan (first touch of 16-24 from above), same signal range t>=20, all and late (t>=225)
S=pd.read_csv(f'{R}/w20_scan.csv',parse_dates=['date']); S=S[(S.L==20)&(S.status=='ok')&(S.t>=20)]; S['grp']=S.ix+'_'+S.set
O=[]
for nm,d in (('OTM-20 all day',S),('OTM-20 from 13:00 (t>=225)',S[S.t>=225])):
    for g,x in d.groupby('grp'): r=freq(x); r['grp']=g; r['what']=nm; r['ret_a']=(x.pnl_a/x.fill).mean(); r['ret_c']=(x.pnl_c/x.fill).mean(); O.append(r)
for g,x in E[E.tvdef=='tv2'].groupby('grp'): r=freq(x); r['grp']=g; r['what']='ITM-20 no time value (tv2)'; r['ret_a']=(x.pnl_a/x.fill).mean(); r['ret_c']=(x.pnl_c/x.fill).mean(); O.append(r)
CMP=pd.DataFrame(O)[['what','grp','n','days','x2','x3','zero','peak_min_med','sig_time_med','ret_a','ret_c']].sort_values(['grp','what'])
# trades: earliest per side per day
T=E.sort_values('t').groupby(['grp','tvdef','date','typ'],as_index=False).first().join(LAB[['L1_VIX','L2_DMA_trend','L3_ER20','L5_NR7','L6_Gap']],on='date')
RN['grp']=RN.ix+'_'+RN.set
def pday(x,col):
    dm=x.groupby('date')[col].mean()
    return stats.ttest_1samp(dm,0).pvalue if len(dm)>=3 and dm.std()>0 else np.nan
def rowstat(x,col,rn=None,spot=None):
    v=x[col].dropna(); r=dict(n=len(v),days=x.date.nunique(),mean=v.mean(),median=v.median(),win=100*(v>0).mean() if len(v) else np.nan,
        no_top3=v.sort_values().iloc[:-3].mean() if len(v)>3 else np.nan,p=pday(x,col))
    if rn is not None and len(rn):
        m=rn.merge(x[['date','typ']],on=['date','typ']).groupby('rep')[col].mean(); r['random_mean']=m.mean(); r['random_pctile']=100*(m<r['mean']).mean() if len(m) else np.nan
    if spot is not None:
        sv=x[spot].dropna(); r['spot_same_dir']=sv.mean(); ov=x['o'+spot]; 
        sims=[np.where(rng.random(len(x))<0.5,x[spot],ov).mean() for _ in range(1000)]; r['spot_coinflip_mean']=np.mean(sims); r['spot_dir_pctile']=100*(np.array(sims)<sv.mean()).mean()
    return r
rows=[]
for (g,tvd),x in T.groupby(['grp','tvdef']):
    for md in ('all','counter','with','neutral'):
        y=x if md=='all' else x[x['mode']==md]
        for sp in ('train','test'):
            z=y[y.split==sp]
            for ex,spc in (('a','sa'),('b','sb'),('c','sc')):
                rn=RN[(RN.grp==g)&(RN.split==sp)] if tvd=='tv2' else None
                if len(z)==0: rows.append(dict(grp=g,tvdef=tvd,mode=md,split=sp,exit=ex,n=0)); continue
                r=rowstat(z,'pnl_'+ex,rn,spc); r.update(grp=g,tvdef=tvd,mode=md,split=sp,exit=ex); rows.append(r)
TS=pd.DataFrame(rows)
def bh(p):
    p=np.asarray(p,float); p=np.where(np.isnan(p),1.0,p); n=len(p); o=np.argsort(p); q=np.empty(n); q[o]=np.minimum.accumulate((p[o]*n/np.arange(1,n+1))[::-1])[::-1]; return np.minimum(q,1)
pm=(TS.split=='test')&(TS.tvdef=='tv2')&TS.grp.isin(['NIFTY_raw','SENSEX_scaled'])&TS['mode'].isin(['all','counter','with'])
TS.loc[pm,'q_bh_primary']=bh(TS.loc[pm,'p']); sm=(TS.split=='test')&~pm; TS.loc[sm,'q_bh_secondary']=bh(TS.loc[sm,'p'])
tr=TS[TS.split=='train'].set_index(['grp','tvdef','mode','exit'])['mean']
TS['train_mean']=[tr.get((a,b,c,d),np.nan) for a,b,c,d in zip(TS.grp,TS.tvdef,TS['mode'],TS.exit)]
TS['counts']=pm&(TS['mean']>0)&(TS.p<0.05)&(TS.q_bh_primary<0.10)&(TS.random_pctile>=90)&(TS.train_mean>0)&(TS.no_top3>0)&(TS['mean']>TS.spot_same_dir)
# regime filter chosen on train
RG=[]
for g in ('NIFTY_raw','SENSEX_scaled'):
    x=T[(T.grp==g)&(T.tvdef=='tv2')]
    for ex in 'abc':
        best=None
        for col in ('L1_VIX','L2_DMA_trend','L3_ER20','L5_NR7','L6_Gap'):
            for val,z in x[x.split=='train'].groupby(col):
                if len(z)>=15 and (best is None or z['pnl_'+ex].mean()>best[2]): best=(col,val,z['pnl_'+ex].mean(),len(z))
        if best is None: RG.append(dict(grp=g,exit=ex,filter='none with n>=15')); continue
        te=x[(x.split=='test')&(x[best[0]]==best[1])]['pnl_'+ex]
        RG.append(dict(grp=g,exit=ex,filter=f'{best[0]}={best[1]}',train_n=best[3],train_mean=best[2],test_n=len(te),test_mean=te.mean() if len(te) else np.nan))
RG=pd.DataFrame(RG)
# spot reachability: P(spot +20f before -20f) proxy = option (a) touches 2x
x=E[E.tvdef=='tv2']
FR=pd.DataFrame([dict(grp=g,**dict(dict(n=len(d),sig_vol_med=d.sig_vol.median(),sig_vol_nan_or0=100*((d.sig_vol.isna())|(d.sig_vol<=0)).mean(),fill_vol_med=d.fill_vol.median(),fill_vol_nan_or0=100*((d.fill_vol.isna())|(d.fill_vol<=0)).mean(),src_opt_panels=100*(d.src=='opt_panels').mean(),src_eod=100*d.src.astype(str).str.contains('eod|atm').mean(),negTV_contracts=int((NEG.ix+"_"+NEG.set==g).sum())))) for g,d in x.groupby('grp')])
for n,d in [('ntv_freq',F),('ntv_freq_mode',FM),('ntv_freq_win',FW),('ntv_vs_otm20',CMP),('ntv_trades',T),('ntv_summary',TS),('ntv_regime',RG),('ntv_fill',FR)]: d.to_csv(f'{R}/{n}.csv',index=False)
pd.set_option('display.width',250); pd.set_option('display.max_columns',30)
print(F.round(1).to_string()); print(FM.round(1).to_string()); print(FW.round(1).to_string()); print(CMP.round(2).to_string())
k=TS[(TS.tvdef=='tv2')&(TS.grp!='SENSEX_raw')]; print(k[['grp','mode','split','exit','n','mean','median','win','no_top3','p','q_bh_primary','random_mean','random_pctile','spot_same_dir','spot_coinflip_mean','train_mean','counts']].round(2).to_string())
print(RG.round(2).to_string()); print(FR.round(1).to_string()); print('primary counts',int(TS.counts.sum()))
print(TS[(TS.tvdef!='tv2')&(TS['mode']=='all')][['grp','tvdef','split','exit','n','mean','p','q_bh_secondary']].round(2).to_string())