import pandas as pd,numpy as np,json,matplotlib; matplotlib.use('Agg'); import matplotlib.pyplot as plt
from scipy import stats
L='/workspace/research/popular-strategies'; R=f'{L}/results'; rng=np.random.default_rng(7)
S=pd.read_csv(f'{R}/w20_scan.csv',parse_dates=['date']); TC=pd.read_csv(f'{R}/w20_touch.csv',parse_dates=['date']); H=pd.read_csv(f'{R}/w20_holds.csv',parse_dates=['date']); HR=pd.read_csv(f'{R}/w20_holds_random.csv',parse_dates=['date'])
for D in (S,H,HR):
    D['x2']=(D.max_ratio>=2).astype(float); D['x3']=(D.max_ratio>=3).astype(float); D['z']=D.near_zero.astype(float)
    for e in 'abc': D[f'r_{e}']=D[f'pnl_{e}']/D.fill
def bh(p):
    p=np.asarray(p,float); n=len(p); o=np.argsort(p); q=np.empty(n); prev=1
    for i in range(n-1,-1,-1): j=o[i]; prev=min(prev,p[j]*n/(i+1)); q[j]=prev
    return q
def boot(g,col,B=1000):
    g=g.dropna(subset=[col]); days=g.date.values; ud=np.unique(days); grp={d:g[col].values[days==d] for d in ud}
    sums=np.array([grp[d].sum() for d in ud]); cnt=np.array([len(grp[d]) for d in ud]); est=sums.sum()/cnt.sum()
    idx=rng.integers(0,len(ud),(B,len(ud))); bs=sums[idx].sum(1)/cnt[idx].sum(1); return est,np.percentile(bs,2.5),np.percentile(bs,97.5)
COLS={'x2':'P(≥2×) %','x3':'P(≥3×) %','z':'P(settle≈0) %','r_a':'ret/premium exit a','r_b':'ret/premium exit b','r_c':'ret/premium exit c'}
rows=[]
for (ix,st,Lv),g in S[S.status=='ok'].groupby(['ix','set','L']):
    r=dict(ix=ix,set=st,L=Lv,n=len(g),days=g.date.nunique(),pnl_a=g.pnl_a.mean(),pnl_b=g.pnl_b.mean(),pnl_c=g.pnl_c.mean())
    for c in COLS:
        e,lo,hi=boot(g,c); k=100 if c in ('x2','x3','z') else 1; r[c]=e*k; r[c+'_lo']=lo*k; r[c+'_hi']=hi*k
    for sp in ('train','test'):
        gg=g[g.split==sp]; r[f'r_a_{sp}']=gg.r_a.mean(); r[f'r_b_{sp}']=gg.r_b.mean(); r[f'r_c_{sp}']=gg.r_c.mean()
    rows.append(r)
SC=pd.DataFrame(rows); SC.to_csv(f'{R}/w20_scan_levels.csv',index=False)
SP=[]
for (ix,st),g in SC.groupby(['ix','set']):
    g=g.set_index('L'); o=g.drop(20)
    for c in COLS:
        cf=np.polyfit(np.log(o.index.values),o[c].values,2); fit=np.polyval(cf,np.log(20)); v=g.loc[20,c]; lo,hi=g.loc[20,c+'_lo'],g.loc[20,c+'_hi']
        d15=v-g.loc[15,c]; d25=v-g.loc[25,c]
        SP.append(dict(ix=ix,set=st,metric=COLS[c],at15=g.loc[15,c],at20=v,at20_ci=f'{lo:.3g}..{hi:.3g}',at25=g.loc[25,c],smooth_fit_at20=fit,ci_excludes_fit=not(lo<=fit<=hi),beats_both_neighbours=bool(np.sign(d15)==np.sign(d25)!=0),special=bool(not(lo<=fit<=hi) and np.sign(d15)==np.sign(d25)!=0)))
SP=pd.DataFrame(SP); SP.to_csv(f'{R}/w20_scan_special.csv',index=False); print(SP.round(3).to_string())
for (ix,st),g in SC.groupby(['ix','set']):
    fig,ax=plt.subplots(1,2,figsize=(11,3.8))
    for c,col in (('x2','C0'),('x3','C1'),('z','C3')):
        ax[0].errorbar(g.L,g[c],yerr=[g[c]-g[c+'_lo'],g[c+'_hi']-g[c]],marker='o',color=col,label=COLS[c],capsize=3)
    ax[0].set_xscale('log'); ax[0].set_xticks(g.L); ax[0].set_xticklabels(g.L); ax[0].axvline(20,color='grey',ls=':'); ax[0].set_xlabel('premium level (band ±20%)'+(' × Sensex/Nifty' if st=='scaled' else '')); ax[0].set_ylabel('%'); ax[0].legend(fontsize=8); ax[0].set_title(f'{ix} {st}: frequency vs level')
    for c,col in (('r_a','C0'),('r_b','C1'),('r_c','C2')):
        ax[1].errorbar(g.L,g[c],yerr=[g[c]-g[c+'_lo'],g[c+'_hi']-g[c]],marker='o',color=col,label=COLS[c],capsize=3)
    ax[1].axhline(0,color='k',lw=.8); ax[1].set_xscale('log'); ax[1].set_xticks(g.L); ax[1].set_xticklabels(g.L); ax[1].axvline(20,color='grey',ls=':'); ax[1].set_ylabel('net return per 1 unit premium'); ax[1].legend(fontsize=8); ax[1].set_title('buy at next open: expectancy vs level')
    fig.tight_layout(); fig.savefig(f'{L}/charts/w20_scan_{ix}_{st}.png',dpi=100); plt.close(fig)
TR=[]
for (ix,st,Lv),g in TC.groupby(['ix','set','L']):
    r=dict(ix=ix,set=st,L=Lv,round=Lv in (10,20,25,30,50),n=len(g))
    for c in ('hold5','hold15','hold15_pct','up15'): g2=g.assign(**{c:g[c].astype(float)}); e,lo,hi=boot(g2,c); r[c]=e*100; r[c+'_ci']=f'{lo*100:.0f}..{hi*100:.0f}'
    TR.append(r)
TR=pd.DataFrame(TR); TR.to_csv(f'{R}/w20_touch_levels.csv',index=False); print(TR.round(1).to_string())
TD=[]
for (ix,st),g0 in TC.groupby(['ix','set']):
    for c in ('hold5','hold15','hold15_pct','up15'):
        g=g0.assign(v=g0[c].astype(float)).dropna(subset=['v']); ud=g.date.unique()
        a=g.groupby(['date','L']).v.agg(['sum','count']).unstack('L').fillna(0)
        RN=[10,20,25,30,50]; NR=[17,18,22,23]
        def stat(ix_):
            s=a['sum'].iloc[ix_].sum(); n=a['count'].iloc[ix_].sum()
            rd=s[RN].sum()/n[RN].sum(); nr=s[NR].sum()/n[NR].sum(); t20=s[20]/n[20]; return rd-nr,t20-nr
        e=stat(np.arange(len(a))); bs=np.array([stat(rng.integers(0,len(a),len(a))) for _ in range(1000)])
        TD.append(dict(ix=ix,set=st,metric=c,round_minus_nonround=e[0]*100,ci=f'{np.percentile(bs[:,0],2.5)*100:.1f}..{np.percentile(bs[:,0],97.5)*100:.1f}',l20_minus_17_18_22_23=e[1]*100,ci20=f'{np.percentile(bs[:,1],2.5)*100:.1f}..{np.percentile(bs[:,1],97.5)*100:.1f}'))
TD=pd.DataFrame(TD); TD.to_csv(f'{R}/w20_touch_round_vs_nonround.csv',index=False); print(TD.round(2).to_string())
EXN={'a':'half 2× / half 3× or close','b':'SL 50% + trail after 2×','c':'hold to settlement'}
def row(x):
    x=np.asarray(x.dropna())
    if len(x)<3: return dict(n=len(x))
    s=np.sort(x); return dict(n=len(x),mean=x.mean(),median=np.median(x),win=(x>0).mean()*100,p=stats.ttest_1samp(x,0).pvalue,no_top3=s[:-3].mean(),best=x.max())
HH=[]
for (ix,st,Lv,N),g in H[H.status=='ok'].groupby(['ix','set','L','N']):
    for e in 'abc':
        for sp in ('train','test'):
            gg=g[g.split==sp]; r=row(gg[f'pnl_{e}']); r.update(ix=ix,set=st,L=Lv,N=N,exit=EXN[e],split=sp,ret_mean=gg[f'r_{e}'].mean(),x2=gg.x2.mean()*100)
            rr=HR[(HR.ix==ix)&(HR.set==st)&(HR.L==Lv)&(HR.N==N)&(HR.split==sp)]
            if len(rr): m=rr.groupby('rep')[f'pnl_{e}'].mean(); r['random_mean']=m.mean(); r['random_pctile']=(m<r.get('mean',np.nan)).mean()*100
            HH.append(r)
HH=pd.DataFrame(HH)
pm=(HH.L==20)&(HH.set=='raw')&(HH.split=='test'); HH.loc[pm,'q_bh_primary']=bh(HH.loc[pm,'p'].fillna(1))
so=(~pm)&(HH.split=='test')&HH.p.notna(); HH.loc[so,'q_bh_sens']=bh(HH.loc[so,'p'])
key=HH.set_index(['ix','set','L','N','exit','split'])['mean']
HH['counts']=False
for i,r in HH[pm].iterrows():
    k=lambda Lv,sp: key.get((r.ix,r.set,Lv,r.N,r.exit,sp),np.nan)
    HH.loc[i,'counts']=bool(r['mean']>0 and r.p<0.05 and r.q_bh_primary<0.10 and r.random_pctile>=90 and k(20,'train')>0 and r.no_top3>0 and r['mean']>k(15,'test') and r['mean']>k(25,'test'))
HH.to_csv(f'{R}/w20_holds_summary.csv',index=False); print(HH[pm|((HH.L==20)&(HH.split=='train')&(HH.set=='raw'))].round(2).to_string())
WD=[]
for (ix,N,w),g in H[(H.status=='ok')&(H.L==20)&(H.set=='raw')].groupby(['ix','N','win']):
    for e in 'abc':
        for sp in ('train','test'):
            r=row(g[g.split==sp][f'pnl_{e}']); r.update(ix=ix,N=N,win=w,exit=EXN[e],split=sp); WD.append(r)
WD=pd.DataFrame(WD); WD.to_csv(f'{R}/w20_holds_windows.csv',index=False)
lv=HH[(HH.set=='raw')].pivot_table(index=['ix','N','exit','split'],columns='L',values='mean').round(2); lv.to_csv(f'{R}/w20_holds_by_level.csv'); print(lv.to_string())