import pandas as pd,numpy as np,json
from scipy import stats
L='/workspace/research/popular-strategies/results'
T=pd.read_csv(f'{L}/ec_signals.csv',parse_dates=['date']); RD=pd.read_csv(f'{L}/ec_random.csv',parse_dates=['date'])
T['year']=T.date.dt.year; T['q']=T.date.dt.to_period('Q').astype(str)
ndays=T.groupby('ix').date.nunique().to_dict()
def bh(p):
    p=np.asarray(p,float); n=len(p); o=np.argsort(p); q=np.empty(n); prev=1
    for i in range(n-1,-1,-1): j=o[i]; prev=min(prev,p[j]*n/(i+1)); q[j]=prev
    return q
def freq(g):
    ok=g[g.ref>0]
    return pd.Series(dict(signals=len(ok),days=ok.date.nunique(),hit2x=(ok.max_ratio>=2).mean()*100,hit3x=(ok.max_ratio>=3).mean()*100,hit5x=(ok.max_ratio>=5).mean()*100,
        med_min_to_peak=ok.peak_min.median(),near_zero_settle=ok.near_zero.astype(float).mean()*100,settle_above_ref=(ok.settle>ok.ref).mean()*100))
sel=lambda b='b15_25',m='counter',w='ALL':T[(T.band==b)&(T['mode']==m)&(T.win==w)]
F=[]
for ix in ('NIFTY','SENSEX'):
    for lab,(b,m,w) in {'counter 15–25':('b15_25','counter','ALL'),'with-trend 15–25':('b15_25','with','ALL'),'counter 10–30':('b10_30','counter','ALL'),'counter DECAY 15–25':('b15_25','decay','ALL'),'counter scaled band':('scaled15_25','counter','ALL'),
                       'counter W1 09:35–11':('b15_25','counter','W1'),'counter W2 11–13':('b15_25','counter','W2'),'counter W3 13–14:30':('b15_25','counter','W3'),'counter W4 14:30–15':('b15_25','counter','W4')}.items():
        g=sel(b,m,w); g=g[g.ix==ix]
        if len(g)==0: continue
        r=freq(g); r['ix']=ix; r['set']=lab; r['expiry_days']=ndays[ix]; F.append(r)
    r=RD[(RD.ix==ix)&(RD.band=='b15_25')&(RD.win=='ALL')]; r=freq(r.assign(date=r.date.astype(str)+r.rep.astype(str))); r['ix']=ix; r['set']='random minute 15–25 (30 reps)'; r['expiry_days']=ndays[ix]; F.append(r)
F=pd.DataFrame(F)[['ix','set','expiry_days','signals','days','hit2x','hit3x','hit5x','med_min_to_peak','near_zero_settle','settle_above_ref']]; F.to_csv(f'{L}/ec_freq.csv',index=False); print(F.round(1).to_string())
# day-level: share of expiry days where ANY counter-trend first signal (either side) reached 2x/3x/5x
P=sel(); D=P.groupby(['ix','date']).max_ratio.max().reset_index(); D['year']=D.date.dt.year; D['q']=D.date.dt.to_period('Q').astype(str)
DY=D.groupby(['ix','year']).agg(days_with_signal=('date','size'),any2x=('max_ratio',lambda s:(s>=2).mean()*100),any3x=('max_ratio',lambda s:(s>=3).mean()*100),any5x=('max_ratio',lambda s:(s>=5).mean()*100)).reset_index()
DY.to_csv(f'{L}/ec_freq_day_year.csv',index=False); print(DY.round(1).to_string())
Q=P.groupby(['ix','q']).apply(lambda g:pd.Series(dict(n=len(g),hit2x=(g.max_ratio>=2).mean()*100,hit3x=(g.max_ratio>=3).mean()*100,near_zero=g.near_zero.astype(float).mean()*100)),include_groups=False).reset_index()
Q.to_csv(f'{L}/ec_freq_quarter.csv',index=False)
# tradable
def row(g,ex):
    x=g[f'pnl_{ex}'].dropna().values
    if len(x)<3: return dict(n=len(x))
    s=np.sort(x)
    return dict(n=len(x),mean=x.mean(),median=np.median(x),win=(x>0).mean()*100,p10=np.percentile(x,10),p90=np.percentile(x,90),best=x.max(),total=x.sum(),t_p=stats.ttest_1samp(x,0).pvalue,no_top3=s[:-3].mean())
EXN={'a':'half 2x / half 3x or close','b':'SL 50% + trail after 2x','c':'hold to settlement'}
PR=[]
for ix in ('NIFTY','SENSEX'):
    for ex in 'abc':
        for sp in ('train','test'):
            g=P[(P.ix==ix)&(P.split==sp)]; r=row(g,ex); r.update(ix=ix,exit=EXN[ex],split=sp)
            w=sel('b15_25','with'); w=w[(w.ix==ix)&(w.split==sp)][f'pnl_{ex}'].dropna(); r['with_trend_mean']=w.mean(); r['with_trend_n']=len(w)
            rr=RD[(RD.ix==ix)&(RD.band=='b15_25')&(RD.win=='ALL')&(RD.split==sp)].groupby('rep')[f'pnl_{ex}'].mean(); r['random_mean']=rr.mean(); r['random_pctile']=(rr<r.get('mean',np.nan)).mean()*100
            if ex=='a': r['gross_mean']=g.gross_a.mean()
            PR.append(r)
PR=pd.DataFrame(PR); te=PR.split=='test'; PR.loc[te,'q_bh']=bh(PR.loc[te,'t_p'])
tr=PR[PR.split=='train'].set_index(['ix','exit'])['mean']
PR['counts']=False
for i,r in PR[te].iterrows():
    PR.loc[i,'counts']=bool(r['mean']>0 and r.t_p<0.05 and r.q_bh<0.10 and r.random_pctile>=90 and r['mean']>r.with_trend_mean and tr[(r.ix,r.exit)]>0 and r.no_top3>0)
PR.to_csv(f'{L}/ec_primary.csv',index=False); print(PR.round(2).to_string())
# sensitivities
SS=[]
for lab,(b,m,w) in {'band 10–30':('b10_30','counter','ALL'),'Sensex scaled band':('scaled15_25','counter','ALL'),'DECAY 15–25':('b15_25','decay','ALL'),'W1 09:35–11':('b15_25','counter','W1'),'W2 11–13':('b15_25','counter','W2'),'W3 13–14:30':('b15_25','counter','W3'),'W4 14:30–15':('b15_25','counter','W4')}.items():
    for ix in ('NIFTY','SENSEX'):
        for ex in 'abc':
            for sp in ('train','test'):
                g=sel(b,m,w); g=g[(g.ix==ix)&(g.split==sp)]
                if len(g)<3: continue
                r=row(g,ex); r.update(set=lab,ix=ix,exit=EXN[ex],split=sp); SS.append(r)
SS=pd.DataFrame(SS); te=(SS.split=='test')&SS.t_p.notna(); SS.loc[te,'q_bh']=bh(SS.loc[te,'t_p']); SS.to_csv(f'{L}/ec_sens.csv',index=False)
print(SS[SS.split=='test'].round(2).to_string())
# regimes chosen on train
RG=[]
for ix in ('NIFTY','SENSEX'):
    for ex in 'abc':
        best=None
        for lc in ['L1_VIX','L6_Gap','L5_NR7','L3_ER20','L2_DMA_trend']:
            for bk,g in P[(P.ix==ix)&(P.split=='train')].groupby(lc):
                x=g[f'pnl_{ex}'].dropna()
                if len(x)>=15 and (best is None or x.mean()>best[2]): best=(lc,bk,x.mean(),len(x))
        if best is None: continue
        g=P[(P.ix==ix)&(P.split=='test')&(P[best[0]]==best[1])]; r=row(g,ex)
        RG.append(dict(ix=ix,exit=EXN[ex],label=best[0],bucket=best[1],train_n=best[3],train_mean=best[2],test_n=r.get('n'),test_mean=r.get('mean'),test_win=r.get('win'),test_p=r.get('t_p'),test_no_top3=r.get('no_top3')))
RG=pd.DataFrame(RG); RG['q_bh']=bh(RG.test_p.fillna(1)); RG.to_csv(f'{L}/ec_regime.csv',index=False); print(RG.round(2).to_string())
# fills
FL=P.assign(has_vol=P.fill_vol.notna()).groupby(['ix','src']).agg(n=('ix','size'),vol_known=('has_vol','mean'),sig_vol_zero_pct=('sig_vol',lambda s:(s.dropna()<=0).mean()*100),fill_vol_zero_pct=('fill_vol',lambda s:(s.dropna()<=0).mean()*100),fill_vol_median=('fill_vol','median'),fill_vol_p10=('fill_vol',lambda s:s.dropna().quantile(.1) if s.notna().any() else np.nan)).reset_index()
FL.to_csv(f'{L}/ec_fills.csv',index=False); print(FL.round(2).to_string())