# Stats for premium-support variants: primary (BH 8), random pctile, periods, Jan-Sep vs Oct 2026, drop-top-3, regime filters chosen on train -> 2026 (BH 8).
import pandas as pd,numpy as np,json
from scipy import stats
L='/workspace/research/popular-strategies'; RS=f'{L}/results'
T=pd.read_csv(f'{RS}/prem_trades.csv',parse_dates=['date']); RR=pd.read_csv(f'{RS}/prem_random.csv',parse_dates=['date','src_date'])
LAB=pd.read_csv(f'{RS}/regime_labels.csv',parse_dates=['date']).set_index('date'); LBS=[c for c in LAB.columns if c[:2] in ('L1','L2','L3','L4','L5','L6','L7','L8') and c[2]=='_']
T=T.join(LAB[LBS],on='date'); RR=RR.join(LAB[LBS],on='date')
RG={k:g.pnl.values for k,g in RR.groupby(['var','ex','src_date'])}
def tp(p): return stats.ttest_1samp(p,0,alternative='greater').pvalue if len(p)>2 else np.nan
def bh(p):
    p=np.where(np.isnan(np.asarray(p,float)),1,np.asarray(p,float)); m=len(p); o=np.argsort(p); q=np.empty(m); q[o]=np.minimum.accumulate((p[o]*m/np.arange(1,m+1))[::-1])[::-1]; return np.minimum(q,1)
rng=np.random.default_rng(2125)
def rp(x):
    arrs=[RG[(v,e,d)] for v,e,d in zip(x['var'],x.ex,x.date) if (v,e,d) in RG]
    if not arrs: return np.nan,np.nan
    mu=np.array([np.mean([a[rng.integers(len(a))] for a in arrs]) for _ in range(2000)]); return (mu<x.pnl.mean()).mean()*100,np.mean(np.concatenate(arrs))
def top(p,k): p=np.sort(np.asarray(p)); return p[:-k].mean() if len(p)>k else np.nan
V=['V1','V2','V3','V4']; X=['X_OLD','X_T50']; rows=[]
for v in V:
    for e in X:
        for sp in ('train','test'):
            x=T[(T['var']==v)&(T.ex==e)&(T.split==sp)]; pc,rm=rp(x)
            rows.append(dict(var=v,ex=e,split=sp,n=len(x),mean=x.pnl.mean(),median=x.pnl.median(),win=(x.pnl>0).mean()*100,total=x.pnl.sum(),rs_total=(x.pnl*np.where(x.date>=pd.Timestamp('2026-01-01'),65,75)).sum(),t_p=tp(x.pnl.values),random_mean=rm,random_pctile=pc,drop_top3=top(x.pnl,3),
                         jan_sep=x[x.date<pd.Timestamp('2026-10-01')].pnl.mean() if sp=='test' else np.nan,oct=x[x.date>=pd.Timestamp('2026-10-01')].pnl.mean() if sp=='test' else np.nan,n_oct=(x.date>=pd.Timestamp('2026-10-01')).sum() if sp=='test' else np.nan))
P=pd.DataFrame(rows); te=P.split=='test'; P.loc[te,'q_bh']=bh(P[te].t_p.values)
P.loc[te,'counts']=np.where((P[te].q_bh<0.10)&(P[te].random_pctile>=90)&(P[te].drop_top3>0),'YES','no')
P.round(3).to_csv(f'{RS}/prem_primary.csv',index=False); print(P.round(2).to_string())
# periods
d=T.copy(); d['year']=d.date.dt.year.astype(str); d['quarter']=d.date.dt.to_period('Q').astype(str); d['month']=d.date.dt.to_period('M').astype(str); out=[]
for g in ('year','quarter','month'):
    z=d.groupby(['var','ex',g]).pnl.agg(n='size',mean='mean',win=lambda s:(s>0).mean()*100).reset_index().rename(columns={g:'period'}); z['level']=g; out.append(z)
pd.concat(out).round(2).to_csv(f'{RS}/prem_by_period.csv',index=False)
# regime buckets + train-chosen filters
rb=[]
for v in V:
    for e in X:
        x=T[(T['var']==v)&(T.ex==e)]
        for lb in LBS:
            for (b,s),g in x.dropna(subset=[lb]).groupby([lb,'split']): rb.append(dict(var=v,ex=e,label=lb,bucket=b,split=s,n=len(g),mean=g.pnl.mean()))
pd.DataFrame(rb).round(2).to_csv(f'{RS}/prem_regime_buckets.csv',index=False)
fr=[]; choices={}
for v in V:
    for e in X:
        x=T[(T['var']==v)&(T.ex==e)]; tr=x[x.split=='train']; ts=x[x.split=='test']; best=None
        for lb in LBS:
            g=tr.groupby(lb).pnl.agg(['size','mean']); good=list(g[(g['size']>=10)&(g['mean']>0)].index)
            f=tr[tr[lb].isin(good)]
            if good and len(f)>=20 and (best is None or f.pnl.sum()>best[2]): best=(lb,good,f.pnl.sum(),len(f),f.pnl.mean())
        if best is None: fr.append(dict(var=v,ex=e,label='none',buckets='',train_n=0,train_mean=np.nan,test_n=0,test_mean=np.nan,always_on_test=ts.pnl.mean(),t_p=1.0,subset_pctile=np.nan,random_same_bucket_mean=np.nan)); continue
        lb,good,_,trn,trm=best; f=ts[ts[lb].isin(good)]; choices[f'{v}_{e}']=dict(label=lb,buckets=good)
        sub=np.array([rng.choice(ts.pnl.values,len(f),replace=False).mean() for _ in range(2000)]) if 0<len(f)<len(ts) else np.array([np.nan])
        rsb=RR[(RR['var']==v)&(RR.ex==e)&(RR.split=='test')]; rsb=rsb[rsb[lb].isin(good)].pnl.mean()
        fr.append(dict(var=v,ex=e,label=lb,buckets='|'.join(map(str,good)),train_n=trn,train_mean=trm,test_n=len(f),test_mean=f.pnl.mean() if len(f) else np.nan,always_on_test=ts.pnl.mean(),t_p=tp(f.pnl.values),subset_pctile=(sub<f.pnl.mean()).mean()*100 if len(f) else np.nan,random_same_bucket_mean=rsb,drop_top3=top(f.pnl,3)))
F=pd.DataFrame(fr); F['q_bh']=bh(F.t_p.values); F['counts']=np.where((F.q_bh<0.10)&(F.test_mean>F.always_on_test)&(F.subset_pctile>=90),'YES','no')
F.round(3).to_csv(f'{RS}/prem_regime_filters.csv',index=False); json.dump(choices,open(f'{RS}/prem_regime_choices.json','w'),indent=1); print(F.round(2).to_string())
q=d[d.ex=='X_T50'].groupby(['var','quarter']).pnl.mean().unstack().round(1); print(q); print(d[d.ex=='X_OLD'].groupby(['var','quarter']).pnl.mean().unstack().round(1))