# Combination / confluence tests (PREREG "COMBINATIONS" section). Reuses the confirming signal's simulated trade + its random set.
import pandas as pd,numpy as np,json,itertools
from scipy import stats
L='/workspace/research/popular-strategies'; RS=f'{L}/results'
C=[f'P{i}' for i in range(1,9)]
X=pd.read_csv(f'{RS}/signals_all.csv',parse_dates=['date'])
T=pd.read_csv(f'{RS}/trades_options.csv',parse_dates=['date']); O=T[(T.ex=='T50')&(T.status=='ok')].set_index(['cand','date'])
SP=pd.read_csv(f'{RS}/trades_spot.csv',parse_dates=['date']).dropna(subset=['spot_pts']).set_index(['cand','date'])
RR=pd.read_csv(f'{RS}/trades_random.csv',parse_dates=['date','src_date']); RG={k:g.pnl.values for k,g in RR.groupby(['cand','src_date'])}
LAB=pd.read_csv(f'{RS}/regime_labels.csv',parse_dates=['date']).set_index('date'); NR7=set(LAB.index[LAB.L5_NR7=='NR7'])
CO,TE=pd.Timestamp('2024-10-21'),pd.Timestamp('2026-01-01')
def tp(p): return stats.ttest_1samp(p,0,alternative='greater').pvalue if len(p)>2 else np.nan
def bh(p):
    p=np.where(np.isnan(np.asarray(p,float)),1,np.asarray(p,float)); m=len(p); o=np.argsort(p); q=np.empty(m); q[o]=np.minimum.accumulate((p[o]*m/np.arange(1,m+1))[::-1])[::-1]; return np.minimum(q,1)
rng=np.random.default_rng(2113)
def rand_pct(ev):
    if ev.empty: return np.nan,np.nan
    arrs=[RG[(c,d)] for c,d in zip(ev.cand,ev.date) if (c,d) in RG]
    if not arrs: return np.nan,np.nan
    mu=np.array([np.mean([a[rng.integers(len(a))] for a in arrs]) for _ in range(2000)]); return (mu<ev.pnl.mean()).mean()*100,np.mean(np.concatenate(arrs))
# ---- combo event builders: return DataFrame(date,cand(confirming),m,dir,members)
def kofn(k,W,pool=C,days=None):
    ev=[]
    for d,g in X[X.cand.isin(pool)].groupby('date'):
        if days is not None and d not in days: continue
        best=None
        for dd,h in g.groupby('dir'):
            h=h.assign(pn=h.cand.str[1:].astype(int)).sort_values(['m','pn']).reset_index(drop=True)
            for i in range(len(h)):
                w=h[(h.index<=i)&(h.m[i]-h.m<=W)]
                if len(w)>=k:
                    if best is None or h.m[i]<best['m']: best=dict(date=d,cand=h.cand[i],m=h.m[i],dir=dd,members='+'.join(sorted(w.cand)))
                    break
        if best: ev.append(best)
    return pd.DataFrame(ev)
def single_on(c,days): x=X[(X.cand==c)&X.date.isin(days)]; return pd.DataFrame(dict(date=x.date,cand=c,m=x.m,dir=x.dir,members=c))
def attach(ev):
    if ev.empty: return pd.DataFrame(columns=['date','cand','m','dir','members','pnl','spot_net','split'])
    ev=ev.copy(); key=list(zip(ev.cand,ev.date))
    ev['pnl']=[O.pnl.get(k,np.nan) for k in key]; ev['spot_net']=[SP.spot_net.get(k,np.nan) for k in key]
    ev['split']=np.where(ev.date<CO,'pre',np.where(ev.date<TE,'train','test')); return ev
# ---- pair selection on TRAIN
pairs=list(itertools.combinations(C,2)); ps=[]
for a,b in pairs:
    e=attach(kofn(2,10**4,pool=[a,b])); tr=e[(e.split=='train')].dropna(subset=['pnl'])
    ps.append(dict(pair=f'{a}+{b}',train_n=len(tr),train_mean=tr.pnl.mean() if len(tr) else np.nan))
PS=pd.DataFrame(ps).round(2); elig=PS[PS.train_n>=25].sort_values('train_mean',ascending=False); PS.to_csv(f'{RS}/combo_pair_selection_train.csv',index=False)
best=elig.iloc[0].pair if len(elig) else None; print('best pair on train:',best,elig.head(5).to_dict('records'))
DEF={'K2_15':kofn(2,15),'K3_30':kofn(3,30),'K2_D':kofn(2,10**4),
     'N2':single_on('P2',NR7),'N3':single_on('P3',NR7),'N4':single_on('P4',NR7),'N234':kofn(2,10**4,pool=['P2','P3','P4'],days=NR7),
     'PAIR_BEST':kofn(2,10**4,pool=best.split('+')),'PAIR_P1P5':kofn(2,10**4,pool=['P1','P5']),'PAIR_P3P4':kofn(2,10**4,pool=['P3','P4']),'PAIR_P2P5':kofn(2,10**4,pool=['P2','P5'])}
CONST={'K2_15':C,'K3_30':C,'K2_D':C,'N2':['P2'],'N3':['P3'],'N4':['P4'],'N234':['P2','P3','P4'],'PAIR_BEST':best.split('+'),'PAIR_P1P5':['P1','P5'],'PAIR_P3P4':['P3','P4'],'PAIR_P2P5':['P2','P5']}
Oall=O.reset_index()
rows=[]; allev=[]
for nm,ev in DEF.items():
    ev=attach(ev); ev['combo']=nm; allev.append(ev)
    for sp in ('train','test'):
        x=ev[(ev.split==sp)].dropna(subset=['pnl']); pc,rm=rand_pct(x)
        base=Oall[Oall.cand.isin(CONST[nm])&((Oall.date>=TE) if sp=='test' else (Oall.date<TE))].pnl
        rows.append(dict(combo=nm,split=sp,n=len(x),mean=x.pnl.mean(),median=x.pnl.median(),win=(x.pnl>0).mean()*100,total=x.pnl.sum(),t_p=tp(x.pnl.values),random_mean=rm,random_pctile=pc,constituents_mean=base.mean()))
R=pd.DataFrame(rows); te=R.split=='test'; R.loc[te,'q_bh']=bh(R[te].t_p.values)
R['counts']=''; R.loc[te,'counts']=np.where((R[te].q_bh<0.10)&(R[te]['mean']>R[te].constituents_mean)&(R[te].random_pctile>=90),'YES','no')
R['contaminated']=R.combo.isin(['N2','N3','N4','N234'])
R.round(3).to_csv(f'{RS}/combo_results.csv',index=False); print(R.round(2).to_string())
E=pd.concat(allev); E['date']=pd.to_datetime(E.date); E.to_csv(f'{RS}/combo_events.csv',index=False)
# breakdowns (options)
Eo=E.dropna(subset=['pnl']).copy(); Eo['year']=Eo.date.dt.year.astype(str); Eo['quarter']=Eo.date.dt.to_period('Q').astype(str)
Eo['seg']=np.where(Eo.date<TE,'train',np.where(Eo.date<pd.Timestamp('2026-10-01'),'2026 Jan-Sep','2026 Oct'))
out=[]
for g in ('year','quarter','seg'):
    z=Eo.groupby(['combo',g]).pnl.agg(n='size',mean='mean',win=lambda v:(v>0).mean()*100).reset_index().rename(columns={g:'period'}); z['level']=g; out.append(z)
pd.concat(out).round(2).to_csv(f'{RS}/combo_by_period.csv',index=False)
# singles Jan-Sep vs Oct for comparison
S1=Oall[Oall.date>=TE].assign(seg=lambda d:np.where(d.date<pd.Timestamp('2026-10-01'),'2026 Jan-Sep','2026 Oct')).groupby(['cand','seg']).pnl.agg(n='size',mean='mean').round(2).reset_index(); S1.to_csv(f'{RS}/singles_2026_seg.csv',index=False)
# spot-level combos 2022-2026
Es=E.dropna(subset=['spot_net']).assign(year=lambda d:d.date.dt.year).groupby(['combo','year']).spot_net.agg(n='size',mean='mean',win=lambda v:(v>0).mean()*100).round(2).reset_index(); Es.to_csv(f'{RS}/combo_spot_by_year.csv',index=False)
# ---- overlap matrix (common window 21-Oct-2024..23-Oct-2026, all 8 live)
W=X[X.date>=CO]; days=sorted(set(pd.read_csv(f'{RS}/regime_labels.csv',parse_dates=['date']).date)); days=[d for d in days if CO<=d<=W.date.max()]
D=W.pivot_table(index='date',columns='cand',values='dir',aggfunc='first').reindex(days).fillna(0)
same=pd.DataFrame(index=C,columns=C,dtype=float); opp=same.copy(); phi=same.copy(); pcor=same.copy(); jac=same.copy()
OP=Oall.pivot_table(index='date',columns='cand',values='pnl',aggfunc='first')
for a in C:
    for b in C:
        fa=D[a]!=0; fb=D[b]!=0
        same.loc[a,b]=((D[a]==D[b])&fa).sum()/fa.sum()*100; opp.loc[a,b]=((D[a]==-D[b])&fa&fb).sum()/fa.sum()*100
        jac.loc[a,b]=((D[a]==D[b])&fa).sum()/((fa|fb).sum())*100
        both=fa&fb; phi.loc[a,b]=np.corrcoef(D[a][both],D[b][both])[0,1] if both.sum()>5 else np.nan
        sd=OP[[a,b]].dropna(); sd=sd[sd.index.isin(D.index[(D[a]==D[b])&fa])]; pcor.loc[a,b]=sd[a].corr(sd[b]) if len(sd)>5 and a!=b else np.nan
for nm,M in (('same',same),('opp',opp),('dircorr',phi),('pnlcorr',pcor),('jaccard',jac)): M.round(2).to_csv(f'{RS}/overlap_{nm}.csv')
fires=(D!=0).sum(); json.dump(dict(days=len(D),fires=fires.to_dict(),any_two_same_dir_days=int(((D>0).sum(1)>=2).sum()+((D<0).sum(1)>=2).sum()),best_pair=best),open(f'{RS}/overlap_meta.json','w'),indent=1)
print(same.round(0)); print(phi.round(2)); print(pcor.round(2)); print(fires.to_dict())