# Pre-registered full-history test of the extracted discretion filter (tod<=36.5 AND basis_chg15>-8.25), first fire per day, exclude May-Jul 2026 (fit window).
import pandas as pd,numpy as np,json
from scipy import stats
RS='/workspace/research/popular-strategies/results'
M=json.load(open(f'{RS}/disc_model.json')); med=json.load(open(f'{RS}/disc_impute_medians.json'))['med']
C=pd.read_csv(f'{RS}/disc_fullhist_candidates.csv',parse_dates=['date']); R=pd.read_csv(f'{RS}/disc_fullhist_random.csv')
m=np.ones(len(C),bool)
for f,op,th in M['rule']:
    x=C[f].fillna(med[f]).values; m&=(x<=th) if op=='<=' else (x>th)
F=C[m&~((C.date>='2026-05-01')&(C.date<'2026-08-01'))].sort_values(['date','m']).groupby('date').head(1)
P={'Oct24-Dec25':(F.date<'2026-01-01'),'2026 (Jan-Apr+Aug-Oct)':(F.date>='2026-01-01')}
rows=[]
for pn,pm in P.items():
    s=F[pm]
    for ex in ('xold','xt50'):
        v=s[ex].dropna(); rr=R[R.cid.isin(s.cid)].groupby('rep')[ex].mean()
        rows.append(dict(period=pn,exit=ex,n=len(v),mean=v.mean(),win=(v>0).mean()*100,total=v.sum(),p=stats.ttest_1samp(v,0,alternative='greater').pvalue if len(v)>2 else np.nan,rand_mean=rr.mean(),rand_pctile=(rr<v.mean()).mean()*100,drop_top3=v.sort_values().iloc[:-3].mean() if len(v)>3 else np.nan))
T=pd.DataFrame(rows); p=T.p.fillna(1).values; o=np.argsort(p); q=np.empty(4); q[o]=np.minimum.accumulate((p[o]*4/np.arange(1,5))[::-1])[::-1]; T['q_bh']=np.minimum(q,1)
T['counted']=(T.q_bh<0.10)&(T.rand_pctile>=90)&(T.drop_top3>0)&(M['lift_ci90'][0]>1)
print(T.round(3).to_string()); T.round(3).to_csv(f'{RS}/disc_filter_fullhist.csv',index=False)
print('fires total',m.sum(),'days',F.date.nunique())