# Trader-discretion reverse-engineering: trader entries vs our candidate setups (V1-V4 all events, P1-P8), features, outcomes. (PREREG YOUTUBE-CALLS; copy of s9 with YouTube calls as 'taken')
import sys,time,re; sys.path.insert(0,'/workspace/research/popular-strategies/scripts')
src=open('/workspace/research/popular-strategies/scripts/s7_premsupport.py').read().split('LAB=pd.read_csv')[0]; exec(src)
t0=time.time(); RS=f'{L}/results'
F=pd.read_csv('/workspace/research/youtube-calls/data/yt_as_fillmatch.csv',parse_dates=['date_used'])
F=F[F.side.isin(['CE','PE'])&F.instrument.isin(['Sensex','Nifty 50'])].copy()
def fam(s):
    s=str(s).lower()
    for k,ws in [('BLACKBOX',['black box','blackbox','premium reversal']),('PM3',['3 pm','3pm','matka','closing spike','3:15','320','315 ']),('BTST',['btst']),('NEWS',['news','geopolit']),('AMD',['amd']),
                 ('SWEEP',['sweep','trap','stop hunt','short covering']),('SUPPORT',['support','resistance','rejection','bounce','reversal','w-pattern']),('BREAK',['breakout','breakdown','break','retest','trendline','range','opening']),('CONT',['continuation','pullback','re-entry','reentry','scalp','momentum','impulse','attempt'])]:
        if any(w in s for w in ws): return k
    return 'OTHER'
F['family']=F.setup.map(fam); F['dir']=np.where(F.side=='CE',1,-1); F['m']=F.time.map(hm2i); F['d']=F.date_used
fut={}
P=pd.read_parquet('/workspace/research/flow-forensics-2026-10/lows/data/minute_panel_v.parquet',columns=['date','hm','fut']); P=P[(P.date>=pd.Timestamp('2024-10-21'))&(P.date<=LAST)]
for d,g in P.groupby('date'):
    a=np.full(NM,np.nan); ix=g.hm.map(hm2i).values; ok=(ix>=0)&(ix<NM); a[ix[ok]]=g.fut.values[ok]; fut[d]=a
LAB=pd.read_csv(f'{RS}/regime_labels.csv',parse_dates=['date']).set_index('date')
EXPD={e for _,e in OPT}
def scan_all(d,var):
    e=nexp(d); a=S[d]; sc=spot5c(a); out=[]; last={}
    for j in range(1,72):
        m=5*j+4
        if m<W0 or m>(W1B if var=='V4' else W1): continue
        for typ in ('CE','PE'):
            if typ in last and m-last[typ]<15: continue
            hit=None
            if var=='V4':
                if np.isnan(sc[j]): continue
                K=float(round(sc[j]/50)*50); oa,Pp=chart(d,e,typ,K)
                if Pp is None or np.isnan(Pp[3,j]): continue
                ph=np.nanmax(Pp[1,:j]) if np.isfinite(Pp[1,:j]).any() else np.nan
                if np.isfinite(ph) and Pp[3,j]>ph and not (j>1 and np.isfinite(Pp[3,j-1]) and Pp[3,j-1]>np.nanmax(Pp[1,:j-1]) if j>1 and np.isfinite(Pp[1,:j-1]).any() else False): hit=(K,Pp[2,j]-1)
            else:
                k=j-1
                if np.isnan(sc[k]): continue
                K=float(round(sc[k]/50)*50); oa,Pp=chart(d,e,typ,K)
                if Pp is None or np.isnan(Pp[3,j]) or np.isnan(Pp[2,k]): continue
                if var=='V1': lvl=pdl(d,e,typ,K)
                elif var=='V2':
                    lvl=np.nan
                    for p in range(k-3,2,-1):
                        w=Pp[2,p-2:p+3]
                        if p-2>=0 and np.isfinite(w).all() and Pp[2,p]==w.min() and (w==Pp[2,p]).sum()==1: lvl=Pp[2,p]; break
                else:
                    lw=Pp[2,k]; lvl=round(lw/25)*25 if 40<=lw<=400 else np.nan
                    if lvl==0: lvl=np.nan
                if touch_ok(Pp,k,lvl) and Pp[3,j]>Pp[1,k] and Pp[3,j]>lvl: hit=(K,Pp[2,k]-1)
            if hit: out.append(dict(date=d,m=m,typ=typ,dir=1 if typ=='CE' else -1,K=hit[0],stop_prem=hit[1],src=var,e=e)); last[typ]=m
    return out
def feats(d,m,dd,K,e,typ):
    a=S[d]; o,h,l,c,v=a; f={}
    f['tod']=m; cm=c[m] if not np.isnan(c[m]) else np.nan
    hh=np.nanmax(h[:m+1]); ll=np.nanmin(l[:m+1]); rng_=hh-ll
    pos=(cm-ll)/rng_ if rng_>0 else np.nan; f['spot_pos']=pos if dd>0 else (1-pos if pos==pos else np.nan)
    f['spot_mv15']=dd*(cm-c[max(0,m-15)]) if m>=15 else np.nan; f['day_range']=rng_
    f['dist_round50']=abs(cm-round(cm/50)*50); f['dist_round100']=abs(cm-round(cm/100)*100)
    lb=LAB.loc[d] if d in LAB.index else None
    f['vix']=lb.vix_open if lb is not None else np.nan; f['gap']=dd*lb.gap_pct if lb is not None else np.nan
    oa=oarr(d,e,typ,K)
    if oa is not None:
        pc=oa[3]; ph=np.nanmax(oa[1,:m+1]) if np.isfinite(oa[1,:m+1]).any() else np.nan; p0=pc[m]
        f['prem']=p0; f['prem_dd']=(ph-p0)/ph*100 if ph==ph and ph>0 else np.nan
        f['prem_chg15']=(p0/pc[m-15]-1)*100 if m>=15 and pc[m-15]>0 else np.nan; f['prem_chg5']=(p0/pc[m-5]-1)*100 if m>=5 and pc[m-5]>0 else np.nan
    O=OPT.get((d,e))
    if O is not None and m>=15:
        oi=O['oi']; ix=list(O.index); hm=i2hm(m); hm15=i2hm(m-15)
        if hm in ix and hm15 in ix:
            r=oi.loc[hm]; r15=oi.loc[hm15]
            if (typ,K) in r.index and r15.get((typ,K),np.nan)>0: f['oi_chg15']=(r[(typ,K)]/r15[(typ,K)]-1)*100
            ks=[K+50*i for i in range(-2,3)]
            def pcr(x): 
                ce=sum(x.get(('CE',k),np.nan) for k in ks); pe=sum(x.get(('PE',k),np.nan) for k in ks); return pe/ce if ce>0 else np.nan
            f['pcr_chg15']=pcr(r)-pcr(r15)
    fu=fut.get(d)
    if fu is not None: f['basis']=fu[m]-cm; f['basis_chg15']=(fu[m]-c[m-15])-(fu[m-15]-c[m-15]) if m>=15 else np.nan
    f['expiry_day']=int(d in EXPD); f['dte']=(e-d).days
    return f
LOGD=sorted(set(F.d)); LOGD=[d for d in LOGD if d in S and nexp(d) is not None]
# trader entries: features + our X_OLD outcome on Nifty ATM
tr=[]
for r in F.itertuples():
    d=r.d
    if d not in S or nexp(d) is None or np.isnan(r.m) or r.m<0 or r.m>=359: continue
    a=S[d]; m=int(r.m); cm=a[3,m]
    if np.isnan(cm): continue
    e=nexp(d); typ=r.side; K=float(round(cm/50)*50); oa=oarr(d,e,typ,K); z=run_exit(d,oa,m,'X_OLD') if oa is not None else None
    tr.append(dict(row_id=r.row_id,date=d,m=m,dir=r.dir,typ=typ,K=K,family=r.family,setup=r.setup,instrument=r.instrument,status_log=r.status_log,log_pts=r.result_pts,replay_pts=r.sim_pts,replay_win=r.sim_win,tier=r.tier,
                   xold_pts=z['pnl'] if z else np.nan,**feats(d,m,r.dir,K,e,typ)))
TR=pd.DataFrame(tr); TR.to_csv(f'{RS}/yt_disc_trader_entries.csv',index=False); print('trader entries',len(TR),round(time.time()-t0),flush=True)
# candidates on trader-active days
X=pd.read_csv(f'{RS}/signals_all.csv',parse_dates=['date']); cand=[]
for d in LOGD:
    for var in ('V1','V2','V3','V4'): cand+=scan_all(d,var)
    e=nexp(d)
    for r in X[X.date==d].itertuples(): cand.append(dict(date=d,m=int(r.m),typ='CE' if r.dir>0 else 'PE',dir=int(r.dir),K=float(round(r.entry/50)*50),stop_prem=np.nan,src=r.cand,e=e,spotR=r.R))
C=pd.DataFrame(cand); print('candidates',len(C),round(time.time()-t0),flush=True)
rows=[]
for r in C.itertuples():
    oa=oarr(r.date,r.e,r.typ,r.K)
    if oa is None or r.m+1>=NM or np.isnan(oa[0,r.m+1]): continue
    raw=oa[0,r.m+1]; Rp=(raw-r.stop_prem) if r.src.startswith('V') else 0.5*r.spotR
    z1=run_exit(r.date,oa,r.m,'X_OLD'); z2=run_exit(r.date,oa,r.m,'X_T50',Rp)
    tk=TR[(TR.date==r.date)&(TR.dir==r.dir)&((TR.m-r.m).abs()<=10)]
    rows.append(dict(date=r.date,m=r.m,typ=r.typ,dir=r.dir,K=r.K,src=r.src,Rp=Rp,taken=int(len(tk)>0),taken_family='|'.join(sorted(set(tk.family))),
                     xold=z1['pnl'] if z1 else np.nan,xt50=z2['pnl'] if z2 else np.nan,**feats(r.date,r.m,r.dir,r.K,r.e,r.typ)))
CD=pd.DataFrame(rows)
cnt=TR.groupby('date').m.apply(list).to_dict(); CD['prior_trades']=[sum(x<m for x in cnt.get(d,[])) for d,m in zip(CD.date,CD.m)]
TR['prior_trades']=[sum(x<m for x in cnt.get(d,[])) for d,m in zip(TR.date,TR.m)]; TR.to_csv(f'{RS}/yt_disc_trader_entries.csv',index=False)
CD.to_csv(f'{RS}/yt_disc_candidates.csv',index=False)
TR['covered']=[int(((CD.date==d)&(CD.dir==dd)&((CD.m-m).abs()<=10)).any()) for d,dd,m in zip(TR.date,TR.dir,TR.m)]
TR['covered_V']=[int(((CD.date==d)&(CD.dir==dd)&((CD.m-m).abs()<=10)&CD.src.str.startswith('V')).any()) for d,dd,m in zip(TR.date,TR.dir,TR.m)]
TR.to_csv(f'{RS}/yt_disc_trader_entries.csv',index=False)
print(CD.groupby('src').taken.agg(['size','sum','mean']).round(3)); print('recall any',TR.covered.mean().round(3),'recall V',TR.covered_V.mean().round(3),round(time.time()-t0))