import sys,glob,json,numpy as np,pandas as pd from math import log,sqrt,exp from scipy.stats import norm from scipy.optimize import brentq sys.path.insert(0,'../scripts'); from load import m1,day,SESS,STK L=pd.read_csv('results/lows_list.csv'); L=L[L.fall>=140].reset_index(drop=True) W=pd.read_csv('../data/implied_weights.csv',index_col=0).pts_per_rupee def hmadd(h,m): return (pd.Timestamp('2000-01-01 '+h)+pd.Timedelta(minutes=m)).strftime('%H:%M') ALLD=sorted(m1('NIFTY').date.unique()) def ratio_frame(df,col='volume'): V=df.pivot_table(index='date',columns='hm',values=col); return V/(V.shift(1).rolling(20,min_periods=15).mean()) # load series N=m1('NIFTY').set_index(['date','hm']); BN=m1('BANKNIFTY').set_index(['date','hm']) NF=m1('NIFTYFUT',clean=False); NN=m1('NIFTYFUT_NOV',clean=False); BF=m1('BANKNIFTYFUT',clean=False) NFr=ratio_frame(NF); BFr=ratio_frame(BF) NF=NF.set_index(['date','hm']); NN=NN.set_index(['date','hm']); BF=BF.set_index(['date','hm']) OI=(NF.oi.add(NN.oi,fill_value=np.nan)) # Oct+Nov units S={s:m1(s) for s in STK}; SR={s:ratio_frame(S[s]) for s in STK}; S={s:S[s].set_index(['date','hm']) for s in STK} flow=pd.read_csv('../results/stock_day_flow.csv').set_index(['date','sym']) DN=day('NIFTY') news=[] for f in glob.glob('/workspace/market-live-stack/feeds/news/2026-10-*.jsonl'): for l in open(f): try: j=json.loads(l) except: continue news.append((pd.Timestamp(j['ts'][:19]),j.get('headline',''),j.get('source',''),j.get('ts_source',''),j.get('importance',''),' '.join(j.get('tags',[]) or []),j.get('india_impact',0) or 0)) news=pd.DataFrame(news,columns=['ts','headline','source','ts_source','importance','tags','impact']).sort_values('ts').drop_duplicates('headline',keep='first') # rule 2: reposts keep earliest timestamp MK=r'sensex|nifty|market|stocks? (fall|rise|jump|crash)|fii|fpi|oil|brent|crude|saudi|iran|houthi|aramco|hormuz|rbi|minister|tax|jio|ipo|rupee|war|ceasefire|trump|fed|nasdaq|bank nifty|vix' news=news[news.headline.str.contains(MK,case=False,regex=True)&~news.headline.str.contains(r'share price|stock price|shares? (jump|rise|fall|surge|slump)s? \d|target price|Q2 results', case=False,regex=True)] # rule (2026-10-25 newsfix): ignore headlines whose quoted index move doesn't match the actual Nifty move that day sys.path.insert(0,'/workspace/research/flow-forensics-2026-10/scripts'); from news_rules import mismatch _mm=[mismatch(t,h) for t,h in zip(news.ts,news.headline)]; news[[x is True for x in _mm]].to_csv('results/news_ignored_move_mismatch.csv',index=False) news=news[[x is not True for x in _mm]] EXP={'2026-10-12':'2026-10-13','2026-10-14':'2026-10-19','2026-10-16':'2026-10-19','2026-10-19':'2026-10-27','2026-10-21':'2026-10-27','2026-10-22':'2026-10-27','2026-10-23':'2026-10-27'} OPT={} for f in glob.glob('data/opt/*.parquet'): d=pd.read_parquet(f); d['hm']=d.ts.dt.strftime('%H:%M') if len(d): OPT[(d.date.iloc[0],d.type.iloc[0],float(d.strike.iloc[0]))]=d.set_index('hm') def ob(d,t,k,hm,col='close'): x=OPT.get((d,t,float(k))); return x[col].get(hm,np.nan) if x is not None else np.nan def b76(F,K,T,sig,cp): d1=(log(F/K)+0.5*sig*sig*T)/(sig*sqrt(T)); d2=d1-sig*sqrt(T) return F*norm.cdf(d1)-K*norm.cdf(d2) if cp=='CE' else K*norm.cdf(-d2)-F*norm.cdf(-d1) def iv(p,F,K,T,cp): try: return brentq(lambda s:b76(F,K,T,s,cp)-p,0.01,3)*100 except Exception: return np.nan rows=[];mins=[];stk_rows=[] for i,r in L.iterrows(): d,h=r.date,r.hm; seq=[hmadd(h,m) for m in range(-15,46)] e=EXP.get(d); K=round(r.low/50)*50 Texp=pd.Timestamp(e+' 15:30') for m,x in zip(range(-15,46),seq): g=lambda ser,col: ser[col].get((d,x),np.nan) ce,pe=ob(d,'CE',K,x),ob(d,'PE',K,x) Fs=K+ce-pe if np.isfinite(ce) and np.isfinite(pe) else np.nan T=max((Texp-pd.Timestamp(d+' '+x)).total_seconds(),60)/(365*86400) mins.append(dict(low_id=i,date=d,low_hm=h,rel=m,hm=x,nifty=g(N,'close'),nifty_low=g(N,'low'),banknifty=g(BN,'close'),fut=g(NF,'close'),bnfut=g(BF,'close'), fut_vol=g(NF,'volume')+g(NN,'volume'),fut_vr=NFr.loc[d].get(x,np.nan) if d in NFr.index else np.nan,bnfut_vr=BFr.loc[d].get(x,np.nan), oi_octnov=OI.get((d,x),np.nan),atm=K,ce=ce,pe=pe,ce_vol=ob(d,'CE',K,x,'volume'),pe_vol=ob(d,'PE',K,x,'volume'), ce_iv=iv(ce,Fs,K,T,'CE') if np.isfinite(Fs) else np.nan,pe_iv=iv(pe,Fs,K,T,'PE') if np.isfinite(Fs) else np.nan, n_stk_3x=sum(1 for s in STK if SR[s].loc[d].get(x,0)>=3))) M=pd.DataFrame([z for z in mins if z['low_id']==i]).set_index('rel') M['basis']=M.fut-M.nifty def at(k,c='nifty'): return M[c].get(k,np.nan) low=r.low; day15=N.xs(d)['close'].loc[:'15:14'] after=N.xs(d).loc[h:'15:14']; nxt=ALLD[ALLD.index(d)+1] if d!=ALLD[-1] else None nlow=N.xs(nxt).low.min() if nxt else np.nan # option OI change at nearby strikes, low-15 -> low+30 (contracts=units/65) def oich(t,k): a,b=ob(d,t,k,hmadd(h,-15),'oi'),ob(d,t,k,hmadd(h,30),'oi'); return (b-a)/65 pe_w=sum(np.nan_to_num(oich('PE',k)) for k in range(K-200,K+1,50)); ce_w=sum(np.nan_to_num(oich('CE',k)) for k in range(K,K+201,50)) pe_oi_low=sum(np.nan_to_num(ob(d,'PE',k,h,'oi')) for k in range(K-200,K+201,50)); ce_oi_low=sum(np.nan_to_num(ob(d,'CE',k,h,'oi')) for k in range(K-200,K+201,50)) pe_oi_30=sum(np.nan_to_num(ob(d,'PE',k,hmadd(h,30),'oi')) for k in range(K-200,K+201,50)); ce_oi_30=sum(np.nan_to_num(ob(d,'CE',k,hmadd(h,30),'oi')) for k in range(K-200,K+201,50)) # stocks st=[] for s in STK: p0=S[s].close.get((d,h),np.nan); p15=S[s].close.get((d,min(hmadd(h,15),'15:14')),np.nan); p30=S[s].close.get((d,min(hmadd(h,30),'15:14')),np.nan) win=S[s].xs(d).loc[hmadd(h,-10):hmadd(h,10)] tmin=win.low.idxmin() if len(win) and win.low.notna().any() else None vr=[SR[s].loc[d].get(hmadd(h,m),np.nan) for m in range(-2,6)] fl=flow.loc[(d,s)] if (d,s) in flow.index else None st.append(dict(low_id=i,date=d,low_hm=h,sym=s,pts15=W[s]*(p15-p0),pts30=W[s]*(p30-p0),own_low_hm=tmin,max_vr_m2_p5=np.nanmax(vr) if np.isfinite(vr).any() else np.nan, deliv=fl.DELIV_PER if fl is not None else np.nan,deliv20=fl.DELIV_PER_20d if fl is not None else np.nan)) st=pd.DataFrame(st); stk_rows.append(st) lead=st.sort_values('pts15',ascending=False).head(5) HW=['HDFCBANK','ICICIBANK','RELIANCE','BHARTIARTL','LT','INFY','SBIN','AXISBANK','KOTAKBANK','TCS'] first=st[st.sym.isin(HW)].sort_values('own_low_hm') nw=news[(news.ts>=pd.Timestamp(d+' '+h)-pd.Timedelta(minutes=10))&(news.ts<=pd.Timestamp(d+' '+h)+pd.Timedelta(minutes=10))] nw2=news[(news.ts>pd.Timestamp(d+' '+h)+pd.Timedelta(minutes=10))&(news.ts<=pd.Timestamp(d+' '+h)+pd.Timedelta(minutes=45))] rec={m:at(m)-low for m in (15,30,60)} if not np.isfinite(rec[60]): t60=hmadd(h,60); rec[60]=N.close.get((d,t60),np.nan)-low mx60=N.xs(d).loc[h:min(hmadd(h,60),'15:14')].high.max()-low oi_d15=(at(15,'oi_octnov')-at(0,'oi_octnov'))/65; oi_dm15=(at(0,'oi_octnov')-at(-15,'oi_octnov'))/65 fut_bounce15=at(15,'fut')-at(0,'fut') vr_low=np.nanmax([at(k,'fut_vr') for k in range(-2,4)]); n3=int(np.nanmax([at(k,'n_stk_3x') for k in range(-2,6)])) lead_deliv=np.nanmean([x.deliv-x.deliv20 for x in lead.itertuples()]) # classification rule (stated in report) pos=nw[(nw.impact>0)&~nw.headline.str.contains(r'falls|crash|drop|slump|plunge|lower|down',case=False,regex=True)] if len(pos)>0 and rec[30]>=40: cls='news-linked' elif oi_d15<=-300: cls='short-covering bounce' elif oi_d15>=300 and lead_deliv>0 and n3>=8: cls='fresh longs + delivery-heavy leaders (real buying)' elif oi_d15>=300: cls='fresh longs (futures); leaders not delivery-heavy' else: cls='no clear OI signal (|dOI|<300 contracts) - mixed' rows.append(dict(low_id=i,date=d,low_hm=h,kind=r.kind,low=low,peak_hm=r.peak_hm,fall=r.fall, rec15=rec[15],rec30=rec[30],rec60=rec[60],max_up60=mx60,retrace60_pct=100*mx60/r.fall, close1514_vs_low=day15.iloc[-1]-low,official_close_vs_low=DN.loc[d,'close']-low, low_held_rest_of_day=bool(after.low.min()>=low-0.01),next_day=nxt,next_day_low_held=(None if nxt is None else bool(nlow>low)), fut_vr_max_m2_p3=vr_low,bnfut_vr_at_low=np.nanmax([at(k,'bnfut_vr') for k in range(-2,4)]),n_stocks_3x_max=n3, oi_chg_15m_before_contracts=oi_dm15,oi_chg_15m_after_contracts=oi_d15,fut_bounce15=fut_bounce15, basis_at_low=at(0,'basis'),basis_chg_15=at(15,'basis')-at(0,'basis'),basis_chg_m15_0=at(0,'basis')-at(-15,'basis'), bn_low_rel=(lambda b: b.idxmin()-0 if b.notna().any() else np.nan)(M.banknifty.loc[-10:10]),bn_rec15=at(15,'banknifty')-at(0,'banknifty'), atm=K,expiry=e,ce_at_low=at(0,'ce'),ce_15=at(15,'ce'),ce_30=at(30,'ce'),pe_at_low=at(0,'pe'),pe_15=at(15,'pe'),pe_30=at(30,'pe'), ce_iv_low=at(0,'ce_iv'),ce_iv_15=at(15,'ce_iv'),pe_iv_low=at(0,'pe_iv'),pe_iv_15=at(15,'pe_iv'),pe_iv_m15=at(-15,'pe_iv'), pe_oi_chg_atm_below_m15_p30=pe_w,ce_oi_chg_atm_above_m15_p30=ce_w,pcr_nearby_low=pe_oi_low/ce_oi_low if ce_oi_low else np.nan,pcr_nearby_p30=pe_oi_30/ce_oi_30 if ce_oi_30 else np.nan, leaders15=', '.join(f'{x.sym} {x.pts15:+.1f}' for x in lead.itertuples()), leaders_volx=', '.join(f'{x.sym} {x.max_vr_m2_p5:.1f}x' for x in lead.itertuples()), first_to_turn=', '.join(f'{x.sym} {x.own_low_hm}' for x in first.itertuples()),n_hw_turned_before_index=int((first.own_low_hm