# minute panel: spot (clean) + Nifty futures (sum of all fetched contracts' volume & OI, front-month price)
import pandas as pd,numpy as np,glob,sys,json
sys.path.insert(0,'../scripts'); from load import m1
sp=pd.read_parquet('/workspace/research/common/data/nifty_1m_clean.parquet')[['timestamp','open','high','low','close','missing']]
x=m1('NIFTY'); x=x[x.date>'2026-10-19']; x=x.rename(columns={'ts':'timestamp'})[['timestamp','open','high','low','close']]; x['missing']=x.close.isna()
sp=pd.concat([sp,x]).drop_duplicates('timestamp').sort_values('timestamp')
sp.loc[sp.missing,['open','high','low','close']]=np.nan
F=[]
for f in glob.glob('data/fut/NIFTY_*.parquet'):
    d=pd.read_parquet(f)
    if len(d): F.append(d[['ts','close','volume','oi','expiry']])
F=pd.concat(F); F['date']=F.ts.dt.normalize(); F['expiry']=pd.to_datetime(F.expiry)
F=F[F.expiry>=F.date]
tot=F.groupby('ts').agg(fvol=('volume','sum'),foi=('oi','sum'),ncon=('expiry','nunique'))
# front month = nearest expiry >= date among ALL monthly expiries (incl. missing Dec-2024)
allexp=sorted(set(pd.to_datetime([c['expiry'] for c in json.load(open('data/NIFTY_fut_contracts.json'))]))|{pd.Timestamp('2026-10-27'),pd.Timestamp('2026-11-24')})
dates=sorted(F.date.unique()); front={d:min(e for e in allexp if e>=d) for d in dates}
F['front']=F.date.map(front); fr=F[F.expiry==F.front].set_index('ts')[['close','volume','oi']].rename(columns={'close':'fut','volume':'fvol_front','oi':'foi_front'})
P=sp.set_index('timestamp').join(tot,how='left').join(fr,how='left')
P['date']=P.index.normalize(); P['hm']=P.index.strftime('%H:%M')
# sessions where front contract is missing (data gap)
gapd=sorted(set(dates)-set(fr.index.normalize().unique()))
P['fut_gap']=P.date.isin(gapd)|P.fvol.isna()
P.loc[P.date.isin(gapd),['fvol','foi']]=np.nan
P=P[P.index>=pd.Timestamp('2022-01-01')]
P.reset_index().to_parquet('data/minute_panel.parquet')
print(P.shape, 'futures sessions',P[P.fvol.notna()].date.nunique(),'gap sessions (front missing):',len(gapd),gapd[:3],gapd[-3:])
print(P[P.fvol.notna()].date.min(),P.date.max())