From 71f9c7849b89435894e4f4e0a6f44ae1977674e7 Mon Sep 17 00:00:00 2001 From: Brent Neale Date: Mon, 16 Feb 2026 15:42:17 +1000 Subject: [PATCH] Fix NaN serialization and upsert response handling in upload Sanitize NaN/inf values that pandas re-introduces into float columns after conversion, preventing JSON serialization errors. Fix upsert response check to detect success via resp.data instead of status_code. Co-Authored-By: Claude Opus 4.6 --- src/supabase_upload.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/supabase_upload.py b/src/supabase_upload.py index 61a3919..7b22683 100644 --- a/src/supabase_upload.py +++ b/src/supabase_upload.py @@ -93,6 +93,15 @@ def df_to_records(df: pd.DataFrame, instrument: str, granularity: str = "M1"): df2[col] = df2[col].apply(_convert_cell) records = df2.to_dict(orient="records") + + # Final pass: ensure no NaN/inf leaks through (pandas can re-introduce + # numpy.nan when storing None back into float columns) + def _sanitize(v): + if isinstance(v, float) and (math.isnan(v) or math.isinf(v)): + return None + return v + + records = [{k: _sanitize(v) for k, v in row.items()} for row in records] return records @@ -131,7 +140,9 @@ def upload_dataframe(df: pd.DataFrame, instrument="EUR_USD", granularity="M1", c batch = records[start:end] resp = supabase.table(TABLE).upsert(batch).execute() - if resp and getattr(resp, "status_code", None) not in (200, 201): + if hasattr(resp, "data") and resp.data is not None: + pass # success + elif getattr(resp, "status_code", None) not in (200, 201, None): print("Upload chunk error:", resp) raise SystemExit("Upsert failed") print(f"Chunk {i+1}/{chunks} uploaded ({len(batch)} rows).")