Fix NaN serialization and upsert response handling in upload

Sanitize NaN/inf values that pandas re-introduces into float columns
after conversion, preventing JSON serialization errors. Fix upsert
response check to detect success via resp.data instead of status_code.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Brent Neale
2026-02-16 15:42:17 +10:00
parent da64544d3a
commit 71f9c7849b
+12 -1
View File
@@ -93,6 +93,15 @@ def df_to_records(df: pd.DataFrame, instrument: str, granularity: str = "M1"):
df2[col] = df2[col].apply(_convert_cell)
records = df2.to_dict(orient="records")
# Final pass: ensure no NaN/inf leaks through (pandas can re-introduce
# numpy.nan when storing None back into float columns)
def _sanitize(v):
if isinstance(v, float) and (math.isnan(v) or math.isinf(v)):
return None
return v
records = [{k: _sanitize(v) for k, v in row.items()} for row in records]
return records
@@ -131,7 +140,9 @@ def upload_dataframe(df: pd.DataFrame, instrument="EUR_USD", granularity="M1", c
batch = records[start:end]
resp = supabase.table(TABLE).upsert(batch).execute()
if resp and getattr(resp, "status_code", None) not in (200, 201):
if hasattr(resp, "data") and resp.data is not None:
pass # success
elif getattr(resp, "status_code", None) not in (200, 201, None):
print("Upload chunk error:", resp)
raise SystemExit("Upsert failed")
print(f"Chunk {i+1}/{chunks} uploaded ({len(batch)} rows).")