mirror of
https://github.com/BrentNeale1/fx-quant.git
synced 2026-07-29 11:27:45 +00:00
Fix NaN serialization and upsert response handling in upload
Sanitize NaN/inf values that pandas re-introduces into float columns after conversion, preventing JSON serialization errors. Fix upsert response check to detect success via resp.data instead of status_code. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
+12
-1
@@ -93,6 +93,15 @@ def df_to_records(df: pd.DataFrame, instrument: str, granularity: str = "M1"):
|
||||
df2[col] = df2[col].apply(_convert_cell)
|
||||
|
||||
records = df2.to_dict(orient="records")
|
||||
|
||||
# Final pass: ensure no NaN/inf leaks through (pandas can re-introduce
|
||||
# numpy.nan when storing None back into float columns)
|
||||
def _sanitize(v):
|
||||
if isinstance(v, float) and (math.isnan(v) or math.isinf(v)):
|
||||
return None
|
||||
return v
|
||||
|
||||
records = [{k: _sanitize(v) for k, v in row.items()} for row in records]
|
||||
return records
|
||||
|
||||
|
||||
@@ -131,7 +140,9 @@ def upload_dataframe(df: pd.DataFrame, instrument="EUR_USD", granularity="M1", c
|
||||
batch = records[start:end]
|
||||
|
||||
resp = supabase.table(TABLE).upsert(batch).execute()
|
||||
if resp and getattr(resp, "status_code", None) not in (200, 201):
|
||||
if hasattr(resp, "data") and resp.data is not None:
|
||||
pass # success
|
||||
elif getattr(resp, "status_code", None) not in (200, 201, None):
|
||||
print("Upload chunk error:", resp)
|
||||
raise SystemExit("Upsert failed")
|
||||
print(f"Chunk {i+1}/{chunks} uploaded ({len(batch)} rows).")
|
||||
|
||||
Reference in New Issue
Block a user