mirror of
https://github.com/BrentNeale1/fx-quant.git
synced 2026-08-18 12:38:06 +00:00
Fix NaN serialization and upsert response handling in upload
Sanitize NaN/inf values that pandas re-introduces into float columns after conversion, preventing JSON serialization errors. Fix upsert response check to detect success via resp.data instead of status_code. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
da64544d3a
commit
71f9c7849b
+12
-1
@@ -93,6 +93,15 @@ def df_to_records(df: pd.DataFrame, instrument: str, granularity: str = "M1"):
|
|||||||
df2[col] = df2[col].apply(_convert_cell)
|
df2[col] = df2[col].apply(_convert_cell)
|
||||||
|
|
||||||
records = df2.to_dict(orient="records")
|
records = df2.to_dict(orient="records")
|
||||||
|
|
||||||
|
# Final pass: ensure no NaN/inf leaks through (pandas can re-introduce
|
||||||
|
# numpy.nan when storing None back into float columns)
|
||||||
|
def _sanitize(v):
|
||||||
|
if isinstance(v, float) and (math.isnan(v) or math.isinf(v)):
|
||||||
|
return None
|
||||||
|
return v
|
||||||
|
|
||||||
|
records = [{k: _sanitize(v) for k, v in row.items()} for row in records]
|
||||||
return records
|
return records
|
||||||
|
|
||||||
|
|
||||||
@@ -131,7 +140,9 @@ def upload_dataframe(df: pd.DataFrame, instrument="EUR_USD", granularity="M1", c
|
|||||||
batch = records[start:end]
|
batch = records[start:end]
|
||||||
|
|
||||||
resp = supabase.table(TABLE).upsert(batch).execute()
|
resp = supabase.table(TABLE).upsert(batch).execute()
|
||||||
if resp and getattr(resp, "status_code", None) not in (200, 201):
|
if hasattr(resp, "data") and resp.data is not None:
|
||||||
|
pass # success
|
||||||
|
elif getattr(resp, "status_code", None) not in (200, 201, None):
|
||||||
print("Upload chunk error:", resp)
|
print("Upload chunk error:", resp)
|
||||||
raise SystemExit("Upsert failed")
|
raise SystemExit("Upsert failed")
|
||||||
print(f"Chunk {i+1}/{chunks} uploaded ({len(batch)} rows).")
|
print(f"Chunk {i+1}/{chunks} uploaded ({len(batch)} rows).")
|
||||||
|
|||||||
Reference in New Issue
Block a user