Prevent dangling TAF fragments in city AI reads

City AI can return a partially streamed JSON string when the provider truncates output. The fallback previously kept an unfinished clause such as '但TAF显示', which made the forecast explanation look broken even though earlier evidence was usable.

Constraint: Provider JSON can be truncated after useful fields have already streamed

Rejected: Drop all partial AI text | would lose valid METAR interpretation already returned before truncation

Confidence: high

Scope-risk: narrow

Tested: pytest city AI truncation regression tests

Tested: npm run build

Not-tested: Live DeepSeek provider response
This commit is contained in:
2569718930@qq.com
2026-04-27 08:28:00 +08:00
parent e4cf570820
commit 4eb50ce880
3 changed files with 179 additions and 8 deletions
+92 -5
View File
@@ -98,7 +98,7 @@ SCAN_CITY_AI_MAX_TOKENS = _env_int(
min_value=800,
max_value=64000,
)
SCAN_CITY_AI_PROMPT_VERSION = "city-observation-read-v4"
SCAN_CITY_AI_PROMPT_VERSION = "city-observation-read-v5"
CITY_AI_REQUIRED_FIELDS = [
"metar_read_zh",
@@ -408,7 +408,7 @@ def _decode_json_string_fragment(fragment: str) -> str:
)
def _extract_json_string_field_from_fragment(raw_text: str, field: str) -> str:
def _extract_json_string_field_fragment(raw_text: str, field: str) -> tuple[str, bool]:
"""Best-effort extraction from a streamed/incomplete JSON object.
DeepSeek may stream useful fields first but still end with truncated JSON.
@@ -419,10 +419,11 @@ def _extract_json_string_field_from_fragment(raw_text: str, field: str) -> str:
text = str(raw_text or "")
match = re.search(rf'"{re.escape(field)}"\s*:\s*"', text)
if not match:
return ""
return "", False
idx = match.end()
chars: List[str] = []
escaped = False
closed = False
while idx < len(text):
char = text[idx]
idx += 1
@@ -434,11 +435,75 @@ def _extract_json_string_field_from_fragment(raw_text: str, field: str) -> str:
escaped = True
continue
if char == '"':
closed = True
break
chars.append(char)
if escaped:
chars.append("\\")
return _decode_json_string_fragment("".join(chars)).strip()
return _decode_json_string_fragment("".join(chars)).strip(), closed
def _extract_json_string_field_from_fragment(raw_text: str, field: str) -> str:
value, _closed = _extract_json_string_field_fragment(raw_text, field)
return value
_CITY_AI_TEXT_FIELDS = {
"metar_read_zh",
"metar_read_en",
"final_judgment_zh",
"final_judgment_en",
"reasoning_zh",
"reasoning_en",
"model_cluster_note_zh",
"model_cluster_note_en",
}
_INCOMPLETE_TAIL_RE = re.compile(
r"(?:(?:[,;]\s*)?(?:但|但是|不过|然而|而且|并且|因为|由于|若|如果|but|however|although|because|if|while|and)\s*)?"
r"(?:TAF|METAR|报文|机场预报|机场报文)?\s*(?:显示|提示|预示|表明|show(?:s)?|indicate(?:s)?|suggest(?:s)?)?\s*$",
re.IGNORECASE,
)
def _strip_incomplete_ai_sentence(value: Any) -> str:
"""Remove dangling provider fragments such as "但TAF显示".
The city AI endpoint can fall back to partially streamed JSON. If the stream
stops in the middle of a string, keeping the whole fragment produces broken
UI text. Prefer the last complete sentence/clause; if the whole field is too
incomplete, let schema completion use the deterministic fallback.
"""
text = re.sub(r"\s+", " ", str(value or "")).strip()
if not text:
return ""
if re.search(r"[。!?.!?]$", text):
return text
stripped = _INCOMPLETE_TAIL_RE.sub("", text).rstrip(" ,;:")
if stripped != text:
text = stripped.strip()
if not text:
return ""
if re.search(r"[。!?.!?]$", text):
return text
return text.rstrip(" ,;:") + ("." if re.search(r"[A-Za-z]$", text) else "")
# If the provider stopped after a semicolon/comma-delimited clause, keep the
# complete preceding clause instead of showing a dangling tail.
for punct in ("", ";", "", "", "", ".", "!", "?"):
idx = text.rfind(punct)
if idx >= 0:
candidate = text[: idx + 1].strip()
if len(candidate) >= 8:
return candidate.rstrip(";") + ("." if punct == ";" else "" if punct == "" else "")
# If no dangling connector is visible, keep the provider text and add only
# terminal punctuation. This avoids replacing otherwise useful long reads
# just because the JSON string missed its closing quote.
return text.rstrip(" ,;:") + ("." if re.search(r"[A-Za-z]$", text) else "")
def _extract_json_number_field_from_fragment(raw_text: str, field: str) -> Optional[float]:
@@ -466,7 +531,9 @@ def _extract_city_ai_partial_fields(raw_text: str) -> Dict[str, Any]:
"confidence",
"unit",
):
value = _extract_json_string_field_from_fragment(text, field)
value, closed = _extract_json_string_field_fragment(text, field)
if value and field in _CITY_AI_TEXT_FIELDS and not closed:
value = _strip_incomplete_ai_sentence(value)
if value:
out[field] = value
for field in ("predicted_max", "range_low", "range_high"):
@@ -731,16 +798,36 @@ def _complete_city_ai_payload(
)
completed: List[str] = []
out = dict(ai_raw)
trimmed: List[str] = []
for field in CITY_AI_REQUIRED_FIELDS:
value = out.get(field)
if isinstance(value, str) and field in _CITY_AI_TEXT_FIELDS:
clean_value = _strip_incomplete_ai_sentence(value)
if clean_value != value:
trimmed.append(field)
value = clean_value
out[field] = clean_value
if value is None or value == "" or value == []:
out[field] = fallback.get(field)
completed.append(field)
for field in ("risks_zh", "risks_en"):
value = out.get(field)
if isinstance(value, list):
cleaned_risks = []
for item in value:
clean_item = _strip_incomplete_ai_sentence(item)
if clean_item:
cleaned_risks.append(clean_item)
if cleaned_risks != value:
trimmed.append(field)
out[field] = cleaned_risks or fallback.get(field)
meta = out.get("_polyweather_meta")
if not isinstance(meta, dict):
meta = {}
if completed:
meta["schema_completed_fields"] = completed
if trimmed:
meta["trimmed_incomplete_fields"] = sorted(set(trimmed))
out["_polyweather_meta"] = meta
return out