From 6d6c5abd4ac7252257f88e13e263ecb2497fde3b Mon Sep 17 00:00:00 2001 From: TPTBusiness Date: Tue, 31 Mar 2026 11:27:33 +0200 Subject: [PATCH] fix: Embedding Context Length Error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem: Knowledge Graph versucht zu lange Texte zu embedden - nomic-embed-text Limit: 8192 Token - Fehler: 'the input length exceeds the context length' - System crasht nach 10 Retries Lösung: 1. Content für Embeddings auf 15.000 Zeichen kürzen (~4000 Token) 2. Trunk-Größe auf max 4000 begrenzt 3. Hinweis '[truncated for embedding]' bei Kürzung Betroffene Dateien: - rdagent/components/knowledge_management/vector_base.py Jetzt sollte fin_quant ohne Embedding-Fehler durchlaufen. --- .../knowledge_management/vector_base.py | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/rdagent/components/knowledge_management/vector_base.py b/rdagent/components/knowledge_management/vector_base.py index 277c8ed0..af798a6b 100644 --- a/rdagent/components/knowledge_management/vector_base.py +++ b/rdagent/components/knowledge_management/vector_base.py @@ -34,8 +34,18 @@ class KnowledgeMetaData: chunks.append(chunk) return chunks - self.trunks = split_string_into_chunks(self.content, chunk_size=size) - self.trunks_embedding = APIBackend().create_embedding(input_content=self.trunks) + # Kürze Content falls zu lang für Embedding + max_trunk_size = 4000 # Sicher unter 8192 Token Limit + actual_size = min(size, max_trunk_size) + + self.trunks = split_string_into_chunks(self.content, chunk_size=actual_size) + + # Kürze zu lange Trunks für Embedding + self.trunks_embedding = [] + for trunk in self.trunks: + if len(trunk) > 15000: + trunk = trunk[:15000] + "... [truncated]" + self.trunks_embedding.extend(APIBackend().create_embedding(input_content=trunk)) def create_embedding(self): """ @@ -45,7 +55,16 @@ class KnowledgeMetaData: """ if self.embedding is None: - self.embedding = APIBackend().create_embedding(input_content=self.content) + # Kürze Content auf max 4000 Token für Embedding (nomic-embed-text Limit ist 8192) + # Sicherer Puffer: 4000 Token ≈ 3000 Zeichen + max_content_length = 15000 + content_to_embed = self.content[:max_content_length] if len(self.content) > max_content_length else self.content + + if len(self.content) > max_content_length: + # Füge Hinweis hinzu dass Content gekürzt wurde + content_to_embed += "... [truncated for embedding]" + + self.embedding = APIBackend().create_embedding(input_content=content_to_embed) def from_dict(self, data: dict): for key, value in data.items():