feat: idea pool integrated to exp_gen & add timer to RD-Agent & pause-resume to RD-loops (#795)

* update all code

* update all code

* dump knowledge base

* rename the tag

* add timer to RD-Agent

* fix CI

* fix CI

* use batch embedding

* fix a small bug

* fix prompt bug

* feat: add pause resume to handle K8S cluster pause (#804)

* add resume to cluster running

* fix non-pickle problem

* fix a small bug

* fix a small bug

* avoid shutil move error

* refine the logic

* move knowledge base out of session

* avoid mistake information to pipeline coding

* avoid load and dump in steps

* archive the right folder

* small improvement

* avoid restart when timer is already started

* fix CI

---------

Co-authored-by: Xu Yang <xuyang1@microsoft.com>

---------

Co-authored-by: Xu Yang <peteryang@vip.qq.com>
Co-authored-by: Xu Yang <xuyang1@microsoft.com>
Co-authored-by: Xu <v-xuminrui@microsoft.com>

rdagent/components/knowledge_management/graph.py

@@ -13,15 +13,17 @@ from rdagent.components.knowledge_management.vector_base import (
     cosine,
 )
 from rdagent.core.knowledge_base import KnowledgeBase
+from rdagent.log import rdagent_logger as logger
 from rdagent.oai.llm_utils import APIBackend
 
 Node = KnowledgeMetaData
 
 
 class UndirectedNode(Node):
-    def __init__(self, content: str = "", label: str = "", embedding: Any = None) -> None:
+    def __init__(self, content: str = "", label: str = "", embedding: Any = None, appendix: Any = None) -> None:
         super().__init__(content, label, embedding)
         self.neighbors: set[UndirectedNode] = set()
+        self.appendix = appendix  # appendix stores any additional information
         assert isinstance(content, str), "content must be a string"
 
     def add_neighbor(self, node: UndirectedNode) -> None:
@@ -86,6 +88,10 @@ class Graph(KnowledgeBase):
         size = 16
         embeddings = []
         for i in range(0, len(contents), size):
+            logger.info(
+                f"Creating embedding for index {i} to {i + size} with {len(contents)} contents",
+                tag="batch embedding",
+            )
             embeddings.extend(
                 APIBackend().create_embedding(input_content=contents[i : i + size]),
             )
@@ -270,7 +276,7 @@ class UndirectedGraph(Graph):
         self,
         node: UndirectedNode | str,
         similarity_threshold: float = 0.0,
-        topk_k: int = 5,
+        topk_k: int = None,
         constraint_labels: list[str] | None = None,
     ) -> list[UndirectedNode]:
         """

rdagent/components/knowledge_management/vector_base.py

@@ -87,7 +87,7 @@ class VectorBase(KnowledgeBase):
         """
         pass
 
-    def search(self, content: str, topk_k: int = 5, similarity_threshold: float = 0) -> List[Document]:
+    def search(self, content: str, topk_k: int | None = None, similarity_threshold: float = 0) -> List[Document]:
         """
         search vector_df by node
         Parameters
@@ -156,7 +156,11 @@ class PDVectorBase(VectorBase):
                 self.add(document=doc)
 
     def search(
-        self, content: str, topk_k: int = 5, similarity_threshold: float = 0, constraint_labels: list[str] | None = None
+        self,
+        content: str,
+        topk_k: int | None = None,
+        similarity_threshold: float = 0,
+        constraint_labels: list[str] | None = None,
     ) -> Tuple[List[Document], List]:
         """
         Search vector by node's embedding.
@@ -192,7 +196,9 @@ class PDVectorBase(VectorBase):
             lambda x: 1 - cosine(x, document.embedding)
         )  # cosine is cosine distance, 1-similarity
 
-        searched_similarities = similarities[similarities > similarity_threshold].nlargest(topk_k)
+        searched_similarities = similarities[similarities > similarity_threshold]
+        if topk_k is not None:
+            searched_similarities = searched_similarities.nlargest(topk_k)
         most_similar_docs = filtered_df.loc[searched_similarities.index]
 
         docs = []