feat: idea pool integrated to exp_gen & add timer to RD-Agent & pause-resume to RD-loops (#795)

* update all code

* update all code

* dump knowledge base

* rename the tag

* add timer to RD-Agent

* fix CI

* fix CI

* use batch embedding

* fix a small bug

* fix prompt bug

* feat: add pause resume to handle K8S cluster pause (#804)

* add resume to cluster running

* fix non-pickle problem

* fix a small bug

* fix a small bug

* avoid shutil move error

* refine the logic

* move knowledge base out of session

* avoid mistake information to pipeline coding

* avoid load and dump in steps

* archive the right folder

* small improvement

* avoid restart when timer is already started

* fix CI

---------

Co-authored-by: Xu Yang <xuyang1@microsoft.com>

---------

Co-authored-by: Xu Yang <peteryang@vip.qq.com>
Co-authored-by: Xu Yang <xuyang1@microsoft.com>
Co-authored-by: Xu <v-xuminrui@microsoft.com>

rdagent/components/knowledge_management/vector_base.py

@@ -87,7 +87,7 @@ class VectorBase(KnowledgeBase):
         """
         pass
 
-    def search(self, content: str, topk_k: int = 5, similarity_threshold: float = 0) -> List[Document]:
+    def search(self, content: str, topk_k: int | None = None, similarity_threshold: float = 0) -> List[Document]:
         """
         search vector_df by node
         Parameters
@@ -156,7 +156,11 @@ class PDVectorBase(VectorBase):
                 self.add(document=doc)
 
     def search(
-        self, content: str, topk_k: int = 5, similarity_threshold: float = 0, constraint_labels: list[str] | None = None
+        self,
+        content: str,
+        topk_k: int | None = None,
+        similarity_threshold: float = 0,
+        constraint_labels: list[str] | None = None,
     ) -> Tuple[List[Document], List]:
         """
         Search vector by node's embedding.
@@ -192,7 +196,9 @@ class PDVectorBase(VectorBase):
             lambda x: 1 - cosine(x, document.embedding)
         )  # cosine is cosine distance, 1-similarity
 
-        searched_similarities = similarities[similarities > similarity_threshold].nlargest(topk_k)
+        searched_similarities = similarities[similarities > similarity_threshold]
+        if topk_k is not None:
+            searched_similarities = searched_similarities.nlargest(topk_k)
         most_similar_docs = filtered_df.loc[searched_similarities.index]
 
         docs = []