first version of model runner and model feedback (#70)

* Implemented model.py

- Need to run within the RDAgent folder (relevant path)
- Each time copy a template & insert code & run qlib & store result back to experiment

* Create model.py

* Create conf.yaml

This is the sample conf.yaml to be copied each time.

This has gone several times of iteration and is now working for both tabular and Time-Series data.

* Create read_exp.py

This is to read the results within Qlib

* Create ReadMe.md

* Update model.py

* Create test_model.py

A testing file that separates model code generation and running&feedback section.

* move the template folder

* help xisen finish the model runner

* help xisen fix improve model feedback generation

* delete debug file

* rename readme.md

---------

Co-authored-by: Xisen Wang <118058822+Xisen-Wang@users.noreply.github.com>

rdagent/core/evolving_agent.py

@@ -32,7 +32,7 @@ class RAGEvoAgent(EvoAgent):
         with_feedback: bool = True,
         knowledge_self_gen: bool = False,
     ) -> EvolvableSubjects:
-        for _ in tqdm(range(self.max_loop), "Implementing factors"):
+        for _ in tqdm(range(self.max_loop), "Implementing"):
             # 1. knowledge self-evolving
             if knowledge_self_gen and self.rag is not None:
                 self.rag.generate_knowledge(self.evolving_trace)

rdagent/core/experiment.py

@@ -124,16 +124,16 @@ class Experiment(ABC, Generic[ASpecificTask, ASpecificImp]):
     The experiment is a sequence of tasks and the implementations of the tasks after generated by the TaskGenerator.
     """
 
-    result_ws: Optional[FBImplementation]
-
     def __init__(self, sub_tasks: Sequence[ASpecificTask]) -> None:
         self.sub_tasks = sub_tasks
         self.sub_implementations: Sequence[ASpecificImp] = [None for _ in self.sub_tasks]
         self.based_experiments: Sequence[Experiment] = []
         self.result: object = None  # The result of the experiment, can be different types in different scenarios.
-        self.result_ws = None
+        self.exp_ws: ASpecificImp = None
 
 
+ASpecificExp = TypeVar("ASpecificExp", bound=Experiment)
+
 TaskOrExperiment = TypeVar("TaskOrExperiment", Task, Experiment)
 
 

rdagent/core/proposal.py

@@ -6,7 +6,7 @@ from abc import ABC, abstractmethod
 from typing import Any, Dict, Generic, List, Tuple, TypeVar
 
 from rdagent.core.evaluation import Feedback
-from rdagent.core.experiment import ASpecificTask, Experiment
+from rdagent.core.experiment import ASpecificExp, ASpecificTask, Experiment
 from rdagent.core.scenario import Scenario
 
 # class data_ana: XXX
@@ -23,7 +23,7 @@ class Hypothesis:
     def __init__(self, hypothesis: str, reason: str) -> None:
         self.hypothesis: str = hypothesis
         self.reason: str = reason
-    
+
     def __str__(self) -> str:
         return f"""Hypothesis: {self.hypothesis}
 Reason: {self.reason}"""
@@ -54,15 +54,15 @@ class Trace(Generic[ASpecificScen]):
         self.scen: ASpecificScen = scen
         self.hist: list[Tuple[Hypothesis, Experiment, HypothesisFeedback]] = []
 
-    def get_last_experiment_info(self) -> Tuple[Hypothesis, ASpecificTask, Any]:
+    def get_SOTA_hypothesis_and_experiment(self) -> Tuple[Hypothesis, Experiment]:
         """Access the last experiment result, sub-task, and the corresponding hypothesis."""
         # TODO: The return value does not align with the signature.
-        if not self.hist:
-            return None
-        last_hypothesis, last_experiment, _ = self.hist[-1]
-        last_task = last_experiment.sub_tasks[-1]
-        last_result = last_experiment.result
-        return last_hypothesis, last_task, last_result
+        for hypothesis, experiment, feedback in self.hist[::-1]:
+            if feedback.decision:
+                return hypothesis, experiment
+
+        return None, None
+
 
 class HypothesisGen:
     def __init__(self, scen: Scenario):
@@ -81,9 +81,6 @@ class HypothesisGen:
         """
 
 
-ASpecificExp = TypeVar("ASpecificExp", bound=Experiment)
-
-
 class Hypothesis2Experiment(ABC, Generic[ASpecificExp]):
     """
     [Abstract description => concrete description] => Code implement

rdagent/core/task_generator.py

@@ -1,11 +1,9 @@
 from abc import ABC, abstractmethod
-from typing import Generic, List, Sequence, TypeVar
+from typing import Generic, List
 
-from rdagent.core.experiment import Experiment
+from rdagent.core.experiment import ASpecificExp
 from rdagent.core.scenario import Scenario
 
-ASpecificExp = TypeVar("ASpecificExp", bound=Experiment)
-
 
 class TaskGenerator(ABC, Generic[ASpecificExp]):
     def __init__(self, scen: Scenario) -> None: