feat: query & cache package_info (#1083)

* feat: add package query in draft.py (not yet enabled)

* feat: integrate package query into task_gen and cache runtime environment

- Remove pkg_query modifications from draft components
- Add package declaration requirement in task_gen prompts
- Add optional packages field to CodingSketch model
- Cache runtime_environment in scenario object for loop-wide reuse
- Parse packages from LLM response and generate runtime environment dynamically

* some refinement

* feat: merge default packages with CLI args in package_info.py

* fix: code style

---------

Co-authored-by: Qizheng Li <jenssenlee@163.com>

rdagent/components/coder/data_science/pipeline/__init__.py

@@ -38,7 +38,7 @@ from rdagent.components.coder.CoSTEER.knowledge_management import (
 )
 from rdagent.components.coder.data_science.conf import DSCoderCoSTEERSettings
 from rdagent.components.coder.data_science.pipeline.eval import PipelineCoSTEEREvaluator
-from rdagent.components.coder.data_science.raw_data_loader.exp import DataLoaderTask
+from rdagent.components.coder.data_science.pipeline.exp import PipelineTask
 from rdagent.components.coder.data_science.share.eval import ModelDumpEvaluator
 from rdagent.core.exception import CoderError
 from rdagent.core.experiment import FBWorkspace
@@ -53,7 +53,7 @@ DIRNAME = Path(__file__).absolute().resolve().parent
 class PipelineMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
     def implement_one_task(
         self,
-        target_task: DataLoaderTask,
+        target_task: PipelineTask,
         queried_knowledge: CoSTEERQueriedKnowledge | None = None,
         workspace: FBWorkspace | None = None,
         prev_task_feedback: CoSTEERSingleFeedback | None = None,
@@ -86,6 +86,7 @@ class PipelineMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
             queried_former_failed_knowledge=queried_former_failed_knowledge[0],
             out_spec=PythonAgentOut.get_spec(),
             runtime_environment=runtime_environment,
+            package_info=target_task.package_info,
             enable_model_dump=DS_RD_SETTING.enable_model_dump,
             enable_debug_mode=DS_RD_SETTING.sample_data_by_LLM,
         )

rdagent/components/coder/data_science/pipeline/exp.py

@@ -3,5 +3,6 @@ from rdagent.components.coder.CoSTEER.task import CoSTEERTask
 
 # Because we use isinstance to distinguish between different types of tasks, we need to use sub classes to represent different types of tasks
 class PipelineTask(CoSTEERTask):
-    def __init__(self, name: str = "Pipeline", *args, **kwargs) -> None:
+    def __init__(self, name: str = "Pipeline", package_info: str | None = None, *args, **kwargs) -> None:
         super().__init__(name=name, *args, **kwargs)
+        self.package_info = package_info

rdagent/components/coder/data_science/pipeline/prompts.yaml

@@ -11,6 +11,14 @@ pipeline_coder:
     
     ## The runtime environment your code will running on
     {{ runtime_environment }}
+
+    {% if package_info is not none %}
+    To help you write the runnable code, the user has provided the package information which contains the package names and versions.
+    You should be careful about the package versions, as the code will be executed in the environment with the specified version and the api might be different from the latest version.
+    The user might provide the packages the environment doesn't have, you should avoid using any of them.
+    ## Package Information
+    {{ package_info }}
+    {% endif %}
     
     ## Hyperparameters Specification
     Follow the hyperparameter choices if they are specified in the task description, unless they are unreasonable or incorrect.