feat: parallel loop running based on asyncio (#932)

* refactor: split workflow into pkg, add WorkflowTracker & wait_retry

* feat: add async LoopBase with parallel workers and step semaphores

* fix: replace pickle with dill and run blocking tasks via joblib wrapper

* feat: add log format settings, dynamic parallelism & pickle-based snapshot

* fix: default step semaphore to 1 and avoid subprocess when single worker

* merge bowen's changes

* merge tim's changes

* refactor: extract component task mapping, add conditional logger setup

* lint

* refactor: add type hints and safer remain_time metric logging in workflow

* lint

* fix: allow BadRequestError to be pickled via custom copyreg reducer

* fix: stop loop when LoopTerminationError is raised in LoopBase

* lint

* refactor: make log tag context-local using ContextVar for thread safety

* feat: add subproc_step flag and helper to decide subprocess execution

* fix: use ./cache path and normalize relative volume bind paths

* fix: reset loop_idx to 0 on loop restart/resume to ensure correct flow

* fix: avoid chmod on cache and input dirs in Env timeout wrapper

* fix: skip chmod on 'cache' and 'input' dirs using find -prune

* fix: restrict chmod to immediate mount dirs excluding cache/input

* fix: chmod cache and input dirs alongside their contents after entry run

* fix: guard chmod with directory checks for cache and input

* fix: prefix mount_path in chmod command for cache/input dirs

* fix: drop quotes from find exclude patterns to ensure chmod executes

* fix: skip chmod on cache/input directories to avoid warning spam

* feat: support string volume mappings and poll subprocess stdout/stderr

* support remove symbolic link

* test: use dynamic home path and code volume in LocalEnv local_simple

* fix: skip trace and progress update when loop step is withdrawn

* refactor: add clean_workspace util and non-destructive workspace backup

* fix: preserve symlinks when backing up workspace with copytree

* fix: prevent AttributeError when _pbar not yet initialized in LoopBase

* perf: replace shutil.copytree with rsync for faster workspace backup

* fix: cast log directory Path to str in tar command of data science loop

* fix: use portable 'cp -r -P' instead of rsync for workspace backup

* fix: add retry and logging to workspace backup for robustness

* refactor: extract backup_folder helper and reuse in DataScienceRDLoop

* fix: propagate backup errors & default _pbar getattr to avoid error

* fix the division by zero bug

* refactor: execute RD loops via asyncio.run and add necessary imports

* lint

* lint

* lint

---------

Co-authored-by: Xu <v-xuminrui@microsoft.com>

rdagent/utils/workflow/tracking.py

@@ -0,0 +1,93 @@
+"""
+Tracking module for experiment tracking using MLflow.
+
+This module provides a clean interface for tracking metrics and parameters
+while keeping the MLflow dependency optional based on configuration.
+"""
+
+import datetime
+from typing import TYPE_CHECKING
+
+import pytz
+
+from rdagent.core.conf import RD_AGENT_SETTINGS
+from rdagent.log.timer import RD_Agent_TIMER_wrapper
+
+if TYPE_CHECKING:
+    # Import here to avoid circular dependency
+    from rdagent.utils.workflow.loop import LoopBase
+
+from rdagent.log import rdagent_logger as logger
+
+# Define a placeholder for mlflow if it's not available
+mlflow = None
+
+# Conditional import to make MLflow optional
+if RD_AGENT_SETTINGS.enable_mlflow:
+    try:
+        import mlflow  # type: ignore[assignment]
+    except ImportError:
+        logger.warning("MLflow is enabled in settings but could not be imported.")
+        RD_AGENT_SETTINGS.enable_mlflow = False
+
+
+class WorkflowTracker:
+    """
+    A workflow-specific tracking system that logs metrics related to workflow execution.
+
+    This class handles metric logging while keeping the MLflow dependency optional.
+    If MLflow is not enabled in settings, tracking calls become no-ops.
+    """
+
+    def __init__(self, loop_base: "LoopBase"):
+        """
+        Initialize a WorkflowTracker with a LoopBase instance.
+
+        Args:
+            loop_base: The LoopBase instance to track metrics for
+        """
+        self.loop_base = loop_base
+
+    @staticmethod
+    def is_enabled() -> bool:
+        """Check if tracking is enabled."""
+        return RD_AGENT_SETTINGS.enable_mlflow
+
+    @staticmethod
+    def _datetime_to_float(dt: datetime.datetime) -> float:
+        """Convert datetime to a structured float representation."""
+        return dt.second + dt.minute * 1e2 + dt.hour * 1e4 + dt.day * 1e6 + dt.month * 1e8 + dt.year * 1e10
+
+    def log_workflow_state(self) -> None:
+        """
+        Log all workflow state metrics from the associated LoopBase instance.
+        """
+        if not RD_AGENT_SETTINGS.enable_mlflow or mlflow is None:
+            return
+
+        # Log workflow progress
+        mlflow.log_metric("loop_index", self.loop_base.loop_idx)
+        mlflow.log_metric("step_index", self.loop_base.step_idx[self.loop_base.loop_idx])
+
+        current_local_datetime = datetime.datetime.now(pytz.timezone("Asia/Shanghai"))
+        float_like_datetime = self._datetime_to_float(current_local_datetime)
+        mlflow.log_metric("current_datetime", float_like_datetime)
+
+        # Log API status
+        mlflow.log_metric("api_fail_count", RD_Agent_TIMER_wrapper.api_fail_count)
+        latest_api_fail_time = RD_Agent_TIMER_wrapper.latest_api_fail_time
+        if latest_api_fail_time is not None:
+            float_like_datetime = self._datetime_to_float(latest_api_fail_time)
+            mlflow.log_metric("lastest_api_fail_time", float_like_datetime)
+
+        # Log timer status if timer is started
+        if self.loop_base.timer.started:
+            remain_time = self.loop_base.timer.remain_time()
+            assert remain_time is not None
+            mlflow.log_metric("remain_time", remain_time.seconds)
+            mlflow.log_metric(
+                "remain_percent",
+                remain_time / self.loop_base.timer.all_duration * 100,
+            )
+
+    # Keep only the log_workflow_state method as it's the primary entry point now