mirror of
https://github.com/NicolasBohn/NexQuant.git
synced 2026-07-27 23:47:46 +00:00
you-n-g
09be71d586
* refactor: split workflow into pkg, add WorkflowTracker & wait_retry * feat: add async LoopBase with parallel workers and step semaphores * fix: replace pickle with dill and run blocking tasks via joblib wrapper * feat: add log format settings, dynamic parallelism & pickle-based snapshot * fix: default step semaphore to 1 and avoid subprocess when single worker * merge bowen's changes * merge tim's changes * refactor: extract component task mapping, add conditional logger setup * lint * refactor: add type hints and safer remain_time metric logging in workflow * lint * fix: allow BadRequestError to be pickled via custom copyreg reducer * fix: stop loop when LoopTerminationError is raised in LoopBase * lint * refactor: make log tag context-local using ContextVar for thread safety * feat: add subproc_step flag and helper to decide subprocess execution * fix: use ./cache path and normalize relative volume bind paths * fix: reset loop_idx to 0 on loop restart/resume to ensure correct flow * fix: avoid chmod on cache and input dirs in Env timeout wrapper * fix: skip chmod on 'cache' and 'input' dirs using find -prune * fix: restrict chmod to immediate mount dirs excluding cache/input * fix: chmod cache and input dirs alongside their contents after entry run * fix: guard chmod with directory checks for cache and input * fix: prefix mount_path in chmod command for cache/input dirs * fix: drop quotes from find exclude patterns to ensure chmod executes * fix: skip chmod on cache/input directories to avoid warning spam * feat: support string volume mappings and poll subprocess stdout/stderr * support remove symbolic link * test: use dynamic home path and code volume in LocalEnv local_simple * fix: skip trace and progress update when loop step is withdrawn * refactor: add clean_workspace util and non-destructive workspace backup * fix: preserve symlinks when backing up workspace with copytree * fix: prevent AttributeError when _pbar not yet initialized in LoopBase * perf: replace shutil.copytree with rsync for faster workspace backup * fix: cast log directory Path to str in tar command of data science loop * fix: use portable 'cp -r -P' instead of rsync for workspace backup * fix: add retry and logging to workspace backup for robustness * refactor: extract backup_folder helper and reuse in DataScienceRDLoop * fix: propagate backup errors & default _pbar getattr to avoid error * fix the division by zero bug * refactor: execute RD loops via asyncio.run and add necessary imports * lint * lint * lint --------- Co-authored-by: Xu <v-xuminrui@microsoft.com>
102 lines
3.5 KiB
Python
102 lines
3.5 KiB
Python
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
from typing import cast
|
|
|
|
from pydantic_settings import (
|
|
BaseSettings,
|
|
EnvSettingsSource,
|
|
PydanticBaseSettingsSource,
|
|
)
|
|
|
|
|
|
class ExtendedBaseSettings(BaseSettings):
|
|
|
|
@classmethod
|
|
def settings_customise_sources(
|
|
cls,
|
|
settings_cls: type[BaseSettings],
|
|
init_settings: PydanticBaseSettingsSource,
|
|
env_settings: PydanticBaseSettingsSource,
|
|
dotenv_settings: PydanticBaseSettingsSource,
|
|
file_secret_settings: PydanticBaseSettingsSource,
|
|
) -> tuple[PydanticBaseSettingsSource, ...]:
|
|
# 1) walk from base class
|
|
def base_iter(settings_cls: type[ExtendedBaseSettings]) -> list[type[ExtendedBaseSettings]]:
|
|
bases = []
|
|
for cl in settings_cls.__bases__:
|
|
if issubclass(cl, ExtendedBaseSettings) and cl is not ExtendedBaseSettings:
|
|
bases.append(cl)
|
|
bases.extend(base_iter(cl))
|
|
return bases
|
|
|
|
# 2) Build EnvSettingsSource from base classes, so we can add parent Env Sources
|
|
parent_env_settings = [
|
|
EnvSettingsSource(
|
|
base_cls,
|
|
case_sensitive=base_cls.model_config.get("case_sensitive"),
|
|
env_prefix=base_cls.model_config.get("env_prefix"),
|
|
env_nested_delimiter=base_cls.model_config.get("env_nested_delimiter"),
|
|
)
|
|
for base_cls in base_iter(cast("type[ExtendedBaseSettings]", settings_cls))
|
|
]
|
|
return init_settings, env_settings, *parent_env_settings, dotenv_settings, file_secret_settings
|
|
|
|
|
|
class RDAgentSettings(ExtendedBaseSettings):
|
|
|
|
# azure document intelligence configs
|
|
azure_document_intelligence_key: str = ""
|
|
azure_document_intelligence_endpoint: str = ""
|
|
# factor extraction conf
|
|
max_input_duplicate_factor_group: int = 300
|
|
max_output_duplicate_factor_group: int = 20
|
|
max_kmeans_group_number: int = 40
|
|
|
|
# workspace conf
|
|
workspace_path: Path = Path.cwd() / "git_ignore_folder" / "RD-Agent_workspace"
|
|
|
|
# multi processing conf
|
|
multi_proc_n: int = 1
|
|
|
|
# pickle cache conf
|
|
cache_with_pickle: bool = True # whether to use pickle cache
|
|
pickle_cache_folder_path_str: str = str(
|
|
Path.cwd() / "pickle_cache/",
|
|
) # the path of the folder to store the pickle cache
|
|
use_file_lock: bool = (
|
|
True # when calling the function with same parameters, whether to use file lock to avoid
|
|
# executing the function multiple times
|
|
)
|
|
|
|
# misc
|
|
"""The limitation of context stdout"""
|
|
stdout_context_len: int = 400
|
|
stdout_line_len: int = 10000
|
|
|
|
enable_mlflow: bool = False
|
|
|
|
initial_fator_library_size: int = 20
|
|
|
|
# parallel loop
|
|
step_semaphore: int | dict[str, int] = 1
|
|
"""the semaphore for each step; you can specify a overall semaphore
|
|
or a step-wise semaphore like {"coding": 3, "running": 2}"""
|
|
|
|
def get_max_parallel(self) -> int:
|
|
"""Based on the setting of semaphore, return the maximum number of parallel loops"""
|
|
if isinstance(self.step_semaphore, int):
|
|
return self.step_semaphore
|
|
else:
|
|
return max(self.step_semaphore.values())
|
|
|
|
# NOTE: for debug
|
|
# the following function only serves as debugging and is necessary in main logic.
|
|
subproc_step: bool = False
|
|
|
|
def is_force_subproc(self) -> bool:
|
|
return self.subproc_step or self.get_max_parallel() > 1
|
|
|
|
|
|
RD_AGENT_SETTINGS = RDAgentSettings()
|