From a126c84c92e69fe098f5bbeaef357cf550f5db73 Mon Sep 17 00:00:00 2001 From: Xu Yang Date: Tue, 18 Jun 2024 11:50:03 +0800 Subject: [PATCH] Refine all the implementation code to higher quality for release (#29) * refine CI script * refine all the code to higher quality * refine the script to factor extraction and implementation * add task loader interface * add a task loader interface && move pdf analysis to pdf task loader * change the name to global variables --------- Co-authored-by: xuyang1 --- .github/workflows/ci.yml | 4 +- .github/workflows/readthedocs-preview.yml | 2 +- Makefile | 14 +- rdagent/app/CI/run.py | 178 +++++++++++------- .../factor_extract_and_implement.py | 39 +--- rdagent/benchmark/eval_method.py | 29 +-- rdagent/core/conf.py | 14 +- rdagent/core/evolving_agent.py | 63 +++++++ rdagent/core/evolving_framework.py | 32 +--- rdagent/core/log.py | 2 +- rdagent/core/task.py | 13 +- rdagent/core/utils.py | 72 +------ .../document_reader.py | 16 +- rdagent/factor_implementation/CoSTEER.py | 43 +++-- .../evolving/evaluators.py | 30 +-- .../evolving/evolving_agent.py | 50 ----- .../evolving/evolving_strategy.py | 32 ++-- .../factor_implementation/evolving/factor.py | 39 ++-- .../evolving/knowledge_management.py | 26 +-- .../evolving/scheduler.py | 30 +-- rdagent/factor_implementation/prompts.yaml | 10 + .../factor_implementation_config.py | 12 +- .../factor_implementation_utils.py | 19 +- .../task_loader/json_loader.py | 31 +++ .../task_loader/pdf_loader.py} | 140 ++++++++------ .../task_loader}/prompts.yaml | 43 +++-- rdagent/knowledge_management/knowledgebase.py | 33 ++-- rdagent/knowledge_management/prompts.yaml | 9 - rdagent/knowledge_management/vector_base.py | 6 +- rdagent/oai/llm_utils.py | 100 +++++----- 30 files changed, 567 insertions(+), 564 deletions(-) create mode 100644 rdagent/core/evolving_agent.py rename rdagent/{document_process => document_reader}/document_reader.py (85%) delete mode 100644 rdagent/factor_implementation/evolving/evolving_agent.py create mode 100644 rdagent/factor_implementation/task_loader/json_loader.py rename rdagent/{document_process/document_analysis.py => factor_implementation/task_loader/pdf_loader.py} (82%) rename rdagent/{document_process => factor_implementation/task_loader}/prompts.yaml (78%) delete mode 100644 rdagent/knowledge_management/prompts.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a18f9cf3..ef6b2a56 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,12 +24,10 @@ jobs: CHAT_AZURE_API_VERSION: ${{ secrets.CHAT_AZURE_API_VERSION }} CHAT_MAX_TOKENS: ${{ secrets.CHAT_MAX_TOKENS }} CHAT_MODEL: ${{ secrets.CHAT_MODEL }} - CHAT_OPENAI_API_KEY: ${{ secrets.CHAT_OPENAI_API_KEY }} CHAT_TEMPERATURE: ${{ secrets.CHAT_TEMPERATURE }} EMBEDDING_AZURE_API_BASE: ${{ secrets.CHAT_AZURE_API_BASE }} EMBEDDING_AZURE_API_VERSION: ${{ secrets.CHAT_AZURE_API_VERSION }} EMBEDDING_MODEL: ${{ secrets.EMBEDDING_MODEL }} - EMBEDDING_OPENAI_API_KEY: ${{ secrets.CHAT_OPENAI_API_KEY }} name: lint test docs and build run: make lint test docs build strategy: @@ -84,4 +82,4 @@ on: - synchronize push: branches: - - master + - main diff --git a/.github/workflows/readthedocs-preview.yml b/.github/workflows/readthedocs-preview.yml index 39d55c32..e3999a3c 100644 --- a/.github/workflows/readthedocs-preview.yml +++ b/.github/workflows/readthedocs-preview.yml @@ -7,7 +7,7 @@ jobs: steps: - uses: readthedocs/actions/preview@v1 with: - project-slug: fincov2 + project-slug: RDAgent name: Read the Docs Pull Request Preview on: pull_request_target: diff --git a/Makefile b/Makefile index 47e73614..5fc4e80b 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ CONSTRAINTS_FILE := constraints/$(PYTHON_VERSION).txt PUBLIC_DIR := $(shell [ "$$READTHEDOCS" = "True" ] && echo "$$READTHEDOCS_OUTPUT/html" || echo "public") # URL and Path of changelog source code. -CHANGELOG_URL := $(shell echo $${CI_PAGES_URL:-https://peteryang1.github.io/fincov2}/_sources/changelog.md.txt) +CHANGELOG_URL := $(shell echo $${CI_PAGES_URL:-https://microsoft.github.io/rdagent}/_sources/changelog.md.txt) CHANGELOG_PATH := docs/changelog.md ######################################################################################## @@ -63,8 +63,8 @@ dev-%: init-qlib-env: # note: You may need to install torch manually # todo: downgrade ruamel.yaml in pyqlib - conda create -n qlibFinCo python=3.8 -y - @source $$(conda info --base)/etc/profile.d/conda.sh && conda activate qlibFinCo && which pip && pip install pyqlib && pip install ruamel-yaml==0.17.21 && pip install torch==2.1.1 && pip install catboost==0.24.3 && conda deactivate + conda create -n qlibRDAgent python=3.8 -y + @source $$(conda info --base)/etc/profile.d/conda.sh && conda activate qlibRDAgent && which pip && pip install pyqlib && pip install ruamel-yaml==0.17.21 && pip install torch==2.1.1 && pip install catboost==0.24.3 && conda deactivate dev: $(PIPRUN) pip install -e .[docs,lint,package,test] -c $(CONSTRAINTS_FILE) @@ -89,11 +89,11 @@ isort: # Check lint with mypy. mypy: - $(PIPRUN) python -m mypy . --exclude FinCo --exclude finco --exclude rdagent/scripts --exclude test/scripts --exclude git_ignore_folder + $(PIPRUN) python -m mypy . --exclude rdagent/scripts --exclude git_ignore_folder # Check lint with ruff. ruff: - $(PIPRUN) ruff check . --exclude FinCo,finco,rdagent/scripts,test/scripts,git_ignore_folder + $(PIPRUN) ruff check . --exclude rdagent/scripts,git_ignore_folder # Check lint with toml-sort. toml-sort: @@ -113,7 +113,7 @@ pre-commit: # Clean and run test with coverage. test-run: $(PIPRUN) python -m coverage erase - $(PIPRUN) python -m coverage run --concurrency=multiprocessing -m pytest --ignore FinCo --ignore test/finco --ignore test/scripts + $(PIPRUN) python -m coverage run --concurrency=multiprocessing -m pytest --ignore test/scripts $(PIPRUN) python -m coverage combine # Generate coverage report for terminal and xml. @@ -162,7 +162,7 @@ docs-gen: # Generate mypy reports. docs-mypy: docs-gen - $(PIPRUN) python -m mypy rdagent test --exclude FinCo --exclude git_ignore_folder --exclude finco --exclude rdagent/scripts --exclude test/scripts --html-report $(PUBLIC_DIR)/reports/mypy + $(PIPRUN) python -m mypy rdagent test --exclude git_ignore_folder --exclude rdagent/scripts --html-report $(PUBLIC_DIR)/reports/mypy # Generate html coverage reports with badge. docs-coverage: test-run docs-gen diff --git a/rdagent/app/CI/run.py b/rdagent/app/CI/run.py index e6e515c5..e73a18fb 100644 --- a/rdagent/app/CI/run.py +++ b/rdagent/app/CI/run.py @@ -37,6 +37,7 @@ from tree_sitter import Language, Node, Parser py_parser = Parser(Language(tree_sitter_python.language())) CI_prompts = Prompts(file_path=Path(__file__).parent / "prompts.yaml") + @dataclass class CIError: raw_str: str @@ -80,8 +81,7 @@ class FixRecord: "directly_fixed_errors": [error.to_dict() for error in self.directly_fixed_errors], "manually_fixed_errors": [error.to_dict() for error in self.manually_fixed_errors], "manual_instructions": { - key: [error.to_dict() for error in errors] - for key, errors in self.manual_instructions.items() + key: [error.to_dict() for error in errors] for key, errors in self.manual_instructions.items() }, } @@ -102,7 +102,6 @@ class CodeFile: return code_with_lineno if isinstance(code, list) else "\n".join(code_with_lineno) - @classmethod def remove_line_number(cls: CodeFile, code: list[str] | str) -> list[str] | str: code_lines = code.split("\n") if isinstance(code, str) else code @@ -124,8 +123,12 @@ class CodeFile: self.code_lines_with_lineno = self.add_line_number(self.code_lines) def get( - self, start: int = 1, end: int | None = None, *, - add_line_number: bool = False, return_list: bool = False, + self, + start: int = 1, + end: int | None = None, + *, + add_line_number: bool = False, + return_list: bool = False, ) -> list[str] | str: """ Retrieves a portion of the code lines. @@ -183,7 +186,7 @@ class CodeFile: return [(node.start_point.row, node.end_point.row + 1)] blocks: list[tuple[int, int]] = [] - block: tuple[int, int] | None = None # [start, end), line number starts from 0 + block: tuple[int, int] | None = None # [start, end), line number starts from 0 for child in node.children: if child.end_point.row + 1 - child.start_point.row > max_lines: @@ -205,7 +208,7 @@ class CodeFile: return blocks # change line number to start from 1 and [start, end) to [start, end] - return [(a+1,b) for a,b in get_blocks_in_node(tree.root_node, max_lines)] + return [(a + 1, b) for a, b in get_blocks_in_node(tree.root_node, max_lines)] def __str__(self) -> str: return f"{self.path}" @@ -221,7 +224,7 @@ class Repo(EvolvableSubjects): excludes = [self.project_path / path for path in excludes] git_ignored_output = subprocess.check_output( - ["/usr/bin/git", "status", "--ignored", "-s"], # noqa: S603 + ["/usr/bin/git", "status", "--ignored", "-s"], # noqa: S603 cwd=str(self.project_path), stderr=subprocess.STDOUT, text=True, @@ -264,6 +267,7 @@ class RuffRule: "preview": false } """ + name: str code: str linter: str @@ -290,7 +294,7 @@ class RuffEvaluator(Evaluator): explain_command = f"ruff rule {error_code} --output-format json" try: out = subprocess.check_output( - shlex.split(explain_command), # noqa: S603 + shlex.split(explain_command), # noqa: S603 stderr=subprocess.STDOUT, text=True, ) @@ -299,12 +303,11 @@ class RuffEvaluator(Evaluator): return RuffRule(**json.loads(out)) - - def evaluate(self, evo: Repo, **kwargs: Any) -> CIFeedback: # noqa: ARG002 + def evaluate(self, evo: Repo, **kwargs: Any) -> CIFeedback: # noqa: ARG002 """Simply run ruff to get the feedbacks.""" try: out = subprocess.check_output( - shlex.split(self.command), # noqa: S603 + shlex.split(self.command), # noqa: S603 cwd=evo.project_path, stderr=subprocess.STDOUT, text=True, @@ -313,7 +316,7 @@ class RuffEvaluator(Evaluator): out = e.output """ruff output format: - src/finco/cli.py:9:5: ANN201 Missing return type annotation for public function `main` + rdagent/cli.py:9:5: ANN201 Missing return type annotation for public function `main` | 9 | def main(prompt=None): | ^^^^ ANN201 @@ -335,19 +338,22 @@ class RuffEvaluator(Evaluator): # TODO @bowen: filter these files when running the check command if evo.project_path / Path(file_path) not in evo.files: continue - error = CIError(raw_str=raw_str, - file_path=file_path, - line=int(line_number), - column=int(column_number), - code=error_code, - msg=error_message, - hint=error_hint, - checker="ruff") + error = CIError( + raw_str=raw_str, + file_path=file_path, + line=int(line_number), + column=int(column_number), + code=error_code, + msg=error_message, + hint=error_hint, + checker="ruff", + ) errors[file_path].append(error) return CIFeedback(errors=errors) + class MypyEvaluator(Evaluator): def __init__(self, command: str | None = None) -> None: @@ -356,10 +362,10 @@ class MypyEvaluator(Evaluator): else: self.command = command - def evaluate(self, evo: Repo, **kwargs: Any) -> CIFeedback: # noqa: ARG002 + def evaluate(self, evo: Repo, **kwargs: Any) -> CIFeedback: # noqa: ARG002 try: out = subprocess.check_output( - shlex.split(self.command), # noqa: S603 + shlex.split(self.command), # noqa: S603 cwd=evo.project_path, stderr=subprocess.STDOUT, text=True, @@ -377,7 +383,10 @@ class MypyEvaluator(Evaluator): error_message = error_message.strip().replace("\n", " ") if re.match(r".*[^\n]*?:\d+:\d+: note:.*", error_hint, flags=re.DOTALL) is not None: error_hint_position = re.split( - pattern=r"[^\n]*?:\d+:\d+: note:", string=error_hint, maxsplit=1, flags=re.DOTALL, + pattern=r"[^\n]*?:\d+:\d+: note:", + string=error_hint, + maxsplit=1, + flags=re.DOTALL, )[0] error_hint_help = re.findall(r"^.*?:\d+:\d+: note: (.*)$", error_hint, flags=re.MULTILINE) error_hint_help = "\n".join(error_hint_help) @@ -385,14 +394,16 @@ class MypyEvaluator(Evaluator): if evo.project_path / Path(file_path) not in evo.files: continue - error = CIError(raw_str=raw_str, - file_path=file_path, - line=int(line_number), - column=int(column_number), - code=error_code, - msg=error_message, - hint=error_hint, - checker="mypy") + error = CIError( + raw_str=raw_str, + file_path=file_path, + line=int(line_number), + column=int(column_number), + code=error_code, + msg=error_message, + hint=error_hint, + checker="mypy", + ) errors[file_path].append(error) @@ -418,13 +429,14 @@ class MultiEvaluator(Evaluator): return CIFeedback(errors=all_errors) + class CIEvoStr(EvolvingStrategy): - def evolve( # noqa: C901, PLR0912, PLR0915 + def evolve( # noqa: C901, PLR0912, PLR0915 self, evo: Repo, evolving_trace: list[EvoStep] | None = None, - knowledge_l: list[Knowledge] | None = None, # noqa: ARG002 - **kwargs: Any, # noqa: ARG002 + knowledge_l: list[Knowledge] | None = None, # noqa: ARG002 + **kwargs: Any, # noqa: ARG002 ) -> Repo: @dataclass @@ -443,13 +455,12 @@ class CIEvoStr(EvolvingStrategy): # print statistics checker_error_counts = { - checker: sum(c_statistics.values()) - for checker, c_statistics in last_feedback.statistics().items() + checker: sum(c_statistics.values()) for checker, c_statistics in last_feedback.statistics().items() } print( f"Found [red]{sum(checker_error_counts.values())}[/red] errors, " - "including: " + - ", ".join( + "including: " + + ", ".join( f"[red]{count}[/red] [magenta]{checker}[/magenta] errors" for checker, count in checker_error_counts.items() ), @@ -496,7 +507,6 @@ class CIEvoStr(EvolvingStrategy): CodeFixGroup(start_line, end_line, group_errors, session_id, []), ) - # Fix errors in each code block with Progress(SpinnerColumn(), *Progress.get_default_columns(), TimeElapsedColumn()) as progress: group_counts = sum([len(groups) for groups in fix_groups.values()]) @@ -509,7 +519,10 @@ class CIEvoStr(EvolvingStrategy): end_line = code_fix_g.end_line group_errors = code_fix_g.errors code_snippet_with_lineno = file.get( - start_line, end_line, add_line_number=True, return_list=False, + start_line, + end_line, + add_line_number=True, + return_list=False, ) errors_str = "\n\n".join(str(e) for e in group_errors) @@ -531,11 +544,16 @@ class CIEvoStr(EvolvingStrategy): advance=1, ) - # Manual inspection and repair for file_path in last_feedback.errors: - print(Rule(f"[bright_blue]Checking[/bright_blue] [cyan]{file_path}[/cyan]", - style="bright_blue", align="left", characters=".")) + print( + Rule( + f"[bright_blue]Checking[/bright_blue] [cyan]{file_path}[/cyan]", + style="bright_blue", + align="left", + characters=".", + ) + ) file = evo.files[evo.project_path / Path(file_path)] @@ -546,17 +564,20 @@ class CIEvoStr(EvolvingStrategy): print(f"[yellow]Checking part {group_id}...[/yellow]") - front_context = file.get(start_line-3, start_line-1) - rear_context = file.get(end_line+1, end_line+3) - front_context_with_lineno = file.get(start_line-3, start_line-1, add_line_number=True) - rear_context_with_lineno = file.get(end_line+1, end_line+3, add_line_number=True) + front_context = file.get(start_line - 3, start_line - 1) + rear_context = file.get(end_line + 1, end_line + 3) + front_context_with_lineno = file.get(start_line - 3, start_line - 1, add_line_number=True) + rear_context_with_lineno = file.get(end_line + 1, end_line + 3, add_line_number=True) code_snippet_with_lineno = file.get(start_line, end_line, add_line_number=True, return_list=False) # print errors printed_errors_str = "\n".join( - [f"[{error.checker}] {error.line: >{file.lineno_width}}:{error.column: <4}" - f" {error.code} {error.msg}" for error in group_errors], + [ + f"[{error.checker}] {error.line: >{file.lineno_width}}:{error.column: <4}" + f" {error.code} {error.msg}" + for error in group_errors + ], ) print( Panel.fit( @@ -611,27 +632,34 @@ class CIEvoStr(EvolvingStrategy): diff_new_lineno = start_line for i in diff: if i.startswith("+"): - table.add_row("", Text(str(diff_new_lineno), style="green bold"), - Text(i, style="green")) + table.add_row( + "", Text(str(diff_new_lineno), style="green bold"), Text(i, style="green") + ) diff_new_lineno += 1 elif i.startswith("-"): - table.add_row(Text(str(diff_original_lineno), style="red bold"), "", - Text(i, style="red")) + table.add_row( + Text(str(diff_original_lineno), style="red bold"), "", Text(i, style="red") + ) diff_original_lineno += 1 elif i.startswith("?"): table.add_row("", "", Text(i, style="yellow")) else: - table.add_row(str(diff_original_lineno), str(diff_new_lineno), - Syntax(i, lexer="python", background_color="default")) + table.add_row( + str(diff_original_lineno), + str(diff_new_lineno), + Syntax(i, lexer="python", background_color="default"), + ) diff_original_lineno += 1 diff_new_lineno += 1 table.add_row("", "", Rule(style="dark_orange")) table.add_row("", "", Syntax(rear_context, lexer="python", background_color="default")) print(Panel.fit(table, title="Repair Status")) - operation = Prompt.ask("Input your operation [ [red]([bold]s[/bold])kip[/red] / " - "[green]([bold]a[/bold])pply[/green] / " - "[yellow]manual instruction[/yellow] ]") + operation = Prompt.ask( + "Input your operation [ [red]([bold]s[/bold])kip[/red] / " + "[green]([bold]a[/bold])pply[/green] / " + "[yellow]manual instruction[/yellow] ]" + ) print() if operation in ("s", "skip"): fix_records[file_path].skipped_errors.extend(group_errors) @@ -692,7 +720,7 @@ while True: fix_records = repo.fix_records filename = f"{DIR.name}_{start_timestamp}_round_{len(ea.evolving_trace)}_fix_records.json" with Path(filename).open("w") as file: - json.dump({k:v.to_dict() for k,v in fix_records.items()}, file, indent=4) + json.dump({k: v.to_dict() for k, v in fix_records.items()}, file, indent=4) # Count the number of skipped errors skipped_errors_count = 0 @@ -735,15 +763,27 @@ while True: total_errors_count = skipped_errors_count + directly_fixed_errors_count + manually_fixed_errors_count table.add_row("Total Errors", "", Text(str(total_errors_count), style="cyan"), "") - table.add_row(Text("Skipped Errors", style="red"), skipped_errors_statistics, - Text(str(skipped_errors_count), style="red"), - Text(f"{skipped_errors_count / total_errors_count:.2%}"), style="red") - table.add_row(Text("Directly Fixed Errors", style="green"), directly_fixed_errors_statistics, - Text(str(directly_fixed_errors_count), style="green"), - Text(f"{directly_fixed_errors_count / total_errors_count:.2%}"), style="green") - table.add_row(Text("Manually Fixed Errors", style="yellow"), manually_fixed_errors_statistics, - Text(str(manually_fixed_errors_count), style="yellow"), - Text(f"{manually_fixed_errors_count / total_errors_count:.2%}"), style="yellow") + table.add_row( + Text("Skipped Errors", style="red"), + skipped_errors_statistics, + Text(str(skipped_errors_count), style="red"), + Text(f"{skipped_errors_count / total_errors_count:.2%}"), + style="red", + ) + table.add_row( + Text("Directly Fixed Errors", style="green"), + directly_fixed_errors_statistics, + Text(str(directly_fixed_errors_count), style="green"), + Text(f"{directly_fixed_errors_count / total_errors_count:.2%}"), + style="green", + ) + table.add_row( + Text("Manually Fixed Errors", style="yellow"), + manually_fixed_errors_statistics, + Text(str(manually_fixed_errors_count), style="yellow"), + Text(f"{manually_fixed_errors_count / total_errors_count:.2%}"), + style="yellow", + ) print(table) operation = Prompt.ask("Start next round? (y/n)", choices=["y", "n"]) diff --git a/rdagent/app/factor_extraction_and_implementation/factor_extract_and_implement.py b/rdagent/app/factor_extraction_and_implementation/factor_extract_and_implement.py index 78e1bd7a..04efa2b6 100644 --- a/rdagent/app/factor_extraction_and_implementation/factor_extract_and_implement.py +++ b/rdagent/app/factor_extraction_and_implementation/factor_extract_and_implement.py @@ -1,43 +1,16 @@ # %% -from pathlib import Path - from dotenv import load_dotenv -from rdagent.document_process.document_analysis import ( - check_factor_viability, - classify_report_from_dict, - deduplicate_factors_by_llm, - extract_factors_from_report_dict, - merge_file_to_factor_dict_to_factor_dict, - classify_report_from_dict, -) -from rdagent.document_process.document_reader import load_and_process_pdfs_by_langchain -from rdagent.factor_implementation.share_modules.factor_implementation_utils import load_data_from_dict from rdagent.factor_implementation.CoSTEER import CoSTEERFG -import pickle -from dotenv import load_dotenv +from rdagent.factor_implementation.task_loader.pdf_loader import FactorImplementationTaskLoaderFromPDFfiles + +assert load_dotenv() def extract_factors_and_implement(report_file_path: str) -> None: - assert load_dotenv() - docs_dict = load_and_process_pdfs_by_langchain(Path(report_file_path)) - - selected_report_dict = classify_report_from_dict(report_dict=docs_dict, vote_time=1) - file_to_factor_result = extract_factors_from_report_dict(docs_dict, selected_report_dict) - factor_dict = merge_file_to_factor_dict_to_factor_dict(file_to_factor_result) - - factor_viability = check_factor_viability(factor_dict) - - factor_dict, duplication_names_list = deduplicate_factors_by_llm(factor_dict, factor_viability) - - factor_tasks = load_data_from_dict(factor_dict) - - factor_generate_method = CoSTEERFG() - - result = factor_generate_method.generate(factor_tasks) - - return result + factor_tasks = FactorImplementationTaskLoaderFromPDFfiles().load(report_file_path) + implementation_result = CoSTEERFG().generate(factor_tasks) + return implementation_result if __name__ == "__main__": extract_factors_and_implement("/home/xuyang1/workspace/report.pdf") - # test_implement() diff --git a/rdagent/benchmark/eval_method.py b/rdagent/benchmark/eval_method.py index 72c57a35..2735e70b 100644 --- a/rdagent/benchmark/eval_method.py +++ b/rdagent/benchmark/eval_method.py @@ -3,7 +3,7 @@ from typing import List, Tuple, Union from tqdm import tqdm from collections import defaultdict -from rdagent.core.conf import RDAgentSettings +from rdagent.core.conf import RD_AGENT_SETTINGS from rdagent.core.exception import ImplementRunException from rdagent.core.task import ( TaskImplementation, @@ -23,10 +23,12 @@ from rdagent.core.implementation import TaskGenerator from rdagent.core.utils import multiprocessing_wrapper from rdagent.factor_implementation.evolving.factor import FileBasedFactorImplementation + class BaseEval: """ The benchmark benchmark evaluation. """ + def __init__( self, evaluator_l: List[FactorImplementationEvaluator], @@ -95,7 +97,8 @@ class BaseEval: else: raise e return eval_res - + + class FactorImplementEval(BaseEval): def __init__( self, @@ -106,15 +109,17 @@ class FactorImplementEval(BaseEval): **kwargs, ): # evaluator collection for online evaluation - online_evaluator_l = [ - FactorImplementationCorrelationEvaluator, - FactorImplementationIndexEvaluator, - FactorImplementationIndexFormatEvaluator, - FactorImplementationMissingValuesEvaluator, - FactorImplementationRowCountEvaluator, - FactorImplementationSingleColumnEvaluator, - FactorImplementationValuesEvaluator, - ], + online_evaluator_l = ( + [ + FactorImplementationCorrelationEvaluator, + FactorImplementationIndexEvaluator, + FactorImplementationIndexFormatEvaluator, + FactorImplementationMissingValuesEvaluator, + FactorImplementationRowCountEvaluator, + FactorImplementationSingleColumnEvaluator, + FactorImplementationValuesEvaluator, + ], + ) super().__init__(online_evaluator_l, test_case, method, *args, **kwargs) self.test_round = test_round @@ -148,7 +153,7 @@ class FactorImplementEval(BaseEval): (self.eval_case, (gt_case.ground_truth, gen_factor)) for gt_case, gen_factor in zip(test_cases_all_rounds, gen_factor_l_all_rounds) ], - n=RDAgentSettings().evo_multi_proc_n, + n=RD_AGENT_SETTINGS.evo_multi_proc_n, ) for gt_case, eval_res, gen_factor in tqdm(zip(test_cases_all_rounds, eval_res_list, gen_factor_l_all_rounds)): diff --git a/rdagent/core/conf.py b/rdagent/core/conf.py index 41cb07a4..88476381 100644 --- a/rdagent/core/conf.py +++ b/rdagent/core/conf.py @@ -19,17 +19,13 @@ class RDAgentSettings(BaseSettings): use_azure_token_provider: bool = False max_retry: int = 10 retry_wait_seconds: int = 1 - continuous_mode: bool = False dump_chat_cache: bool = False use_chat_cache: bool = False dump_embedding_cache: bool = False use_embedding_cache: bool = False - workspace: str = "./finco_workspace" prompt_cache_path: str = str(Path.cwd() / "prompt_cache.db") session_cache_folder_location: str = str(Path.cwd() / "session_cache_folder/") max_past_message_include: int = 10 - - use_vector_only: bool = False log_llm_chat_content: bool = True # Chat configs @@ -43,11 +39,9 @@ class RDAgentSettings(BaseSettings): chat_seed: int | None = None chat_frequency_penalty: float = 0.0 chat_presence_penalty: float = 0.0 - chat_token_limit: int = ( 100000 # 100000 is the maximum limit of gpt4, which might increase in the future version of gpt ) - default_system_prompt: str = "You are an AI assistant who helps to answer user's questions about finance." # Embedding configs @@ -56,17 +50,17 @@ class RDAgentSettings(BaseSettings): embedding_azure_api_version: str = "" embedding_model: str = "" - # llama2 related config + # offline llama2 related config use_llama2: bool = False llama2_ckpt_dir: str = "Llama-2-7b-chat" llama2_tokenizer_path: str = "Llama-2-7b-chat/tokenizer.model" llams2_max_batch_size: int = 8 - # finco v2 configs + # azure document intelligence configs azure_document_intelligence_key: str = "" azure_document_intelligence_endpoint: str = "" - # fincov2 llama2 endpoint + # server served endpoints use_gcr_endpoint: bool = False gcr_endpoint_type: str = "llama2_70b" # or "llama3_70b", "phi2", "phi3_4k", "phi3_128k" @@ -99,3 +93,5 @@ class RDAgentSettings(BaseSettings): max_input_duplicate_factor_group: int = 600 max_output_duplicate_factor_group: int = 20 + +RD_AGENT_SETTINGS = RDAgentSettings() diff --git a/rdagent/core/evolving_agent.py b/rdagent/core/evolving_agent.py new file mode 100644 index 00000000..30a8f729 --- /dev/null +++ b/rdagent/core/evolving_agent.py @@ -0,0 +1,63 @@ +from rdagent.core.evaluation import Evaluator +from rdagent.core.evolving_framework import Feedback, EvolvableSubjects, EvoStep +from tqdm import tqdm +from abc import ABC, abstractmethod +from typing import Any + + +class EvoAgent(ABC): + def __init__(self, max_loop, evolving_strategy) -> None: + self.max_loop = max_loop + self.evolving_strategy = evolving_strategy + + @abstractmethod + def multistep_evolve(self, evo: EvolvableSubjects, eva: Evaluator | Feedback, **kwargs: Any) -> EvolvableSubjects: + pass + + +class RAGEvoAgent(EvoAgent): + def __init__(self, max_loop, evolving_strategy, rag) -> None: + super().__init__(max_loop, evolving_strategy) + self.rag = rag + self.evolving_trace = [] + + def multistep_evolve( + self, + evo: EvolvableSubjects, + eva: Evaluator | Feedback, + *, + with_knowledge: bool = False, + with_feedback: bool = True, + knowledge_self_gen: bool = False, + ) -> EvolvableSubjects: + + for _ in tqdm(range(self.max_loop), "Implementing factors"): + # 1. knowledge self-evolving + if knowledge_self_gen and self.rag is not None: + self.rag.generate_knowledge(self.evolving_trace) + # 2. RAG + queried_knowledge = None + if with_knowledge and self.rag is not None: + # TODO: Putting the evolving trace in here doesn't actually work + queried_knowledge = self.rag.query(evo, self.evolving_trace) + + # 3. evolve + evo = self.evolving_strategy.evolve( + evo=evo, + evolving_trace=self.evolving_trace, + queried_knowledge=queried_knowledge, + ) + + # 4. Pack evolve results + es = EvoStep(evo, queried_knowledge) + + # 5. Evaluation + if with_feedback: + es.feedback = ( + eva if isinstance(eva, Feedback) else eva.evaluate(evo, queried_knowledge=queried_knowledge) + ) + + # 6. update trace + self.evolving_trace.append(es) + + return evo diff --git a/rdagent/core/evolving_framework.py b/rdagent/core/evolving_framework.py index eb1dcad0..06486a65 100644 --- a/rdagent/core/evolving_framework.py +++ b/rdagent/core/evolving_framework.py @@ -33,27 +33,7 @@ class EvolvableSubjects: return copy.deepcopy(self) -class QlibEvolvableSubjects(EvolvableSubjects): - ... - - -class Evaluator(ABC): - """Both external EvolvableSubjects and internal evovler, it is - - FAQ: - - Q: If we have a external whitebox evaluator, do we need a - intenral EvolvableSubjects? - A: When the external evovler is very complex, maybe a internal LLM-based evovler - may provide more understandable feedbacks. - """ - - @abstractmethod - def evaluate(self, evo: EvolvableSubjects, **kwargs: Any) -> Feedback: - raise NotImplementedError - - -class SelfEvaluator(Evaluator): - pass +class QlibEvolvableSubjects(EvolvableSubjects): ... @dataclass @@ -91,16 +71,6 @@ class EvolvingStrategy(ABC): """ -class EvoAgent(ABC): - def __init__(self, max_loop, evolving_strategy) -> None: - self.max_loop = max_loop - self.evolving_strategy = evolving_strategy - - @abstractmethod - def multistep_evolve(self, evo: EvolvableSubjects, eva: Evaluator | Feedback, **kwargs: Any) -> EvolvableSubjects: - pass - - class RAGStrategy(ABC): """Retrival Augmentation Generation Strategy""" diff --git a/rdagent/core/log.py b/rdagent/core/log.py index 87630fa9..9c6e2b74 100644 --- a/rdagent/core/log.py +++ b/rdagent/core/log.py @@ -72,7 +72,7 @@ class LogColors: return f"{style}{text}{self.END}" -class FinCoLog: +class RDAgentLog: # logger.add(loguru_handler, level="INFO") # you can add use storage as a loguru handler def __init__(self) -> None: diff --git a/rdagent/core/task.py b/rdagent/core/task.py index e2fb6ac0..ebae4c67 100644 --- a/rdagent/core/task.py +++ b/rdagent/core/task.py @@ -2,13 +2,16 @@ from abc import ABC, abstractmethod from typing import Tuple import pandas as pd -''' +""" This file contains the all the data class for rdagent task. -''' +""" + + class BaseTask(ABC): # 把name放在这里作为主键 pass + class TaskImplementation(ABC): def __init__(self, target_task: BaseTask) -> None: self.target_task = target_task @@ -17,6 +20,7 @@ class TaskImplementation(ABC): def execute(self, *args, **kwargs) -> Tuple[str, pd.DataFrame]: raise NotImplementedError("__call__ method is not implemented.") + class TestCase: def __init__( self, @@ -26,3 +30,8 @@ class TestCase: self.ground_truth = ground_truth self.target_task = target_task + +class TaskLoader: + @abstractmethod + def load(self, *args, **kwargs) -> BaseTask | list[BaseTask]: + raise NotImplementedError("load method is not implemented.") diff --git a/rdagent/core/utils.py b/rdagent/core/utils.py index 69932504..e72b17eb 100644 --- a/rdagent/core/utils.py +++ b/rdagent/core/utils.py @@ -14,7 +14,7 @@ import yaml from fuzzywuzzy import fuzz -class RDAgentException(Exception): # noqa: N818 +class RDAgentException(Exception): # noqa: N818 pass @@ -31,6 +31,7 @@ class SingletonMeta(type): cls._instance_dict[kwargs_hash] = super().__call__(**kwargs) return cls._instance_dict[kwargs_hash] + class SingletonBaseClass(metaclass=SingletonMeta): """ Because we try to support defining Singleton with `class A(SingletonBaseClass)` @@ -57,67 +58,6 @@ def similarity(text1: str, text2: str) -> int: return fuzz.ratio(text1, text2) -def random_string(length: int = 10) -> str: - letters = string.ascii_letters + string.digits - return "".join(random.SystemRandom().choice(letters) for _ in range(length)) - - -def remove_uncommon_keys(new_dict: dict, org_dict: dict) -> None: - keys_to_remove = [] - - for key in new_dict: - if key not in org_dict: - keys_to_remove.append(key) - elif isinstance(new_dict[key], dict) and isinstance(org_dict[key], dict): - remove_uncommon_keys(new_dict[key], org_dict[key]) - elif isinstance(new_dict[key], dict) and isinstance(org_dict[key], str): - new_dict[key] = org_dict[key] - - for key in keys_to_remove: - del new_dict[key] - - -def crawl_the_folder(folder_path: Path) -> list: - yaml_files = [] - for root, _, files in os.walk(folder_path.as_posix()): - for file in files: - if file.endswith((".yaml", ".yml")): - yaml_file_path = Path(root) / file - yaml_files.append(str(yaml_file_path.relative_to(folder_path))) - return sorted(yaml_files) - - -def compare_yaml(file1: Path | str, file2: Path | str) -> bool: - with Path(file1).open() as stream: - data1 = yaml.safe_load(stream) - with Path(file2).open() as stream: - data2 = yaml.safe_load(stream) - return data1 == data2 - - -def remove_keys(valid_keys: set[Any], ori_dict: dict[Any, Any]) -> dict[Any, Any]: - for key in list(ori_dict.keys()): - if key not in valid_keys: - ori_dict.pop(key) - return ori_dict - - -class YamlConfigCache(SingletonBaseClass): - def __init__(self) -> None: - super().__init__() - self.path_to_config = {} - - def load(self, path: str) -> None: - with Path(path).open() as stream: - data = yaml.safe_load(stream) - self.path_to_config[path] = data - - def __getitem__(self, path: str) -> Any: - if path not in self.path_to_config: - self.load(path) - return self.path_to_config[path] - - def import_class(class_path: str) -> Any: """ Parameters @@ -156,11 +96,3 @@ def multiprocessing_wrapper(func_calls: list[tuple[Callable, tuple]], n: int) -> with mp.Pool(processes=n) as pool: results = [pool.apply_async(f, args) for f, args in func_calls] return [result.get() for result in results] - - -# You can test the above function -# def f(x): -# return x**2 -# -# if __name__ == "__main__": -# print(multiprocessing_wrapper([(f, (i,)) for i in range(10)], 4)) diff --git a/rdagent/document_process/document_reader.py b/rdagent/document_reader/document_reader.py similarity index 85% rename from rdagent/document_process/document_reader.py rename to rdagent/document_reader/document_reader.py index e028fa33..db5cc570 100644 --- a/rdagent/document_process/document_reader.py +++ b/rdagent/document_reader/document_reader.py @@ -9,7 +9,7 @@ from langchain.document_loaders import PyPDFDirectoryLoader, PyPDFLoader if TYPE_CHECKING: from langchain_core.documents import Document -from rdagent.core.conf import RDAgentSettings as Config +from rdagent.core.conf import RD_AGENT_SETTINGS def load_documents_by_langchain(path: Path) -> list: @@ -74,10 +74,8 @@ def load_and_process_one_pdf_by_azure_document_intelligence( def load_and_process_pdfs_by_azure_document_intelligence(path: Path) -> dict[str, str]: - config = Config() - - assert config.azure_document_intelligence_key is not None - assert config.azure_document_intelligence_endpoint is not None + assert RD_AGENT_SETTINGS.azure_document_intelligence_key is not None + assert RD_AGENT_SETTINGS.azure_document_intelligence_endpoint is not None content_dict = {} ab_path = path.resolve() @@ -86,15 +84,15 @@ def load_and_process_pdfs_by_azure_document_intelligence(path: Path) -> dict[str proc = load_and_process_one_pdf_by_azure_document_intelligence content_dict[str(ab_path)] = proc( ab_path, - config.azure_document_intelligence_key, - config.azure_document_intelligence_endpoint, + RD_AGENT_SETTINGS.azure_document_intelligence_key, + RD_AGENT_SETTINGS.azure_document_intelligence_endpoint, ) else: for file_path in ab_path.rglob("*"): if file_path.is_file() and ".pdf" in file_path.suffixes: content_dict[str(file_path)] = load_and_process_one_pdf_by_azure_document_intelligence( file_path, - config.azure_document_intelligence_key, - config.azure_document_intelligence_endpoint, + RD_AGENT_SETTINGS.azure_document_intelligence_key, + RD_AGENT_SETTINGS.azure_document_intelligence_endpoint, ) return content_dict diff --git a/rdagent/factor_implementation/CoSTEER.py b/rdagent/factor_implementation/CoSTEER.py index 5f820ec4..2519da6b 100644 --- a/rdagent/factor_implementation/CoSTEER.py +++ b/rdagent/factor_implementation/CoSTEER.py @@ -5,13 +5,20 @@ from rdagent.core.implementation import TaskGenerator from rdagent.core.task import TaskImplementation from rdagent.factor_implementation.evolving.knowledge_management import FactorImplementationKnowledgeBaseV1 from rdagent.factor_implementation.evolving.factor import FactorImplementTask, FactorEvovlingItem -from rdagent.knowledge_management.knowledgebase import FactorImplementationGraphKnowledgeBase, FactorImplementationGraphRAGStrategy -from rdagent.factor_implementation.evolving.evolving_strategy import FactorEvolvingStrategyWithGraph -from rdagent.factor_implementation.evolving.evaluators import FactorImplementationsMultiEvaluator, FactorImplementationEvaluatorV1 -from rdagent.factor_implementation.evolving.evolving_agent import RAGEvoAgent -from rdagent.factor_implementation.share_modules.factor_implementation_config import ( - FactorImplementSettings, +from rdagent.factor_implementation.evolving.knowledge_management import ( + FactorImplementationGraphKnowledgeBase, + FactorImplementationGraphRAGStrategy, ) +from rdagent.factor_implementation.evolving.evolving_strategy import FactorEvolvingStrategyWithGraph +from rdagent.factor_implementation.evolving.evaluators import ( + FactorImplementationsMultiEvaluator, + FactorImplementationEvaluatorV1, +) +from rdagent.core.evolving_agent import RAGEvoAgent +from rdagent.factor_implementation.share_modules.factor_implementation_config import ( + FACTOR_IMPLEMENT_SETTINGS, +) + class CoSTEERFG(TaskGenerator): def __init__( @@ -20,9 +27,17 @@ class CoSTEERFG(TaskGenerator): with_feedback: bool = True, knowledge_self_gen: bool = True, ) -> None: - self.max_loop = FactorImplementSettings().max_loop - self.knowledge_base_path = Path(FactorImplementSettings().knowledge_base_path) if FactorImplementSettings().knowledge_base_path is not None else None - self.new_knowledge_base_path = Path(FactorImplementSettings().new_knowledge_base_path) if FactorImplementSettings().new_knowledge_base_path is not None else None + self.max_loop = FACTOR_IMPLEMENT_SETTINGS.max_loop + self.knowledge_base_path = ( + Path(FACTOR_IMPLEMENT_SETTINGS.knowledge_base_path) + if FACTOR_IMPLEMENT_SETTINGS.knowledge_base_path is not None + else None + ) + self.new_knowledge_base_path = ( + Path(FACTOR_IMPLEMENT_SETTINGS.new_knowledge_base_path) + if FACTOR_IMPLEMENT_SETTINGS.new_knowledge_base_path is not None + else None + ) self.with_knowledge = with_knowledge self.with_feedback = with_feedback self.knowledge_self_gen = knowledge_self_gen @@ -32,7 +47,7 @@ class CoSTEERFG(TaskGenerator): self.evolving_version = 2 def load_or_init_knowledge_base(self, former_knowledge_base_path: Path = None, component_init_list: list = []): - + if former_knowledge_base_path is not None and former_knowledge_base_path.exists(): factor_knowledge_base = pickle.load(open(former_knowledge_base_path, "rb")) if self.evolving_version == 1 and not isinstance( @@ -53,7 +68,7 @@ class CoSTEERFG(TaskGenerator): else FactorImplementationKnowledgeBaseV1() ) return factor_knowledge_base - + def generate(self, tasks: List[FactorImplementTask]) -> List[TaskImplementation]: # init knowledge base factor_knowledge_base = self.load_or_init_knowledge_base( @@ -61,9 +76,7 @@ class CoSTEERFG(TaskGenerator): component_init_list=[], ) # init rag method - self.rag = ( - FactorImplementationGraphRAGStrategy(factor_knowledge_base) - ) + self.rag = FactorImplementationGraphRAGStrategy(factor_knowledge_base) # init indermediate items factor_implementations = FactorEvovlingItem(target_factor_tasks=tasks) @@ -83,4 +96,4 @@ class CoSTEERFG(TaskGenerator): pickle.dump(factor_knowledge_base, open(self.new_knowledge_base_path, "wb")) self.knowledge_base = factor_knowledge_base self.latest_factor_implementations = tasks - return factor_implementations \ No newline at end of file + return factor_implementations diff --git a/rdagent/factor_implementation/evolving/evaluators.py b/rdagent/factor_implementation/evolving/evaluators.py index d0280925..11a3d330 100644 --- a/rdagent/factor_implementation/evolving/evaluators.py +++ b/rdagent/factor_implementation/evolving/evaluators.py @@ -8,21 +8,23 @@ import pandas as pd from jinja2 import Template from rdagent.oai.llm_utils import APIBackend -from rdagent.core.log import FinCoLog +from rdagent.core.log import RDAgentLog from rdagent.factor_implementation.evolving.evolving_strategy import FactorImplementTask, FactorEvovlingItem from rdagent.core.task import ( TaskImplementation, ) from typing import List, Tuple -from rdagent.core.evolving_framework import QueriedKnowledge,Feedback +from rdagent.core.evolving_framework import QueriedKnowledge, Feedback from rdagent.core.evaluation import Evaluator from rdagent.core.prompts import Prompts -from rdagent.factor_implementation.share_modules.factor_implementation_config import FactorImplementSettings +from rdagent.core.conf import RD_AGENT_SETTINGS +from rdagent.factor_implementation.share_modules.factor_implementation_config import FACTOR_IMPLEMENT_SETTINGS from rdagent.core.utils import multiprocessing_wrapper from pathlib import Path evaluate_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml") + class FactorImplementationEvaluator(Evaluator): # TODO: # I think we should have unified interface for all evaluates, for examples. @@ -58,6 +60,7 @@ class FactorImplementationEvaluator(Evaluator): gt_df = gt_df.to_frame("gt_factor") return gt_df, gen_df + class FactorImplementationCodeEvaluator(Evaluator): def evaluate( self, @@ -89,7 +92,7 @@ class FactorImplementationCodeEvaluator(Evaluator): system_prompt=system_prompt, former_messages=[], ) - > FactorImplementSettings().chat_token_limit + > RD_AGENT_SETTINGS.chat_token_limit ): execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :] user_prompt = Template( @@ -109,6 +112,7 @@ class FactorImplementationCodeEvaluator(Evaluator): return critic_response + class FactorImplementationSingleColumnEvaluator(FactorImplementationEvaluator): def evaluate( self, @@ -442,7 +446,7 @@ class FactorImplementationValueEvaluator(Evaluator): ) except Exception as e: - FinCoLog().warning(f"Error occurred when calculating the correlation: {str(e)}") + RDAgentLog().warning(f"Error occurred when calculating the correlation: {str(e)}") conclusions.append( f"Some error occurred when calculating the correlation. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent. Error: {e}", ) @@ -474,7 +478,9 @@ class FactorImplementationFinalDecisionEvaluator(Evaluator): code_feedback: str, **kwargs, ) -> Tuple: - system_prompt = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")["evaluator_final_decision_v1_system"] + system_prompt = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[ + "evaluator_final_decision_v1_system" + ] execution_feedback_to_render = execution_feedback user_prompt = Template( evaluate_prompts["evaluator_final_decision_v1_user"], @@ -494,7 +500,7 @@ class FactorImplementationFinalDecisionEvaluator(Evaluator): system_prompt=system_prompt, former_messages=[], ) - > FactorImplementSettings().chat_token_limit + > RD_AGENT_SETTINGS.chat_token_limit ): execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :] user_prompt = Template( @@ -522,6 +528,7 @@ class FactorImplementationFinalDecisionEvaluator(Evaluator): final_evaluation_dict["final_feedback"], ) + class FactorImplementationSingleFeedback: """This class is a feedback to single implementation which is generated from an evaluator.""" @@ -556,12 +563,14 @@ class FactorImplementationSingleFeedback: This implementation is {'SUCCESS' if self.final_decision else 'FAIL'}. """ + class FactorImplementationsMultiFeedback( Feedback, List[FactorImplementationSingleFeedback], ): """Feedback contains a list, each element is the corresponding feedback for each factor implementation.""" + class FactorImplementationEvaluatorV1(FactorImplementationEvaluator): """This class is the v1 version of evaluator for a single factor implementation. It calls several evaluators in share modules to evaluate the factor implementation. @@ -633,7 +642,7 @@ class FactorImplementationEvaluatorV1(FactorImplementationEvaluator): value_decision, ) = self.value_evaluator.evaluate(source_df=source_df, gt_df=gt_df) except Exception as e: - FinCoLog().warning("Value evaluation failed with exception: %s", e) + RDAgentLog().warning("Value evaluation failed with exception: %s", e) factor_feedback.factor_value_feedback = "Value evaluation failed." value_decision = False @@ -713,13 +722,12 @@ class FactorImplementationsMultiEvaluator(Evaluator): ), ), ) - multi_implementation_feedback = multiprocessing_wrapper(calls, n=FactorImplementSettings().evo_multi_proc_n) + multi_implementation_feedback = multiprocessing_wrapper(calls, n=FACTOR_IMPLEMENT_SETTINGS.evo_multi_proc_n) final_decision = [ None if single_feedback is None else single_feedback.final_decision for single_feedback in multi_implementation_feedback ] - print(f"Final decisions: {final_decision} True count: {final_decision.count(True)}") + RDAgentLog().info(f"Final decisions: {final_decision} True count: {final_decision.count(True)}") return multi_implementation_feedback - diff --git a/rdagent/factor_implementation/evolving/evolving_agent.py b/rdagent/factor_implementation/evolving/evolving_agent.py deleted file mode 100644 index a67d4e89..00000000 --- a/rdagent/factor_implementation/evolving/evolving_agent.py +++ /dev/null @@ -1,50 +0,0 @@ -from rdagent.core.evolving_framework import Feedback, EvolvableSubjects, Evaluator, EvoStep -from rdagent.core.evolving_framework import EvoAgent -from tqdm import tqdm - -class RAGEvoAgent(EvoAgent): - def __init__(self, max_loop, evolving_strategy, rag) -> None: - super().__init__(max_loop, evolving_strategy) - self.rag = rag - self.evolving_trace = [] - - def multistep_evolve( - self, - evo: EvolvableSubjects, - eva: Evaluator | Feedback, - *, - with_knowledge: bool = False, - with_feedback: bool = True, - knowledge_self_gen: bool = False) -> EvolvableSubjects: - - for _ in tqdm(range(self.max_loop), "Implementing factors"): - # 1. knowledge self-evolving - if knowledge_self_gen and self.rag is not None: - self.rag.generate_knowledge(self.evolving_trace) - # 2. 检索需要的Knowledge - queried_knowledge = None - if with_knowledge and self.rag is not None: - # TODO: 这里放了evolving_trace实际上没有作用 - queried_knowledge = self.rag.query(evo, self.evolving_trace) - - # 3. evolve - evo = self.evolving_strategy.evolve( - evo=evo, - evolving_trace=self.evolving_trace, - queried_knowledge=queried_knowledge, - ) - - # 4. 封装Evolve结果 - es = EvoStep(evo, queried_knowledge) - - # 5. 环境评测反馈 - if with_feedback: - es.feedback = eva if isinstance(eva, Feedback) else eva.evaluate(evo, queried_knowledge=queried_knowledge) - - # 6. 更新trace - self.evolving_trace.append(es) - for index, feedback in enumerate(es.feedback): - if feedback is not None: - evo.evolve_trace[evo.target_factor_tasks[index].factor_name][-1].feedback = feedback - - return evo \ No newline at end of file diff --git a/rdagent/factor_implementation/evolving/evolving_strategy.py b/rdagent/factor_implementation/evolving/evolving_strategy.py index 623bb71e..71dde026 100644 --- a/rdagent/factor_implementation/evolving/evolving_strategy.py +++ b/rdagent/factor_implementation/evolving/evolving_strategy.py @@ -8,10 +8,11 @@ from typing import TYPE_CHECKING from jinja2 import Template +from rdagent.core.conf import RD_AGENT_SETTINGS from rdagent.core.evolving_framework import EvolvingStrategy, QueriedKnowledge from rdagent.oai.llm_utils import APIBackend from rdagent.factor_implementation.share_modules.factor_implementation_config import ( - FactorImplementSettings, + FACTOR_IMPLEMENT_SETTINGS, ) from rdagent.core.task import ( @@ -81,18 +82,18 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy): # 2. 选择selection方法 # if the number of factors to be implemented is larger than the limit, we need to select some of them - if FactorImplementSettings().select_ratio < 1: + if FACTOR_IMPLEMENT_SETTINGS.select_ratio < 1: # if the number of loops is equal to the select_loop, we need to select some of them implementation_factors_per_round = int( - FactorImplementSettings().select_ratio * len(to_be_finished_task_index) + FACTOR_IMPLEMENT_SETTINGS.select_ratio * len(to_be_finished_task_index) ) - if FactorImplementSettings().select_method == "random": + if FACTOR_IMPLEMENT_SETTINGS.select_method == "random": to_be_finished_task_index = RandomSelect( to_be_finished_task_index, implementation_factors_per_round, ) - if FactorImplementSettings().select_method == "scheduler": + if FACTOR_IMPLEMENT_SETTINGS.select_method == "scheduler": to_be_finished_task_index = LLMSelect( to_be_finished_task_index, implementation_factors_per_round, @@ -105,17 +106,18 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy): (self.implement_one_factor, (new_evo.target_factor_tasks[target_index], queried_knowledge)) for target_index in to_be_finished_task_index ], - n=FactorImplementSettings().evo_multi_proc_n, + n=FACTOR_IMPLEMENT_SETTINGS.evo_multi_proc_n, ) for index, target_index in enumerate(to_be_finished_task_index): new_evo.corresponding_implementations[target_index] = result[index] - if result[index].target_task.factor_name in new_evo.evolve_trace: - new_evo.evolve_trace[result[index].target_task.factor_name].append(result[index]) - else: - new_evo.evolve_trace[result[index].target_task.factor_name] = [result[index]] - new_evo.corresponding_selection.append(to_be_finished_task_index) + # for target_index in to_be_finished_task_index: + # new_evo.corresponding_implementations[target_index] = self.implement_one_factor( + # new_evo.target_factor_tasks[target_index], queried_knowledge + # ) + + new_evo.corresponding_selection = to_be_finished_task_index return new_evo @@ -172,7 +174,7 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy): session.build_chat_completion_message_and_calculate_token( user_prompt, ) - < FactorImplementSettings().chat_token_limit + < RD_AGENT_SETTINGS.chat_token_limit ): break elif len(queried_former_failed_knowledge_to_render) > 1: @@ -205,7 +207,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy): target_task: FactorImplementTask, queried_knowledge, ) -> TaskImplementation: - error_summary = FactorImplementSettings().v2_error_summary + error_summary = FACTOR_IMPLEMENT_SETTINGS.v2_error_summary # 1. 提取因子的背景信息 target_factor_task_information = target_task.get_factor_information() @@ -284,7 +286,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy): ) if ( session_summary.build_chat_completion_message_and_calculate_token(error_summary_user_prompt) - < FactorImplementSettings().chat_token_limit + < RD_AGENT_SETTINGS.chat_token_limit ): break elif len(queried_similar_error_knowledge_to_render) > 0: @@ -311,7 +313,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy): session.build_chat_completion_message_and_calculate_token( user_prompt, ) - < FactorImplementSettings().chat_token_limit + < RD_AGENT_SETTINGS.chat_token_limit ): break elif len(queried_former_failed_knowledge_to_render) > 1: diff --git a/rdagent/factor_implementation/evolving/factor.py b/rdagent/factor_implementation/evolving/factor.py index 010a8dad..29887af3 100644 --- a/rdagent/factor_implementation/evolving/factor.py +++ b/rdagent/factor_implementation/evolving/factor.py @@ -1,6 +1,6 @@ from __future__ import annotations from rdagent.factor_implementation.share_modules.factor_implementation_config import ( - FactorImplementSettings, + FACTOR_IMPLEMENT_SETTINGS, ) from rdagent.core.task import ( @@ -9,7 +9,7 @@ from rdagent.core.task import ( TestCase, ) from rdagent.core.evolving_framework import EvolvableSubjects -from rdagent.core.log import FinCoLog +from rdagent.core.log import RDAgentLog from pathlib import Path @@ -35,7 +35,7 @@ class FactorImplementTask(BaseTask): factor_name, factor_description, factor_formulation, - factor_formulation_description: str = '', + factor_formulation_description: str = "", variables: dict = {}, resource: str = None, ) -> None: @@ -68,20 +68,17 @@ class FactorEvovlingItem(EvolvableSubjects): def __init__( self, target_factor_tasks: list[FactorImplementTask], - corresponding_gt: list[TestCase] = None, corresponding_gt_implementations: list[TaskImplementation] = None, ): super().__init__() self.target_factor_tasks = target_factor_tasks self.corresponding_implementations: list[TaskImplementation] = [None for _ in target_factor_tasks] - self.corresponding_selection: list[list] = [] - self.evolve_trace = {} - self.corresponding_gt = corresponding_gt + self.corresponding_selection: list = None if corresponding_gt_implementations is not None and len( corresponding_gt_implementations, ) != len(target_factor_tasks): self.corresponding_gt_implementations = None - FinCoLog.warning( + RDAgentLog().warning( "The length of corresponding_gt_implementations is not equal to the length of target_factor_tasks, set corresponding_gt_implementations to None", ) else: @@ -112,10 +109,10 @@ class FileBasedFactorImplementation(TaskImplementation): super().__init__(target_task) self.code = code self.executed_factor_value_dataframe = executed_factor_value_dataframe - self.logger = FinCoLog() + self.logger = RDAgentLog() self.raise_exception = raise_exception self.workspace_path = Path( - FactorImplementSettings().file_based_execution_workspace, + FACTOR_IMPLEMENT_SETTINGS.file_based_execution_workspace, ) / str(uuid.uuid4()) @staticmethod @@ -151,17 +148,14 @@ class FileBasedFactorImplementation(TaskImplementation): # TODO: to make the interface compatible with previous code. I kept the original behavior. raise ValueError(self.FB_CODE_NOT_SET) with FileLock(self.workspace_path / "execution.lock"): - (Path.cwd() / "git_ignore_folder" / "factor_implementation_execution_cache").mkdir( - exist_ok=True, parents=True - ) - if FactorImplementSettings().enable_execution_cache: + if FACTOR_IMPLEMENT_SETTINGS.enable_execution_cache: # NOTE: cache the result for the same code target_file_name = md5_hash(self.code) cache_file_path = ( - Path.cwd() - / "git_ignore_folder" - / "factor_implementation_execution_cache" - / f"{target_file_name}.pkl" + Path(FACTOR_IMPLEMENT_SETTINGS.implementation_execution_cache_location) / f"{target_file_name}.pkl" + ) + Path(FACTOR_IMPLEMENT_SETTINGS.implementation_execution_cache_location).mkdir( + exist_ok=True, parents=True ) if cache_file_path.exists() and not self.raise_exception: cached_res = pickle.load(open(cache_file_path, "rb")) @@ -173,7 +167,7 @@ class FileBasedFactorImplementation(TaskImplementation): return self.FB_FROM_CACHE, self.executed_factor_value_dataframe source_data_path = Path( - FactorImplementSettings().file_based_execution_data_folder, + FACTOR_IMPLEMENT_SETTINGS.file_based_execution_data_folder, ) self.workspace_path.mkdir(exist_ok=True, parents=True) @@ -189,7 +183,7 @@ class FileBasedFactorImplementation(TaskImplementation): shell=True, cwd=self.workspace_path, stderr=subprocess.STDOUT, - timeout=FactorImplementSettings().file_based_execution_timeout, + timeout=FACTOR_IMPLEMENT_SETTINGS.file_based_execution_timeout, ) except subprocess.CalledProcessError as e: import site @@ -206,7 +200,7 @@ class FileBasedFactorImplementation(TaskImplementation): if self.raise_exception: raise RuntimeErrorException(execution_feedback) except subprocess.TimeoutExpired: - execution_feedback += f"Execution timeout error and the timeout is set to {FactorImplementSettings().file_based_execution_timeout} seconds." + execution_feedback += f"Execution timeout error and the timeout is set to {FACTOR_IMPLEMENT_SETTINGS.file_based_execution_timeout} seconds." if self.raise_exception: raise RuntimeErrorException(execution_feedback) @@ -227,7 +221,7 @@ class FileBasedFactorImplementation(TaskImplementation): if store_result and executed_factor_value_dataframe is not None: self.executed_factor_value_dataframe = executed_factor_value_dataframe - if FactorImplementSettings().enable_execution_cache: + if FACTOR_IMPLEMENT_SETTINGS.enable_execution_cache: pickle.dump( (execution_feedback, executed_factor_value_dataframe), open(cache_file_path, "wb"), @@ -249,4 +243,3 @@ class FileBasedFactorImplementation(TaskImplementation): with factor_path.open("r") as f: code = f.read() return FileBasedFactorImplementation(task, code=code, **kwargs) - diff --git a/rdagent/factor_implementation/evolving/knowledge_management.py b/rdagent/factor_implementation/evolving/knowledge_management.py index 62c7529e..d8437d4f 100644 --- a/rdagent/factor_implementation/evolving/knowledge_management.py +++ b/rdagent/factor_implementation/evolving/knowledge_management.py @@ -18,7 +18,7 @@ from rdagent.core.evolving_framework import ( QueriedKnowledge, RAGStrategy, ) -from rdagent.core.log import FinCoLog +from rdagent.core.log import RDAgentLog from rdagent.core.prompts import Prompts from rdagent.factor_implementation.evolving.evaluators import FactorImplementationSingleFeedback from rdagent.core.task import ( @@ -30,9 +30,10 @@ from rdagent.knowledge_management.graph import UndirectedGraph, UndirectedNode from rdagent.oai.llm_utils import APIBackend, calculate_embedding_distance_between_str_list from rdagent.factor_implementation.share_modules.factor_implementation_config import ( - FactorImplementSettings, + FACTOR_IMPLEMENT_SETTINGS, ) + class FactorImplementationKnowledge(Knowledge): def __init__( self, @@ -138,9 +139,9 @@ class FactorImplementationRAGStrategyV1(RAGStrategy): evo: EvolvableSubjects, evolving_trace: list[EvoStep], ) -> QueriedKnowledge | None: - v1_query_former_trace_limit = FactorImplementSettings().v1_query_former_trace_limit - v1_query_similar_success_limit = FactorImplementSettings().v1_query_similar_success_limit - fail_task_trial_limit = FactorImplementSettings().fail_task_trial_limit + v1_query_former_trace_limit = FACTOR_IMPLEMENT_SETTINGS.v1_query_former_trace_limit + v1_query_similar_success_limit = FACTOR_IMPLEMENT_SETTINGS.v1_query_similar_success_limit + fail_task_trial_limit = FACTOR_IMPLEMENT_SETTINGS.fail_task_trial_limit queried_knowledge = FactorImplementationQueriedKnowledgeV1() for target_factor_task in evo.target_factor_tasks: @@ -280,7 +281,7 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy): return None def query(self, evo: EvolvableSubjects, evolving_trace: list[EvoStep]) -> QueriedKnowledge | None: - conf_knowledge_sampler = FactorImplementSettings().v2_knowledge_sampler + conf_knowledge_sampler = FACTOR_IMPLEMENT_SETTINGS.v2_knowledge_sampler factor_implementation_queried_graph_knowledge = FactorImplementationQueriedGraphKnowledge( success_task_to_knowledge_dict=self.knowledgebase.success_task_to_knowledge_dict, ) @@ -288,18 +289,18 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy): factor_implementation_queried_graph_knowledge = self.former_trace_query( evo, factor_implementation_queried_graph_knowledge, - FactorImplementSettings().v2_query_former_trace_limit, + FACTOR_IMPLEMENT_SETTINGS.v2_query_former_trace_limit, ) factor_implementation_queried_graph_knowledge = self.component_query( evo, factor_implementation_queried_graph_knowledge, - FactorImplementSettings().v2_query_component_limit, + FACTOR_IMPLEMENT_SETTINGS.v2_query_component_limit, knowledge_sampler=conf_knowledge_sampler, ) factor_implementation_queried_graph_knowledge = self.error_query( evo, factor_implementation_queried_graph_knowledge, - FactorImplementSettings().v2_query_error_limit, + FACTOR_IMPLEMENT_SETTINGS.v2_query_error_limit, knowledge_sampler=conf_knowledge_sampler, ) return factor_implementation_queried_graph_knowledge @@ -327,7 +328,7 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy): )["component_no_list"] return [all_component_nodes[index - 1] for index in sorted(list(set(component_no_list)))] except: - FinCoLog.warning("Error when analyzing components.") + RDAgentLog().warning("Error when analyzing components.") analyze_component_user_prompt = "Your response is not a valid component index list." return [] @@ -383,7 +384,7 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy): """ Query the former trace knowledge of the working trace, and find all the failed task information which tried more than fail_task_trial_limit times """ - fail_task_trial_limit = FactorImplementSettings().fail_task_trial_limit + fail_task_trial_limit = FACTOR_IMPLEMENT_SETTINGS.fail_task_trial_limit for target_factor_task in evo.target_factor_tasks: target_factor_task_information = target_factor_task.get_factor_information() @@ -705,7 +706,7 @@ class FactorImplementationGraphKnowledgeBase(KnowledgeBase): Load knowledge, offer brief information of knowledge and common handle interfaces """ self.graph: UndirectedGraph = UndirectedGraph.load(Path.cwd() / "graph.pkl") - FinCoLog().info(f"Knowledge Graph loaded, size={self.graph.size()}") + RDAgentLog().info(f"Knowledge Graph loaded, size={self.graph.size()}") if init_component_list: for component in init_component_list: @@ -901,4 +902,3 @@ class FactorImplementationGraphKnowledgeBase(KnowledgeBase): intersection_node_list_sort_by_freq.append(node) return intersection_node_list_sort_by_freq - diff --git a/rdagent/factor_implementation/evolving/scheduler.py b/rdagent/factor_implementation/evolving/scheduler.py index 86c5a942..5c3263f6 100644 --- a/rdagent/factor_implementation/evolving/scheduler.py +++ b/rdagent/factor_implementation/evolving/scheduler.py @@ -1,27 +1,32 @@ from rdagent.oai.llm_utils import APIBackend from jinja2 import Template -from rdagent.factor_implementation.share_modules.factor_implementation_config import FactorImplementSettings import json from rdagent.factor_implementation.share_modules.factor_implementation_utils import get_data_folder_intro from rdagent.factor_implementation.evolving.factor import FactorEvovlingItem from rdagent.core.prompts import Prompts +from rdagent.core.log import RDAgentLog +from rdagent.core.conf import RD_AGENT_SETTINGS from pathlib import Path scheduler_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml") + def RandomSelect(to_be_finished_task_index, implementation_factors_per_round): import random + to_be_finished_task_index = random.sample( to_be_finished_task_index, implementation_factors_per_round, ) - print("The random selection is:",to_be_finished_task_index) + + RDAgentLog().info(f"The random selection is: {to_be_finished_task_index}") return to_be_finished_task_index -def LLMSelect(to_be_finished_task_index, implementation_factors_per_round, evo:FactorEvovlingItem, former_trace): + +def LLMSelect(to_be_finished_task_index, implementation_factors_per_round, evo: FactorEvovlingItem, former_trace): tasks = [] for i in to_be_finished_task_index: - # find corresponding former trace for each task + # find corresponding former trace for each task target_factor_task_information = evo.target_factor_tasks[i].get_factor_information() if target_factor_task_information in former_trace: tasks.append((i, evo.target_factor_tasks[i], former_trace[target_factor_task_information])) @@ -37,20 +42,17 @@ def LLMSelect(to_be_finished_task_index, implementation_factors_per_round, evo:F ) while True: - user_prompt = ( - Template( - scheduler_prompts["select_implementable_factor_user"], - ) - .render( - factor_num = implementation_factors_per_round, - target_factor_tasks=tasks, - ) + user_prompt = Template( + scheduler_prompts["select_implementable_factor_user"], + ).render( + factor_num=implementation_factors_per_round, + target_factor_tasks=tasks, ) if ( session.build_chat_completion_message_and_calculate_token( user_prompt, ) - < FactorImplementSettings().chat_token_limit + < RD_AGENT_SETTINGS.chat_token_limit ): break @@ -65,5 +67,5 @@ def LLMSelect(to_be_finished_task_index, implementation_factors_per_round, evo:F selection_index = [x for x in selection if isinstance(x, int)] except: return to_be_finished_task_index - + return selection_index diff --git a/rdagent/factor_implementation/prompts.yaml b/rdagent/factor_implementation/prompts.yaml index a3cef645..d843a107 100644 --- a/rdagent/factor_implementation/prompts.yaml +++ b/rdagent/factor_implementation/prompts.yaml @@ -217,3 +217,13 @@ select_implementable_factor_user: |- {% endfor %} {% endif %} {% endfor %} + +analyze_component_prompt_v1_system: |- + User is getting a new task that might consist of the components below (given in component_index: component_description): + {{all_component_content}} + + You should find out what components does the new task have, and put their indices in a list. + Please response the critic in the json format. Here is an example structure for the JSON output, please strictly follow the format: + { + "component_no_list": the list containing indices of components. + } \ No newline at end of file diff --git a/rdagent/factor_implementation/share_modules/factor_implementation_config.py b/rdagent/factor_implementation/share_modules/factor_implementation_config.py index e4f46ff8..515e1193 100644 --- a/rdagent/factor_implementation/share_modules/factor_implementation_config.py +++ b/rdagent/factor_implementation/share_modules/factor_implementation_config.py @@ -9,13 +9,13 @@ SELECT_METHOD = Literal["random", "scheduler"] class FactorImplementSettings(BaseSettings): file_based_execution_data_folder: str = str( - (Path().cwd() / "git_ignore_folder" / "factor_implementation_source_data").absolute(), + (Path().cwd() / "factor_implementation_source_data").absolute(), ) file_based_execution_workspace: str = str( - (Path().cwd() / "git_ignore_folder" / "factor_implementation_workspace").absolute(), + (Path().cwd() / "factor_implementation_workspace").absolute(), ) implementation_execution_cache_location: str = str( - (Path().cwd() / "git_ignore_folder" / "factor_implementation_execution_cache.pkl").absolute(), + (Path().cwd() / "factor_implementation_execution_cache").absolute(), ) enable_execution_cache: bool = True # whether to enable the execution cache @@ -45,9 +45,5 @@ class FactorImplementSettings(BaseSettings): knowledge_base_path: Union[str, None] = None new_knowledge_base_path: Union[str, None] = None - chat_token_limit: int = ( - 100000 # 100000 is the maximum limit of gpt4, which might increase in the future version of gpt - ) - -FIS = FactorImplementSettings() +FACTOR_IMPLEMENT_SETTINGS = FactorImplementSettings() diff --git a/rdagent/factor_implementation/share_modules/factor_implementation_utils.py b/rdagent/factor_implementation/share_modules/factor_implementation_utils.py index a0b1f2e6..8056cad7 100644 --- a/rdagent/factor_implementation/share_modules/factor_implementation_utils.py +++ b/rdagent/factor_implementation/share_modules/factor_implementation_utils.py @@ -4,7 +4,7 @@ import pandas as pd # render it with jinja from jinja2 import Template -from rdagent.factor_implementation.share_modules.factor_implementation_config import FIS +from rdagent.factor_implementation.share_modules.factor_implementation_config import FACTOR_IMPLEMENT_SETTINGS from rdagent.factor_implementation.evolving.factor import FactorImplementTask @@ -17,12 +17,13 @@ TPL = """ # Create a Jinja template from the string JJ_TPL = Template(TPL) + def get_data_folder_intro(): """Direclty get the info of the data folder. It is for preparing prompting message. """ content_l = [] - for p in Path(FIS.file_based_execution_data_folder).iterdir(): + for p in Path(FACTOR_IMPLEMENT_SETTINGS.file_based_execution_data_folder).iterdir(): if p.name.endswith(".h5"): df = pd.read_hdf(p) # get df.head() as string with full width @@ -49,17 +50,3 @@ def get_data_folder_intro(): f"file type {p.name} is not supported. Please implement its description function.", ) return "\n ----------------- file spliter -------------\n".join(content_l) - -def load_data_from_dict(factor_dict:dict) -> list[FactorImplementTask]: - """Load data from a dict. - """ - task_l = [] - for factor_name, factor_data in factor_dict.items(): - task = FactorImplementTask( - factor_name=factor_name, - factor_description=factor_data["description"], - factor_formulation=factor_data["formulation"], - variables=factor_data["variables"], - ) - task_l.append(task) - return task_l \ No newline at end of file diff --git a/rdagent/factor_implementation/task_loader/json_loader.py b/rdagent/factor_implementation/task_loader/json_loader.py new file mode 100644 index 00000000..722b2ecc --- /dev/null +++ b/rdagent/factor_implementation/task_loader/json_loader.py @@ -0,0 +1,31 @@ +import json +from pathlib import Path +from rdagent.core.task import TaskLoader +from rdagent.factor_implementation.evolving.factor import FactorImplementTask + + +class FactorImplementationTaskLoaderFromDict(TaskLoader): + def load(self, factor_dict: dict) -> list: + """Load data from a dict.""" + task_l = [] + for factor_name, factor_data in factor_dict.items(): + task = FactorImplementTask( + factor_name=factor_name, + factor_description=factor_data["description"], + factor_formulation=factor_data["formulation"], + variables=factor_data["variables"], + ) + task_l.append(task) + return task_l + + +class FactorImplementationTaskLoaderFromJsonFile(TaskLoader): + def load(self, json_file_path: Path) -> list: + factor_dict = json.load(json_file_path) + return FactorImplementationTaskLoaderFromDict().load(factor_dict) + + +class FactorImplementationTaskLoaderFromJsonString(TaskLoader): + def load(self, json_string: str) -> list: + factor_dict = json.loads(json_string) + return FactorImplementationTaskLoaderFromDict().load(factor_dict) diff --git a/rdagent/document_process/document_analysis.py b/rdagent/factor_implementation/task_loader/pdf_loader.py similarity index 82% rename from rdagent/document_process/document_analysis.py rename to rdagent/factor_implementation/task_loader/pdf_loader.py index d033536f..aa964f8c 100644 --- a/rdagent/document_process/document_analysis.py +++ b/rdagent/factor_implementation/task_loader/pdf_loader.py @@ -10,9 +10,12 @@ import numpy as np import pandas as pd import tiktoken from jinja2 import Template -from rdagent.core.conf import RDAgentSettings as Config -from rdagent.core.log import FinCoLog +from rdagent.core.conf import RD_AGENT_SETTINGS +from rdagent.core.log import RDAgentLog from rdagent.core.prompts import Prompts +from rdagent.core.task import TaskLoader +from rdagent.document_reader.document_reader import load_and_process_pdfs_by_langchain +from rdagent.factor_implementation.task_loader.json_loader import FactorImplementationTaskLoaderFromDict from rdagent.oai.llm_utils import APIBackend, create_embedding_with_multiprocessing from sklearn.cluster import KMeans from sklearn.metrics.pairwise import cosine_similarity @@ -23,7 +26,6 @@ document_process_prompts = Prompts(file_path=Path(__file__).parent / "prompts.ya def classify_report_from_dict( report_dict: Mapping[str, str], - input_max_token: int = 128000, vote_time: int = 1, substrings: tuple[str] = (), ) -> dict[str, dict[str, str]]: @@ -41,21 +43,19 @@ def classify_report_from_dict( with a single key 'class' and its value being the classification result (0 or 1). """ - if len(substrings) == 0: - substrings = ( - "FinCo", - "金融工程", - "金工", - "回测", - "因子", - "机器学习", - "深度学习", - "量化", - ) + # if len(substrings) == 0: + # substrings = ( + # "金融工程", + # "金工", + # "回测", + # "因子", + # "机器学习", + # "深度学习", + # "量化", + # ) res_dict = {} classify_prompt = document_process_prompts["classify_system"] - enc = tiktoken.encoding_for_model("gpt-4-turbo") for key, value in report_dict.items(): if not key.endswith(".pdf"): @@ -65,44 +65,47 @@ def classify_report_from_dict( if isinstance(value, str): content = value else: - FinCoLog().warning(f"输入格式不符合要求: {file_name}") + RDAgentLog().warning(f"输入格式不符合要求: {file_name}") res_dict[file_name] = {"class": 0} continue - if not any(substring in content for substring in substrings) and False: - res_dict[file_name] = {"class": 0} - else: - while ( - APIBackend().build_messages_and_calculate_token( - user_prompt=content, - system_prompt=classify_prompt, - ) - > Config().chat_token_limit - ): - content = content[: -(Config().chat_token_limit // 100)] + # pre-filter document with key words is not necessary, skip this check for now + # if ( + # not any(substring in content for substring in substrings) and False + # ): + # res_dict[file_name] = {"class": 0} + # else: + while ( + APIBackend().build_messages_and_calculate_token( + user_prompt=content, + system_prompt=classify_prompt, + ) + > RD_AGENT_SETTINGS.chat_token_limit + ): + content = content[: -(RD_AGENT_SETTINGS.chat_token_limit // 100)] - vote_list = [] - for _ in range(vote_time): - user_prompt = content - system_prompt = classify_prompt - res = APIBackend().build_messages_and_create_chat_completion( - user_prompt=user_prompt, - system_prompt=system_prompt, - json_mode=True, - ) - try: - res = json.loads(res) - vote_list.append(int(res["class"])) - except json.JSONDecodeError: - FinCoLog().warning(f"返回值无法解析: {file_name}") - res_dict[file_name] = {"class": 0} - count_0 = vote_list.count(0) - count_1 = vote_list.count(1) - if max(count_0, count_1) > int(vote_time / 2): - break + vote_list = [] + for _ in range(vote_time): + user_prompt = content + system_prompt = classify_prompt + res = APIBackend().build_messages_and_create_chat_completion( + user_prompt=user_prompt, + system_prompt=system_prompt, + json_mode=True, + ) + try: + res = json.loads(res) + vote_list.append(int(res["class"])) + except json.JSONDecodeError: + RDAgentLog().warning(f"返回值无法解析: {file_name}") + res_dict[file_name] = {"class": 0} + count_0 = vote_list.count(0) + count_1 = vote_list.count(1) + if max(count_0, count_1) > int(vote_time / 2): + break - result = 1 if count_1 > count_0 else 0 - res_dict[file_name] = {"class": result} + result = 1 if count_1 > count_0 else 0 + res_dict[file_name] = {"class": result} return res_dict @@ -235,7 +238,7 @@ def extract_factors_from_report_dict( if int(value.get("class")) == 1: useful_report_dict[key] = report_dict[key] else: - FinCoLog().warning(f"输入格式不符合要求: {key}") + RDAgentLog().warning(f"Invalid input format: {key}") final_report_factor_dict = {} # for file_name, content in useful_report_dict.items(): @@ -265,7 +268,7 @@ def extract_factors_from_report_dict( file_name = file_names[index] final_report_factor_dict.setdefault(file_name, {}) final_report_factor_dict[file_name] = result.get() - FinCoLog().info(f"已经完成{len(final_report_factor_dict)}个报告的因子提取") + RDAgentLog().info(f"已经完成{len(final_report_factor_dict)}个报告的因子提取") return final_report_factor_dict @@ -409,7 +412,7 @@ def __kmeans_embeddings(embeddings: np.ndarray, k: int = 20) -> list[list[str]]: random_state=42, ) - # KMeans算法使用欧氏距离, 我们需要自定义一个函数来找到最相似的簇中心 + # KMeans algorithm uses Euclidean distance, and we need to customize a function to find the most similar cluster center def find_closest_cluster_cosine_similarity( data: np.ndarray, centroids: np.ndarray, @@ -417,25 +420,25 @@ def __kmeans_embeddings(embeddings: np.ndarray, k: int = 20) -> list[list[str]]: similarity = cosine_similarity(data, centroids) return np.argmax(similarity, axis=1) - # 初始化簇中心 + # Initializes the cluster center rng = np.random.default_rng() centroids = rng.choice(x_normalized, size=k, replace=False) - # 迭代直到收敛或达到最大迭代次数 + # Iterate until convergence or the maximum number of iterations is reached for _ in range(kmeans.max_iter): - # 分配样本到最近的簇中心 + # Assign the sample to the nearest cluster center closest_clusters = find_closest_cluster_cosine_similarity( x_normalized, centroids, ) - # 更新簇中心 + # update the cluster center new_centroids = np.array( [x_normalized[closest_clusters == i].mean(axis=0) for i in range(k)], ) new_centroids = normalize(new_centroids) # 归一化新的簇中心 - # 检查簇中心是否发生变化 + # Check whether the cluster center has changed if np.allclose(centroids, new_centroids): break @@ -477,18 +480,18 @@ Factor variables: {variables} embeddings = create_embedding_with_multiprocessing(full_str_list) target_k = None - if len(full_str_list) < Config().max_input_duplicate_factor_group: + if len(full_str_list) < RD_AGENT_SETTINGS.max_input_duplicate_factor_group: kmeans_index_group = [list(range(len(full_str_list)))] target_k = 1 else: for k in range( - len(full_str_list) // Config().max_input_duplicate_factor_group, + len(full_str_list) // RD_AGENT_SETTINGS.max_input_duplicate_factor_group, 30, ): kmeans_index_group = __kmeans_embeddings(embeddings=embeddings, k=k) - if len(kmeans_index_group[0]) < Config().max_input_duplicate_factor_group: + if len(kmeans_index_group[0]) < RD_AGENT_SETTINGS.max_input_duplicate_factor_group: target_k = k - FinCoLog().info(f"K-means group number: {k}") + RDAgentLog().info(f"K-means group number: {k}") break factor_name_groups = [[factor_names[index] for index in index_group] for index_group in kmeans_index_group] @@ -519,7 +522,7 @@ Factor variables: {variables} return duplication_names_list -def deduplicate_factors_by_llm( # noqa: C901, PLR0912 +def deduplicate_factors_by_llm( # noqa: C901, PLR0912 factor_dict: dict[str, dict[str, str]], factor_viability_dict: dict[str, dict[str, str]] | None = None, ) -> list[list[str]]: @@ -530,7 +533,7 @@ def deduplicate_factors_by_llm( # noqa: C901, PLR0912 new_round_names = [] for duplication_names in duplication_names_list: - if len(duplication_names) < Config().max_output_duplicate_factor_group: + if len(duplication_names) < RD_AGENT_SETTINGS.max_output_duplicate_factor_group: final_duplication_names_list.append(duplication_names) else: new_round_names.extend(duplication_names) @@ -566,3 +569,16 @@ def deduplicate_factors_by_llm( # noqa: C901, PLR0912 llm_deduplicated_factor_dict[factor_name] = factor_dict[factor_name] return llm_deduplicated_factor_dict, final_duplication_names_list + + +class FactorImplementationTaskLoaderFromPDFfiles(TaskLoader): + def load(self, file_or_folder_path: Path) -> dict: + docs_dict = load_and_process_pdfs_by_langchain(Path(file_or_folder_path)) + + selected_report_dict = classify_report_from_dict(report_dict=docs_dict, vote_time=1) + file_to_factor_result = extract_factors_from_report_dict(docs_dict, selected_report_dict) + factor_dict = merge_file_to_factor_dict_to_factor_dict(file_to_factor_result) + + factor_viability = check_factor_viability(factor_dict) + factor_dict, duplication_names_list = deduplicate_factors_by_llm(factor_dict, factor_viability) + return FactorImplementationTaskLoaderFromDict().load(factor_dict) diff --git a/rdagent/document_process/prompts.yaml b/rdagent/factor_implementation/task_loader/prompts.yaml similarity index 78% rename from rdagent/document_process/prompts.yaml rename to rdagent/factor_implementation/task_loader/prompts.yaml index 72ee3721..682898dc 100644 --- a/rdagent/document_process/prompts.yaml +++ b/rdagent/factor_implementation/task_loader/prompts.yaml @@ -35,40 +35,45 @@ extract_factors_follow_user: |- ``` extract_factor_formulation_system: |- - 用户会提供一篇金融工程研报，和用户从中提取到的因子列表，请结合文章和用户提供的因子名称和因子描述，按照要求抽取： - 1. 因子的计算公式，使用latex格式，公式中的变量名称不能包含空格，可用下划线连接，公式中的因子名称与用户提供的因子名称保持一致; - 2. 因子公式中的变量和函数解释，请使用英文描述，变量名和函数名请与公式中的名称对齐 + I have a financial engineering research report and a list of factors extracted from it. I need assistance in extracting specific information based on the report and the provided list of factors. The tasks are as follows: - User has several source data: - 1. The Stock Trade Data Table containing information about stock trades, such as daily open, close, high, low, vwap prices, volume, and turnover; - 2. The Financial Data Table containing company financial statements such as the balance sheet, income statement, and cash flow statement; - 3. The Stock Fundamental Data Table containing basic information about stocks, like total shares outstanding, free float shares, industry classification, market classification, etc; - 4. The high frequency data containing price and volume of each stock containing open close high low volume vwap in each minute. - Please try to expand the formulation to using the source data provided by user. + 1. For each factor, I need its calculation formula in LaTeX format. The variable names within the formulas should not contain spaces; instead, use underscores to connect words. Ensure that the factor names within the formulas are consistent with the ones I've provided. + 2. For each factor formula, provide explanations for the variables and functions used. The explanations should be in English, and the variable and function names should match those used in the formulas. + + Here are the sources of data I have: + + 1. Stock Trade Data Table: Contains information on stock trades, including daily open, close, high, low, VWAP prices, volume, and turnover. + 2. Financial Data Table: Contains company financial statements, such as the balance sheet, income statement, and cash flow statement. + 3. Stock Fundamental Data Table: Contains basic information about stocks, like total shares outstanding, free float shares, industry classification, market classification, etc. + 4. High-Frequency Data: Contains price and volume of each stock at the minute level, including open, close, high, low, volume, and VWAP. + + Please expand the formulation to use the source data I have provided. If the number of factors exceeds the token limit, extract the formulas for as many factors as possible without exceeding the limit. Ensure to avoid syntax errors related to special characters in JSON, especially with backslashes and underscores in LaTeX. + + Provide your analysis in JSON format, using the following schema: - user will treat your factor name as key to store the factor, don't put any interaction message in the content. Just response the output without any interaction and explanation. - You can extract part of the user's input factors if token is not enough. To avoid the situation that you don't respond in the valid format, don't extract more than thirty factors in one response. - Be caution of the "\" in your formulation because In JSON, certain characters like the backslash need to be escaped with another backslash. Especially, _ and \_ are different in latex so use \_ to represent _ in latex. - Respond with your analysis in JSON format. The JSON schema should include: ```json { - "name of factor 1": { + "factor name 1": { "formulation": "latex formulation of factor 1", "variables": { - "Name to variable or function 1": "Description to variable or function 1", - "Name to variable or function 2": "Description to variable or function 2" + "variable or function name 1": "description of variable or function 1", + "variable or function name 2": "description of variable or function 2" } }, - "name of factor 2": { + "factor name 2": { "formulation": "latex formulation of factor 2", "variables": { - "Name to variable or function 1": "Description to variable or function 1", - "Name to variable or function 2": "Description to variable or function 2" + "variable or function name 1": "description of variable or function 1", + "variable or function name 2": "description of variable or function 2" } } + # Don't add ellipsis (...) or any filler text that might cause JSON parsing errors here! } ``` + Please only include the factors you've extracted in your response, without any additional messages or explanations. If you can only provide part of the factors, do not use ellipsis or any filler text that might cause JSON parsing errors. + + extract_factor_formulation_user: |- ===========================Report content:============================= {{ report_content }} diff --git a/rdagent/knowledge_management/knowledgebase.py b/rdagent/knowledge_management/knowledgebase.py index 61f75032..cffdb8da 100644 --- a/rdagent/knowledge_management/knowledgebase.py +++ b/rdagent/knowledge_management/knowledgebase.py @@ -15,12 +15,15 @@ from rdagent.core.evolving_framework import ( KnowledgeBase, ) from rdagent.core.evolving_framework import EvoStep, EvolvableSubjects, RAGStrategy, Knowledge, QueriedKnowledge -from rdagent.factor_implementation.evolving.knowledge_management import FactorImplementationKnowledge, FactorImplementationQueriedGraphKnowledge -from rdagent.factor_implementation.share_modules.factor_implementation_config import FactorImplementSettings +from rdagent.factor_implementation.evolving.knowledge_management import ( + FactorImplementationKnowledge, + FactorImplementationQueriedGraphKnowledge, +) +from rdagent.factor_implementation.share_modules.factor_implementation_config import FACTOR_IMPLEMENT_SETTINGS from rdagent.knowledge_management.graph import UndirectedGraph, UndirectedNode from rdagent.core.prompts import Prompts -from rdagent.core.log import FinCoLog +from rdagent.core.log import RDAgentLog from rdagent.oai.llm_utils import APIBackend, calculate_embedding_distance_between_str_list @@ -30,7 +33,7 @@ class FactorImplementationGraphKnowledgeBase(KnowledgeBase): Load knowledge, offer brief information of knowledge and common handle interfaces """ self.graph: UndirectedGraph = UndirectedGraph.load(Path.cwd() / "graph.pkl") - FinCoLog().info(f"Knowledge Graph loaded, size={self.graph.size()}") + RDAgentLog().info(f"Knowledge Graph loaded, size={self.graph.size()}") if init_component_list: for component in init_component_list: @@ -228,7 +231,6 @@ class FactorImplementationGraphKnowledgeBase(KnowledgeBase): return intersection_node_list_sort_by_freq - class FactorImplementationGraphRAGStrategy(RAGStrategy): def __init__(self, knowledgebase: FactorImplementationGraphKnowledgeBase) -> None: super().__init__(knowledgebase) @@ -302,26 +304,26 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy): return None def query(self, evo: EvolvableSubjects, evolving_trace: list[EvoStep]) -> QueriedKnowledge | None: - conf_knowledge_sampler = FactorImplementSettings().v2_knowledge_sampler + conf_knowledge_sampler = FACTOR_IMPLEMENT_SETTINGS.v2_knowledge_sampler factor_implementation_queried_graph_knowledge = FactorImplementationQueriedGraphKnowledge( success_task_to_knowledge_dict=self.knowledgebase.success_task_to_knowledge_dict, ) - + factor_implementation_queried_graph_knowledge = self.former_trace_query( evo, factor_implementation_queried_graph_knowledge, - FactorImplementSettings().v2_query_former_trace_limit, + FACTOR_IMPLEMENT_SETTINGS.v2_query_former_trace_limit, ) factor_implementation_queried_graph_knowledge = self.component_query( evo, factor_implementation_queried_graph_knowledge, - FactorImplementSettings().v2_query_component_limit, + FACTOR_IMPLEMENT_SETTINGS.v2_query_component_limit, knowledge_sampler=conf_knowledge_sampler, ) factor_implementation_queried_graph_knowledge = self.error_query( evo, factor_implementation_queried_graph_knowledge, - FactorImplementSettings().v2_query_error_limit, + FACTOR_IMPLEMENT_SETTINGS.v2_query_error_limit, knowledge_sampler=conf_knowledge_sampler, ) return factor_implementation_queried_graph_knowledge @@ -349,7 +351,7 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy): )["component_no_list"] return [all_component_nodes[index - 1] for index in sorted(list(set(component_no_list)))] except: - FinCoLog.warning("Error when analyzing components.") + RDAgentLog().warning("Error when analyzing components.") analyze_component_user_prompt = "Your response is not a valid component index list." return [] @@ -405,7 +407,7 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy): """ Query the former trace knowledge of the working trace, and find all the failed task information which tried more than fail_task_trial_limit times """ - fail_task_trial_limit = FactorImplementSettings().fail_task_trial_limit + fail_task_trial_limit = FACTOR_IMPLEMENT_SETTINGS.fail_task_trial_limit for target_factor_task in evo.target_factor_tasks: target_factor_task_information = target_factor_task.get_factor_information() @@ -438,9 +440,9 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy): else: current_index += 1 - factor_implementation_queried_graph_knowledge.former_traces[ - target_factor_task_information - ] = former_trace_knowledge[-v2_query_former_trace_limit:] + factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = ( + former_trace_knowledge[-v2_query_former_trace_limit:] + ) else: factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = [] @@ -719,4 +721,3 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy): ] = same_error_success_knowledge_pair_list return factor_implementation_queried_graph_knowledge - diff --git a/rdagent/knowledge_management/prompts.yaml b/rdagent/knowledge_management/prompts.yaml deleted file mode 100644 index f5d73d67..00000000 --- a/rdagent/knowledge_management/prompts.yaml +++ /dev/null @@ -1,9 +0,0 @@ -analyze_component_prompt_v1_system: |- - User is getting a new task that might consist of the components below (given in component_index: component_description): - {{all_component_content}} - - You should find out what components does the new task have, and put their indices in a list. - Please response the critic in the json format. Here is an example structure for the JSON output, please strictly follow the format: - { - "component_no_list": the list containing indices of components. - } \ No newline at end of file diff --git a/rdagent/knowledge_management/vector_base.py b/rdagent/knowledge_management/vector_base.py index dcbf2167..b377a740 100644 --- a/rdagent/knowledge_management/vector_base.py +++ b/rdagent/knowledge_management/vector_base.py @@ -6,7 +6,7 @@ import pandas as pd from scipy.spatial.distance import cosine from rdagent.oai.llm_utils import APIBackend -from rdagent.core.log import FinCoLog +from rdagent.core.log import RDAgentLog class KnowledgeMetaData: @@ -127,7 +127,7 @@ class PDVectorBase(VectorBase): else: self.vector_df = pd.DataFrame(columns=["id", "label", "content", "embedding"]) - FinCoLog().info(f"VectorBase loaded, shape={self.vector_df.shape}") + RDAgentLog().info(f"VectorBase loaded, shape={self.vector_df.shape}") def shape(self): return self.vector_df.shape @@ -205,4 +205,4 @@ class PDVectorBase(VectorBase): def save(self, vector_df_path, **kwargs): self.vector_df.to_pickle(vector_df_path) - FinCoLog().info(f"Save vectorBase vector_df to: {vector_df_path}") + RDAgentLog().info(f"Save vectorBase vector_df to: {vector_df_path}") diff --git a/rdagent/oai/llm_utils.py b/rdagent/oai/llm_utils.py index fab029d9..2d1c8c01 100644 --- a/rdagent/oai/llm_utils.py +++ b/rdagent/oai/llm_utils.py @@ -18,8 +18,8 @@ from typing import Any import numpy as np import tiktoken -from rdagent.core.conf import RDAgentSettings as Config -from rdagent.core.log import FinCoLog, LogColors +from rdagent.core.conf import RD_AGENT_SETTINGS +from rdagent.core.log import RDAgentLog, LogColors from rdagent.core.utils import SingletonBaseClass DEFAULT_QLIB_DOT_PATH = Path("./") @@ -35,17 +35,17 @@ def md5_hash(input_string: str) -> str: try: from azure.identity import DefaultAzureCredential, get_bearer_token_provider except ImportError: - FinCoLog().warning("azure.identity is not installed.") + RDAgentLog().warning("azure.identity is not installed.") try: import openai except ImportError: - FinCoLog().warning("openai is not installed.") + RDAgentLog().warning("openai is not installed.") try: from llama import Llama except ImportError: - FinCoLog().warning("llama is not installed.") + RDAgentLog().warning("llama is not installed.") class ConvManager: @@ -133,7 +133,6 @@ class SQliteLazyCache(SingletonBaseClass): ) self.conn.commit() - def embedding_set(self, content_to_embedding_dict: dict) -> None: for key, value in content_to_embedding_dict.items(): md5_key = md5_hash(key) @@ -147,15 +146,15 @@ class SQliteLazyCache(SingletonBaseClass): class SessionChatHistoryCache(SingletonBaseClass): def __init__(self) -> None: """load all history conversation json file from self.session_cache_location""" - self.cfg = Config() + self.cfg = RD_AGENT_SETTINGS self.session_cache_location = Path(self.cfg.session_cache_folder_location) self.cache = {} if not self.session_cache_location.exists(): - FinCoLog.warning(f"Directory {self.session_cache_location} does not exist.") + RDAgentLog().warning(f"Directory {self.session_cache_location} does not exist.") self.session_cache_location.mkdir(parents=True, exist_ok=True) json_files = [f for f in self.session_cache_location.iterdir() if f.suffix == ".json"] if not json_files: - FinCoLog.info(f"No JSON files found in {self.session_cache_location}.") + RDAgentLog().info(f"No JSON files found in {self.session_cache_location}.") for file_path in json_files: conversation_id = file_path.stem with file_path.open("r") as f: @@ -177,7 +176,7 @@ class SessionChatHistoryCache(SingletonBaseClass): class ChatSession: def __init__(self, api_backend: Any, conversation_id: str | None = None, system_prompt: str | None = None) -> None: self.conversation_id = str(uuid.uuid4()) if conversation_id is None else conversation_id - self.cfg = Config() + self.cfg = RD_AGENT_SETTINGS self.system_prompt = system_prompt if system_prompt is not None else self.cfg.default_system_prompt self.api_backend = api_backend @@ -205,8 +204,10 @@ class ChatSession: """ messages = self.build_chat_completion_message(user_prompt) - response = self.api_backend._try_create_chat_completion_or_embedding( # noqa: SLF001 - messages=messages, chat_completion=True, **kwargs, + response = self.api_backend._try_create_chat_completion_or_embedding( # noqa: SLF001 + messages=messages, + chat_completion=True, + **kwargs, ) messages.append( { @@ -226,7 +227,7 @@ class ChatSession: class APIBackend: - def __init__( # noqa: C901, PLR0912, PLR0915 + def __init__( # noqa: C901, PLR0912, PLR0915 self, *, chat_api_key: str | None = None, @@ -242,7 +243,7 @@ class APIBackend: use_embedding_cache: bool | None = None, dump_embedding_cache: bool | None = None, ) -> None: - self.cfg = Config() + self.cfg = RD_AGENT_SETTINGS if self.cfg.use_llama2: self.generator = Llama.build( ckpt_dir=self.cfg.llama2_ckpt_dir, @@ -286,7 +287,7 @@ class APIBackend: self.gcr_endpoint_do_sample = self.cfg.gcr_endpoint_do_sample self.gcr_endpoint_max_token = self.cfg.gcr_endpoint_max_token if not os.environ.get("PYTHONHTTPSVERIFY", "") and hasattr(ssl, "_create_unverified_context"): - ssl._create_default_https_context = ssl._create_unverified_context # noqa: SLF001 + ssl._create_default_https_context = ssl._create_unverified_context # noqa: SLF001 self.encoder = None else: self.use_azure = self.cfg.use_azure @@ -315,7 +316,8 @@ class APIBackend: if self.use_azure_token_provider: credential = DefaultAzureCredential() token_provider = get_bearer_token_provider( - credential, "https://cognitiveservices.azure.com/.default", + credential, + "https://cognitiveservices.azure.com/.default", ) self.chat_client = openai.AzureOpenAI( azure_ad_token_provider=token_provider, @@ -414,7 +416,9 @@ class APIBackend: ) -> str: if former_messages is None: former_messages = [] - messages = self.build_messages(user_prompt, system_prompt, former_messages, shrink_multiple_break=shrink_multiple_break) + messages = self.build_messages( + user_prompt, system_prompt, former_messages, shrink_multiple_break=shrink_multiple_break + ) return self._try_create_chat_completion_or_embedding( messages=messages, chat_completion=True, @@ -422,12 +426,12 @@ class APIBackend: **kwargs, ) - - def create_embedding(self, input_content: str | list[str], **kwargs: Any) -> list[Any] | Any: input_content_list = [input_content] if isinstance(input_content, str) else input_content resp = self._try_create_chat_completion_or_embedding( - input_content_list=input_content_list, embedding=True, **kwargs, + input_content_list=input_content_list, + embedding=True, + **kwargs, ) if isinstance(input_content, str): return resp[0] @@ -454,7 +458,12 @@ class APIBackend: return response def _try_create_chat_completion_or_embedding( - self, max_retry: int = 10, *, chat_completion: bool = False, embedding: bool = False, **kwargs: Any, + self, + max_retry: int = 10, + *, + chat_completion: bool = False, + embedding: bool = False, + **kwargs: Any, ) -> Any: assert not (chat_completion and embedding), "chat_completion and embedding cannot be True at the same time" max_retry = self.cfg.max_retry if self.cfg.max_retry is not None else max_retry @@ -464,23 +473,25 @@ class APIBackend: return self._create_embedding_inner_function(**kwargs) if chat_completion: return self._create_chat_completion_auto_continue(**kwargs) - except openai.BadRequestError as e: # noqa: PERF203 - print(e) - print(f"Retrying {i+1}th time...") + except openai.BadRequestError as e: # noqa: PERF203 + RDAgentLog().warning(e) + RDAgentLog().warning(f"Retrying {i+1}th time...") if "'messages' must contain the word 'json' in some form" in e.message: kwargs["add_json_in_prompt"] = True elif embedding and "maximum context length" in e.message: kwargs["input_content_list"] = [ content[: len(content) // 2] for content in kwargs.get("input_content_list", []) ] - except Exception as e: # noqa: BLE001 - print(e) - print(f"Retrying {i+1}th time...") + except Exception as e: # noqa: BLE001 + RDAgentLog().warning(e) + RDAgentLog().warning(f"Retrying {i+1}th time...") time.sleep(self.retry_wait_seconds) error_message = f"Failed to create chat completion after {max_retry} retries." raise RuntimeError(error_message) - def _create_embedding_inner_function(self, input_content_list: list[str], **kwargs: Any) -> list[Any]: # noqa: ARG002 + def _create_embedding_inner_function( + self, input_content_list: list[str], **kwargs: Any + ) -> list[Any]: # noqa: ARG002 content_to_embedding_dict = {} filtered_input_content_list = [] if self.use_embedding_cache: @@ -511,7 +522,6 @@ class APIBackend: self.cache.embedding_set(content_to_embedding_dict) return [content_to_embedding_dict[content] for content in input_content_list] - def _build_messages(self, messages: list[dict]) -> str: log_messages = "" for m in messages: @@ -525,16 +535,16 @@ class APIBackend: def log_messages(self, messages: list[dict]) -> None: if self.cfg.log_llm_chat_content: - FinCoLog().info(self._build_messages(messages)) + RDAgentLog().info(self._build_messages(messages)) def log_response(self, response: str | None = None, *, stream: bool = False) -> None: if self.cfg.log_llm_chat_content: if stream: - FinCoLog().info(f"\n{LogColors.CYAN}Response:{LogColors.END}") + RDAgentLog().info(f"\n{LogColors.CYAN}Response:{LogColors.END}") else: - FinCoLog().info(f"\n{LogColors.CYAN}Response:{response}{LogColors.END}") + RDAgentLog().info(f"\n{LogColors.CYAN}Response:{response}{LogColors.END}") - def _create_chat_completion_inner_function( # noqa: C901, PLR0912, PLR0915 + def _create_chat_completion_inner_function( # noqa: C901, PLR0912, PLR0915 self, messages: list[dict], temperature: float | None = None, @@ -592,8 +602,8 @@ class APIBackend: ), ) - req = urllib.request.Request(self.gcr_endpoint, body, self.headers) # noqa: S310 - response = urllib.request.urlopen(req) # noqa: S310 + req = urllib.request.Request(self.gcr_endpoint, body, self.headers) # noqa: S310 + response = urllib.request.urlopen(req) # noqa: S310 resp = json.loads(response.read().decode())["output"] self.log_response(resp) else: @@ -662,7 +672,7 @@ class APIBackend: def calculate_token_from_messages(self, messages: list[dict]) -> int: if self.use_llama2 or self.use_gcr_endpoint: - FinCoLog().warning("num_tokens_from_messages() is not implemented for model llama2.") + RDAgentLog().warning("num_tokens_from_messages() is not implemented for model llama2.") return 0 # TODO implement this function for llama2 if "gpt4" in self.chat_model or "gpt-4" in self.chat_model: @@ -691,7 +701,9 @@ class APIBackend: ) -> int: if former_messages is None: former_messages = [] - messages = self.build_messages(user_prompt, system_prompt, former_messages, shrink_multiple_break=shrink_multiple_break) + messages = self.build_messages( + user_prompt, system_prompt, former_messages, shrink_multiple_break=shrink_multiple_break + ) return self.calculate_token_from_messages(messages) @@ -703,8 +715,10 @@ def create_embedding_with_multiprocessing(str_list: list, slice_count: int = 50, embeddings = [] pool = multiprocessing.Pool(nproc) - result_list = [pool.apply_async(calculate_embedding_process, (str_list[index : index + slice_count],)) - for index in range(0, len(str_list), slice_count)] + result_list = [ + pool.apply_async(calculate_embedding_process, (str_list[index : index + slice_count],)) + for index in range(0, len(str_list), slice_count) + ] pool.close() pool.join() @@ -713,16 +727,16 @@ def create_embedding_with_multiprocessing(str_list: list, slice_count: int = 50, return embeddings - def calculate_embedding_distance_between_str_list( - source_str_list: list[str], target_str_list: list[str], + source_str_list: list[str], + target_str_list: list[str], ) -> list[list[float]]: if not source_str_list or not target_str_list: return [[]] embeddings = create_embedding_with_multiprocessing(source_str_list + target_str_list, slice_count=50, nproc=8) - source_embeddings = embeddings[:len(source_str_list)] - target_embeddings = embeddings[len(source_str_list):] + source_embeddings = embeddings[: len(source_str_list)] + target_embeddings = embeddings[len(source_str_list) :] source_embeddings_np = np.array(source_embeddings) target_embeddings_np = np.array(target_embeddings)