diff --git a/Makefile b/Makefile index 8c2b4cf8..a0726a32 100644 --- a/Makefile +++ b/Makefile @@ -93,14 +93,14 @@ mypy: # Check lint with ruff. ruff: - $(PIPRUN) python -m ruff . --exclude FinCo,finco,rdagent/scripts,test/scripts,git_ignore_folder --ignore ANN101,ANN401,TCH003,D,ERA001,PLR0913,S101 --line-length 120 + $(PIPRUN) ruff check . --exclude FinCo,finco,rdagent/scripts,test/scripts,git_ignore_folder # Check lint with toml-sort. toml-sort: $(PIPRUN) toml-sort --check pyproject.toml # Check lint with all linters. -lint: black isort mypy ruff toml-sort +lint: mypy ruff toml-sort # Run pre-commit with autofix against all files. pre-commit: diff --git a/constraints/3.10.txt b/constraints/3.10.txt index e11a59f0..9441e17c 100644 --- a/constraints/3.10.txt +++ b/constraints/3.10.txt @@ -111,7 +111,7 @@ rfc3986==2.0.0 rich==13.7.0 ruamel.yaml==0.18.5 ruamel.yaml.clib==0.2.8 -ruff==0.1.9 +ruff==0.4.5 scipy==1.11.4 SecretStorage==3.3.3 semver==3.0.2 diff --git a/constraints/3.11.txt b/constraints/3.11.txt index f8da2921..d4914162 100644 --- a/constraints/3.11.txt +++ b/constraints/3.11.txt @@ -109,7 +109,7 @@ rfc3986==2.0.0 rich==13.7.0 ruamel.yaml==0.18.5 ruamel.yaml.clib==0.2.8 -ruff==0.1.9 +ruff==0.4.5 scipy==1.11.4 SecretStorage==3.3.3 semver==3.0.2 diff --git a/constraints/3.8.txt b/constraints/3.8.txt index bd6a80d1..f022bd1f 100644 --- a/constraints/3.8.txt +++ b/constraints/3.8.txt @@ -112,7 +112,7 @@ rfc3986==2.0.0 rich==13.7.0 ruamel.yaml==0.18.5 ruamel.yaml.clib==0.2.8 -ruff==0.1.9 +ruff==0.4.5 scipy==1.10.1 SecretStorage==3.3.3 semver==3.0.2 diff --git a/constraints/3.9.txt b/constraints/3.9.txt index e11a59f0..9441e17c 100644 --- a/constraints/3.9.txt +++ b/constraints/3.9.txt @@ -111,7 +111,7 @@ rfc3986==2.0.0 rich==13.7.0 ruamel.yaml==0.18.5 ruamel.yaml.clib==0.2.8 -ruff==0.1.9 +ruff==0.4.5 scipy==1.11.4 SecretStorage==3.3.3 semver==3.0.2 diff --git a/pyproject.toml b/pyproject.toml index 2122ffdb..00634562 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,32 +73,28 @@ log_format = "%(asctime)s %(levelname)s %(message)s" minversion = "6.0" [tool.ruff] -fix = true -ignore = [ - # https://docs.astral.sh/ruff/rules/#pydocstyle-d - "D203", - "D204", - "D213", - "D215", - "D400", - "D404", - "D406", - "D407", - "D408", - "D409", - "D413", - "ANN101", - "ANN401", - "TCH003", - "D", - "ERA001", - "PLR0913", - "S101", -] -select = ["ALL"] +line-length = 120 src = ["rdagent"] -[tool.ruff.per-file-ignores] +[tool.ruff.lint] +ignore = [ + # https://docs.astral.sh/ruff/rules/#pydocstyle-d + "ANN101", + "ANN401", + "D", + "ERA001", + "FIX", + "INP001", + "PGH", + "PLR0913", + "S101", + "T20", + "TCH003", + "TD", +] +select = ["ALL"] + +[tool.ruff.lint.per-file-ignores] "docs/conf.py" = ["INP001"] "test/*" = ["S101"] diff --git a/rdagent/app/CI/prompts.py b/rdagent/app/CI/prompts.py new file mode 100644 index 00000000..cfb9548e --- /dev/null +++ b/rdagent/app/CI/prompts.py @@ -0,0 +1,126 @@ +linting_system_prompt_template = "You are a software engineer. \ +You can write code to a high standard and are adept at solving {language} linting problems." + + +user_get_makefile_lint_commands_template = """ +You get a Makefile which contains some linting rules. Here are its content ```{file_text}``` +Please find executable commands about linting from it. + +Please response with following json template: +{{ + "commands": , +}} +""" + +user_get_files_contain_lint_commands_template = """ +You get a file list of a repository. \ +Some file maybe contain linting rules or linting commands which defined by repo authors. +Here are the file list: +``` +{file_list} +``` + +Please find all files maybe correspond to linting from it. +Please response with following json template: +{{ + "files": , +}} +""" + + +generate_lint_command_template = """ +Please generate a command to lint or format a {language} repository. +Here are some information about different linting tools ```{linting_tools}``` +""" + + +suffix2language_template = """ +Here are the files suffix in one code repo: {suffix}. +Please tell me the programming language used in this repo and which language has linting-tools. +Your response should follow this template: +{{ + "languages": , + "languages_with_linting_tools": +}} +""" + + +session_start_template = """ +Please modify the Python code based on the lint info. +Due to the length of the code, I will first tell you the entire code, and then each time I ask a question, \ +I will extract a portion of the code and tell you the error information contained in this code segment. +You need to fix the corresponding error in the code segment \ +and return the code that can replace the corresponding code segment. + +The Python code is from a complete Python project file. Each line of the code is annotated with a line number, \ +separated from the original code by three characters ("|"). The vertical bars are aligned. +Here is the complete code, please be prepared to fix it: +```Python +{code} +``` +""" + + +session_normal_template = """Please modify this code snippet based on the lint info. Here is the code snippet: +```Python +{code} +``` + +-----Lint info----- +{lint_info} +------------------- + +The lint info contains one or more errors. \ +Different errors are separated by blank lines. Each error follows this format: +-----Lint info format----- +: + + +-------------------------- +The error code is an abbreviation set by the checker for ease of describing the error. \ +The error context includes the relevant code around the error, and the helpful information suggests possible fixes. + +Please simply reply the code after you fix all linting errors. +The code you return does not require line numbers, \ +and should just replace the code I provided you, and does not require comments. +Please wrap your code with following format: + +```python + +``` +""" + + +user_template_for_code_snippet = """Please modify the Python code based on the lint info. +-----Python Code----- +{code} +--------------------- + +-----Lint info----- +{lint_info} +------------------- + +The Python code is a snippet from a complete Python project file. \ +Each line of the code is annotated with a line number, \ +separated from the original code by three characters ("|"). \ +The vertical bars are aligned. + +The lint info contains one or more errors. Different errors are separated by blank lines. \ +Each error follows this format: +-----Lint info format----- +: + + +-------------------------- +The error code is an abbreviation set by the checker for ease of describing the error. \ +The error context includes the relevant code around the error, and the helpful information suggests possible fixes. + +Please simply reply the code after you fix all linting errors. +The code you return does not require line numbers, \ +and should just replace the code I provided you, and does not require comments. +Please wrap your code with following format: + +```python + +``` +""" diff --git a/rdagent/app/CI/run.py b/rdagent/app/CI/run.py new file mode 100644 index 00000000..b47bf15f --- /dev/null +++ b/rdagent/app/CI/run.py @@ -0,0 +1,426 @@ +""" + +""" +import json +import re +import subprocess +import time +from collections import defaultdict +from dataclasses import dataclass +from difflib import IS_LINE_JUNK, ndiff +from pathlib import Path +from typing import Dict, List, Tuple, Union, cast + +from rdagent.core.evolving_framework import ( + Evaluator, + EvoAgent, + EvolvableSubjects, + EvolvingStrategy, + EvoStep, + Feedback, + Knowledge, +) +from rdagent.oai.llm_utils import APIBackend +from rich import print +from rich.panel import Panel +from rich.prompt import Prompt +from rich.rule import Rule +from rich.syntax import Syntax +from rich.table import Table +from rich.text import Text + +from .prompts import ( + linting_system_prompt_template, + session_normal_template, + session_start_template, +) + + +@dataclass +class CIError: + raw_str: str + file_path: Union[Path, str] + line: int + column: int + code: str + msg: str + hint: str + + +@dataclass +class CIFeedback(Feedback): + errors: Dict[str, List[CIError]] + + +@dataclass +class FixRecord: + skipped_errors: List[CIError] + directly_fixed_errors: List[CIError] + manually_fixed_errors: List[CIError] + + +class CodeFile: + def __init__(self, path: Union[Path, str]): + self.path = Path(path) + self.load() + + + def load(self) -> None: + code = self.path.read_text(encoding="utf-8") + self.code_lines = code.split("\n") + + # add line number + self.lineno = len(self.code_lines) + self.lineno_width = len(str(self.lineno)) + self.code_lines_with_lineno = [] + for i, code_line in enumerate(self.code_lines): + self.code_lines_with_lineno.append(f"{i+1: >{self.lineno_width}} | {code_line}") + + + def get(self, start = 0, end = None, add_line_number: bool = False, return_list: bool = False) -> Union[List[str], str]: + start -= 1 + if start < 0: start = 0 + end = self.lineno if end is None else end-1 + + res = self.code_lines_with_lineno[start:end] if add_line_number else self.code_lines[start:end] + + return res if return_list else "\n".join(res) + + + def apply_changes(self, changes: List[Tuple[int, int, str]]) -> None: + offset = 0 + for start, end, code in changes: + start -= 1 + if start < 0: start = 0 + end -= 1 + + new_code = code.split("\n") + self.code_lines[start+offset:end+offset] = new_code + offset += len(new_code) - (end - start) + + self.path.write_text("\n".join(self.code_lines), encoding="utf-8") + self.load() + + + def __str__(self): + return f"{self.path}" + + +class Repo(EvolvableSubjects): + def __init__(self, project_path: Union[Path, str], **kwargs): + self.params = kwargs + self.project_path = Path(project_path) + git_ignored_output = subprocess.check_output( + "git status --ignored -s", + shell=True, + cwd=project_path, + stderr=subprocess.STDOUT, + ).decode("utf-8") + git_ignored_files = [ + (self.project_path / Path(line[3:])).resolve() + for line in git_ignored_output.split("\n") + if line.startswith("!!") + ] + + files = [ + file + for file in self.project_path.glob("**/*") + if file.is_file() + and not any(str(file).startswith(str(path)) for path in git_ignored_files) + and ".git/" not in str(file) + and file.suffix == ".py" + ] + self.files = {file: CodeFile(file) for file in files} + + self.fix_records: Dict[str, FixRecord] | None = None + + +@dataclass +class RuffRule: + """ + { + "name": "missing-trailing-comma", + "code": "COM812", + "linter": "flake8-commas", + "summary": "Trailing comma missing", + "message_formats": [ + "Trailing comma missing" + ], + "fix": "Fix is always available.", + "explanation": "## What it does\nChecks for the absence of trailing commas.\n\n## Why is this bad?\nThe presence of a trailing comma can reduce diff size when parameters or\nelements are added or removed from function calls, function definitions,\nliterals, etc.\n\n## Example\n```python\nfoo = {\n \"bar\": 1,\n \"baz\": 2\n}\n```\n\nUse instead:\n```python\nfoo = {\n \"bar\": 1,\n \"baz\": 2,\n}\n```\n", + "preview": false + } + """ + name: str + code: str + linter: str + summary: str + message_formats: List[str] + fix: str + explanation: str + preview: bool + + +class RuffEvaluator(Evaluator): + """The error message are generated by + `python -m ruff . --exclude FinCo,finco,fincov1 --ignore ANN101,TCH003,D,ERA001` + """ + + def __init__(self, command: str = None): + if command is None: + self.command = "ruff check . --no-fix --output-format full" + else: + self.command = command + + + def explain_rule(self, error_code: str) -> RuffRule: + explain_command = "ruff rule {error_code} --output-format json" + try: + out = subprocess.check_output( + explain_command.format(error_code=error_code), + shell=True, + stderr=subprocess.STDOUT, + ) + except subprocess.CalledProcessError as e: + out = e.output + + return json.loads(out.decode()) + + + def evaluate(self, evo: Repo, **kwargs) -> CIFeedback: + """Simply run ruff to get the feedbacks.""" + try: + out = subprocess.check_output( + self.command, + shell=True, + cwd=evo.project_path, + stderr=subprocess.STDOUT, + ) + except subprocess.CalledProcessError as e: + out = e.output + + """ruff output format: + src/finco/cli.py:9:5: ANN201 Missing return type annotation for public function `main` + | + 9 | def main(prompt=None): + | ^^^^ ANN201 + 10 | load_dotenv(verbose=True, override=True) + 11 | wm = WorkflowManager() + | + = help: Add return type annotation: `None` + """ + + # extract error info + pattern = r"(([^\n]*):(\d+):(\d+): (\w+) ([^\n]*)\n(.*?))\n\n" + matches = re.findall(pattern, out.decode(), re.DOTALL) + + errors = defaultdict(list) + for match in matches: + raw_str, file_path, line_number, column_number, error_code, error_message, error_hint = match + error = CIError(raw_str=raw_str, + file_path=file_path, + line=int(line_number), + column=int(column_number), + code=error_code, + msg=error_message, + hint=error_hint) + errors[file_path].append(error) + + return CIFeedback(errors=errors) + +class MypyEvaluator(Evaluator): + + def __init__(self, command: str = None): + if command is None: + self.command = "mypy . --explicit-package-bases" + else: + self.command = command + + def evaluate(self, evo: Repo, **kwargs) -> CIFeedback: + try: + out = subprocess.check_output( + self.command, + shell=True, + cwd=evo.project_path, + stderr=subprocess.STDOUT, + ) + except subprocess.CalledProcessError as e: + out = e.output + + return CIFeedback(cast(str, out).decode("utf-8")) + + +class CIEvoStr(EvolvingStrategy): + + def evolve( + self, + evo: Repo, + evolving_trace: List[EvoStep] = [], + knowledge_l: List[Knowledge] = [], + **kwargs, + ) -> Repo: + api = APIBackend() + system_prompt = linting_system_prompt_template.format(language="Python") + + if len(evolving_trace) > 0: + last_feedback: CIFeedback = evolving_trace[-1].feedback + fix_records: Dict[str, FixRecord] = defaultdict(lambda: FixRecord([], [], [])) + # iterate by file + for file_path, errors in last_feedback.errors.items(): + print(Rule(f"[cyan]Fixing {file_path}[/cyan]", style="bold cyan", align="left", characters=".")) + + file = evo.files[evo.project_path / Path(file_path)] + + # Group errors based on position + # TODO @bowen: Crossover between different groups after adding 3 lines of context + groups: List[List[CIError]] = [] + near_errors = [errors[0]] + for error in errors[1:]: + if error.line - near_errors[-1].line <= 6: + near_errors.append(error) + else: + groups.append(near_errors) + near_errors = [error] + groups.append(near_errors) + + changes = [] + + # generate changes + for group_id, group in enumerate(groups, start=1): + session = api.build_chat_session(session_system_prompt=system_prompt) + session.build_chat_completion(session_start_template.format(code=file.get(add_line_number=True))) + + print(f"[yellow]Fixing part {group_id}...[/yellow]\n") + + start_line = group[0].line - 3 + end_line = group[-1].line + 3 + 1 + code_snippet_with_lineno = file.get(start_line, end_line, add_line_number=True, return_list=False) + code_snippet_lines = file.get(start_line, end_line, add_line_number=False, return_list=True) + + # front_anchor_code = file.get(start_line-3, start_line, add_line_number=False, return_list=False) + # rear_anchor_code = file.get(end_line+1, end_line+3+1, add_line_number=False, return_list=False) + + errors_str = "\n".join([f"{error.raw_str}\n" for error in group]) + + print(Panel.fit(Syntax("\n".join([f"{error.line}: {error.msg}" for error in group]), lexer="python", background_color="default"), title=f"{len(group)} Errors")) + # print(f"[bold yellow]original code:[/bold yellow]\n\n{code_snippet_with_lineno}") + print(Panel.fit(Syntax(code_snippet_with_lineno, lexer="python", background_color="default"), title="Original Code")) + user_prompt = session_normal_template.format( + code=code_snippet_with_lineno, + lint_info=errors_str, + ) + res = session.build_chat_completion(user_prompt) + + manual_fix_flag = False + + while True: + new_code = re.search(r".*```[Pp]ython\n(.*)\n```.*", res, re.DOTALL).group(1) + + # print repair status (code diff) + diff = ndiff(code_snippet_lines, new_code.split("\n"), linejunk=IS_LINE_JUNK) + table = Table(show_header=False, box=None) + table.add_column() + for i in diff: + if i.startswith("+"): table.add_row(Text(i, style="green")) + elif i.startswith("-"): table.add_row(Text(i, style="red")) + elif i.startswith("?"): table.add_row(Text(i, style="yellow")) + else: table.add_row(Syntax(i, lexer="python", background_color="default")) + print(Panel.fit(table, title="Repair Status")) + + operation = input("Input your operation: ") + if operation == "s" or operation == "skip": + fix_records[file_path].skipped_errors.extend(group) + break + if operation == "a" or operation == "apply": + if manual_fix_flag: + fix_records[file_path].manually_fixed_errors.extend(group) + else: + fix_records[file_path].directly_fixed_errors.extend(group) + + changes.append((start_line, end_line, new_code)) + break + + manual_fix_flag = True + res = session.build_chat_completion(operation) + + # apply changes + file.apply_changes(changes) + + evo.fix_records = fix_records + + return evo + + +# DIR = "/home/bowen/workspace/fincov2_test/" +DIR = "/home/bowen/workspace/RD-Agent/" +PY = "/home/bowen/miniconda3/envs/cr/bin/python" + +start_time = time.time() + +evo = Repo(DIR, python_path=PY) + +eval = RuffEvaluator() +estr = CIEvoStr() +rag = None # RAG is not enable firstly. +ea = EvoAgent(estr, rag=rag) +ea.step_evolving(evo, eval) +while True: + print(Rule(f"Round {len(ea.evolving_trace)} repair", style="blue")) + evo: Repo = ea.step_evolving(evo, eval) + + fix_records = evo.fix_records + + # Count the number of skipped errors + skipped_errors_count = 0 + directly_fixed_errors_count = 0 + manually_fixed_errors_count = 0 + skipped_errors_code_count = defaultdict(int) + directly_fixed_errors_code_count = defaultdict(int) + manually_fixed_errors_code_count = defaultdict(int) + for record in fix_records.values(): + skipped_errors_count += len(record.skipped_errors) + directly_fixed_errors_count += len(record.directly_fixed_errors) + manually_fixed_errors_count += len(record.manually_fixed_errors) + for error in record.skipped_errors: + skipped_errors_code_count[error.code] += 1 + for error in record.directly_fixed_errors: + directly_fixed_errors_code_count[error.code] += 1 + for error in record.manually_fixed_errors: + manually_fixed_errors_code_count[error.code] += 1 + + skipped_errors_statistics = "" + directly_fixed_errors_statistics = "" + manually_fixed_errors_statistics = "" + for code, count in sorted(skipped_errors_code_count.items(), key=lambda x: x[1], reverse=True): + skipped_errors_statistics += f"{count: >5} {code: >10} {eval.explain_rule(code).summary}\n" + for code, count in sorted(directly_fixed_errors_code_count.items(), key=lambda x: x[1], reverse=True): + directly_fixed_errors_statistics += f"{count: >5} {code: >10} {eval.explain_rule(code).summary}\n" + for code, count in sorted(manually_fixed_errors_code_count.items(), key=lambda x: x[1], reverse=True): + manually_fixed_errors_statistics += f"{count: >5} {code: >10} {eval.explain_rule(code).summary}\n" + + # Create a table to display the counts and ratios + table = Table(title="Error Fix Statistics") + table.add_column("Type") + table.add_column("Statistics") + table.add_column("Count") + table.add_column("Ratio") + + total_errors_count = skipped_errors_count + directly_fixed_errors_count + manually_fixed_errors_count + table.add_row("Total Errors", "", str(total_errors_count), "") + table.add_row("Skipped Errors", skipped_errors_statistics, str(skipped_errors_count), f"{skipped_errors_count / total_errors_count:.2%}") + table.add_row("Directly Fixed Errors", directly_fixed_errors_statistics, str(directly_fixed_errors_count), f"{directly_fixed_errors_count / total_errors_count:.2%}") + table.add_row("Manually Fixed Errors", manually_fixed_errors_statistics, str(manually_fixed_errors_count), f"{manually_fixed_errors_count / total_errors_count:.2%}") + + print(table) + operation = Prompt.ask("Start next round? (y/n): ", choices=["y", "n"]) + if operation == "n": break + + +end_time = time.time() +execution_time = end_time - start_time +print(f"Execution time: {execution_time} seconds") + +""" Please commit it by hand... and then run the next round +git add -u +git commit --no-verify -v +""" diff --git a/rdagent/app/factor_extraction_and_implementation/factor_extract_and_implement.py b/rdagent/app/factor_extraction_and_implementation/factor_extract_and_implement.py index aa99ca46..b995c47b 100644 --- a/rdagent/app/factor_extraction_and_implementation/factor_extract_and_implement.py +++ b/rdagent/app/factor_extraction_and_implementation/factor_extract_and_implement.py @@ -1,14 +1,16 @@ # %% -from document_process.document_reader import load_and_process_pdfs_by_langchain, classify_report_from_dict -from dotenv import load_dotenv -from oai.llm_utils import APIBackend -from pathlib import Path import json +from pathlib import Path +from dotenv import load_dotenv -from document_process.document_analysis import extract_factors_from_report_dict_and_classify_result -from document_process.document_analysis import check_factor_dict_viability -from document_process.document_analysis import deduplicate_factors_several_times +from document_process.document_analysis import ( + check_factor_dict_viability, + deduplicate_factors_several_times, + extract_factors_from_report_dict_and_classify_result, +) +from document_process.document_reader import classify_report_from_dict, load_and_process_pdfs_by_langchain +from oai.llm_utils import APIBackend def extract_factors_and_implement(report_file_path: str): @@ -29,7 +31,7 @@ def extract_factors_and_implement(report_file_path: str): for factor_name in factor_dict: if len(factor_dict[factor_name]) > 1: factor_dict_simple_deduplication[factor_name] = max( - factor_dict[factor_name], key=lambda x: len(x["formulation"]) + factor_dict[factor_name], key=lambda x: len(x["formulation"]), ) else: factor_dict_simple_deduplication[factor_name] = factor_dict[factor_name][0] @@ -115,11 +117,11 @@ def extract_factors_and_implement(report_file_path: str): for factor_name in target_dict: formulation = target_dict[factor_name]["formulation"] if factor_name in formulation: - target_factor_name = factor_name.replace("_", "\_") + target_factor_name = factor_name.replace("_", r"\_") formulation = formulation.replace(factor_name, target_factor_name) for variable in target_dict[factor_name]["variables"]: if variable in formulation: - target_variable = variable.replace("_", "\_") + target_variable = variable.replace("_", r"\_") formulation = formulation.replace(variable, target_variable) fw.write(f"## {current_index}. 因子名称：{factor_name}\n") @@ -130,9 +132,9 @@ def extract_factors_and_implement(report_file_path: str): fw.write(f"### formulation string: {formulation}\n") # write a table of variable and its description - fw.write(f"### variable tables: \n") - fw.write(f"| variable | description |\n") - fw.write(f"| -------- | ----------- |\n") + fw.write("### variable tables: \n") + fw.write("| variable | description |\n") + fw.write("| -------- | ----------- |\n") for variable in target_dict[factor_name]["variables"]: fw.write(f"| {variable} | {target_dict[factor_name]['variables'][variable]} |\n") diff --git a/rdagent/core/conf.py b/rdagent/core/conf.py index 577dd325..cbf033b0 100644 --- a/rdagent/core/conf.py +++ b/rdagent/core/conf.py @@ -20,10 +20,12 @@ class FincoSettings(BaseSettings): use_chat_cache: bool = False dump_embedding_cache: bool = False use_embedding_cache: bool = False + workspace: str = "./finco_workspace" prompt_cache_path: str = os.getcwd() + "/prompt_cache.db" session_cache_folder_location: str = os.getcwd() + "/session_cache_folder/" max_past_message_include: int = 10 + use_vector_only: bool = False log_llm_chat_content: bool = True # Chat configs @@ -45,3 +47,46 @@ class FincoSettings(BaseSettings): embedding_azure_api_base: str = "" embedding_azure_api_version: str = "" embedding_model: str = "" + + # llama2 related config + use_llama2: bool = False + llama2_ckpt_dir: str = "Llama-2-7b-chat" + llama2_tokenizer_path: str = "Llama-2-7b-chat/tokenizer.model" + llams2_max_batch_size: int = 8 + + # finco v2 configs + azure_document_intelligence_key: str = "" + azure_document_intelligence_endpoint: str = "" + + # fincov2 llama2 endpoint + use_gcr_endpoint: bool = False + gcr_endpoint_type: str = "llama2_70b" # or "llama3_70b", "phi2", "phi3_4k", "phi3_128k" + + llama2_70b_endpoint: str = "" + llama2_70b_endpoint_key: str = "" + llama2_70b_endpoint_deployment: str = "" + + llama3_70b_endpoint: str = "" + llama3_70b_endpoint_key: str = "" + llama3_70b_endpoint_deployment: str = "" + + phi2_endpoint: str = "" + phi2_endpoint_key: str = "" + phi2_endpoint_deployment: str = "" + + phi3_4k_endpoint: str = "" + phi3_4k_endpoint_key: str = "" + phi3_4k_endpoint_deployment: str = "" + + phi3_128k_endpoint: str = "" + phi3_128k_endpoint_key: str = "" + phi3_128k_endpoint_deployment: str = "" + + gcr_endpoint_temperature: float = 0.7 + gcr_endpoint_top_p: float = 0.9 + gcr_endpoint_do_sample: bool = False + gcr_endpoint_max_token: int = 100 + + # factor extraction conf + max_input_duplicate_factor_group: int = 600 + max_output_duplicate_factor_group: int = 20 diff --git a/rdagent/core/log.py b/rdagent/core/log.py index 535edd07..7c421ac1 100644 --- a/rdagent/core/log.py +++ b/rdagent/core/log.py @@ -1,11 +1,9 @@ from __future__ import annotations -from contextlib import contextmanager -from typing import TYPE_CHECKING, Generator, Sequence +from typing import TYPE_CHECKING, Sequence from loguru import logger - if TYPE_CHECKING: from loguru import Logger diff --git a/rdagent/core/utils.py b/rdagent/core/utils.py index ace5d83f..85628358 100644 --- a/rdagent/core/utils.py +++ b/rdagent/core/utils.py @@ -1,20 +1,18 @@ from __future__ import annotations +import importlib import json +import multiprocessing as mp import os import random import string -import uuid +from collections.abc import Callable from pathlib import Path -import importlib from typing import Any import yaml from fuzzywuzzy import fuzz -import multiprocessing as mp -from collections.abc import Callable - class FincoException(Exception): pass @@ -87,9 +85,9 @@ def crawl_the_folder(folder_path: Path): def compare_yaml(file1, file2): - with open(file1, "r") as stream: + with open(file1) as stream: data1 = yaml.safe_load(stream) - with open(file2, "r") as stream: + with open(file2) as stream: data2 = yaml.safe_load(stream) return data1 == data2 @@ -107,7 +105,7 @@ class YamlConfigCache(SingletonBaseClass): self.path_to_config = dict() def load(self, path): - with open(path, "r") as stream: + with open(path) as stream: data = yaml.safe_load(stream) self.path_to_config[path] = data diff --git a/rdagent/document_process/document_analysis.py b/rdagent/document_process/document_analysis.py index 20d92dc4..724c6f73 100644 --- a/rdagent/document_process/document_analysis.py +++ b/rdagent/document_process/document_analysis.py @@ -18,8 +18,8 @@ from sklearn.metrics.pairwise import cosine_similarity from sklearn.preprocessing import normalize from core.conf import FincoSettings as Config -from oai.llm_utils import APIBackend, create_embedding_with_multiprocessing from core.log import FinCoLog +from oai.llm_utils import APIBackend, create_embedding_with_multiprocessing if TYPE_CHECKING: from langchain_core.documents import Document @@ -229,7 +229,7 @@ def __extract_factors_name_and_desc_from_content( parse_success = False if ret_json_str is None or not parse_success: current_user_prompt = ( - "Your response didn't follow the instruction" " might be wrong json format. Try again." + "Your response didn't follow the instruction might be wrong json format. Try again." ) else: factors = ret_dict["factors"] @@ -273,7 +273,7 @@ def __extract_factors_formulation_from_content( parse_success = False if ret_json_str is None or not parse_success: current_user_prompt = ( - "Your response didn't follow the instruction" " might be wrong json format. Try again." + "Your response didn't follow the instruction might be wrong json format. Try again." ) else: for name, formulation_and_description in ret_dict.items(): @@ -393,7 +393,7 @@ def check_factor_dict_viability_simulate_json_mode( parse_success = False if ret_json_str is None or not parse_success: current_user_prompt = ( - "Your response didn't follow the " "instruction might be wrong json format. Try again." + "Your response didn't follow the instruction might be wrong json format. Try again." ) else: return ret_dict diff --git a/rdagent/document_process/document_reader.py b/rdagent/document_process/document_reader.py index 88f1fe45..6a93eb1c 100644 --- a/rdagent/document_process/document_reader.py +++ b/rdagent/document_process/document_reader.py @@ -5,10 +5,7 @@ from pathlib import Path import yaml from azure.ai.formrecognizer import DocumentAnalysisClient from azure.core.credentials import AzureKeyCredential - from finco.conf import FincoSettings as Config -from oai.llm_utils import APIBackend, create_embedding_with_multiprocessing -from core.log import FinCoLog if TYPE_CHECKING: from langchain_core.documents import Document diff --git a/rdagent/factor_implementation/evolving/evaluators.py b/rdagent/factor_implementation/evolving/evaluators.py index 882564d8..421f9eef 100644 --- a/rdagent/factor_implementation/evolving/evaluators.py +++ b/rdagent/factor_implementation/evolving/evaluators.py @@ -8,6 +8,7 @@ from pandas.core.api import DataFrame as DataFrame from core.evolving_framework import Evaluator as EvolvingEvaluator from core.evolving_framework import Feedback, QueriedKnowledge from core.log import FinCoLog +from core.utils import multiprocessing_wrapper from factor_implementation.evolving.evolvable_subjects import ( FactorImplementationList, ) @@ -24,7 +25,6 @@ from factor_implementation.share_modules.factor import ( FactorImplementation, FactorImplementationTask, ) -from core.utils import multiprocessing_wrapper class FactorImplementationSingleFeedback: diff --git a/rdagent/factor_implementation/evolving/evolvable_subjects.py b/rdagent/factor_implementation/evolving/evolvable_subjects.py index af4ee731..43e91c0e 100644 --- a/rdagent/factor_implementation/evolving/evolvable_subjects.py +++ b/rdagent/factor_implementation/evolving/evolvable_subjects.py @@ -1,7 +1,5 @@ from __future__ import annotations -import pandas as pd - from core.evolving_framework import EvolvableSubjects from core.log import FinCoLog from factor_implementation.share_modules.factor import ( diff --git a/rdagent/factor_implementation/evolving/evolving_strategy.py b/rdagent/factor_implementation/evolving/evolving_strategy.py index f07b5a8d..b414f258 100644 --- a/rdagent/factor_implementation/evolving/evolving_strategy.py +++ b/rdagent/factor_implementation/evolving/evolving_strategy.py @@ -9,7 +9,7 @@ from typing import TYPE_CHECKING from jinja2 import Template from core.evolving_framework import EvolvingStrategy, QueriedKnowledge -from oai.llm_utils import APIBackend +from core.utils import multiprocessing_wrapper from factor_implementation.share_modules.conf import FactorImplementSettings from factor_implementation.share_modules.factor import ( FactorImplementation, @@ -20,7 +20,7 @@ from factor_implementation.share_modules.prompt import ( FactorImplementationPrompts, ) from factor_implementation.share_modules.utils import get_data_folder_intro -from core.utils import multiprocessing_wrapper +from oai.llm_utils import APIBackend if TYPE_CHECKING: from factor_implementation.evolving.evolvable_subjects import ( diff --git a/rdagent/factor_implementation/evolving/factor_implementation_evolving_cli.py b/rdagent/factor_implementation/evolving/factor_implementation_evolving_cli.py index fd60b187..705b6730 100644 --- a/rdagent/factor_implementation/evolving/factor_implementation_evolving_cli.py +++ b/rdagent/factor_implementation/evolving/factor_implementation_evolving_cli.py @@ -8,6 +8,7 @@ from fire.core import Fire from tqdm import tqdm from core.evolving_framework import EvoAgent, KnowledgeBase +from core.utils import multiprocessing_wrapper from factor_implementation.evolving.evaluators import ( FactorImplementationEvaluatorV1, FactorImplementationsMultiEvaluator, @@ -29,7 +30,6 @@ from factor_implementation.share_modules.factor import ( FactorImplementationTask, FileBasedFactorImplementation, ) -from core.utils import multiprocessing_wrapper ALPHA101_INIT_COMPONENTS = [ "1. abs(): absolute value to certain columns", @@ -108,10 +108,8 @@ class FactorImplementationEvolvingCli: if former_knowledge_base_path is not None and former_knowledge_base_path.exists(): factor_knowledge_base = pickle.load(open(former_knowledge_base_path, "rb")) if self.evolving_version == 1 and not isinstance( - factor_knowledge_base, FactorImplementationKnowledgeBaseV1 - ): - raise ValueError("The former knowledge base is not compatible with the current version") - elif self.evolving_version == 2 and not isinstance( + factor_knowledge_base, FactorImplementationKnowledgeBaseV1, + ) or self.evolving_version == 2 and not isinstance( factor_knowledge_base, FactorImplementationGraphKnowledgeBase, ): @@ -261,7 +259,7 @@ class FactorImplementationEvolvingCli: print([feedback.final_decision if feedback is not None else None for feedback in feedbacks].count(True)) def implement_amc( - self, evo_sub_path_str, former_knowledge_base_path_str, implementation_dump_path_str, slice_index + self, evo_sub_path_str, former_knowledge_base_path_str, implementation_dump_path_str, slice_index, ): factor_implementations: FactorImplementationList = pickle.load(open(evo_sub_path_str, "rb")) factor_implementations.target_factor_tasks = factor_implementations.target_factor_tasks[ diff --git a/rdagent/factor_implementation/evolving/knowledge_management.py b/rdagent/factor_implementation/evolving/knowledge_management.py index f76ca63f..bfd8e7fc 100644 --- a/rdagent/factor_implementation/evolving/knowledge_management.py +++ b/rdagent/factor_implementation/evolving/knowledge_management.py @@ -8,6 +8,7 @@ from itertools import combinations from pathlib import Path from typing import Union +from finco.graph import UndirectedGraph, UndirectedNode from jinja2 import Template from core.evolving_framework import ( @@ -18,8 +19,6 @@ from core.evolving_framework import ( QueriedKnowledge, RAGStrategy, ) -from finco.graph import UndirectedGraph, UndirectedNode -from oai.llm_utils import APIBackend, calculate_embedding_distance_between_str_list from core.log import FinCoLog from factor_implementation.evolving.evaluators import ( FactorImplementationSingleFeedback, @@ -32,6 +31,7 @@ from factor_implementation.share_modules.factor import ( from factor_implementation.share_modules.prompt import ( FactorImplementationPrompts, ) +from oai.llm_utils import APIBackend, calculate_embedding_distance_between_str_list class FactorImplementationKnowledge(Knowledge): @@ -150,47 +150,46 @@ class FactorImplementationRAGStrategyV1(RAGStrategy): queried_knowledge.success_task_to_knowledge_dict[target_factor_task_information] = ( self.knowledgebase.implementation_trace[target_factor_task_information][-1] ) + elif ( + len( + self.knowledgebase.implementation_trace.setdefault( + target_factor_task_information, + [], + ), + ) + >= fail_task_trial_limit + ): + queried_knowledge.failed_task_info_set.add(target_factor_task_information) else: - if ( - len( - self.knowledgebase.implementation_trace.setdefault( - target_factor_task_information, - [], - ), - ) - >= fail_task_trial_limit - ): - queried_knowledge.failed_task_info_set.add(target_factor_task_information) - else: - queried_knowledge.working_task_to_former_failed_knowledge_dict[target_factor_task_information] = ( - self.knowledgebase.implementation_trace.setdefault( - target_factor_task_information, - [], - )[-v1_query_former_trace_limit:] - ) + queried_knowledge.working_task_to_former_failed_knowledge_dict[target_factor_task_information] = ( + self.knowledgebase.implementation_trace.setdefault( + target_factor_task_information, + [], + )[-v1_query_former_trace_limit:] + ) - knowledge_base_success_task_list = list( - self.knowledgebase.success_task_info_set, - ) - similarity = calculate_embedding_distance_between_str_list( - [target_factor_task_information], - knowledge_base_success_task_list, - )[0] - similar_indexes = sorted( - range(len(similarity)), - key=lambda i: similarity[i], - reverse=True, - )[:v1_query_similar_success_limit] - similar_successful_knowledge = [ - self.knowledgebase.implementation_trace.setdefault( - knowledge_base_success_task_list[index], - [], - )[-1] - for index in similar_indexes - ] - queried_knowledge.working_task_to_similar_successful_knowledge_dict[ - target_factor_task_information - ] = similar_successful_knowledge + knowledge_base_success_task_list = list( + self.knowledgebase.success_task_info_set, + ) + similarity = calculate_embedding_distance_between_str_list( + [target_factor_task_information], + knowledge_base_success_task_list, + )[0] + similar_indexes = sorted( + range(len(similarity)), + key=lambda i: similarity[i], + reverse=True, + )[:v1_query_similar_success_limit] + similar_successful_knowledge = [ + self.knowledgebase.implementation_trace.setdefault( + knowledge_base_success_task_list[index], + [], + )[-1] + for index in similar_indexes + ] + queried_knowledge.working_task_to_similar_successful_knowledge_dict[ + target_factor_task_information + ] = similar_successful_knowledge return queried_knowledge @@ -889,7 +888,7 @@ class FactorImplementationGraphKnowledgeBase(KnowledgeBase): for possible_combination in possible_combinations: node_list = list(possible_combination) intersection_node_list.extend( - self.graph.get_nodes_intersection(node_list, steps=steps, constraint_labels=constraint_labels) + self.graph.get_nodes_intersection(node_list, steps=steps, constraint_labels=constraint_labels), ) if output_intersection_origin: for _ in range(len(intersection_node_list)): diff --git a/rdagent/factor_implementation/share_modules/evaluator.py b/rdagent/factor_implementation/share_modules/evaluator.py index 0596525a..5f985c46 100644 --- a/rdagent/factor_implementation/share_modules/evaluator.py +++ b/rdagent/factor_implementation/share_modules/evaluator.py @@ -3,10 +3,9 @@ from abc import ABC, abstractmethod from typing import Tuple import pandas as pd +from finco.log import FinCoLog from jinja2 import Template -from oai.llm_utils import APIBackend -from finco.log import FinCoLog from factor_implementation.share_modules.conf import FactorImplementSettings from factor_implementation.share_modules.factor import ( FactorImplementation, @@ -15,6 +14,7 @@ from factor_implementation.share_modules.factor import ( from factor_implementation.share_modules.prompt import ( FactorImplementationPrompts, ) +from oai.llm_utils import APIBackend class Evaluator(ABC): @@ -454,7 +454,7 @@ class FactorImplementationValueEvaluator(Evaluator): ) except Exception as e: - FinCoLog().warning(f"Error occurred when calculating the correlation: {str(e)}") + FinCoLog().warning(f"Error occurred when calculating the correlation: {e!s}") conclusions.append( f"Some error occurred when calculating the correlation. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent. Error: {e}", ) diff --git a/rdagent/factor_implementation/share_modules/factor.py b/rdagent/factor_implementation/share_modules/factor.py index 1a37ca38..1b81af93 100644 --- a/rdagent/factor_implementation/share_modules/factor.py +++ b/rdagent/factor_implementation/share_modules/factor.py @@ -7,15 +7,15 @@ from typing import Tuple, Union import pandas as pd from filelock import FileLock - -from oai.llm_utils import md5_hash from finco.log import FinCoLog + from factor_implementation.share_modules.conf import FactorImplementSettings from factor_implementation.share_modules.exception import ( CodeFormatException, NoOutputException, RuntimeErrorException, ) +from oai.llm_utils import md5_hash class FactorImplementationTask: @@ -122,7 +122,7 @@ class FileBasedFactorImplementation(FactorImplementation): raise ValueError(self.FB_CODE_NOT_SET) with FileLock(self.workspace_path / "execution.lock"): (Path.cwd() / "git_ignore_folder" / "factor_implementation_execution_cache").mkdir( - exist_ok=True, parents=True + exist_ok=True, parents=True, ) if FactorImplementSettings().enable_execution_cache: # NOTE: cache the result for the same code diff --git a/rdagent/factor_implementation/share_modules/prompt.py b/rdagent/factor_implementation/share_modules/prompt.py index 765c5d94..33cea23b 100644 --- a/rdagent/factor_implementation/share_modules/prompt.py +++ b/rdagent/factor_implementation/share_modules/prompt.py @@ -2,7 +2,6 @@ from pathlib import Path from typing import Dict import yaml - from finco.utils import SingletonBaseClass diff --git a/rdagent/knowledge_management/graph.py b/rdagent/knowledge_management/graph.py index 1031df60..b3f4bf19 100644 --- a/rdagent/knowledge_management/graph.py +++ b/rdagent/knowledge_management/graph.py @@ -98,11 +98,11 @@ class Graph: embeddings = [] for i in range(0, len(contents), size): embeddings.extend( - APIBackend().create_embedding(input_content=contents[i : i + size]) + APIBackend().create_embedding(input_content=contents[i : i + size]), ) assert len(nodes) == len( - embeddings + embeddings, ), "nodes' length must equals embeddings' length" for node, embedding in zip(nodes, embeddings): node.embedding = embedding @@ -238,7 +238,7 @@ class UndirectedGraph(Graph): result.append(node) for neighbor in sorted( - list(self.get_node(node.id).neighbors), key=lambda x: x.content + list(self.get_node(node.id).neighbors), key=lambda x: x.content, ): # to make sure the result is deterministic if neighbor not in visited: if not (block and neighbor.label not in constraint_labels): @@ -275,12 +275,12 @@ class UndirectedGraph(Graph): for node in nodes: if intersection is None: intersection = self.get_nodes_within_steps( - node, steps=steps, constraint_labels=constraint_labels + node, steps=steps, constraint_labels=constraint_labels, ) intersection = self.intersection( nodes1=intersection, nodes2=self.get_nodes_within_steps( - node, steps=steps, constraint_labels=constraint_labels + node, steps=steps, constraint_labels=constraint_labels, ), ) @@ -393,7 +393,7 @@ class UndirectedGraph(Graph): res_list = [] for query in content: similar_nodes = self.semantic_search( - content=query, topk_k=topk_k, similarity_threshold=similarity_threshold + content=query, topk_k=topk_k, similarity_threshold=similarity_threshold, ) connected_nodes = [] @@ -411,13 +411,13 @@ class UndirectedGraph(Graph): node for node in graph_query_node_res if node not in connected_nodes - ] + ], ) if len(connected_nodes) >= topk_k: break res_list.extend( - [node for node in connected_nodes[:topk_k] if node not in res_list] + [node for node in connected_nodes[:topk_k] if node not in res_list], ) return res_list @@ -455,7 +455,7 @@ def graph_to_edges(graph: Dict[str, List[str]]): def assign_random_coordinate_to_node( - nodes: List, scope: float = 1.0, origin: Tuple = (0.0, 0.0) + nodes: List, scope: float = 1.0, origin: Tuple = (0.0, 0.0), ) -> Dict: coordinates = {} @@ -468,7 +468,7 @@ def assign_random_coordinate_to_node( def assign_isometric_coordinate_to_node( - nodes: List, x_step: float = 1.0, x_origin: float = 0.0, y_origin: float = 0.0 + nodes: List, x_step: float = 1.0, x_origin: float = 0.0, y_origin: float = 0.0, ) -> Dict: coordinates = {} @@ -481,7 +481,7 @@ def assign_isometric_coordinate_to_node( def curly_node_coordinate( - coordinates: Dict, center_y: float = 1.0, r: float = 1.0 + coordinates: Dict, center_y: float = 1.0, r: float = 1.0, ) -> Dict: # noto: this method can only curly < 90 degree, and the curl line is circle. # the original funtion is: x**2 + (y-m)**2 = r**2 diff --git a/rdagent/oai/llm_utils.py b/rdagent/oai/llm_utils.py index 73eecf8f..2e6b5e1e 100644 --- a/rdagent/oai/llm_utils.py +++ b/rdagent/oai/llm_utils.py @@ -15,11 +15,9 @@ from typing import List, Optional, Tuple, Union import numpy as np import tiktoken -from scipy.spatial.distance import cosine - -from core.conf import FincoSettings as Config -from core.log import FinCoLog, LogColors -from core.utils import SingletonBaseClass +from rdagent.core.conf import FincoSettings as Config +from rdagent.core.log import FinCoLog, LogColors +from rdagent.core.utils import SingletonBaseClass DEFAULT_QLIB_DOT_PATH = Path("./") @@ -65,7 +63,6 @@ class ConvManager: if m is not None: n = int(m.group(1)) pairs.append((n, f)) - pass pairs.sort(key=lambda x: x[0]) for n, f in pairs[: self.recent_n][::-1]: if Path(self.path / f"{n+1}.json").exists(): @@ -92,7 +89,7 @@ class SQliteLazyCache(SingletonBaseClass): md5_key TEXT PRIMARY KEY, chat TEXT ) - """ + """, ) self.c.execute( """ @@ -100,7 +97,7 @@ class SQliteLazyCache(SingletonBaseClass): md5_key TEXT PRIMARY KEY, embedding TEXT ) - """ + """, ) self.conn.commit() @@ -186,7 +183,7 @@ class ChatSession: { "role": "user", "content": user_prompt, - } + }, ) return messages @@ -202,13 +199,13 @@ class ChatSession: messages = self.build_chat_completion_message(user_prompt, **kwargs) response = self.api_backend._try_create_chat_completion_or_embedding( - messages=messages, chat_completion=True, **kwargs + messages=messages, chat_completion=True, **kwargs, ) messages.append( { "role": "assistant", "content": response, - } + }, ) SessionChatHistoryCache().message_set(self.conversation_id, messages) return response @@ -362,14 +359,14 @@ class APIBackend: { "role": "system", "content": system_prompt, - } + }, ] messages.extend(former_messages[-1 * self.cfg.max_past_message_include :]) messages.append( { "role": "user", "content": user_prompt, - } + }, ) return messages @@ -400,7 +397,7 @@ class APIBackend: elif isinstance(input_content, list): input_content_list = input_content resp = self._try_create_chat_completion_or_embedding( - input_content_list=input_content_list, embedding=True, **kwargs + input_content_list=input_content_list, embedding=True, **kwargs, ) if isinstance(input_content, str): return resp[0] @@ -421,7 +418,7 @@ class APIBackend: { "role": "user", "content": "continue the former output with no overlap", - } + }, ) new_response, finish_reason = self._create_chat_completion_inner_function(messages=new_message, **kwargs) return response + new_response @@ -563,9 +560,9 @@ class APIBackend: "do_sample": self.gcr_endpoint_do_sample, "max_new_tokens": self.gcr_endpoint_max_token, }, - } - } - ) + }, + }, + ), ) req = urllib.request.Request(self.gcr_endpoint, body, self.headers)