gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-07-27 15:37:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

add CI fix tool to app (#10)

Browse Source 
* add CI fix tool to app

* perform ruff safe fix

* wrap too long lines in prompts.py
This commit is contained in:
XianBW
2024-05-30 10:33:07 +08:00
committed by GitHub
parent c6833b0858
commit 62a2f7a742
24 changed files with 729 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+2 -2
View File
@@ -93,14 +93,14 @@ mypy:
# Check lint with ruff.
ruff:
$(PIPRUN) python -m ruff . --exclude FinCo,finco,rdagent/scripts,test/scripts,git_ignore_folder --ignore ANN101,ANN401,TCH003,D,ERA001,PLR0913,S101 --line-length 120
$(PIPRUN) ruff check . --exclude FinCo,finco,rdagent/scripts,test/scripts,git_ignore_folder
# Check lint with toml-sort.
toml-sort:
$(PIPRUN) toml-sort --check pyproject.toml
# Check lint with all linters.
lint: black isort mypy ruff toml-sort
lint: mypy ruff toml-sort
# Run pre-commit with autofix against all files.
pre-commit:
constraints/3.10.txt
+1 -1
View File
@@ -111,7 +111,7 @@ rfc3986==2.0.0
rich==13.7.0
ruamel.yaml==0.18.5
ruamel.yaml.clib==0.2.8
ruff==0.1.9
ruff==0.4.5
scipy==1.11.4
SecretStorage==3.3.3
semver==3.0.2
constraints/3.11.txt
+1 -1
View File
@@ -109,7 +109,7 @@ rfc3986==2.0.0
rich==13.7.0
ruamel.yaml==0.18.5
ruamel.yaml.clib==0.2.8
ruff==0.1.9
ruff==0.4.5
scipy==1.11.4
SecretStorage==3.3.3
semver==3.0.2
constraints/3.8.txt
+1 -1
View File
@@ -112,7 +112,7 @@ rfc3986==2.0.0
rich==13.7.0
ruamel.yaml==0.18.5
ruamel.yaml.clib==0.2.8
ruff==0.1.9
ruff==0.4.5
scipy==1.10.1
SecretStorage==3.3.3
semver==3.0.2
constraints/3.9.txt
+1 -1
View File
@@ -111,7 +111,7 @@ rfc3986==2.0.0
rich==13.7.0
ruamel.yaml==0.18.5
ruamel.yaml.clib==0.2.8
ruff==0.1.9
ruff==0.4.5
scipy==1.11.4
SecretStorage==3.3.3
semver==3.0.2
pyproject.toml
+20 -24
View File
@@ -73,32 +73,28 @@ log_format = "%(asctime)s %(levelname)s %(message)s"
minversion = "6.0"
[tool.ruff]
fix = true
ignore = [
# https://docs.astral.sh/ruff/rules/#pydocstyle-d
"D203",
"D204",
"D213",
"D215",
"D400",
"D404",
"D406",
"D407",
"D408",
"D409",
"D413",
"ANN101",
"ANN401",
"TCH003",
"D",
"ERA001",
"PLR0913",
"S101",
]
select = ["ALL"]
line-length = 120
src = ["rdagent"]
[tool.ruff.per-file-ignores]
[tool.ruff.lint]
ignore = [
# https://docs.astral.sh/ruff/rules/#pydocstyle-d
"ANN101",
"ANN401",
"D",
"ERA001",
"FIX",
"INP001",
"PGH",
"PLR0913",
"S101",
"T20",
"TCH003",
"TD",
]
select = ["ALL"]
[tool.ruff.lint.per-file-ignores]
"docs/conf.py" = ["INP001"]
"test/*" = ["S101"]
rdagent/app/CI/prompts.py
+126
View File
@@ -0,0 +1,126 @@
linting_system_prompt_template = "You are a software engineer. \
You can write code to a high standard and are adept at solving {language} linting problems."
user_get_makefile_lint_commands_template = """
You get a Makefile which contains some linting rules. Here are its content ```{file_text}```
Please find executable commands about linting from it.
Please response with following json template:
{{
"commands": <python -m xxx --params >,
}}
"""
user_get_files_contain_lint_commands_template = """
You get a file list of a repository. \
Some file maybe contain linting rules or linting commands which defined by repo authors.
Here are the file list:
```
{file_list}
```
Please find all files maybe correspond to linting from it.
Please response with following json template:
{{
"files": </path/to/file>,
}}
"""
generate_lint_command_template = """
Please generate a command to lint or format a {language} repository.
Here are some information about different linting tools ```{linting_tools}```
"""
suffix2language_template = """
Here are the files suffix in one code repo: {suffix}.
Please tell me the programming language used in this repo and which language has linting-tools.
Your response should follow this template:
{{
"languages": <languages list>,
"languages_with_linting_tools": <languages with lingting tools list>
}}
"""
session_start_template = """
Please modify the Python code based on the lint info.
Due to the length of the code, I will first tell you the entire code, and then each time I ask a question, \
I will extract a portion of the code and tell you the error information contained in this code segment.
You need to fix the corresponding error in the code segment \
and return the code that can replace the corresponding code segment.
The Python code is from a complete Python project file. Each line of the code is annotated with a line number, \
separated from the original code by three characters ("<white space>|<white space>"). The vertical bars are aligned.
Here is the complete code, please be prepared to fix it:
```Python
{code}
```
"""
session_normal_template = """Please modify this code snippet based on the lint info. Here is the code snippet:
```Python
{code}
```
-----Lint info-----
{lint_info}
-------------------
The lint info contains one or more errors. \
Different errors are separated by blank lines. Each error follows this format:
-----Lint info format-----
<Line Number>:<Error Start Position> <Error Code> <Error Message>
<Error Context (multiple lines)>
<Helpful Information (last line)>
--------------------------
The error code is an abbreviation set by the checker for ease of describing the error. \
The error context includes the relevant code around the error, and the helpful information suggests possible fixes.
Please simply reply the code after you fix all linting errors.
The code you return does not require line numbers, \
and should just replace the code I provided you, and does not require comments.
Please wrap your code with following format:
```python
<your code..>
```
"""
user_template_for_code_snippet = """Please modify the Python code based on the lint info.
-----Python Code-----
{code}
---------------------
-----Lint info-----
{lint_info}
-------------------
The Python code is a snippet from a complete Python project file. \
Each line of the code is annotated with a line number, \
separated from the original code by three characters ("<white space>|<white space>"). \
The vertical bars are aligned.
The lint info contains one or more errors. Different errors are separated by blank lines. \
Each error follows this format:
-----Lint info format-----
<Line Number>:<Error Start Position> <Error Code> <Error Message>
<Error Context (multiple lines)>
<Helpful Information (last line)>
--------------------------
The error code is an abbreviation set by the checker for ease of describing the error. \
The error context includes the relevant code around the error, and the helpful information suggests possible fixes.
Please simply reply the code after you fix all linting errors.
The code you return does not require line numbers, \
and should just replace the code I provided you, and does not require comments.
Please wrap your code with following format:
```python
<your code..>
```
"""
rdagent/app/CI/run.py
+426
View File
@@ -0,0 +1,426 @@
"""
"""
import json
import re
import subprocess
import time
from collections import defaultdict
from dataclasses import dataclass
from difflib import IS_LINE_JUNK, ndiff
from pathlib import Path
from typing import Dict, List, Tuple, Union, cast
from rdagent.core.evolving_framework import (
Evaluator,
EvoAgent,
EvolvableSubjects,
EvolvingStrategy,
EvoStep,
Feedback,
Knowledge,
)
from rdagent.oai.llm_utils import APIBackend
from rich import print
from rich.panel import Panel
from rich.prompt import Prompt
from rich.rule import Rule
from rich.syntax import Syntax
from rich.table import Table
from rich.text import Text
from .prompts import (
linting_system_prompt_template,
session_normal_template,
session_start_template,
)
@dataclass
class CIError:
raw_str: str
file_path: Union[Path, str]
line: int
column: int
code: str
msg: str
hint: str
@dataclass
class CIFeedback(Feedback):
errors: Dict[str, List[CIError]]
@dataclass
class FixRecord:
skipped_errors: List[CIError]
directly_fixed_errors: List[CIError]
manually_fixed_errors: List[CIError]
class CodeFile:
def __init__(self, path: Union[Path, str]):
self.path = Path(path)
self.load()
def load(self) -> None:
code = self.path.read_text(encoding="utf-8")
self.code_lines = code.split("\n")
# add line number
self.lineno = len(self.code_lines)
self.lineno_width = len(str(self.lineno))
self.code_lines_with_lineno = []
for i, code_line in enumerate(self.code_lines):
self.code_lines_with_lineno.append(f"{i+1: >{self.lineno_width}} | {code_line}")
def get(self, start = 0, end = None, add_line_number: bool = False, return_list: bool = False) -> Union[List[str], str]:
start -= 1
if start < 0: start = 0
end = self.lineno if end is None else end-1
res = self.code_lines_with_lineno[start:end] if add_line_number else self.code_lines[start:end]
return res if return_list else "\n".join(res)
def apply_changes(self, changes: List[Tuple[int, int, str]]) -> None:
offset = 0
for start, end, code in changes:
start -= 1
if start < 0: start = 0
end -= 1
new_code = code.split("\n")
self.code_lines[start+offset:end+offset] = new_code
offset += len(new_code) - (end - start)
self.path.write_text("\n".join(self.code_lines), encoding="utf-8")
self.load()
def __str__(self):
return f"{self.path}"
class Repo(EvolvableSubjects):
def __init__(self, project_path: Union[Path, str], **kwargs):
self.params = kwargs
self.project_path = Path(project_path)
git_ignored_output = subprocess.check_output(
"git status --ignored -s",
shell=True,
cwd=project_path,
stderr=subprocess.STDOUT,
).decode("utf-8")
git_ignored_files = [
(self.project_path / Path(line[3:])).resolve()
for line in git_ignored_output.split("\n")
if line.startswith("!!")
]
files = [
file
for file in self.project_path.glob("**/*")
if file.is_file()
and not any(str(file).startswith(str(path)) for path in git_ignored_files)
and ".git/" not in str(file)
and file.suffix == ".py"
]
self.files = {file: CodeFile(file) for file in files}
self.fix_records: Dict[str, FixRecord] | None = None
@dataclass
class RuffRule:
"""
{
"name": "missing-trailing-comma",
"code": "COM812",
"linter": "flake8-commas",
"summary": "Trailing comma missing",
"message_formats": [
"Trailing comma missing"
],
"fix": "Fix is always available.",
"explanation": "## What it does\nChecks for the absence of trailing commas.\n\n## Why is this bad?\nThe presence of a trailing comma can reduce diff size when parameters or\nelements are added or removed from function calls, function definitions,\nliterals, etc.\n\n## Example\n```python\nfoo = {\n \"bar\": 1,\n \"baz\": 2\n}\n```\n\nUse instead:\n```python\nfoo = {\n \"bar\": 1,\n \"baz\": 2,\n}\n```\n",
"preview": false
}
"""
name: str
code: str
linter: str
summary: str
message_formats: List[str]
fix: str
explanation: str
preview: bool
class RuffEvaluator(Evaluator):
"""The error message are generated by
`python -m ruff . --exclude FinCo,finco,fincov1 --ignore ANN101,TCH003,D,ERA001`
"""
def __init__(self, command: str = None):
if command is None:
self.command = "ruff check . --no-fix --output-format full"
else:
self.command = command
def explain_rule(self, error_code: str) -> RuffRule:
explain_command = "ruff rule {error_code} --output-format json"
try:
out = subprocess.check_output(
explain_command.format(error_code=error_code),
shell=True,
stderr=subprocess.STDOUT,
)
except subprocess.CalledProcessError as e:
out = e.output
return json.loads(out.decode())
def evaluate(self, evo: Repo, **kwargs) -> CIFeedback:
"""Simply run ruff to get the feedbacks."""
try:
out = subprocess.check_output(
self.command,
shell=True,
cwd=evo.project_path,
stderr=subprocess.STDOUT,
)
except subprocess.CalledProcessError as e:
out = e.output
"""ruff output format:
src/finco/cli.py:9:5: ANN201 Missing return type annotation for public function `main`
|
9 | def main(prompt=None):
| ^^^^ ANN201
10 | load_dotenv(verbose=True, override=True)
11 | wm = WorkflowManager()
|
= help: Add return type annotation: `None`
"""
# extract error info
pattern = r"(([^\n]*):(\d+):(\d+): (\w+) ([^\n]*)\n(.*?))\n\n"
matches = re.findall(pattern, out.decode(), re.DOTALL)
errors = defaultdict(list)
for match in matches:
raw_str, file_path, line_number, column_number, error_code, error_message, error_hint = match
error = CIError(raw_str=raw_str,
file_path=file_path,
line=int(line_number),
column=int(column_number),
code=error_code,
msg=error_message,
hint=error_hint)
errors[file_path].append(error)
return CIFeedback(errors=errors)
class MypyEvaluator(Evaluator):
def __init__(self, command: str = None):
if command is None:
self.command = "mypy . --explicit-package-bases"
else:
self.command = command
def evaluate(self, evo: Repo, **kwargs) -> CIFeedback:
try:
out = subprocess.check_output(
self.command,
shell=True,
cwd=evo.project_path,
stderr=subprocess.STDOUT,
)
except subprocess.CalledProcessError as e:
out = e.output
return CIFeedback(cast(str, out).decode("utf-8"))
class CIEvoStr(EvolvingStrategy):
def evolve(
self,
evo: Repo,
evolving_trace: List[EvoStep] = [],
knowledge_l: List[Knowledge] = [],
**kwargs,
) -> Repo:
api = APIBackend()
system_prompt = linting_system_prompt_template.format(language="Python")
if len(evolving_trace) > 0:
last_feedback: CIFeedback = evolving_trace[-1].feedback
fix_records: Dict[str, FixRecord] = defaultdict(lambda: FixRecord([], [], []))
# iterate by file
for file_path, errors in last_feedback.errors.items():
print(Rule(f"[cyan]Fixing {file_path}[/cyan]", style="bold cyan", align="left", characters="."))
file = evo.files[evo.project_path / Path(file_path)]
# Group errors based on position
# TODO @bowen: Crossover between different groups after adding 3 lines of context
groups: List[List[CIError]] = []
near_errors = [errors[0]]
for error in errors[1:]:
if error.line - near_errors[-1].line <= 6:
near_errors.append(error)
else:
groups.append(near_errors)
near_errors = [error]
groups.append(near_errors)
changes = []
# generate changes
for group_id, group in enumerate(groups, start=1):
session = api.build_chat_session(session_system_prompt=system_prompt)
session.build_chat_completion(session_start_template.format(code=file.get(add_line_number=True)))
print(f"[yellow]Fixing part {group_id}...[/yellow]\n")
start_line = group[0].line - 3
end_line = group[-1].line + 3 + 1
code_snippet_with_lineno = file.get(start_line, end_line, add_line_number=True, return_list=False)
code_snippet_lines = file.get(start_line, end_line, add_line_number=False, return_list=True)
# front_anchor_code = file.get(start_line-3, start_line, add_line_number=False, return_list=False)
# rear_anchor_code = file.get(end_line+1, end_line+3+1, add_line_number=False, return_list=False)
errors_str = "\n".join([f"{error.raw_str}\n" for error in group])
print(Panel.fit(Syntax("\n".join([f"{error.line}: {error.msg}" for error in group]), lexer="python", background_color="default"), title=f"{len(group)} Errors"))
# print(f"[bold yellow]original code:[/bold yellow]\n\n{code_snippet_with_lineno}")
print(Panel.fit(Syntax(code_snippet_with_lineno, lexer="python", background_color="default"), title="Original Code"))
user_prompt = session_normal_template.format(
code=code_snippet_with_lineno,
lint_info=errors_str,
)
res = session.build_chat_completion(user_prompt)
manual_fix_flag = False
while True:
new_code = re.search(r".*```[Pp]ython\n(.*)\n```.*", res, re.DOTALL).group(1)
# print repair status (code diff)
diff = ndiff(code_snippet_lines, new_code.split("\n"), linejunk=IS_LINE_JUNK)
table = Table(show_header=False, box=None)
table.add_column()
for i in diff:
if i.startswith("+"): table.add_row(Text(i, style="green"))
elif i.startswith("-"): table.add_row(Text(i, style="red"))
elif i.startswith("?"): table.add_row(Text(i, style="yellow"))
else: table.add_row(Syntax(i, lexer="python", background_color="default"))
print(Panel.fit(table, title="Repair Status"))
operation = input("Input your operation: ")
if operation == "s" or operation == "skip":
fix_records[file_path].skipped_errors.extend(group)
break
if operation == "a" or operation == "apply":
if manual_fix_flag:
fix_records[file_path].manually_fixed_errors.extend(group)
else:
fix_records[file_path].directly_fixed_errors.extend(group)
changes.append((start_line, end_line, new_code))
break
manual_fix_flag = True
res = session.build_chat_completion(operation)
# apply changes
file.apply_changes(changes)
evo.fix_records = fix_records
return evo
# DIR = "/home/bowen/workspace/fincov2_test/"
DIR = "/home/bowen/workspace/RD-Agent/"
PY = "/home/bowen/miniconda3/envs/cr/bin/python"
start_time = time.time()
evo = Repo(DIR, python_path=PY)
eval = RuffEvaluator()
estr = CIEvoStr()
rag = None # RAG is not enable firstly.
ea = EvoAgent(estr, rag=rag)
ea.step_evolving(evo, eval)
while True:
print(Rule(f"Round {len(ea.evolving_trace)} repair", style="blue"))
evo: Repo = ea.step_evolving(evo, eval)
fix_records = evo.fix_records
# Count the number of skipped errors
skipped_errors_count = 0
directly_fixed_errors_count = 0
manually_fixed_errors_count = 0
skipped_errors_code_count = defaultdict(int)
directly_fixed_errors_code_count = defaultdict(int)
manually_fixed_errors_code_count = defaultdict(int)
for record in fix_records.values():
skipped_errors_count += len(record.skipped_errors)
directly_fixed_errors_count += len(record.directly_fixed_errors)
manually_fixed_errors_count += len(record.manually_fixed_errors)
for error in record.skipped_errors:
skipped_errors_code_count[error.code] += 1
for error in record.directly_fixed_errors:
directly_fixed_errors_code_count[error.code] += 1
for error in record.manually_fixed_errors:
manually_fixed_errors_code_count[error.code] += 1
skipped_errors_statistics = ""
directly_fixed_errors_statistics = ""
manually_fixed_errors_statistics = ""
for code, count in sorted(skipped_errors_code_count.items(), key=lambda x: x[1], reverse=True):
skipped_errors_statistics += f"{count: >5} {code: >10} {eval.explain_rule(code).summary}\n"
for code, count in sorted(directly_fixed_errors_code_count.items(), key=lambda x: x[1], reverse=True):
directly_fixed_errors_statistics += f"{count: >5} {code: >10} {eval.explain_rule(code).summary}\n"
for code, count in sorted(manually_fixed_errors_code_count.items(), key=lambda x: x[1], reverse=True):
manually_fixed_errors_statistics += f"{count: >5} {code: >10} {eval.explain_rule(code).summary}\n"
# Create a table to display the counts and ratios
table = Table(title="Error Fix Statistics")
table.add_column("Type")
table.add_column("Statistics")
table.add_column("Count")
table.add_column("Ratio")
total_errors_count = skipped_errors_count + directly_fixed_errors_count + manually_fixed_errors_count
table.add_row("Total Errors", "", str(total_errors_count), "")
table.add_row("Skipped Errors", skipped_errors_statistics, str(skipped_errors_count), f"{skipped_errors_count / total_errors_count:.2%}")
table.add_row("Directly Fixed Errors", directly_fixed_errors_statistics, str(directly_fixed_errors_count), f"{directly_fixed_errors_count / total_errors_count:.2%}")
table.add_row("Manually Fixed Errors", manually_fixed_errors_statistics, str(manually_fixed_errors_count), f"{manually_fixed_errors_count / total_errors_count:.2%}")
print(table)
operation = Prompt.ask("Start next round? (y/n): ", choices=["y", "n"])
if operation == "n": break
end_time = time.time()
execution_time = end_time - start_time
print(f"Execution time: {execution_time} seconds")
""" Please commit it by hand... and then run the next round
git add -u
git commit --no-verify -v
"""
rdagent/app/factor_extraction_and_implementation/factor_extract_and_implement.py
+15 -13
View File
@@ -1,14 +1,16 @@
# %%
from document_process.document_reader import load_and_process_pdfs_by_langchain, classify_report_from_dict
from dotenv import load_dotenv
from oai.llm_utils import APIBackend
from pathlib import Path
import json
from pathlib import Path
from dotenv import load_dotenv
from document_process.document_analysis import extract_factors_from_report_dict_and_classify_result
from document_process.document_analysis import check_factor_dict_viability
from document_process.document_analysis import deduplicate_factors_several_times
from document_process.document_analysis import (
check_factor_dict_viability,
deduplicate_factors_several_times,
extract_factors_from_report_dict_and_classify_result,
)
from document_process.document_reader import classify_report_from_dict, load_and_process_pdfs_by_langchain
from oai.llm_utils import APIBackend
def extract_factors_and_implement(report_file_path: str):
@@ -29,7 +31,7 @@ def extract_factors_and_implement(report_file_path: str):
for factor_name in factor_dict:
if len(factor_dict[factor_name]) > 1:
factor_dict_simple_deduplication[factor_name] = max(
factor_dict[factor_name], key=lambda x: len(x["formulation"])
factor_dict[factor_name], key=lambda x: len(x["formulation"]),
)
else:
factor_dict_simple_deduplication[factor_name] = factor_dict[factor_name][0]
@@ -115,11 +117,11 @@ def extract_factors_and_implement(report_file_path: str):
for factor_name in target_dict:
formulation = target_dict[factor_name]["formulation"]
if factor_name in formulation:
target_factor_name = factor_name.replace("_", "\_")
target_factor_name = factor_name.replace("_", r"\_")
formulation = formulation.replace(factor_name, target_factor_name)
for variable in target_dict[factor_name]["variables"]:
if variable in formulation:
target_variable = variable.replace("_", "\_")
target_variable = variable.replace("_", r"\_")
formulation = formulation.replace(variable, target_variable)
fw.write(f"## {current_index}. 因子名称：{factor_name}\n")
@@ -130,9 +132,9 @@ def extract_factors_and_implement(report_file_path: str):
fw.write(f"### formulation string: {formulation}\n")
# write a table of variable and its description
fw.write(f"### variable tables: \n")
fw.write(f"| variable | description |\n")
fw.write(f"| -------- | ----------- |\n")
fw.write("### variable tables: \n")
fw.write("| variable | description |\n")
fw.write("| -------- | ----------- |\n")
for variable in target_dict[factor_name]["variables"]:
fw.write(f"| {variable} | {target_dict[factor_name]['variables'][variable]} |\n")
rdagent/core/conf.py
+45
View File
@@ -20,10 +20,12 @@ class FincoSettings(BaseSettings):
use_chat_cache: bool = False
dump_embedding_cache: bool = False
use_embedding_cache: bool = False
workspace: str = "./finco_workspace"
prompt_cache_path: str = os.getcwd() + "/prompt_cache.db"
session_cache_folder_location: str = os.getcwd() + "/session_cache_folder/"
max_past_message_include: int = 10
use_vector_only: bool = False
log_llm_chat_content: bool = True
# Chat configs
@@ -45,3 +47,46 @@ class FincoSettings(BaseSettings):
embedding_azure_api_base: str = ""
embedding_azure_api_version: str = ""
embedding_model: str = ""
# llama2 related config
use_llama2: bool = False
llama2_ckpt_dir: str = "Llama-2-7b-chat"
llama2_tokenizer_path: str = "Llama-2-7b-chat/tokenizer.model"
llams2_max_batch_size: int = 8
# finco v2 configs
azure_document_intelligence_key: str = ""
azure_document_intelligence_endpoint: str = ""
# fincov2 llama2 endpoint
use_gcr_endpoint: bool = False
gcr_endpoint_type: str = "llama2_70b" # or "llama3_70b", "phi2", "phi3_4k", "phi3_128k"
llama2_70b_endpoint: str = ""
llama2_70b_endpoint_key: str = ""
llama2_70b_endpoint_deployment: str = ""
llama3_70b_endpoint: str = ""
llama3_70b_endpoint_key: str = ""
llama3_70b_endpoint_deployment: str = ""
phi2_endpoint: str = ""
phi2_endpoint_key: str = ""
phi2_endpoint_deployment: str = ""
phi3_4k_endpoint: str = ""
phi3_4k_endpoint_key: str = ""
phi3_4k_endpoint_deployment: str = ""
phi3_128k_endpoint: str = ""
phi3_128k_endpoint_key: str = ""
phi3_128k_endpoint_deployment: str = ""
gcr_endpoint_temperature: float = 0.7
gcr_endpoint_top_p: float = 0.9
gcr_endpoint_do_sample: bool = False
gcr_endpoint_max_token: int = 100
# factor extraction conf
max_input_duplicate_factor_group: int = 600
max_output_duplicate_factor_group: int = 20
rdagent/core/log.py
+1 -3
View File
@@ -1,11 +1,9 @@
from __future__ import annotations
from contextlib import contextmanager
from typing import TYPE_CHECKING, Generator, Sequence
from typing import TYPE_CHECKING, Sequence
from loguru import logger
if TYPE_CHECKING:
from loguru import Logger
rdagent/core/utils.py
+6 -8
View File
@@ -1,20 +1,18 @@
from __future__ import annotations
import importlib
import json
import multiprocessing as mp
import os
import random
import string
import uuid
from collections.abc import Callable
from pathlib import Path
import importlib
from typing import Any
import yaml
from fuzzywuzzy import fuzz
import multiprocessing as mp
from collections.abc import Callable
class FincoException(Exception):
pass
@@ -87,9 +85,9 @@ def crawl_the_folder(folder_path: Path):
def compare_yaml(file1, file2):
with open(file1, "r") as stream:
with open(file1) as stream:
data1 = yaml.safe_load(stream)
with open(file2, "r") as stream:
with open(file2) as stream:
data2 = yaml.safe_load(stream)
return data1 == data2
@@ -107,7 +105,7 @@ class YamlConfigCache(SingletonBaseClass):
self.path_to_config = dict()
def load(self, path):
with open(path, "r") as stream:
with open(path) as stream:
data = yaml.safe_load(stream)
self.path_to_config[path] = data
rdagent/document_process/document_analysis.py
+4 -4
View File
@@ -18,8 +18,8 @@ from sklearn.metrics.pairwise import cosine_similarity
from sklearn.preprocessing import normalize
from core.conf import FincoSettings as Config
from oai.llm_utils import APIBackend, create_embedding_with_multiprocessing
from core.log import FinCoLog
from oai.llm_utils import APIBackend, create_embedding_with_multiprocessing
if TYPE_CHECKING:
from langchain_core.documents import Document
@@ -229,7 +229,7 @@ def __extract_factors_name_and_desc_from_content(
parse_success = False
if ret_json_str is None or not parse_success:
current_user_prompt = (
"Your response didn't follow the instruction" " might be wrong json format. Try again."
"Your response didn't follow the instruction might be wrong json format. Try again."
)
else:
factors = ret_dict["factors"]
@@ -273,7 +273,7 @@ def __extract_factors_formulation_from_content(
parse_success = False
if ret_json_str is None or not parse_success:
current_user_prompt = (
"Your response didn't follow the instruction" " might be wrong json format. Try again."
"Your response didn't follow the instruction might be wrong json format. Try again."
)
else:
for name, formulation_and_description in ret_dict.items():
@@ -393,7 +393,7 @@ def check_factor_dict_viability_simulate_json_mode(
parse_success = False
if ret_json_str is None or not parse_success:
current_user_prompt = (
"Your response didn't follow the " "instruction might be wrong json format. Try again."
"Your response didn't follow the instruction might be wrong json format. Try again."
)
else:
return ret_dict
rdagent/document_process/document_reader.py
-3
View File
@@ -5,10 +5,7 @@ from pathlib import Path
import yaml
from azure.ai.formrecognizer import DocumentAnalysisClient
from azure.core.credentials import AzureKeyCredential
from finco.conf import FincoSettings as Config
from oai.llm_utils import APIBackend, create_embedding_with_multiprocessing
from core.log import FinCoLog
if TYPE_CHECKING:
from langchain_core.documents import Document
rdagent/factor_implementation/evolving/evaluators.py
+1 -1
View File
@@ -8,6 +8,7 @@ from pandas.core.api import DataFrame as DataFrame
from core.evolving_framework import Evaluator as EvolvingEvaluator
from core.evolving_framework import Feedback, QueriedKnowledge
from core.log import FinCoLog
from core.utils import multiprocessing_wrapper
from factor_implementation.evolving.evolvable_subjects import (
FactorImplementationList,
)
@@ -24,7 +25,6 @@ from factor_implementation.share_modules.factor import (
FactorImplementation,
FactorImplementationTask,
)
from core.utils import multiprocessing_wrapper
class FactorImplementationSingleFeedback:
rdagent/factor_implementation/evolving/evolvable_subjects.py
-2
View File
@@ -1,7 +1,5 @@
from __future__ import annotations
import pandas as pd
from core.evolving_framework import EvolvableSubjects
from core.log import FinCoLog
from factor_implementation.share_modules.factor import (
rdagent/factor_implementation/evolving/evolving_strategy.py
+2 -2
View File
@@ -9,7 +9,7 @@ from typing import TYPE_CHECKING
from jinja2 import Template
from core.evolving_framework import EvolvingStrategy, QueriedKnowledge
from oai.llm_utils import APIBackend
from core.utils import multiprocessing_wrapper
from factor_implementation.share_modules.conf import FactorImplementSettings
from factor_implementation.share_modules.factor import (
FactorImplementation,
@@ -20,7 +20,7 @@ from factor_implementation.share_modules.prompt import (
FactorImplementationPrompts,
)
from factor_implementation.share_modules.utils import get_data_folder_intro
from core.utils import multiprocessing_wrapper
from oai.llm_utils import APIBackend
if TYPE_CHECKING:
from factor_implementation.evolving.evolvable_subjects import (
rdagent/factor_implementation/evolving/factor_implementation_evolving_cli.py
+4 -6
View File
@@ -8,6 +8,7 @@ from fire.core import Fire
from tqdm import tqdm
from core.evolving_framework import EvoAgent, KnowledgeBase
from core.utils import multiprocessing_wrapper
from factor_implementation.evolving.evaluators import (
FactorImplementationEvaluatorV1,
FactorImplementationsMultiEvaluator,
@@ -29,7 +30,6 @@ from factor_implementation.share_modules.factor import (
FactorImplementationTask,
FileBasedFactorImplementation,
)
from core.utils import multiprocessing_wrapper
ALPHA101_INIT_COMPONENTS = [
"1. abs(): absolute value to certain columns",
@@ -108,10 +108,8 @@ class FactorImplementationEvolvingCli:
if former_knowledge_base_path is not None and former_knowledge_base_path.exists():
factor_knowledge_base = pickle.load(open(former_knowledge_base_path, "rb"))
if self.evolving_version == 1 and not isinstance(
factor_knowledge_base, FactorImplementationKnowledgeBaseV1
):
raise ValueError("The former knowledge base is not compatible with the current version")
elif self.evolving_version == 2 and not isinstance(
factor_knowledge_base, FactorImplementationKnowledgeBaseV1,
) or self.evolving_version == 2 and not isinstance(
factor_knowledge_base,
FactorImplementationGraphKnowledgeBase,
):
@@ -261,7 +259,7 @@ class FactorImplementationEvolvingCli:
print([feedback.final_decision if feedback is not None else None for feedback in feedbacks].count(True))
def implement_amc(
self, evo_sub_path_str, former_knowledge_base_path_str, implementation_dump_path_str, slice_index
self, evo_sub_path_str, former_knowledge_base_path_str, implementation_dump_path_str, slice_index,
):
factor_implementations: FactorImplementationList = pickle.load(open(evo_sub_path_str, "rb"))
factor_implementations.target_factor_tasks = factor_implementations.target_factor_tasks[
rdagent/factor_implementation/evolving/knowledge_management.py
+41 -42
View File
@@ -8,6 +8,7 @@ from itertools import combinations
from pathlib import Path
from typing import Union
from finco.graph import UndirectedGraph, UndirectedNode
from jinja2 import Template
from core.evolving_framework import (
@@ -18,8 +19,6 @@ from core.evolving_framework import (
QueriedKnowledge,
RAGStrategy,
)
from finco.graph import UndirectedGraph, UndirectedNode
from oai.llm_utils import APIBackend, calculate_embedding_distance_between_str_list
from core.log import FinCoLog
from factor_implementation.evolving.evaluators import (
FactorImplementationSingleFeedback,
@@ -32,6 +31,7 @@ from factor_implementation.share_modules.factor import (
from factor_implementation.share_modules.prompt import (
FactorImplementationPrompts,
)
from oai.llm_utils import APIBackend, calculate_embedding_distance_between_str_list
class FactorImplementationKnowledge(Knowledge):
@@ -150,47 +150,46 @@ class FactorImplementationRAGStrategyV1(RAGStrategy):
queried_knowledge.success_task_to_knowledge_dict[target_factor_task_information] = (
self.knowledgebase.implementation_trace[target_factor_task_information][-1]
)
elif (
len(
self.knowledgebase.implementation_trace.setdefault(
target_factor_task_information,
[],
),
)
>= fail_task_trial_limit
):
queried_knowledge.failed_task_info_set.add(target_factor_task_information)
else:
if (
len(
self.knowledgebase.implementation_trace.setdefault(
target_factor_task_information,
[],
),
)
>= fail_task_trial_limit
):
queried_knowledge.failed_task_info_set.add(target_factor_task_information)
else:
queried_knowledge.working_task_to_former_failed_knowledge_dict[target_factor_task_information] = (
self.knowledgebase.implementation_trace.setdefault(
target_factor_task_information,
[],
)[-v1_query_former_trace_limit:]
)
queried_knowledge.working_task_to_former_failed_knowledge_dict[target_factor_task_information] = (
self.knowledgebase.implementation_trace.setdefault(
target_factor_task_information,
[],
)[-v1_query_former_trace_limit:]
)
knowledge_base_success_task_list = list(
self.knowledgebase.success_task_info_set,
)
similarity = calculate_embedding_distance_between_str_list(
[target_factor_task_information],
knowledge_base_success_task_list,
)[0]
similar_indexes = sorted(
range(len(similarity)),
key=lambda i: similarity[i],
reverse=True,
)[:v1_query_similar_success_limit]
similar_successful_knowledge = [
self.knowledgebase.implementation_trace.setdefault(
knowledge_base_success_task_list[index],
[],
)[-1]
for index in similar_indexes
]
queried_knowledge.working_task_to_similar_successful_knowledge_dict[
target_factor_task_information
] = similar_successful_knowledge
knowledge_base_success_task_list = list(
self.knowledgebase.success_task_info_set,
)
similarity = calculate_embedding_distance_between_str_list(
[target_factor_task_information],
knowledge_base_success_task_list,
)[0]
similar_indexes = sorted(
range(len(similarity)),
key=lambda i: similarity[i],
reverse=True,
)[:v1_query_similar_success_limit]
similar_successful_knowledge = [
self.knowledgebase.implementation_trace.setdefault(
knowledge_base_success_task_list[index],
[],
)[-1]
for index in similar_indexes
]
queried_knowledge.working_task_to_similar_successful_knowledge_dict[
target_factor_task_information
] = similar_successful_knowledge
return queried_knowledge
@@ -889,7 +888,7 @@ class FactorImplementationGraphKnowledgeBase(KnowledgeBase):
for possible_combination in possible_combinations:
node_list = list(possible_combination)
intersection_node_list.extend(
self.graph.get_nodes_intersection(node_list, steps=steps, constraint_labels=constraint_labels)
self.graph.get_nodes_intersection(node_list, steps=steps, constraint_labels=constraint_labels),
)
if output_intersection_origin:
for _ in range(len(intersection_node_list)):
rdagent/factor_implementation/share_modules/evaluator.py
+3 -3
View File
@@ -3,10 +3,9 @@ from abc import ABC, abstractmethod
from typing import Tuple
import pandas as pd
from finco.log import FinCoLog
from jinja2 import Template
from oai.llm_utils import APIBackend
from finco.log import FinCoLog
from factor_implementation.share_modules.conf import FactorImplementSettings
from factor_implementation.share_modules.factor import (
FactorImplementation,
@@ -15,6 +14,7 @@ from factor_implementation.share_modules.factor import (
from factor_implementation.share_modules.prompt import (
FactorImplementationPrompts,
)
from oai.llm_utils import APIBackend
class Evaluator(ABC):
@@ -454,7 +454,7 @@ class FactorImplementationValueEvaluator(Evaluator):
)
except Exception as e:
FinCoLog().warning(f"Error occurred when calculating the correlation: {str(e)}")
FinCoLog().warning(f"Error occurred when calculating the correlation: {e!s}")
conclusions.append(
f"Some error occurred when calculating the correlation. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent. Error: {e}",
)
rdagent/factor_implementation/share_modules/factor.py
+3 -3
View File
@@ -7,15 +7,15 @@ from typing import Tuple, Union
import pandas as pd
from filelock import FileLock
from oai.llm_utils import md5_hash
from finco.log import FinCoLog
from factor_implementation.share_modules.conf import FactorImplementSettings
from factor_implementation.share_modules.exception import (
CodeFormatException,
NoOutputException,
RuntimeErrorException,
)
from oai.llm_utils import md5_hash
class FactorImplementationTask:
@@ -122,7 +122,7 @@ class FileBasedFactorImplementation(FactorImplementation):
raise ValueError(self.FB_CODE_NOT_SET)
with FileLock(self.workspace_path / "execution.lock"):
(Path.cwd() / "git_ignore_folder" / "factor_implementation_execution_cache").mkdir(
exist_ok=True, parents=True
exist_ok=True, parents=True,
)
if FactorImplementSettings().enable_execution_cache:
# NOTE: cache the result for the same code
rdagent/factor_implementation/share_modules/prompt.py
-1
View File
@@ -2,7 +2,6 @@ from pathlib import Path
from typing import Dict
import yaml
from finco.utils import SingletonBaseClass
rdagent/knowledge_management/graph.py
+11 -11
View File
@@ -98,11 +98,11 @@ class Graph:
embeddings = []
for i in range(0, len(contents), size):
embeddings.extend(
APIBackend().create_embedding(input_content=contents[i : i + size])
APIBackend().create_embedding(input_content=contents[i : i + size]),
)
assert len(nodes) == len(
embeddings
embeddings,
), "nodes' length must equals embeddings' length"
for node, embedding in zip(nodes, embeddings):
node.embedding = embedding
@@ -238,7 +238,7 @@ class UndirectedGraph(Graph):
result.append(node)
for neighbor in sorted(
list(self.get_node(node.id).neighbors), key=lambda x: x.content
list(self.get_node(node.id).neighbors), key=lambda x: x.content,
): # to make sure the result is deterministic
if neighbor not in visited:
if not (block and neighbor.label not in constraint_labels):
@@ -275,12 +275,12 @@ class UndirectedGraph(Graph):
for node in nodes:
if intersection is None:
intersection = self.get_nodes_within_steps(
node, steps=steps, constraint_labels=constraint_labels
node, steps=steps, constraint_labels=constraint_labels,
)
intersection = self.intersection(
nodes1=intersection,
nodes2=self.get_nodes_within_steps(
node, steps=steps, constraint_labels=constraint_labels
node, steps=steps, constraint_labels=constraint_labels,
),
)
@@ -393,7 +393,7 @@ class UndirectedGraph(Graph):
res_list = []
for query in content:
similar_nodes = self.semantic_search(
content=query, topk_k=topk_k, similarity_threshold=similarity_threshold
content=query, topk_k=topk_k, similarity_threshold=similarity_threshold,
)
connected_nodes = []
@@ -411,13 +411,13 @@ class UndirectedGraph(Graph):
node
for node in graph_query_node_res
if node not in connected_nodes
]
],
)
if len(connected_nodes) >= topk_k:
break
res_list.extend(
[node for node in connected_nodes[:topk_k] if node not in res_list]
[node for node in connected_nodes[:topk_k] if node not in res_list],
)
return res_list
@@ -455,7 +455,7 @@ def graph_to_edges(graph: Dict[str, List[str]]):
def assign_random_coordinate_to_node(
nodes: List, scope: float = 1.0, origin: Tuple = (0.0, 0.0)
nodes: List, scope: float = 1.0, origin: Tuple = (0.0, 0.0),
) -> Dict:
coordinates = {}
@@ -468,7 +468,7 @@ def assign_random_coordinate_to_node(
def assign_isometric_coordinate_to_node(
nodes: List, x_step: float = 1.0, x_origin: float = 0.0, y_origin: float = 0.0
nodes: List, x_step: float = 1.0, x_origin: float = 0.0, y_origin: float = 0.0,
) -> Dict:
coordinates = {}
@@ -481,7 +481,7 @@ def assign_isometric_coordinate_to_node(
def curly_node_coordinate(
coordinates: Dict, center_y: float = 1.0, r: float = 1.0
coordinates: Dict, center_y: float = 1.0, r: float = 1.0,
) -> Dict:
# noto: this method can only curly < 90 degree, and the curl line is circle.
# the original funtion is: x**2 + (y-m)**2 = r**2
rdagent/oai/llm_utils.py
+15 -18
View File
@@ -15,11 +15,9 @@ from typing import List, Optional, Tuple, Union
import numpy as np
import tiktoken
from scipy.spatial.distance import cosine
from core.conf import FincoSettings as Config
from core.log import FinCoLog, LogColors
from core.utils import SingletonBaseClass
from rdagent.core.conf import FincoSettings as Config
from rdagent.core.log import FinCoLog, LogColors
from rdagent.core.utils import SingletonBaseClass
DEFAULT_QLIB_DOT_PATH = Path("./")
@@ -65,7 +63,6 @@ class ConvManager:
if m is not None:
n = int(m.group(1))
pairs.append((n, f))
pass
pairs.sort(key=lambda x: x[0])
for n, f in pairs[: self.recent_n][::-1]:
if Path(self.path / f"{n+1}.json").exists():
@@ -92,7 +89,7 @@ class SQliteLazyCache(SingletonBaseClass):
md5_key TEXT PRIMARY KEY,
chat TEXT
)
"""
""",
)
self.c.execute(
"""
@@ -100,7 +97,7 @@ class SQliteLazyCache(SingletonBaseClass):
md5_key TEXT PRIMARY KEY,
embedding TEXT
)
"""
""",
)
self.conn.commit()
@@ -186,7 +183,7 @@ class ChatSession:
{
"role": "user",
"content": user_prompt,
}
},
)
return messages
@@ -202,13 +199,13 @@ class ChatSession:
messages = self.build_chat_completion_message(user_prompt, **kwargs)
response = self.api_backend._try_create_chat_completion_or_embedding(
messages=messages, chat_completion=True, **kwargs
messages=messages, chat_completion=True, **kwargs,
)
messages.append(
{
"role": "assistant",
"content": response,
}
},
)
SessionChatHistoryCache().message_set(self.conversation_id, messages)
return response
@@ -362,14 +359,14 @@ class APIBackend:
{
"role": "system",
"content": system_prompt,
}
},
]
messages.extend(former_messages[-1 * self.cfg.max_past_message_include :])
messages.append(
{
"role": "user",
"content": user_prompt,
}
},
)
return messages
@@ -400,7 +397,7 @@ class APIBackend:
elif isinstance(input_content, list):
input_content_list = input_content
resp = self._try_create_chat_completion_or_embedding(
input_content_list=input_content_list, embedding=True, **kwargs
input_content_list=input_content_list, embedding=True, **kwargs,
)
if isinstance(input_content, str):
return resp[0]
@@ -421,7 +418,7 @@ class APIBackend:
{
"role": "user",
"content": "continue the former output with no overlap",
}
},
)
new_response, finish_reason = self._create_chat_completion_inner_function(messages=new_message, **kwargs)
return response + new_response
@@ -563,9 +560,9 @@ class APIBackend:
"do_sample": self.gcr_endpoint_do_sample,
"max_new_tokens": self.gcr_endpoint_max_token,
},
}
}
)
},
},
),
)
req = urllib.request.Request(self.gcr_endpoint, body, self.headers)