diff --git a/rdagent/app/general_model/general_model.py b/rdagent/app/general_model/general_model.py index 196c88c4..4c4d7b47 100644 --- a/rdagent/app/general_model/general_model.py +++ b/rdagent/app/general_model/general_model.py @@ -20,6 +20,22 @@ from rdagent.scenarios.qlib.developer.model_coder import QlibModelCoSTEER def extract_models_and_implement( report_file_path: str, ) -> None: + """ + Extracts models from a given PDF report file and implements the necessary operations. + + Parameters: + report_file_path (str): The path to the report file. The file must be a PDF file. + + Example URLs of PDF reports: + - https://arxiv.org/pdf/2210.09789 + - https://arxiv.org/pdf/2305.10498 + - https://arxiv.org/pdf/2110.14446 + - https://arxiv.org/pdf/2205.12454 + - https://arxiv.org/pdf/2210.16518 + + Returns: + None + """ with logger.tag("init"): scenario = GeneralModelScenario() logger.log_object(scenario, tag="scenario") diff --git a/rdagent/app/qlib_rd_loop/factor_from_report_w_sc.py b/rdagent/app/qlib_rd_loop/factor_from_report_w_sc.py index fa9e61c4..a9999478 100644 --- a/rdagent/app/qlib_rd_loop/factor_from_report_w_sc.py +++ b/rdagent/app/qlib_rd_loop/factor_from_report_w_sc.py @@ -85,7 +85,7 @@ def extract_hypothesis_and_exp_from_reports(report_file_path: str) -> Tuple[Qlib pdf_screenshot = extract_first_page_screenshot_from_pdf(report_file_path) logger.log_object(pdf_screenshot) - docs_dict = load_and_process_pdfs_by_langchain(Path(report_file_path)) + docs_dict = load_and_process_pdfs_by_langchain(report_file_path) factor_result = { task.factor_name: { diff --git a/rdagent/components/coder/model_coder/task_loader.py b/rdagent/components/coder/model_coder/task_loader.py index 0e0a06ce..a0867745 100644 --- a/rdagent/components/coder/model_coder/task_loader.py +++ b/rdagent/components/coder/model_coder/task_loader.py @@ -115,8 +115,8 @@ class ModelExperimentLoaderFromDict(ModelTaskLoader): class ModelExperimentLoaderFromPDFfiles(ModelTaskLoader): - def load(self, file_or_folder_path: Path) -> dict: - docs_dict = load_and_process_pdfs_by_langchain(Path(file_or_folder_path)) # dict{file_path:content} + def load(self, file_or_folder_path: str) -> dict: + docs_dict = load_and_process_pdfs_by_langchain(file_or_folder_path) # dict{file_path:content} model_dict = extract_model_from_docs( docs_dict ) # dict{file_name: dict{model_name: dict{description, formulation, variables}}} @@ -124,14 +124,3 @@ class ModelExperimentLoaderFromPDFfiles(ModelTaskLoader): model_dict ) # dict {model_name: dict{description, formulation, variables}} return ModelExperimentLoaderFromDict().load(model_dict) - - -def main(path="../test_doc"): - doc_dict = load_and_process_pdfs_by_langchain(Path(path)) - print(doc_dict.keys()) # if you run code like "python -u", the print content will be truncated - - -import fire - -if __name__ == "__main__": - fire.Fire(main) diff --git a/rdagent/components/document_reader/document_reader.py b/rdagent/components/document_reader/document_reader.py index d6147691..258a5975 100644 --- a/rdagent/components/document_reader/document_reader.py +++ b/rdagent/components/document_reader/document_reader.py @@ -6,7 +6,11 @@ from typing import TYPE_CHECKING import fitz from azure.ai.formrecognizer import DocumentAnalysisClient from azure.core.credentials import AzureKeyCredential -from langchain.document_loaders import PyPDFDirectoryLoader, PyPDFLoader +from langchain.document_loaders import ( + OnlinePDFLoader, + PyPDFDirectoryLoader, + PyPDFLoader, +) from PIL import Image if TYPE_CHECKING: @@ -15,7 +19,7 @@ if TYPE_CHECKING: from rdagent.core.conf import RD_AGENT_SETTINGS -def load_documents_by_langchain(path: Path) -> list: +def load_documents_by_langchain(path: str) -> list: """Load documents from the specified path. Args: @@ -24,7 +28,10 @@ def load_documents_by_langchain(path: Path) -> list: Returns: list: A list of loaded documents. """ - loader = PyPDFDirectoryLoader(str(path), silent_errors=True) if path.is_dir() else PyPDFLoader(str(path)) + if Path(path).is_dir(): + loader = PyPDFDirectoryLoader(path, silent_errors=True) + else: + loader = PyPDFLoader(path) return loader.load() @@ -41,7 +48,10 @@ def process_documents_by_langchain(docs: list[Document]) -> dict[str, str]: content_dict = {} for doc in docs: - doc_name = str(Path(doc.metadata["source"]).resolve()) + if Path(doc.metadata["source"]).exists(): + doc_name = str(Path(doc.metadata["source"]).resolve()) + else: + doc_name = doc.metadata["source"] doc_content = doc.page_content if doc_name not in content_dict: @@ -52,7 +62,7 @@ def process_documents_by_langchain(docs: list[Document]) -> dict[str, str]: return content_dict -def load_and_process_pdfs_by_langchain(path: Path) -> dict[str, str]: +def load_and_process_pdfs_by_langchain(path: str) -> dict[str, str]: return process_documents_by_langchain(load_documents_by_langchain(path)) diff --git a/rdagent/scenarios/general_model/test_doc1.pdf b/rdagent/scenarios/general_model/test_doc1.pdf deleted file mode 100644 index e8346d67..00000000 Binary files a/rdagent/scenarios/general_model/test_doc1.pdf and /dev/null differ diff --git a/rdagent/scenarios/general_model/test_doc2.pdf b/rdagent/scenarios/general_model/test_doc2.pdf deleted file mode 100644 index c3fccd8f..00000000 Binary files a/rdagent/scenarios/general_model/test_doc2.pdf and /dev/null differ diff --git a/rdagent/scenarios/general_model/test_doc3.pdf b/rdagent/scenarios/general_model/test_doc3.pdf deleted file mode 100644 index f66de054..00000000 Binary files a/rdagent/scenarios/general_model/test_doc3.pdf and /dev/null differ diff --git a/rdagent/scenarios/general_model/test_doc4.pdf b/rdagent/scenarios/general_model/test_doc4.pdf deleted file mode 100644 index e53adaf3..00000000 Binary files a/rdagent/scenarios/general_model/test_doc4.pdf and /dev/null differ diff --git a/rdagent/scenarios/general_model/test_doc5.pdf b/rdagent/scenarios/general_model/test_doc5.pdf deleted file mode 100644 index bffc098a..00000000 Binary files a/rdagent/scenarios/general_model/test_doc5.pdf and /dev/null differ diff --git a/rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py b/rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py index b0c5e9ee..71e7b312 100644 --- a/rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py +++ b/rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py @@ -509,9 +509,9 @@ def deduplicate_factors_by_llm( # noqa: C901, PLR0912 class FactorExperimentLoaderFromPDFfiles(FactorExperimentLoader): - def load(self, file_or_folder_path: Path) -> dict: + def load(self, file_or_folder_path: str) -> dict: with logger.tag("docs"): - docs_dict = load_and_process_pdfs_by_langchain(Path(file_or_folder_path)) + docs_dict = load_and_process_pdfs_by_langchain(file_or_folder_path) logger.log_object(docs_dict) selected_report_dict = classify_report_from_dict(report_dict=docs_dict, vote_time=1)