feat: remove pdfs and enable online pdf readings (#183)

* remove pdfs and enable online pdf readings

* update doc format

* use url as key

rdagent/components/document_reader/document_reader.py

@@ -6,7 +6,11 @@ from typing import TYPE_CHECKING
 import fitz
 from azure.ai.formrecognizer import DocumentAnalysisClient
 from azure.core.credentials import AzureKeyCredential
-from langchain.document_loaders import PyPDFDirectoryLoader, PyPDFLoader
+from langchain.document_loaders import (
+    OnlinePDFLoader,
+    PyPDFDirectoryLoader,
+    PyPDFLoader,
+)
 from PIL import Image
 
 if TYPE_CHECKING:
@@ -15,7 +19,7 @@ if TYPE_CHECKING:
 from rdagent.core.conf import RD_AGENT_SETTINGS
 
 
-def load_documents_by_langchain(path: Path) -> list:
+def load_documents_by_langchain(path: str) -> list:
     """Load documents from the specified path.
 
     Args:
@@ -24,7 +28,10 @@ def load_documents_by_langchain(path: Path) -> list:
     Returns:
         list: A list of loaded documents.
     """
-    loader = PyPDFDirectoryLoader(str(path), silent_errors=True) if path.is_dir() else PyPDFLoader(str(path))
+    if Path(path).is_dir():
+        loader = PyPDFDirectoryLoader(path, silent_errors=True)
+    else:
+        loader = PyPDFLoader(path)
     return loader.load()
 
 
@@ -41,7 +48,10 @@ def process_documents_by_langchain(docs: list[Document]) -> dict[str, str]:
     content_dict = {}
 
     for doc in docs:
-        doc_name = str(Path(doc.metadata["source"]).resolve())
+        if Path(doc.metadata["source"]).exists():
+            doc_name = str(Path(doc.metadata["source"]).resolve())
+        else:
+            doc_name = doc.metadata["source"]
         doc_content = doc.page_content
 
         if doc_name not in content_dict:
@@ -52,7 +62,7 @@ def process_documents_by_langchain(docs: list[Document]) -> dict[str, str]:
     return content_dict
 
 
-def load_and_process_pdfs_by_langchain(path: Path) -> dict[str, str]:
+def load_and_process_pdfs_by_langchain(path: str) -> dict[str, str]:
     return process_documents_by_langchain(load_documents_by_langchain(path))