gavindiaz
125470b24a
Synced from github/main: aebcfa2d copybot: live paper feed [skip ci]
Our additions: serve_dashboard.py, bot_dashboard.html, mihomo-healthcheck.sh,
中文文档, 我们的钱包配置
87 lines
2.3 KiB
Python
87 lines
2.3 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Scrape Polymarket documentation using web_fetch tool.
|
|
"""
|
|
|
|
import subprocess
|
|
import json
|
|
import os
|
|
import time
|
|
from pathlib import Path
|
|
|
|
def fetch_url(url):
|
|
"""Use web_fetch to get markdown content from a URL."""
|
|
cmd = [
|
|
"npx", "tsx", "-e",
|
|
f'''
|
|
import tool from "./tool.mjs";
|
|
const result = await tool.read({{path: "/home/himalaya/clawd/PolymarketDocumentation/scrape_single.mjs", limit: 200}});
|
|
console.log(JSON.stringify(result));
|
|
'''
|
|
]
|
|
|
|
# Use subprocess to call a Node.js script that uses the web_fetch
|
|
result = subprocess.run(
|
|
["node", "-e", f'''
|
|
const {{ web_fetch }} = require('./node_modules/openclaw/node_tools.js');
|
|
web_fetch({{url: "{url}"}}).then(r => console.log(JSON.stringify(r, null, 2)));
|
|
'''"],
|
|
capture_output=True,
|
|
text=True,
|
|
cwd="/home/himalaya/clawd"
|
|
)
|
|
return result.stdout
|
|
|
|
def main():
|
|
# Read TARGET.md
|
|
with open("TARGET.md", "r") as f:
|
|
lines = f.readlines()
|
|
|
|
# Extract URLs
|
|
urls = []
|
|
for line in lines:
|
|
line = line.strip()
|
|
if line.startswith("https://"):
|
|
urls.append(line)
|
|
|
|
print(f"Found {len(urls)} URLs to scrape")
|
|
|
|
# Create docs directory
|
|
Path("docs").mkdir(exist_ok=True)
|
|
|
|
# Process each URL
|
|
for i, url in enumerate(urls, 1):
|
|
print(f"[{i}/{len(urls)}] Fetching: {url}")
|
|
|
|
# Extract path
|
|
path = url.replace("https://docs.polymarket.com/", "")
|
|
|
|
# Create directory structure
|
|
dir_path = Path("docs") / Path(path).parent
|
|
dir_path.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Output file
|
|
output_file = Path("docs") / path
|
|
|
|
# Use curl to fetch and web_fetch for markdown extraction
|
|
import subprocess
|
|
result = subprocess.run(
|
|
["curl", "-s", "-L", "-A", "Mozilla/5.0", url],
|
|
capture_output=True,
|
|
text=True
|
|
)
|
|
|
|
content = result.stdout
|
|
|
|
# Save content
|
|
with open(output_file, "w") as f:
|
|
f.write(content)
|
|
|
|
print(f" -> {output_file}")
|
|
time.sleep(0.5)
|
|
|
|
print("Done!")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|