gavindiaz
125470b24a
Synced from github/main: aebcfa2d copybot: live paper feed [skip ci]
Our additions: serve_dashboard.py, bot_dashboard.html, mihomo-healthcheck.sh,
中文文档, 我们的钱包配置
38 lines
922 B
Bash
38 lines
922 B
Bash
#!/bin/bash
|
|
# Fetch Polymarket documentation as markdown
|
|
|
|
OUTPUT_DIR="docs"
|
|
mkdir -p "$OUTPUT_DIR"
|
|
|
|
# Read URLs from TARGET.md
|
|
grep -v '^#' TARGET.md | grep -v '^$' | grep 'https://' | while read -r url; do
|
|
# Extract the path portion
|
|
path="${url#https://docs.polymarket.com/}"
|
|
|
|
# Create directory
|
|
dir=$(dirname "$path")
|
|
mkdir -p "$OUTPUT_DIR/$dir"
|
|
|
|
# Output file
|
|
output_file="$OUTPUT_DIR/$path"
|
|
|
|
# Fetch the page with proper headers
|
|
echo "Fetching: $url"
|
|
content=$(curl -s -L -A "Mozilla/5.0 (compatible; Bot/1.0)" \
|
|
-H "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" \
|
|
-H "Accept-Language: en-US,en;q=0.5" \
|
|
"$url")
|
|
|
|
if [ -n "$content" ]; then
|
|
echo "$content" > "$output_file"
|
|
echo " -> Saved to $output_file ($(echo "$content" | wc -c) bytes)"
|
|
else
|
|
echo " -> Failed to fetch"
|
|
fi
|
|
|
|
sleep 0.3
|
|
done
|
|
|
|
echo ""
|
|
echo "Scraping complete!"
|