Add scrape script
This commit is contained in:
@@ -0,0 +1,34 @@
|
||||
#!/bin/bash
|
||||
BASE_URL="https://docs.polymarket.com"
|
||||
OUTPUT_DIR="docs"
|
||||
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
# Read URLs from TARGET.md (skip comments and empty lines)
|
||||
grep -v '^#' TARGET.md | grep -v '^$' | grep 'https://' | while read -r url; do
|
||||
# Extract the path portion (remove https://docs.polymarket.com/)
|
||||
path="${url#https://docs.polymarket.com/}"
|
||||
|
||||
# Create directory
|
||||
dir=$(dirname "$path")
|
||||
mkdir -p "$OUTPUT_DIR/$dir"
|
||||
|
||||
# Determine output file path
|
||||
output_file="$OUTPUT_DIR/$path"
|
||||
|
||||
# Fetch the page
|
||||
echo "Fetching: $url -> $output_file"
|
||||
curl -s "$url" -o "$output_file"
|
||||
|
||||
# Check if successful
|
||||
if [ $? -eq 0 ] && [ -s "$output_file" ]; then
|
||||
echo " ✅ Success"
|
||||
else
|
||||
echo " ❌ Failed"
|
||||
fi
|
||||
|
||||
# Rate limit
|
||||
sleep 0.3
|
||||
done
|
||||
|
||||
echo "Done!"
|
||||
Reference in New Issue
Block a user