Update Polymarket documentation (2026-02-19)
- Added new documentation URLs from llms.txt index - Updated TARGET.md with 244 total documentation pages - Scraped new pages for trading, concepts, and API reference sections - Updated changelog and new index pages
This commit is contained in:
Executable
+37
@@ -0,0 +1,37 @@
|
||||
#!/bin/bash
|
||||
# Fetch Polymarket documentation as markdown
|
||||
|
||||
OUTPUT_DIR="docs"
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
# Read URLs from TARGET.md
|
||||
grep -v '^#' TARGET.md | grep -v '^$' | grep 'https://' | while read -r url; do
|
||||
# Extract the path portion
|
||||
path="${url#https://docs.polymarket.com/}"
|
||||
|
||||
# Create directory
|
||||
dir=$(dirname "$path")
|
||||
mkdir -p "$OUTPUT_DIR/$dir"
|
||||
|
||||
# Output file
|
||||
output_file="$OUTPUT_DIR/$path"
|
||||
|
||||
# Fetch the page with proper headers
|
||||
echo "Fetching: $url"
|
||||
content=$(curl -s -L -A "Mozilla/5.0 (compatible; Bot/1.0)" \
|
||||
-H "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" \
|
||||
-H "Accept-Language: en-US,en;q=0.5" \
|
||||
"$url")
|
||||
|
||||
if [ -n "$content" ]; then
|
||||
echo "$content" > "$output_file"
|
||||
echo " -> Saved to $output_file ($(echo "$content" | wc -c) bytes)"
|
||||
else
|
||||
echo " -> Failed to fetch"
|
||||
fi
|
||||
|
||||
sleep 0.3
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Scraping complete!"
|
||||
Reference in New Issue
Block a user