From 26483fda452c932da925d029088cc00f47efe998 Mon Sep 17 00:00:00 2001
From: Etherdrake <bodewescoding@protonmail.com>
Date: Sat, 14 Feb 2026 13:00:30 +0100
Subject: [PATCH] Add scrape script

---
 scrape.sh | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100755 scrape.sh

diff --git a/scrape.sh b/scrape.sh
new file mode 100755
index 0000000..983b961
--- /dev/null
+++ b/scrape.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+BASE_URL="https://docs.polymarket.com"
+OUTPUT_DIR="docs"
+
+mkdir -p "$OUTPUT_DIR"
+
+# Read URLs from TARGET.md (skip comments and empty lines)
+grep -v '^#' TARGET.md | grep -v '^$' | grep 'https://' | while read -r url; do
+  # Extract the path portion (remove https://docs.polymarket.com/)
+  path="${url#https://docs.polymarket.com/}"
+  
+  # Create directory
+  dir=$(dirname "$path")
+  mkdir -p "$OUTPUT_DIR/$dir"
+  
+  # Determine output file path
+  output_file="$OUTPUT_DIR/$path"
+  
+  # Fetch the page
+  echo "Fetching: $url -> $output_file"
+  curl -s "$url" -o "$output_file"
+  
+  # Check if successful
+  if [ $? -eq 0 ] && [ -s "$output_file" ]; then
+    echo "  ✅ Success"
+  else
+    echo "  ❌ Failed"
+  fi
+  
+  # Rate limit
+  sleep 0.3
+done
+
+echo "Done!"