7 Web Scraping Challenges in 2025 (And How to Overcome Them)

# What happens when you ignore CAPTCHAs
import requests
from bs4 import BeautifulSoup

url = "https://target-site.com/products"
response = requests.get(url)
soup = BeautifulSoup(response.text, 'html.parser')

products = soup.find_all('div', class_='product')
print(f"Found {len(products)} products")

# Output: Found 0 products
# Why? You got served a CAPTCHA challenge page instead of product data
# Your script has no idea it's been blocked

# ScrapingBot - CAPTCHAs simply don't appear
curl "https://api.scrapingbot.io/v1/scrape" \
  -H "x-api-key: YOUR_KEY" \
  -d "url=https://target-site.com/products" \
  -d "render_js=true" \
  -d "premium_proxy=true"

{
  "success": true,
  "html": "... actual product data ...",
  "statusCode": 200
}

# No CAPTCHA. No delays. Just clean data.

# Managing proxies manually = nightmare fuel
import requests
import random
import time
from datetime import datetime

proxies = load_proxy_list()  # 1000 proxies you paid for
working_proxies = proxies.copy()
failed_proxies = []

def test_proxy(proxy):
    """Test if a proxy is still working"""
    try:
        response = requests.get(
            'https://httpbin.org/ip', 
            proxies={'http': proxy, 'https': proxy},
            timeout=5
        )
        return response.status_code == 200
    except:
        return False

for url in urls_to_scrape:
    attempt = 0
    success = False
    
    while attempt < 5 and not success:
        if len(working_proxies) == 0:
            print("CRITICAL: No working proxies left!")
            break
            
        proxy = random.choice(working_proxies)
        try:
            response = requests.get(
                url, 
                proxies={'http': proxy, 'https': proxy},
                timeout=10,
                headers={'User-Agent': random_user_agent()}
            )
            
            if response.status_code == 200:
                success = True
                parse_data(response)
            elif response.status_code == 403:
                # Proxy is banned, remove it
                working_proxies.remove(proxy)
                failed_proxies.append({
                    'proxy': proxy,
                    'failed_at': datetime.now(),
                    'reason': 'banned'
                })
            else:
                # Other error, try different proxy
                pass
                
        except requests.exceptions.Timeout:
            # Proxy too slow, mark as unreliable
            working_proxies.remove(proxy)
        except Exception as e:
            # Connection error, proxy likely dead
            working_proxies.remove(proxy)
        
        attempt += 1
        time.sleep(random.uniform(3, 8))
    
    if not success:
        print(f"Failed to scrape {url} after 5 attempts")
    
    # Monitor proxy health
    proxy_health = len(working_proxies) / len(proxies) * 100
    print(f"Proxy pool health: {proxy_health:.1f}%")
    
    if len(working_proxies) < 50:
        print("WARNING: Running out of proxies! Need to buy more...")
        # Emergency: buy more proxies or pause scraping

# This code requires:
# - Constant monitoring
# - Regular proxy replacement
# - Error handling for dozens of edge cases
# - Database to track proxy performance
# - Alerts when proxy pool degrades

# Basic Puppeteer scraper - will get detected
const puppeteer = require('puppeteer');

async function scrapeWithPuppeteer(url) {
    const browser = await puppeteer.launch({
        headless: true,  // Detectable!
        args: ['--no-sandbox']
    });
    
    const page = await browser.newPage();
    
    // Set basic user agent
    await page.setUserAgent(
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
    );
    
    try {
        await page.goto(url, { waitUntil: 'networkidle0' });
        
        // Wait for content to load
        await page.waitForSelector('.product-list', { timeout: 10000 });
        
        // Extract data
        const data = await page.evaluate(() => {
            const products = [];
            document.querySelectorAll('.product-item').forEach(item => {
                products.push({
                    name: item.querySelector('.product-name')?.textContent,
                    price: item.querySelector('.price')?.textContent
                });
            });
            return products;
        });
        
        await browser.close();
        return data;
        
    } catch (error) {
        await browser.close();
        // Common errors you'll see:
        // - TimeoutError: Waiting for selector timed out
        // - Navigation failed because page crashed
        // - Access denied (you've been detected)
        throw error;
    }
}

// Problems with this approach:
// 1. navigator.webdriver is true (instant detection)
// 2. Missing browser plugins (Chrome extensions, PDF viewer)
// 3. No WebGL fingerprint
// 4. Wrong canvas fingerprint
// 5. Inconsistent screen dimensions
// 6. No audio context
// 7. Suspicious timing (too fast/consistent)

# Stealth Puppeteer - much better, but still complex
const puppeteer = require('puppeteer-extra');
const StealthPlugin = require('puppeteer-extra-plugin-stealth');
const AdblockerPlugin = require('puppeteer-extra-plugin-adblocker');

// Add stealth plugins
puppeteer.use(StealthPlugin());
puppeteer.use(AdblockerPlugin({ blockTrackers: true }));

async function scrapeWithStealth(url) {
    const browser = await puppeteer.launch({
        headless: 'new',
        args: [
            '--no-sandbox',
            '--disable-setuid-sandbox',
            '--disable-dev-shm-usage',
            '--disable-accelerated-2d-canvas',
            '--disable-gpu',
            '--window-size=1920x1080',
            '--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
        ]
    });
    
    const page = await browser.newPage();
    
    // Set viewport to match window size
    await page.setViewport({ width: 1920, height: 1080 });
    
    // Set additional headers
    await page.setExtraHTTPHeaders({
        'Accept-Language': 'en-US,en;q=0.9',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        'Accept-Encoding': 'gzip, deflate, br',
        'Connection': 'keep-alive',
    });
    
    // Randomize some behaviors to appear human
    await page.evaluateOnNewDocument(() => {
        // Override the navigator properties
        Object.defineProperty(navigator, 'webdriver', {
            get: () => false,
        });
        
        // Add Chrome runtime
        window.chrome = {
            runtime: {},
        };
        
        // Override permissions
        const originalQuery = window.navigator.permissions.query;
        window.navigator.permissions.query = (parameters) => (
            parameters.name === 'notifications' ?
                Promise.resolve({ state: Notification.permission }) :
                originalQuery(parameters)
        );
    });
    
    try {
        // Navigate with random delays
        await page.goto(url, { 
            waitUntil: 'networkidle2',
            timeout: 30000
        });
        
        // Random mouse movements (appear human)
        await page.mouse.move(100, 100);
        await page.mouse.move(200, 200);
        
        // Random scroll
        await page.evaluate(() => {
            window.scrollBy(0, Math.floor(Math.random() * 500) + 100);
        });
        
        // Wait with random delay
        await page.waitForTimeout(Math.random() * 2000 + 1000);
        
        const data = await page.evaluate(() => {
            // Extract data...
            return extractProducts();
        });
        
        await browser.close();
        return data;
        
    } catch (error) {
        await browser.close();
        throw error;
    }
}

// Better, but still requires:
// - Managing browser instances
// - Handling crashes and memory leaks
// - Load balancing across multiple browsers
// - Monitoring and auto-restart on failures
// - Keeping stealth plugins updated

# Naive approach - instant ban
import requests

urls = [f"https://site.com/page/{i}" for i in range(1000)]

for url in urls:
    response = requests.get(url)
    parse_data(response)

# Banned after request #15
# Why? 15 requests in 3 seconds is obviously a bot

# Adaptive rate limiter that learns from responses
import time
import random
from datetime import datetime

class AdaptiveRateLimiter:
    def __init__(self, initial_delay=2.0):
        self.delay = initial_delay
        self.success_count = 0
        self.failure_count = 0
        self.last_request_time = None
        
    def record_success(self):
        self.success_count += 1
        self.failure_count = 0
        
        # Speed up if consistently successful
        if self.success_count > 10:
            self.delay = max(1.0, self.delay * 0.95)
            self.success_count = 0
    
    def record_failure(self, status_code):
        self.failure_count += 1
        self.success_count = 0
        
        if status_code == 429:  # Too Many Requests
            self.delay = min(30.0, self.delay * 2.0)
        elif status_code == 403:  # Forbidden
            self.delay = min(60.0, self.delay * 3.0)
        
        if self.failure_count > 3:
            self.delay = min(120.0, self.delay * 2.0)
    
    def wait(self):
        if self.last_request_time:
            elapsed = time.time() - self.last_request_time
            sleep_time = max(0, self.delay - elapsed)
            if sleep_time > 0:
                # Add random jitter (±20%)
                jitter = random.uniform(-0.2, 0.2) * sleep_time
                time.sleep(sleep_time + jitter)
        
        self.last_request_time = time.time()

# Usage
rate_limiter = AdaptiveRateLimiter(initial_delay=3.0)

for url in urls_to_scrape:
    rate_limiter.wait()
    
    try:
        response = requests.get(url)
        
        if response.status_code == 200:
            rate_limiter.record_success()
            process_data(response)
        else:
            rate_limiter.record_failure(response.status_code)
            
    except Exception as e:
        rate_limiter.record_failure(500)

# Still requires:
# - Per-domain rate limiting
# - IP-based tracking
# - Retry-After header handling
# - Exponential backoff

# AI extraction - works even when HTML changes
curl "https://api.scrapingbot.io/v1/scrape" \
  -H "x-api-key: YOUR_KEY" \
  -d "url=https://ecommerce-site.com/product/12345" \
  -d "render_js=true" \
  -d "ai_query=extract the product name, price, rating, and availability"

{
  "success": true,
  "ai_result": {
    "product_name": "Wireless Bluetooth Headphones",
    "price": "$89.99",
    "rating": "4.5",
    "availability": "In Stock"
  }
}

# Site changes HTML? AI adapts automatically.
# No code updates needed.

# Managing sessions manually
import requests
from requests.cookies import RequestsCookieJar

class SessionManager:
    def __init__(self):
        self.sessions = {}
    
    def get_or_create_session(self, site_id):
        if site_id not in self.sessions:
            session = requests.Session()
            
            # Set headers that persist across requests
            session.headers.update({
                'User-Agent': 'Mozilla/5.0 ...',
                'Accept': 'text/html,application/xhtml+xml...',
                'Accept-Language': 'en-US,en;q=0.9',
            })
            
            self.sessions[site_id] = session
        
        return self.sessions[site_id]
    
    def scrape_with_session(self, url, site_id):
        session = self.get_or_create_session(site_id)
        
        # First request might set cookies
        response = session.get(url)
        
        # Subsequent requests automatically include cookies
        if 'login' in response.url:
            # Need to handle login flow
            csrf_token = extract_csrf_token(response.text)
            
            login_data = {
                'username': 'user',
                'password': 'pass',
                'csrf_token': csrf_token
            }
            
            login_response = session.post(
                'https://site.com/login',
                data=login_data
            )
            
            # Now cookies are set for authenticated requests
            response = session.get(url)
        
        return response

# Challenges:
# - Sessions expire and need refresh
# - CSRF tokens change frequently
# - Some sites use LocalStorage (not accessible via requests)
# - Rate limiting applies per-session
# - Need to detect when session is invalid

# ScrapingBot handles sessions easily
curl "https://api.scrapingbot.io/v1/scrape" \
  -H "x-api-key: YOUR_KEY" \
  -d "url=https://site.com/account/orders" \
  -d "render_js=true" \
  -d "cookies=session_id=abc123;user_token=xyz789"

# Cookies are automatically maintained across requests
# No session management infrastructure needed

# Complete DIY e-commerce scraper (simplified)
import requests
from bs4 import BeautifulSoup
from selenium import webdriver
import time
import random
from datetime import datetime
import logging

class ProductScraper:
    def __init__(self):
        self.proxy_pool = load_proxies()  # $300/month
        self.working_proxies = self.proxy_pool.copy()
        self.rate_limiter = AdaptiveRateLimiter()
        self.session_manager = SessionManager()
        self.captcha_solver = CaptchaSolver()  # $100/month
        
    def scrape_product(self, url, retries=3):
        for attempt in range(retries):
            try:
                # Get working proxy
                if len(self.working_proxies) == 0:
                    logging.error("No working proxies!")
                    return None
                    
                proxy = random.choice(self.working_proxies)
                
                # Respect rate limits
                self.rate_limiter.wait()
                
                # Launch browser with proxy
                options = webdriver.ChromeOptions()
                options.add_argument(f'--proxy-server={proxy}')
                options.add_argument('--headless')
                driver = webdriver.Chrome(options=options)
                
                try:
                    driver.get(url)
                    time.sleep(random.uniform(2, 5))
                    
                    # Check for CAPTCHA
                    if 'captcha' in driver.page_source.lower():
                        logging.warning(f"CAPTCHA detected on {url}")
                        captcha_solution = self.captcha_solver.solve(driver)
                        if not captcha_solution:
                            self.working_proxies.remove(proxy)
                            driver.quit()
                            continue
                    
                    # Wait for dynamic content
                    WebDriverWait(driver, 10).until(
                        EC.presence_of_element_located((By.CLASS_NAME, "product"))
                    )
                    
                    # Extract data
                    html = driver.page_source
                    soup = BeautifulSoup(html, 'html.parser')
                    
                    product = {
                        'name': soup.select_one('.product-name')?.text,
                        'price': soup.select_one('.price')?.text,
                        'availability': soup.select_one('.stock')?.text,
                        'scraped_at': datetime.now()
                    }
                    
                    driver.quit()
                    self.rate_limiter.record_success()
                    return product
                    
                except TimeoutException:
                    logging.error(f"Timeout on {url}")
                    driver.quit()
                    self.working_proxies.remove(proxy)
                    
                except Exception as e:
                    logging.error(f"Error: {e}")
                    driver.quit()
                    
            except Exception as e:
                logging.error(f"Fatal error: {e}")
                
        return None
    
    def scrape_catalog(self, urls):
        results = []
        failed = []
        
        for url in urls:
            product = self.scrape_product(url)
            if product:
                results.append(product)
            else:
                failed.append(url)
            
            # Monitor proxy health
            health = len(self.working_proxies) / len(self.proxy_pool)
            if health < 0.2:
                logging.critical("Proxy pool critically low!")
                # Need to buy more proxies...
        
        return results, failed

# This requires:
# - 800+ lines of additional code (error handling, retry logic, monitoring)
# - Database to track proxy performance and failures
# - Monitoring dashboard and alerts
# - Regular updates when sites change
# - Dedicated server (m5.2xlarge: $280/month)
# - 20+ hours/month maintenance

# Complete ScrapingBot e-commerce scraper
import requests

API_KEY = "your_key"
BASE_URL = "https://api.scrapingbot.io/v1/scrape"

def scrape_product(url):
    response = requests.get(BASE_URL, params={
        "url": url,
        "render_js": "true",
        "premium_proxy": "true",
        "ai_query": "extract product name, price, and availability"
    }, headers={"x-api-key": API_KEY})
    
    data = response.json()
    return data["ai_result"] if data["success"] else None

def scrape_catalog(urls):
    results = []
    for url in urls:
        product = scrape_product(url)
        if product:
            results.append(product)
    return results

# That's it. 20 lines vs 800+.
# No proxies to manage
# No browsers to run
# No CAPTCHAs to solve
# No infrastructure to maintain
# Works reliably at scale