Web Scraping Best Practices with Rotating Proxies
Web scraping at scale requires more than just sending requests — you need to avoid detection, handle rate limits, and maintain efficiency. This guide covers essential techniques when using rotating residential proxies.
1. Request Rate Optimization
The Problem
Sending too many requests too fast triggers anti-bot systems. Going too slow wastes time and money.
The Solution: Adaptive Rate Limiting
import time
from collections import deque
class AdaptiveRateLimiter:
def __init__(self, initial_delay=1.0, max_delay=10.0):
self.delay = initial_delay
self.max_delay = max_delay
self.success_streak = 0
self.recent_errors = deque(maxlen=10)
def on_success(self):
self.success_streak += 1
# Speed up after 5 consecutive successes
if self.success_streak >= 5:
self.delay = max(0.5, self.delay * 0.9)
self.success_streak = 0
def on_error(self, error_code):
self.recent_errors.append(error_code)
self.success_streak = 0
# Slow down on rate limit errors
if error_code in [429, 503]:
self.delay = min(self.max_delay, self.delay * 2)
# Back off if seeing many errors
if len(self.recent_errors) >= 5:
recent_429s = sum(1 for e in self.recent_errors if e == 429)
if recent_429s >= 3:
self.delay = min(self.max_delay, self.delay * 1.5)
def wait(self):
time.sleep(self.delay)
# Usage
limiter = AdaptiveRateLimiter()
for url in urls:
try:
response = scrape(url, proxy=get_rotating_proxy())
limiter.on_success()
except RateLimitError as e:
limiter.on_error(429)
limiter.wait()
Key insight: Start conservative, speed up on success, back off on errors.
2. Session Management
Sticky Sessions for Multi-Step Flows
When scraping requires login or shopping cart operations:
import requests
class StickySession:
def __init__(self, proxy_url):
self.session = requests.Session()
# Add session_id to keep same IP
self.session.proxies = {
'http': f'{proxy_url}?session=user_{hash(time.time())}',
'https': f'{proxy_url}?session=user_{hash(time.time())}'
}
def login(self, credentials):
# First request establishes session
response = self.session.post('/login', data=credentials)
return response.ok
def scrape_protected_page(self, url):
# Subsequent requests use same IP
return self.session.get(url)
def close(self):
self.session.close()
# Usage
with StickySession(PROXY_URL) as session:
session.login({'user': 'bot', 'pass': 'secret'})
data = session.scrape_protected_page('/account/orders')
When to use sticky sessions:
- ✅ Login-protected content
- ✅ Shopping cart flows
- ✅ Multi-step forms
- ❌ Simple public data scraping (wastes sticky session overhead)
3. User-Agent Rotation
Combine proxy rotation with User-Agent rotation:
import random
USER_AGENTS = [
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36',
]
def get_headers():
return {
'User-Agent': random.choice(USER_AGENTS),
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9',
'Accept-Language': 'en-US,en;q=0.9',
'Accept-Encoding': 'gzip, deflate, br',
'DNT': '1',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1'
}
Pro tip: Match User-Agent complexity to your target. E-commerce sites need full browser fingerprinting; public APIs may only check User-Agent.
4. Error Handling Strategy
Not all errors mean "retry immediately":
def scrape_with_retry(url, max_retries=3):
for attempt in range(max_retries):
try:
response = requests.get(url,
proxies=get_rotating_proxy(),
timeout=10)
if response.status_code == 200:
return response
elif response.status_code == 404:
# Don't retry 404s
return None
elif response.status_code == 429:
# Rate limited - exponential backoff
wait_time = (2 ** attempt) * 5
time.sleep(wait_time)
elif response.status_code in [500, 502, 503]:
# Server error - short retry
time.sleep(2)
else:
# Unknown error - log and skip
log_error(response.status_code, url)
return None
except requests.Timeout:
# Timeout - try different proxy
continue
except requests.ConnectionError:
# Connection failed - exponential backoff
time.sleep(2 ** attempt)
return None
5. Geographic Targeting
Match proxy location to target site's region:
# Scraping US e-commerce site
response = requests.get(url,
proxies={'http': f'{PROXY_URL}?country=us'})
# Scraping Japanese content
response = requests.get(url,
proxies={'http': f'{PROXY_URL}?country=jp'})
Why this matters:
- Some sites serve different content by region
- Geo-blocked content requires local IPs
- Reduces latency (faster scraping)
6. Monitoring & Metrics
Track these metrics to optimize your scraping:
import time
from dataclasses import dataclass
from typing import Dict
@dataclass
class ScrapingMetrics:
total_requests: int = 0
successful_requests: int = 0
failed_requests: int = 0
total_data_mb: float = 0
blocked_ips: set = field(default_factory=set)
status_codes: Dict[int, int] = field(default_factory=dict)
avg_response_time: float = 0
def success_rate(self):
if self.total_requests == 0:
return 0
return self.successful_requests / self.total_requests * 100
def cost_estimate(self, price_per_gb=2.5):
gb_used = self.total_data_mb / 1024
return gb_used * price_per_gb
metrics = ScrapingMetrics()
def scrape_with_metrics(url):
metrics.total_requests += 1
start = time.time()
try:
response = requests.get(url, proxies=get_proxy())
elapsed = time.time() - start
metrics.avg_response_time = (
(metrics.avg_response_time * (metrics.total_requests - 1) + elapsed)
/ metrics.total_requests
)
metrics.status_codes[response.status_code] = \
metrics.status_codes.get(response.status_code, 0) + 1
if response.ok:
metrics.successful_requests += 1
metrics.total_data_mb += len(response.content) / (1024 * 1024)
else:
metrics.failed_requests += 1
return response
except Exception as e:
metrics.failed_requests += 1
raise
# Print metrics every 100 requests
if metrics.total_requests % 100 == 0:
print(f"Success rate: {metrics.success_rate():.1f}%")
print(f"Avg response time: {metrics.avg_response_time:.2f}s")
print(f"Est. cost: ${metrics.cost_estimate():.2f}")
7. Common Pitfalls
❌ Don't: Scrape too aggressively
# BAD: No delays, no error handling
for url in urls:
data = requests.get(url).json()
✅ Do: Respect rate limits
# GOOD: Adaptive delays, error handling
for url in urls:
try:
data = scrape_with_retry(url)
limiter.wait()
except Exception as e:
log_error(e)
continue
❌ Don't: Ignore Content-Type
# BAD: Assumes all responses are HTML
soup = BeautifulSoup(response.text)
✅ Do: Check response type
# GOOD: Handle different content types
if 'application/json' in response.headers.get('Content-Type', ''):
data = response.json()
elif 'text/html' in response.headers.get('Content-Type', ''):
soup = BeautifulSoup(response.text, 'html.parser')
Checklist for Production Scraping
- Rate limiting implemented
- Retry logic with exponential backoff
- Session management for multi-step flows
- User-Agent rotation
- Geographic targeting configured
- Metrics tracking
- Error logging
- Graceful shutdown handling
- Data validation
- Cost monitoring
Next Steps
Questions? Reach out via the console — we help customers optimize their scraping setups daily.