5e893db025
- Stealth mode: playwright-stealth, random fingerprints, human delays - Retry logic: exponential backoff (3 attempts) - Logging: rotating logs to /root/.hermes/logs/gmb/ - Validation: phone/website/rating validation + dedup - Pain detection: 12 signals, scoring, service matching - Review scraper: extract reviews + pain keyword detection - Website health: SSL, speed, mobile, contact form checks - Pitch generator: Apex pitches (SMS, email, call, Gumtree) - Docker containerization - .env for secrets (no hardcoded API keys) - Integration with Pipecat voice dialer (gmb_to_voice.py)
96 lines
2.8 KiB
Python
96 lines
2.8 KiB
Python
"""
|
|
Retry Logic Module
|
|
==================
|
|
Exponential backoff retry decorator for resilient scraping.
|
|
"""
|
|
|
|
import time
|
|
import random
|
|
from functools import wraps
|
|
from .logger import get_logger
|
|
|
|
|
|
def retry_with_backoff(
|
|
max_attempts=3,
|
|
base_delay=2.0,
|
|
max_delay=30.0,
|
|
exponential_base=2.0,
|
|
jitter=True,
|
|
retry_on=(Exception,),
|
|
on_retry=None
|
|
):
|
|
"""
|
|
Decorator for retrying functions with exponential backoff.
|
|
|
|
Args:
|
|
max_attempts: Maximum number of retry attempts
|
|
base_delay: Initial delay in seconds
|
|
max_delay: Maximum delay in seconds
|
|
exponential_base: Base for exponential growth
|
|
jitter: Add random jitter to prevent thundering herd
|
|
retry_on: Tuple of exception types to retry on
|
|
on_retry: Callback function(attempt, exception, delay)
|
|
|
|
Returns:
|
|
Decorated function with retry logic
|
|
"""
|
|
def decorator(func):
|
|
@wraps(func)
|
|
def wrapper(*args, **kwargs):
|
|
logger = get_logger()
|
|
last_exception = None
|
|
|
|
for attempt in range(max_attempts):
|
|
try:
|
|
return func(*args, **kwargs)
|
|
except retry_on as e:
|
|
last_exception = e
|
|
|
|
if attempt == max_attempts - 1:
|
|
logger.error(
|
|
f"{func.__name__} failed after {max_attempts} attempts: {e}"
|
|
)
|
|
raise
|
|
|
|
# Calculate delay with exponential backoff
|
|
delay = min(
|
|
base_delay * (exponential_base ** attempt),
|
|
max_delay
|
|
)
|
|
|
|
# Add jitter to prevent thundering herd
|
|
if jitter:
|
|
delay *= (0.5 + random.random())
|
|
|
|
logger.warning(
|
|
f"{func.__name__} attempt {attempt + 1}/{max_attempts} failed: {e}. "
|
|
f"Retrying in {delay:.1f}s..."
|
|
)
|
|
|
|
if on_retry:
|
|
on_retry(attempt + 1, e, delay)
|
|
|
|
time.sleep(delay)
|
|
|
|
raise last_exception
|
|
|
|
return wrapper
|
|
return decorator
|
|
|
|
|
|
def retry_simple(max_attempts=3, delay=2.0):
|
|
"""
|
|
Simple retry without exponential backoff.
|
|
Good for quick operations.
|
|
|
|
Args:
|
|
max_attempts: Maximum number of attempts
|
|
delay: Fixed delay between attempts
|
|
"""
|
|
return retry_with_backoff(
|
|
max_attempts=max_attempts,
|
|
base_delay=delay,
|
|
max_delay=delay,
|
|
exponential_base=1.0,
|
|
jitter=False
|
|
)
|