Using AI and LLMs to Supercharge Web Scraping

I need to scrape product data from this HTML structure:

<div class="product-card">
  <h2 class="title">Wireless Headphones</h2>
  <span class="price">$79.99</span>
  <div class="rating">4.5 stars (230 reviews)</div>
  <p class="description">Premium sound quality...</p>
</div>

Write a Python scraper using BeautifulSoup that extracts:
- Product name
- Price (as float)
- Rating (as float)
- Number of reviews (as int)
- Description

import requests
from bs4 import BeautifulSoup
import re

def scrape_product(url):
    response = requests.get(url)
    soup = BeautifulSoup(response.text, 'html.parser')

    products = []
    for card in soup.select('.product-card'):
        # Extract rating and reviews from combined text
        rating_text = card.select_one('.rating').text
        rating_match = re.search(r'([\d.]+)\s*stars?\s*\((\d+)', rating_text)

        product = {
            'name': card.select_one('.title').text.strip(),
            'price': float(card.select_one('.price').text.replace('$', '')),
            'rating': float(rating_match.group(1)) if rating_match else None,
            'reviews': int(rating_match.group(2)) if rating_match else None,
            'description': card.select_one('.description').text.strip()
        }
        products.append(product)

    return products

"The scraper crashes when a product has no reviews.
The rating div shows 'No reviews yet' instead of the star rating.
Update the code to handle this edge case."

from openai import OpenAI

client = OpenAI()

def extract_product_specs(description):
    """Use GPT to extract structured specs from product descriptions."""

    response = client.chat.completions.create(
        model="gpt-4-turbo",
        messages=[
            {
                "role": "system",
                "content": """Extract product specifications from the description.
                Return JSON with these fields (null if not found):
                - dimensions (object with width, height, depth, unit)
                - weight (object with value, unit)
                - material
                - color
                - warranty_months
                - key_features (array of strings)"""
            },
            {
                "role": "user",
                "content": description
            }
        ],
        response_format={"type": "json_object"}
    )

    return json.loads(response.choices[0].message.content)

# Example usage
description = """
The ProMax 3000 wireless speaker delivers room-filling sound in a
compact 6" x 4" x 4" package. Weighing just 2.3 lbs, it features
premium aluminum construction in midnight black. Includes 24-month
warranty. Key features: Bluetooth 5.0, 20-hour battery, IPX7 waterproof.
"""

specs = extract_product_specs(description)
# Returns structured JSON with all specs extracted

def parse_contact_info(raw_text):
    """Extract structured contact info from messy text."""

    response = client.chat.completions.create(
        model="gpt-4-turbo",
        messages=[
            {
                "role": "system",
                "content": """Parse contact information from text. Return JSON:
                {
                    "name": "string or null",
                    "email": "string or null",
                    "phone": "string or null",
                    "address": {
                        "street": "string",
                        "city": "string",
                        "state": "string",
                        "zip": "string",
                        "country": "string"
                    }
                }"""
            },
            {"role": "user", "content": raw_text}
        ],
        response_format={"type": "json_object"}
    )

    return json.loads(response.choices[0].message.content)

# Works with messy input like:
text = "Contact John Smith at john@example.com or call (555) 123-4567.
        Our office: 123 Main St, Suite 400, San Francisco CA 94102"

import anthropic

client = anthropic.Anthropic()

def find_selector(html_snippet, target_description):
    """Use Claude to find the best selector for a target element."""

    message = client.messages.create(
        model="claude-sonnet-4-20250514",
        max_tokens=500,
        messages=[
            {
                "role": "user",
                "content": f"""Given this HTML:

{html_snippet}

Find the best CSS selector to extract: {target_description}

Return ONLY the CSS selector, nothing else. Prefer:
1. Unique IDs
2. Semantic class names
3. Data attributes
4. Structural selectors as last resort"""
            }
        ]
    )

    return message.content[0].text.strip()

def scrape_with_fallback(soup, selectors, target_description, html_context):
    """Try multiple selectors, use AI to find new one if all fail."""

    for selector in selectors:
        element = soup.select_one(selector)
        if element:
            return element.text.strip()

    # All selectors failed - ask AI for help
    new_selector = find_selector(html_context, target_description)
    element = soup.select_one(new_selector)

    if element:
        # Log the new selector for future use
        print(f"AI found new selector: {new_selector}")
        return element.text.strip()

    return None

def enrich_product_data(product):
    """Add AI-generated insights to scraped product data."""

    enrichment_prompt = f"""Analyze this product and return JSON:
    {{
        "category": "main product category",
        "subcategory": "specific subcategory",
        "target_audience": "who would buy this",
        "price_tier": "budget/mid-range/premium/luxury",
        "key_selling_points": ["point1", "point2", "point3"],
        "potential_competitors": ["competitor1", "competitor2"]
    }}

    Product: {product['name']}
    Price: ${product['price']}
    Description: {product['description']}"""

    response = client.chat.completions.create(
        model="gpt-4-turbo",
        messages=[{"role": "user", "content": enrichment_prompt}],
        response_format={"type": "json_object"}
    )

    enrichment = json.loads(response.choices[0].message.content)
    return {**product, **enrichment}

# Batch processing with rate limiting
async def enrich_products_batch(products, batch_size=10):
    enriched = []
    for i in range(0, len(products), batch_size):
        batch = products[i:i+batch_size]
        tasks = [enrich_product_data(p) for p in batch]
        results = await asyncio.gather(*tasks)
        enriched.extend(results)
        await asyncio.sleep(1)  # Rate limiting
    return enriched

import hashlib
from functools import lru_cache

# Cache LLM responses by content hash
@lru_cache(maxsize=10000)
def cached_extract(content_hash, extraction_type):
    # Only call API if not cached
    return call_llm_api(...)

def extract_with_cache(content, extraction_type):
    content_hash = hashlib.md5(content.encode()).hexdigest()
    return cached_extract(content_hash, extraction_type)

# Batch similar extractions
def batch_extract(items, batch_size=20):
    """Send multiple items in one API call."""
    batched_prompt = "Extract data from each item:\n\n"
    for i, item in enumerate(items):
        batched_prompt += f"Item {i+1}: {item}\n\n"
    batched_prompt += "Return a JSON array with results for each item."

    # One API call instead of 20
    return call_llm_api(batched_prompt)

import requests
from bs4 import BeautifulSoup
from openai import OpenAI
import json

class AIAssistedScraper:
    def __init__(self):
        self.client = OpenAI()
        self.selector_cache = {}

    def scrape_page(self, url, schema):
        """Scrape a page using AI-assisted extraction."""
        response = requests.get(url)
        soup = BeautifulSoup(response.text, 'html.parser')

        result = {}
        for field, config in schema.items():
            # Try cached/configured selectors first
            if config.get('selector'):
                element = soup.select_one(config['selector'])
                if element:
                    result[field] = self.clean_value(element.text, config)
                    continue

            # Fall back to AI extraction
            result[field] = self.ai_extract(
                str(soup),
                config['description'],
                config.get('type', 'string')
            )

        return result

    def ai_extract(self, html, description, value_type):
        """Use AI to extract a specific value from HTML."""
        response = self.client.chat.completions.create(
            model="gpt-3.5-turbo",  # Cost-effective for simple extraction
            messages=[
                {
                    "role": "system",
                    "content": f"Extract the {description} from the HTML. Return only the value as {value_type}. If not found, return null."
                },
                {"role": "user", "content": html[:4000]}  # Truncate for token limits
            ]
        )
        return response.choices[0].message.content.strip()

    def clean_value(self, value, config):
        """Clean and convert extracted value."""
        value = value.strip()
        if config.get('type') == 'float':
            return float(re.sub(r'[^\d.]', '', value))
        if config.get('type') == 'int':
            return int(re.sub(r'[^\d]', '', value))
        return value

# Usage
scraper = AIAssistedScraper()

schema = {
    'title': {
        'selector': 'h1.product-title',
        'description': 'product title/name',
        'type': 'string'
    },
    'price': {
        'selector': '.price-current',
        'description': 'current price in dollars',
        'type': 'float'
    },
    'availability': {
        'description': 'whether the product is in stock',
        'type': 'string'
    }
}

product = scraper.scrape_page('https://example.com/product', schema)