from typing import List, Optional
import logging
from fastapi import APIRouter, Depends, HTTPException, Query, status, BackgroundTasks
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select

from app.api import deps
from app.models.product import Product
from app.schemas.product import (
    ProductCreate,
    ProductUpdate,
    ProductResponse,
    ProductListResponse,
)
from app.schemas.scraping import ScrapingResponse
from app.services.product_service import ProductService
from app.services.scraper_service import ScraperService
from datetime import datetime

router = APIRouter()
logger = logging.getLogger(__name__)


async def _perform_background_scraping(
    product_id: int,
    headless_mode: bool,
    timeout_seconds: int,
    enable_discovery: bool,
):
    """
    Background task to perform scraping after product creation.
    This runs asynchronously without blocking the API response.
    Executes UNIFIED SCRAPING: Google Search + Shopping Light (same as /run-scraping endpoint)
    """
    try:
        from app.db.session import AsyncSessionLocal
        from datetime import datetime
        
        logger.info(f"🚀 [Background] Starting UNIFIED first-time scraping for product ID: {product_id}")
        
        async with AsyncSessionLocal() as db:
            # Fetch product
            result = await db.execute(select(Product).where(Product.id == product_id))
            product = result.scalars().first()
            
            if not product:
                logger.error(f"❌ [Background] Product {product_id} not found")
                return
            
            # Cache product attributes
            product_name_cached = product.product_name
            product_barcode_cached = product.barcode
            product_msp_cached = float(product.msp)
            
            logger.info(f"[Background] Product: {product_name_cached} (ID: {product_id})")
            logger.info(f"[Background] Using UNIFIED approach: Google Search + Shopping Light")
            
            all_results = []
            all_violations = []
            search_summary = {
                "google_search": {"results": 0, "violations": 0, "status": "pending"},
                "shopping_light": {"results": 0, "violations": 0, "status": "pending"}
            }
            
            # ===== STEP 1: Google Search =====
            logger.info(f"[Background] [STEP 1/2] 🔍 Google Search (Organic Results)...")
            try:
                google_results, google_violations = await ScraperService.search_google_serp(
                    product_name=product.product_name,
                    barcode=product.barcode,
                    msp=product.msp,
                    product=product,
                    db=db
                )
                all_results.extend(google_results)
                all_violations.extend(google_violations)
                
                search_summary["google_search"]["results"] = len(google_results)
                search_summary["google_search"]["violations"] = len(google_violations)
                search_summary["google_search"]["status"] = "completed"
                
                logger.info(f"[Background] ✅ Google Search: {len(google_results)} results, {len(google_violations)} violations")
            except Exception as google_err:
                search_summary["google_search"]["status"] = f"failed - {str(google_err)}"
                logger.error(f"[Background] ⚠️ Google Search failed: {str(google_err)}", exc_info=False)
            
            # ===== STEP 2: Google Shopping Light =====
            logger.info(f"[Background] [STEP 2/2] 🛒 Google Shopping Light + Registered Vendors...")
            try:
                shopping_result = await ScraperService.scrape_product_serp(
                    db,
                    product_id=product_id,
                    enable_discovery=enable_discovery,
                    headless_mode=headless_mode,
                    timeout_seconds=timeout_seconds
                )
                
                shopping_results = shopping_result.get("results", [])
                shopping_violations = shopping_result.get("violations", [])
                
                all_results.extend(shopping_results)
                all_violations.extend(shopping_violations)
                
                search_summary["shopping_light"]["results"] = len(shopping_results)
                search_summary["shopping_light"]["violations"] = len(shopping_violations)
                search_summary["shopping_light"]["status"] = "completed"
                
                logger.info(f"[Background] ✅ Shopping Light: {len(shopping_results)} results, {len(shopping_violations)} violations")
            except Exception as shopping_err:
                search_summary["shopping_light"]["status"] = f"failed - {str(shopping_err)}"
                logger.error(f"[Background] ⚠️ Shopping Light failed: {str(shopping_err)}", exc_info=False)
            
            # ===== DEDUPLICATION =====
            seen_violations = set()
            deduped_violations = []
            for v in all_violations:
                violation_key = (
                    v.get("vendor_name"),
                    v.get("product_name"),
                    v.get("msp"),
                    round(v.get("scraped_price", 0), 2)
                )
                if violation_key not in seen_violations:
                    seen_violations.add(violation_key)
                    deduped_violations.append(v)
            
            # ===== FINAL SUMMARY =====
            execution_time_str = datetime.utcnow().isoformat()
            
            # Update product with execution timestamp
            product.last_execution_time = execution_time_str
            await db.commit()
            
            logger.info(f"\n{'='*80}")
            logger.info(f"✅ [Background] UNIFIED SCRAPING COMPLETED for product ID: {product_id}")
            logger.info(f"{'='*80}")
            logger.info(f"   • Product: {product_name_cached}")
            logger.info(f"   • Total Results: {len(all_results)}")
            logger.info(f"   • Total Violations (deduplicated): {len(deduped_violations)}")
            logger.info(f"   • Execution Time: {execution_time_str}")
            logger.info(f"   • Google Search: {search_summary['google_search']['status']} ({search_summary['google_search']['results']} results, {search_summary['google_search']['violations']} violations)")
            logger.info(f"   • Shopping Light: {search_summary['shopping_light']['status']} ({search_summary['shopping_light']['results']} results, {search_summary['shopping_light']['violations']} violations)")
            logger.info(f"{'='*80}\n")
            
    except Exception as e:
        logger.error(f"⚠️ [Background] UNIFIED scraping failed for product ID {product_id}: {str(e)}", exc_info=True)


@router.get("/", response_model=ProductListResponse)
async def get_products(
    db: AsyncSession = Depends(deps.get_db),
    page: int = Query(1, ge=1),
    limit: int = Query(10, ge=1, le=100),
    sortBy: str = Query("product_name", regex="^(product_name|msp|last_scraped_date)$"),
    search: Optional[str] = None,
    current_user=Depends(deps.get_current_user),
):
    """
    Retrieve a list of products with pagination, sorting, and searching.
    """
    products, total = await ProductService.get_products(
        db, page=page, limit=limit, sort_by=sortBy, search=search
    )
    return ProductListResponse(items=products, total=total, page=page, limit=limit)


@router.post("/", response_model=dict, status_code=status.HTTP_201_CREATED)
async def add_product(
    product_in: ProductCreate,
    background_tasks: BackgroundTasks,
    headless: bool = Query(True, description="Run browser in headless mode for initial scraping"),
    timeout: int = Query(15, ge=5, le=60, description="Timeout in seconds for page load during initial scraping"),
    enable_discovery: bool = Query(True, description="Discover alternative vendors via SERP API during initial scraping"),
    db: AsyncSession = Depends(deps.get_db),
    current_user=Depends(deps.get_current_user),
):
    """
    Add a new product and automatically trigger scraping in the background.
    
    **Returns immediately** (HTTP 201) without waiting for scraping to complete.
    Scraping happens asynchronously in the background using SERP API.
    
    Parameters:
    - product_in: Product creation data
    - headless: Whether to run browser in headless mode (default: True)
    - timeout: Page load timeout in seconds (default: 15)
    - enable_discovery: Discover alternative vendors via SERP API (default: True)
    
    **Response (Immediate):**
    - message: Product creation success
    - product_id: ID of created product
    - scraping_status: "in_progress" - indicates scraping is running in background
    
    **Note:** Scraping continues in background. You can check results via GET /products/{id} 
    or monitor logs for completion message: "First-time scraping completed for product ID: {id}"
    """
    # Create the product (immediate)
    product = await ProductService.create_product(db, product_in)
    logger.info(f"✅ Product created: {product.product_name} (ID: {product.id})")
    
    # Schedule scraping as background task (non-blocking)
    background_tasks.add_task(
        _perform_background_scraping,
        product_id=product.id,
        headless_mode=headless,
        timeout_seconds=timeout,
        enable_discovery=enable_discovery
    )
    logger.info(f"📌 Background scraping scheduled for product ID: {product.id}")
    
    # Return immediately to user
    return {
        "message": "Product added successfully! Scraping is starting in the background.",
        "product_id": product.id,
        "scraping_status": "in_progress",
        "note": "You can start working with the product while scraping happens. Check logs or GET /products/{id} for scraping results."
    }


@router.put("/{id}", response_model=dict)
async def update_product(
    id: int,
    product_in: ProductUpdate,
    db: AsyncSession = Depends(deps.get_db),
    current_user=Depends(deps.get_current_user),
):
    """
    Edit an existing product. (Admin only)
    """
    await ProductService.update_product(db, id, product_in)
    return {"message": "Product updated successfully"}


@router.delete("/{id}", response_model=dict)
async def delete_product(
    id: int,
    db: AsyncSession = Depends(deps.get_db),
    current_user=Depends(deps.get_current_user),
):
    """
    Delete a product. (Admin only)
    """
    await ProductService.delete_product(db, id)
    return {"message": "Product deleted successfully"}


@router.get("/{id}", response_model=ProductResponse)
async def get_product(
    id: int,
    db: AsyncSession = Depends(deps.get_db),
    current_user=Depends(deps.get_current_user),
) -> ProductResponse:
    """
    Retrieve a single product by its ID.
    """
    product = await ProductService.get_product_by_id(db, id)
    return product


@router.post("/{id}/scrape", response_model=ScrapingResponse)
async def trigger_scraping(
    id: int,
    headless: bool = Query(True, description="Run browser in headless mode"),
    timeout: int = Query(15, ge=5, le=60, description="Timeout in seconds for page load"),
    enable_discovery: bool = Query(True, description="Also discover alternative vendors via Google search"),
    db: AsyncSession = Depends(deps.get_db),
    current_user=Depends(deps.get_current_user),
):
    """
    Manually trigger scraping for a specific product.
    Scrapes the product against all registered active vendors.
    When discovery is enabled, also searches Google for alternative sellers and scrapes their prices.
    
    Parameters:
    - id: Product ID to scrape
    - headless: Whether to run browser in headless mode (default: True)
    - timeout: Page load timeout in seconds (default: 15)
    - enable_discovery: if true, discover and scrape alternative vendors (default: true)
    """
    product = await ProductService.get_product_by_id(db, id)
    if not product.status:
        raise HTTPException(
            status_code=status.HTTP_400_BAD_REQUEST,
            detail="Scraping can only be triggered for active products",
        )
    
    # Run scraping service
    result = await ScraperService.scrape_product(
        db,
        product_id=id,
        enable_discovery=enable_discovery,
        headless_mode=headless,
        timeout_seconds=timeout
    )
    
    return result


@router.post("/scrape-all", response_model=dict, status_code=status.HTTP_200_OK)
async def scrape_all_products(
    headless: bool = Query(True, description="Run browser in headless mode"),
    timeout: int = Query(15, ge=5, le=60, description="Timeout in seconds for page load"),
    enable_discovery: bool = Query(True, description="Also discover alternative vendors via Google search"),
    db: AsyncSession = Depends(deps.get_db),
    current_user=Depends(deps.get_current_user),
):
    """
    Manually trigger scraping for ALL active products.
    Scrapes EACH product against ALL registered active vendors.
    For each vendor's website URL: Extract price using search and price detection.
    Then if discovery enabled: Google search for alternative vendors and scrape them too.
    
    This is a SYNCHRONOUS operation - waits for all products to complete before returning.
    Products are scraped sequentially to avoid browser and connection issues.
    
    Parameters:
    - headless: Whether to run browser in headless mode (default: True)
    - timeout: Page load timeout in seconds (default: 15)
    - enable_discovery: Also discover and scrape alternative vendors via Google search (default: True)
    
    Returns:
    - Complete results including all scraped vendors and violations for each product
    """
    # Get all active products
    stmt = select(Product).where(Product.status == True)
    result = await db.execute(stmt)
    products = result.scalars().all()
    
    if not products:
        raise HTTPException(
            status_code=status.HTTP_400_BAD_REQUEST,
            detail="No active products found for scraping"
        )
    
    all_products_results = []
    total_scraped = 0
    total_violations = 0
    failed_products = []
    
    logger.info(f"\n\n{'='*80}")
    logger.info(f"🚀 [SCRAPE ALL] Starting scraping for {len(products)} products")
    logger.info(f"{'='*80}\n")
    
    try:
        for idx, product in enumerate(products, 1):
            fresh_db = None
            try:
                # Create a fresh session for this product
                from app.db.session import AsyncSessionLocal
                fresh_db = AsyncSessionLocal()
                
                logger.info(f"\n{'─'*80}")
                logger.info(f"[{idx}/{len(products)}] 📦 PRODUCT: {product.product_name} (ID: {product.id})")
                logger.info(f"{'─'*80}")
                logger.info(f"        Barcode: {product.barcode}")
                logger.info(f"        MSP: ₹{product.msp}")
                logger.info(f"        Discovery Enabled: {enable_discovery}")
                
                # Scrape this product against all vendors + discovery
                product_result = await ScraperService.scrape_product(
                    fresh_db,
                    product_id=product.id,
                    enable_discovery=enable_discovery,
                    headless_mode=headless,
                    timeout_seconds=timeout
                )
                
                scraped_count = product_result.get("scraped_count", 0)
                violation_count = product_result.get("violation_count", 0)
                
                logger.info(f"\n✅ [RESULT] Product {product.id}:")
                logger.info(f"   • Vendors Scraped: {scraped_count}")
                logger.info(f"   • Violations Found: {violation_count}")
                
                # Log individual violations
                violations = product_result.get("violations", [])
                if violations:
                    logger.warning(f"\n   🚨 VIOLATIONS FOR {product.product_name}:")
                    for v in violations:
                        logger.warning(f"      - {v['vendor_name']}: ₹{v['scraped_price']:.3f} (MSP: ₹{v['msp']:.3f})")
                
                total_scraped += scraped_count
                total_violations += violation_count
                all_products_results.append(product_result)
                
            except HTTPException as he:
                logger.error(f"\n❌ [ERROR] Product {product.id}: {he.detail}", exc_info=False)
                failed_products.append({
                    "product_id": product.id,
                    "product_name": product.product_name,
                    "error": he.detail
                })
            except Exception as e:
                logger.error(f"\n❌ [ERROR] Product {product.id}: {str(e)}", exc_info=True)
                failed_products.append({
                    "product_id": product.id,
                    "product_name": product.product_name,
                    "error": str(e)
                })
            finally:
                # Ensure session is properly closed
                if fresh_db:
                    try:
                        await fresh_db.close()
                    except Exception as e:
                        logger.warning(f"Warning closing session: {str(e)}")
        
        # Final summary
        logger.info(f"\n\n{'='*80}")
        logger.info(f"📊 [SCRAPE ALL] FINAL SUMMARY")
        logger.info(f"{'='*80}")
        logger.info(f"Total Products: {len(products)}")
        logger.info(f"Successfully Scraped: {len(all_products_results)}")
        logger.info(f"Failed: {len(failed_products)}")
        logger.info(f"Total Vendor URLs Scraped: {total_scraped}")
        logger.info(f"Total Violations Found: {total_violations}")
        logger.info(f"{'='*80}\n")
        
        return {
            "status": "completed",
            "message": f"Scraping completed for {len(all_products_results)} products",
            "total_products": len(products),
            "successful_products": len(all_products_results),
            "failed_products": len(failed_products),
            "total_vendors_scraped": total_scraped,
            "total_violations": total_violations,
            "products": all_products_results,
            "failed_list": failed_products if failed_products else None,
            "enable_discovery": enable_discovery
        }
        
    except Exception as e:
        logger.error(f"\n\n❌ [FATAL ERROR] Scrape All failed: {str(e)}", exc_info=True)
        raise HTTPException(
            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
            detail=f"Scraping failed: {str(e)}"
        )


@router.post("/{id}/scrape-serp", response_model=dict)
async def trigger_scraping_serp(
    id: int,
    headless: bool = Query(True, description="Run browser in headless mode"),
    timeout: int = Query(15, ge=5, le=60, description="Timeout in seconds for page load"),
    enable_discovery: bool = Query(True, description="Also discover alternative vendors via SERP API Google Shopping"),
    db: AsyncSession = Depends(deps.get_db),
    current_user=Depends(deps.get_current_user),
):
    """
    Manually trigger scraping for a specific product using SERP API for discovery.
    Same functionality as /{id}/scrape, but uses SERP API Google Shopping Light Engine
    instead of Tavily API for discovering alternative vendors.
    
    Scrapes the product against all registered active vendors.
    When discovery is enabled, also searches SERP API Google Shopping for alternative sellers 
    and scrapes their prices.
    
    Parameters:
    - id: Product ID to scrape
    - headless: Whether to run browser in headless mode (default: True)
    - timeout: Page load timeout in seconds (default: 15)
    - enable_discovery: if true, discover and scrape alternative vendors via SERP API (default: true)
    """
    product = await ProductService.get_product_by_id(db, id)
    if not product.status:
        raise HTTPException(
            status_code=status.HTTP_400_BAD_REQUEST,
            detail="Scraping can only be triggered for active products",
        )
    
    # Run scraping service with SERP API discovery
    result = await ScraperService.scrape_product_serp(
        db,
        product_id=id,
        enable_discovery=enable_discovery,
        headless_mode=headless,
        timeout_seconds=timeout
    )
    
    return result


@router.post("/scrape-all-serp", response_model=dict, status_code=status.HTTP_200_OK)
async def scrape_all_products_serp(
    headless: bool = Query(True, description="Run browser in headless mode"),
    timeout: int = Query(15, ge=5, le=60, description="Timeout in seconds for page load"),
    enable_discovery: bool = Query(True, description="Also discover alternative vendors via SERP API Google Shopping"),
    db: AsyncSession = Depends(deps.get_db),
    current_user=Depends(deps.get_current_user),
):
    """
    Manually trigger scraping for ALL active products using SERP API for discovery.
    Same functionality as /scrape-all, but uses SERP API Google Shopping Light Engine
    instead of Tavily API for discovering alternative vendors.
    
    Scrapes EACH product against ALL registered active vendors.
    For each vendor's website URL: Extract price using search and price detection.
    Then if discovery enabled: SERP API search for alternative vendors and scrape them too.
    
    This is a SYNCHRONOUS operation - waits for all products to complete before returning.
    Products are scraped sequentially to avoid browser and connection issues.
    
    Parameters:
    - headless: Whether to run browser in headless mode (default: True)
    - timeout: Page load timeout in seconds (default: 15)
    - enable_discovery: Also discover and scrape alternative vendors via SERP API (default: True)
    
    Returns:
    - Complete results including all scraped vendors and violations for each product
    """
    # Get all active products
    stmt = select(Product).where(Product.status == True)
    result = await db.execute(stmt)
    products = result.scalars().all()
    
    if not products:
        raise HTTPException(
            status_code=status.HTTP_400_BAD_REQUEST,
            detail="No active products found for scraping"
        )
    
    all_products_results = []
    total_scraped = 0
    total_violations = 0
    failed_products = []
    
    logger.info(f"\n\n{'='*80}")
    logger.info(f"🚀 [SCRAPE ALL SERP] Starting scraping for {len(products)} products using SERP API")
    logger.info(f"{'='*80}\n")
    
    try:
        for idx, product in enumerate(products, 1):
            fresh_db = None
            try:
                # Create a fresh session for this product
                from app.db.session import AsyncSessionLocal
                fresh_db = AsyncSessionLocal()
                
                logger.info(f"\n{'─'*80}")
                logger.info(f"[{idx}/{len(products)}] 📦 PRODUCT: {product.product_name} (ID: {product.id})")
                logger.info(f"{'─'*80}")
                logger.info(f"        Barcode: {product.barcode}")
                logger.info(f"        MSP: ₹{product.msp}")
                logger.info(f"        Discovery Method: SERP API Google Shopping")
                logger.info(f"        Discovery Enabled: {enable_discovery}")
                
                # Scrape this product against all vendors with SERP API discovery
                product_result = await ScraperService.scrape_product_serp(
                    fresh_db,
                    product_id=product.id,
                    enable_discovery=enable_discovery,
                    headless_mode=headless,
                    timeout_seconds=timeout
                )
                
                total_results = product_result.get("total_results", 0)
                total_violation_count = product_result.get("total_violations", 0)
                
                logger.info(f"\n✅ [RESULT] Product {product.id}:")
                logger.info(f"   • Total Results: {total_results}")
                logger.info(f"   • Total Violations Found: {total_violation_count}")
                
                # Log individual violations
                violations = product_result.get("violations", [])
                if violations:
                    logger.warning(f"\n   🚨 VIOLATIONS FOR {product.product_name}:")
                    for v in violations:
                        logger.warning(f"      - {v.get('vendor_name', 'Unknown')}: ₹{v.get('scraped_price', 'N/A')} (MSP: ₹{v.get('msp', 'N/A')})")
                
                total_scraped += total_results
                total_violations += total_violation_count
                all_products_results.append(product_result)
                
            except HTTPException as he:
                logger.error(f"\n❌ [ERROR] Product {product.id}: {he.detail}", exc_info=False)
                failed_products.append({
                    "product_id": product.id,
                    "product_name": product.product_name,
                    "error": he.detail
                })
            except Exception as e:
                logger.error(f"\n❌ [ERROR] Product {product.id}: {str(e)}", exc_info=True)
                failed_products.append({
                    "product_id": product.id,
                    "product_name": product.product_name,
                    "error": str(e)
                })
            finally:
                # Ensure session is properly closed
                if fresh_db:
                    try:
                        await fresh_db.close()
                    except Exception as e:
                        logger.warning(f"Warning closing session: {str(e)}")
        
        # Final summary
        logger.info(f"\n\n{'='*80}")
        logger.info(f"📊 [SCRAPE ALL SERP] FINAL SUMMARY")
        logger.info(f"{'='*80}")
        logger.info(f"Total Products: {len(products)}")
        logger.info(f"Successfully Scraped: {len(all_products_results)}")
        logger.info(f"Failed: {len(failed_products)}")
        logger.info(f"Total Results: {total_scraped}")
        logger.info(f"Total Violations Found: {total_violations}")
        logger.info(f"Discovery Method: SERP API Google Shopping Light Engine")
        logger.info(f"{'='*80}\n")
        
        return {
            "status": "completed",
            "message": f"Scraping completed for {len(all_products_results)} products using SERP API",
            "total_products": len(products),
            "successful_products": len(all_products_results),
            "failed_products": len(failed_products),
            "total_results": total_scraped,
            "total_violations": total_violations,
            "products": all_products_results,
            "failed_list": failed_products if failed_products else None,
            "enable_discovery": enable_discovery,
            "discovery_method": "serp_api_google_shopping",
            "source_type": "browser_with_serp_api_discovery"
        }
        
    except Exception as e:
        logger.error(f"\n\n❌ [FATAL ERROR] Scrape All SERP failed: {str(e)}", exc_info=True)
        raise HTTPException(
            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
            detail=f"Scraping failed: {str(e)}"
        )


@router.post("/{id}/search-google", response_model=dict)
async def search_google_product(
    id: int,
    db: AsyncSession = Depends(deps.get_db),
    current_user=Depends(deps.get_current_user),
):
    """
    Search for a product using Google Search (SERP API - regular search engine).
    Creates violations in the database following the same schema as SERP Shopping Light API.
    
    Parameters:
    - id: Product ID to search for
    
    Returns:
    - Search results from Google Search (organic_results)
    - Violations found and stored in database
    """
    try:
        # Get product
        product = await ProductService.get_product_by_id(db, id)
        
        logger.info(f"\n{'='*80}")
        logger.info(f"🔍 [Google Search] Searching product: {product.product_name}")
        logger.info(f"{'='*80}\n")
        
        # Call Google Search function with db session for violation storage
        results, violations = await ScraperService.search_google_serp(
            product_name=product.product_name,
            barcode=product.barcode,
            msp=product.msp,
            product=product,
            db=db
        )
        
        logger.info(f"[Google Search] ✓ Search completed. Found {len(results)} results, {len(violations)} violations")
        
        return {
            "status": "completed",
            "product_id": id,
            "product_name": product.product_name,
            "barcode": product.barcode,
            "msp": float(product.msp),
            "total_results": len(results),
            "results": results,
            "violations": violations,
            "total_violations": len(violations),
            "search_engine": "google_search",
            "source": "serp_api",
            "note": "Results and violations are stored in database following same schema as SERP Shopping Light API"
        }
        
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"❌ Google Search failed: {str(e)}", exc_info=True)
        raise HTTPException(
            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
            detail=f"Search failed: {str(e)}"
        )


async def _run_scraping_unified_background(product_id: int):
    from app.db.session import AsyncSessionLocal
    from datetime import datetime

    async with AsyncSessionLocal() as db:
        try:
            product = await ProductService.get_product_by_id(db, product_id)

            # MARK STARTED
            product.scraping_status = "in_progress"
            product.scraping_started_at = datetime.utcnow()
            product.scraping_error = None
            await db.commit()

            product_name_cached = product.product_name
            product_barcode_cached = product.barcode
            product_msp_cached = float(product.msp)

            logger.info(f"\n\n{'='*80}")
            logger.info(f"🎯 [BACKGROUND UNIFIED SCRAPING] Starting")
            logger.info(f"Product: {product_name_cached} (ID: {product_id})")
            logger.info(f"{'='*80}\n")

            all_results = []
            all_violations = []
            search_summary = {
                "google_search": {"results": 0, "violations": 0, "status": "pending"},
                "shopping_light": {"results": 0, "violations": 0, "status": "pending"}
            }

            # ===== STEP 1 =====
            try:
                google_results, google_violations = await ScraperService.search_google_serp(
                    product_name=product.product_name,
                    barcode=product.barcode,
                    msp=product.msp,
                    product=product,
                    db=db
                )
                all_results.extend(google_results)
                all_violations.extend(google_violations)

                search_summary["google_search"]["results"] = len(google_results)
                search_summary["google_search"]["violations"] = len(google_violations)
                search_summary["google_search"]["status"] = "completed"

            except Exception as e:
                search_summary["google_search"]["status"] = f"failed - {str(e)}"

            # ===== STEP 2 =====
            try:
                shopping_result = await ScraperService.scrape_product_serp(
                    db,
                    product_id=product_id,
                    enable_discovery=True,
                    headless_mode=True,
                    timeout_seconds=15
                )

                shopping_results = shopping_result.get("results", [])
                shopping_violations = shopping_result.get("violations", [])

                all_results.extend(shopping_results)
                all_violations.extend(shopping_violations)

                search_summary["shopping_light"]["results"] = len(shopping_results)
                search_summary["shopping_light"]["violations"] = len(shopping_violations)
                search_summary["shopping_light"]["status"] = "completed"

            except Exception as e:
                search_summary["shopping_light"]["status"] = f"failed - {str(e)}"

            # ===== DEDUP =====
            seen = set()
            deduped = []
            for v in all_violations:
                key = (
                    v.get("vendor_name"),
                    v.get("product_name"),
                    v.get("msp"),
                    round(v.get("scraped_price", 0), 2)
                )
                if key not in seen:
                    seen.add(key)
                    deduped.append(v)

            execution_time = datetime.utcnow().isoformat()

            logger.info(f"✅ [BACKGROUND DONE] Product {product_id}")
            logger.info(f"Results: {len(all_results)}, Violations: {len(deduped)}")

            # MARK COMPLETED
            product.scraping_status = "completed"
            product.scraping_completed_at = datetime.utcnow()
            await db.commit()

        except Exception as e:
            logger.error(f"❌ Background unified scraping failed: {str(e)}", exc_info=True)
            # MARK FAILED
            product.scraping_status = "failed"
            product.scraping_error = str(e)
            product.scraping_completed_at = datetime.utcnow()
            await db.commit()

@router.post("/{id}/run-scraping", response_model=dict)
async def run_scraping_unified(
    id: int,
    background_tasks: BackgroundTasks,
    db: AsyncSession = Depends(deps.get_db),
    current_user=Depends(deps.get_current_user),
):
    product = await ProductService.get_product_by_id(db, id)

    if not product.status:
        raise HTTPException(
            status_code=status.HTTP_400_BAD_REQUEST,
            detail="Scraping can only be triggered for active products",
        )

    # Run in background
    background_tasks.add_task(
        _run_scraping_unified_background,
        product_id=id
    )

    logger.info(f"Unified scraping scheduled (background) for product {id}")

    # immediate response
    return {
        "status": "in_progress",
        "product_id": id,
        "message": "Unified scraping started in background"
    }

@router.get("/{id}/scraping-status", response_model=dict)
async def get_scraping_status(
    id: int,
    db: AsyncSession = Depends(deps.get_db),
    current_user=Depends(deps.get_current_user),
):
    product = await ProductService.get_product_by_id(db, id)

    if not product:
        raise HTTPException(
            status_code=status.HTTP_404_NOT_FOUND,
            detail="Product not found"
        )

    return {
        "product_id": id,
        "product_name": product.product_name,
        "status": product.scraping_status or "idle",
        "started_at": product.scraping_started_at,
        "completed_at": product.scraping_completed_at,
        "error": product.scraping_error,
    }