import logging import os from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from sqlalchemy import delete from sqlalchemy import text from sqlalchemy.exc import SQLAlchemyError from app.api.routes.products import router as product_router from app.api.routes.auth import router as auth_router from app.api.routes.vendors import router as vendor_router from app.api.routes.violations import router as violations_router from app.api.routes.dashboard import router as dashboard_router from app.api.routes.admin_settings import router as admin_settings_router #from app.core.config import settings from app.db.base import Base from app.db.session import engine #from app import models # noqa: F401 from app.core.security import utcnow from app.models.password_reset_token import PasswordResetToken # Import all models to ensure they are registered with SQLAlchemy from app.models import User, RefreshToken, Vendor, Violation, Product, ScrapingResult, SiteSettings # noqa: F401 from app.services.scheduler_service import SchedulerService # below is for windows only # import asyncio # import sys # if sys.platform == "win32": # asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) # Configure logging to show INFO level and above logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' ) # Set log level for all app loggers to INFO for logger_name in ['app', 'app.services', 'app.api']: logging.getLogger(logger_name).setLevel(logging.INFO) logger = logging.getLogger(__name__) def create_app() -> FastAPI: app = FastAPI(title="MARCO Scraping") # app.add_middleware( # CORSMiddleware, # allow_origin_regex=r"^(https:\/\/marco-fe\.testyourapp\.online\/?|\/?)$", # allow_credentials=True, # allow_methods=["*"], # allow_headers=["*"], # ) app.add_middleware( CORSMiddleware, allow_origins=[ "https://marco-fe.testyourapp.online", ], allow_origin_regex=r"^https?:\/\/(localhost|127\.0\.0\.1)(:\d+)?$", allow_credentials=True, allow_methods=["*"], allow_headers=["*"], ) @app.on_event("startup") async def _create_tables() -> None: try: async with engine.begin() as conn: await conn.run_sync(Base.metadata.create_all) await conn.execute( text( "ALTER TABLE users " "ADD COLUMN IF NOT EXISTS tokens_invalid_before TIMESTAMPTZ NULL" ) ) await conn.execute( delete(PasswordResetToken).where( (PasswordResetToken.expires_at <= utcnow()) | (PasswordResetToken.used_at.is_not(None)) ) ) await conn.execute( text( "ALTER TABLE violations ADD COLUMN IF NOT EXISTS barcode_number VARCHAR(255);") ) await conn.execute( text( "ALTER TABLE violations ADD COLUMN IF NOT EXISTS reference_id VARCHAR(255);") ) # Add new columns for violation tracking await conn.execute( text( "ALTER TABLE violations ADD COLUMN IF NOT EXISTS confirmation_count INTEGER DEFAULT 1;") ) await conn.execute( text( "ALTER TABLE violations ADD COLUMN IF NOT EXISTS source_type VARCHAR(50) DEFAULT 'registered';") ) await conn.execute( text( "ALTER TABLE violations ADD COLUMN IF NOT EXISTS last_confirmed_date TIMESTAMPTZ NULL;") ) await conn.execute( text( "ALTER TABLE violations ADD COLUMN IF NOT EXISTS notification_sent_at TIMESTAMPTZ NULL;") ) await conn.execute( text( "ALTER TABLE violations ADD COLUMN IF NOT EXISTS vendor_name VARCHAR(512);") ) await conn.execute( text( "ALTER TABLE violations ADD COLUMN IF NOT EXISTS marketplace VARCHAR(255);") ) # make vendor_id nullable for discovered violations and scraping results await conn.execute( text( "ALTER TABLE violations ALTER COLUMN vendor_id DROP NOT NULL;" ) ) await conn.execute( text( "ALTER TABLE scraping_results ALTER COLUMN vendor_id DROP NOT NULL;" ) ) # Add price tracking columns to violations table await conn.execute( text( "ALTER TABLE violations ADD COLUMN IF NOT EXISTS price_difference NUMERIC;") ) await conn.execute( text( "ALTER TABLE violations ADD COLUMN IF NOT EXISTS percentage_difference NUMERIC;") ) # Add pack-based pricing columns to products table await conn.execute( text( "ALTER TABLE products ADD COLUMN IF NOT EXISTS price_1_pack NUMERIC(10, 2) DEFAULT 0;" ) ) await conn.execute( text( "ALTER TABLE products ADD COLUMN IF NOT EXISTS price_2_pack NUMERIC(10, 2) DEFAULT 0;" ) ) await conn.execute( text( "ALTER TABLE products ADD COLUMN IF NOT EXISTS price_3_pack NUMERIC(10, 2) DEFAULT 0;" ) ) await conn.execute( text( "ALTER TABLE products ADD COLUMN IF NOT EXISTS price_4_pack NUMERIC(10, 2) DEFAULT 0;" ) ) await conn.execute( text( "ALTER TABLE products ADD COLUMN IF NOT EXISTS price_5_pack NUMERIC(10, 2) DEFAULT 0;" ) ) await conn.execute( text( "ALTER TABLE products ADD COLUMN IF NOT EXISTS price_6_pack NUMERIC(10, 2) DEFAULT 0;" ) ) await conn.execute( text( "ALTER TABLE products ADD COLUMN IF NOT EXISTS price_12_pack NUMERIC(10, 2) DEFAULT 0;" ) ) # Add packs column to violations table await conn.execute( text( "ALTER TABLE violations ADD COLUMN IF NOT EXISTS packs INTEGER DEFAULT 1;" ) ) # Add last_scraped_date to products table await conn.execute( text( "ALTER TABLE products ADD COLUMN IF NOT EXISTS last_scraped_date TIMESTAMPTZ NULL;") ) # Add last_execution_time to products table await conn.execute( text( "ALTER TABLE products ADD COLUMN IF NOT EXISTS last_execution_time VARCHAR(255);") ) # Add scraping status tracking columns await conn.execute( text( "ALTER TABLE products ADD COLUMN IF NOT EXISTS scraping_status VARCHAR(50) DEFAULT 'idle';" ) ) await conn.execute( text( "ALTER TABLE products ADD COLUMN IF NOT EXISTS scraping_started_at TIMESTAMPTZ NULL;" ) ) await conn.execute( text( "ALTER TABLE products ADD COLUMN IF NOT EXISTS scraping_completed_at TIMESTAMPTZ NULL;" ) ) await conn.execute( text( "ALTER TABLE products ADD COLUMN IF NOT EXISTS scraping_error VARCHAR(500) NULL;" ) ) # Add domain_name to scraping_results table await conn.execute( text( "ALTER TABLE scraping_results ADD COLUMN IF NOT EXISTS domain_name VARCHAR(255);") ) # Site settings table columns (if they don't exist) await conn.execute( text( "ALTER TABLE site_settings ADD COLUMN IF NOT EXISTS scraping_enabled BOOLEAN DEFAULT FALSE;" ) ) await conn.execute( text( "ALTER TABLE site_settings ADD COLUMN IF NOT EXISTS scraping_frequency VARCHAR(50) DEFAULT 'daily';" ) ) await conn.execute( text( "ALTER TABLE site_settings ADD COLUMN IF NOT EXISTS scraping_time VARCHAR(5) DEFAULT '02:00';" ) ) # Make scraping_time nullable for custom frequency mode await conn.execute( text( "ALTER TABLE site_settings ALTER COLUMN scraping_time DROP NOT NULL;" ) ) await conn.execute( text( "ALTER TABLE site_settings ADD COLUMN IF NOT EXISTS scraping_day_of_week VARCHAR(50) DEFAULT 'monday';" ) ) await conn.execute( text( "ALTER TABLE site_settings ADD COLUMN IF NOT EXISTS scraping_custom_cron VARCHAR(255) NULL;" ) ) await conn.execute( text( "ALTER TABLE site_settings ADD COLUMN IF NOT EXISTS enable_discovery BOOLEAN DEFAULT TRUE;" ) ) await conn.execute( text( "ALTER TABLE site_settings ADD COLUMN IF NOT EXISTS headless_mode BOOLEAN DEFAULT TRUE;" ) ) await conn.execute( text( "ALTER TABLE site_settings ADD COLUMN IF NOT EXISTS timeout_seconds INTEGER DEFAULT 15;" ) ) await conn.execute( text( "ALTER TABLE site_settings ADD COLUMN IF NOT EXISTS last_scheduled_run TIMESTAMPTZ NULL;" ) ) await conn.execute( text( "ALTER TABLE site_settings ADD COLUMN IF NOT EXISTS last_scheduled_run_status VARCHAR(50) NULL;" ) ) await conn.execute( text( "ALTER TABLE site_settings ADD COLUMN IF NOT EXISTS last_scheduled_run_summary VARCHAR(500) NULL;" ) ) await conn.execute( text( "ALTER TABLE site_settings ADD COLUMN IF NOT EXISTS client_timezone VARCHAR(50) DEFAULT 'UTC';" ) ) # Add product_id foreign key to violations table with cascade delete await conn.execute( text( "ALTER TABLE violations ADD COLUMN IF NOT EXISTS product_id INTEGER;" ) ) # Add foreign key constraint if it doesn't exist await conn.execute( text( """ DO $$ BEGIN IF NOT EXISTS ( SELECT 1 FROM information_schema.table_constraints WHERE table_name = 'violations' AND constraint_name = 'fk_violations_product_id' ) THEN ALTER TABLE violations ADD CONSTRAINT fk_violations_product_id FOREIGN KEY (product_id) REFERENCES products(id) ON DELETE CASCADE; END IF; END $$; """ ) ) # Create index on product_id for better query performance await conn.execute( text( "CREATE INDEX IF NOT EXISTS idx_violations_product_id ON violations(product_id);" ) ) # Data migration: Link existing violations to products by matching barcode # This ensures cascade delete works for all violations, not just new ones await conn.execute( text( """ UPDATE violations v SET product_id = p.id FROM products p WHERE v.product_id IS NULL AND v.barcode_number IS NOT NULL AND p.barcode = v.barcode_number; """ ) ) except SQLAlchemyError as e: raise RuntimeError( "Database connection failed during startup. " "Check DATABASE_URL in .env (username/password/host/port/db) and ensure Postgres is running." ) from e @app.on_event("startup") async def _start_scheduler() -> None: """Start the scheduler when the app starts (only if ENABLE_SCHEDULER=true)""" # In production with multiple Gunicorn workers, set ENABLE_SCHEDULER=true # on only ONE worker to prevent duplicate scheduled jobs enable_scheduler = os.getenv("ENABLE_SCHEDULER", "false").lower() in ("true", "1", "yes") if not enable_scheduler: logger.info("[Scheduler] ⊘ Scheduler disabled (set ENABLE_SCHEDULER=true to enable)") return try: logger.info("\n" + "="*80) logger.info("🚀 [Scheduler] Starting scheduler at app startup...") logger.info("="*80) await SchedulerService.start_scheduler() except Exception as e: logger.error(f"❌ [Scheduler] Failed to start scheduler: {str(e)}", exc_info=True) # Don't re-raise - app should still work without scheduler @app.on_event("shutdown") async def _stop_scheduler() -> None: """Stop the scheduler when the app shuts down""" try: await SchedulerService.stop_scheduler() except Exception as e: logger.error(f"⚠️ [Scheduler] Error stopping scheduler: {str(e)}") app.include_router(auth_router, prefix="/auth", tags=["auth"]) app.include_router(vendor_router, tags=["vendors"]) app.include_router(product_router, prefix="/api/products", tags=["products"]) app.include_router(violations_router, prefix="/api/violations", tags=["violations"]) app.include_router(dashboard_router, prefix="/api", tags=["dashboard"]) app.include_router(admin_settings_router) return app app = create_app()