
    1iT                         d Z ddlZddlmZ ddlmZ ddlmZ ddlmZ ddl	m
Z
mZ ddlmZmZmZ ddlZdd	lmZ dd
lmZ ddlmZ ddlmZ  ej2                  e      Z G d d      Zy)zz
Scheduler service for managing automated product scraping.
Handles job scheduling, execution, and configuration updates.
    N)datetime)AsyncIOScheduler)CronTrigger)AsyncSession)selectupdate)OptionalDictAny)SiteSettings)Product)ScraperService)AsyncSessionLocalc                       e Zd ZU dZdZee   ed<   dZe	defd       Z
e	d        Ze	d        Ze	d	        Ze	d
eddfd       Ze	d
eddfd       Ze	d        Ze	deeef   fd       Zedededefd       Zy)SchedulerServicez1Service for managing automated scraping schedulerN
_schedulerscheduled_scraping_jobreturnc                 z    | j                   $t               | _         t        j                  d       | j                   S )z0Get or create the scheduler instance (in-memory)zN[Scheduler] Created in-memory async scheduler (jobs reload from DB on startup))r   r   loggerinfo)clss    D/var/www/html/marco-python-backend/app/services/scheduler_service.pyget_schedulerzSchedulerService.get_scheduler   s/     >>!-/CNKKhi~~    c                   K   	 | j                         }|j                  s>|j                          t        j	                  d       | j                          d{    yt        j	                  d       | j                          d{    y7 37 # t        $ r)}t        j                  dt        |       d        d}~ww xY ww)zStart the scheduleru.   ✅ [Scheduler] Scheduler started successfullyNz%[Scheduler] Scheduler already runningu+   ❌ [Scheduler] Failed to start scheduler: Texc_info)	r   runningstartr   r   _apply_stored_settings	Exceptionerrorstrr   	scheduleres      r   start_schedulerz SchedulerService.start_scheduler$   s     	))+I$$!LM 00222CD00222	 3 3 	LLFs1vhOZ^L_	sY   CAB BB C(B BB CB B 	C$B==CCc                   K   	 | j                         }|j                  r&|j                          t        j	                  d       yy# t
        $ r-}t        j                  dt        |       d       Y d}~yd}~ww xY ww)zStop the scheduleru!   ✅ [Scheduler] Scheduler stoppedu*   ❌ [Scheduler] Failed to stop scheduler: Tr   N)r   r   shutdownr   r   r"   r#   r$   r%   s      r   stop_schedulerzSchedulerService.stop_scheduler7   st     	_))+I  ""$?@ !  	_LLEc!fXNY]L^^	_s/   B AA B 	A=#A83B 8A==B c                 j  K   	 t               4 d{   }|j                  t        t              j	                  t        j
                  dk(               d{   }|j                         j                         }|r5t        j                  d|j                   d|j                          |j                  rt        j                  d       | j                  ||       d{    | j                         }|j                  | j                        }|rZt        j                  d       t        j                  d|j                           t        j                  d|j"                          nYt        j%                  d	       nCt        j                  d
       | j'                          d{    nt        j%                  d       ddd      d{    y7 7 7 7 37 # 1 d{  7  sw Y   yxY w# t(        $ r-}t        j+                  dt-        |       d       Y d}~yd}~ww xY ww)z*Load settings from database and apply themN   z)[Scheduler] Found site settings: enabled=z, frequency=u7   [Scheduler] ✅ Scraping is ENABLED - scheduling job...u+   [Scheduler] ✅ Job scheduled successfully!z[Scheduler]   Next run: z[Scheduler]   Trigger: u=   [Scheduler] ⚠️ Job was not found after scheduling attemptu:   [Scheduler] ⚠️ Scraping is DISABLED - no job scheduleduL   [Scheduler] ⚠️ No site settings found in database - creating defaults...u0   [Scheduler] ❌ Error applying stored settings: Tr   )r   executer   r   whereidscalarsfirstr   r   scraping_enabledscraping_frequency_schedule_scrapingr   get_job_job_idnext_run_timetriggerwarningstop_scraping_jobr"   r#   r$   )r   dbresultsettingsr&   jobr'   s          r   r!   z'SchedulerService._apply_stored_settingsB   s    	e(* s sb!zz&*>*D*D\__XYEY*Z[[!>>+113KK"KHLeLeKffrs{  tO  tO  sP  !Q  R00$]^!44RBBB %($5$5$7	'//<"KK*UW"KK*B3CTCTBU(VW"KK*A#++(OP"NN+jk$`a!33555NN#qr1s s s[ C 6-s s s s4  	eLLKCPQF8T_cLdd	es   H3G: GG: AG%GB
G%#G$C	G%-G!.G%G: G#G: H3G: G%G%!G%#G: %G7+G.,G73G: 6H37G: :	H0#H+&H3+H00H3r<   r>   r   c                 R  K   	 |j                   r0t        j                  d       | j                  ||       d{    yt        j                  d       | j	                          d{    y7 37 # t
        $ r)}t        j                  dt        |       d        d}~ww xY ww)zr
        Update scheduler based on new site settings.
        Called when admin updates settings via API.
        z-[Scheduler] Scraping ENABLED - scheduling jobNz6[Scheduler] Scraping DISABLED - removing scheduled jobu)   ❌ [Scheduler] Error updating settings: Tr   )r3   r   r   r5   r;   r"   r#   r$   )r   r<   r>   r'   s       r   update_settingsz SchedulerService.update_settingsc   s     		((KL,,R:::TU++--- ; . 	LLDSVHMX\L]	sU   B'6A2 A.A2 B' (A2 (A0)A2 -B'.A2 0A2 2	B$;$BB$$B'c           
        K   	 | j                         }|j                  | j                        r0|j                  | j                         t        j                  d       |j                  rt        j                  |j                        nt        j                  }|j                  dk(  r|j                  st        j                  d       y|j                  j                  d      }t        |d         t        |d         }}t        |||      }t        j                  d	|j                   d
|j                          n|j                  dk(  r|j                  st        j                  d       y|j                   st        j                  d       y|j                  j                  d      }t        |d         t        |d         }}|j                   j#                         }	dddddddd}
|
j%                  |	d      }t        ||||      }t        j                  d|	j'                          d|j                   d
|j                          n|j                  dk(  rt|j(                  st        j                  d       yt        j*                  |j(                  |      }t        j                  d|j(                   d|j                   d       n#t        j                  d|j                          y|j-                  | j.                  || j                  d|j0                  |j2                  |j4                  gddd       }t        j                  d!       t        j                  d"|j                   d#|j0                          y# t6        $ r)}t        j                  d$t9        |       d%        d}~ww xY ww)&zk
        Schedule the scraping job based on frequency settings.
        Replaces any existing job.
        z)[Scheduler] Removed previous scraping jobdailyz9[Scheduler] scraping_time is required for daily frequencyN:r   r-   )hourminutetimezonez([Scheduler] Scheduled DAILY scraping at  weeklyz:[Scheduler] scraping_time is required for weekly frequencyzA[Scheduler] scraping_day_of_week is required for weekly frequency               )mondaytuesday	wednesdaythursdayfridaysaturdaysunday)day_of_weekrE   rF   rG   z,[Scheduler] Scheduled WEEKLY scraping every z at customzA[Scheduler] scraping_custom_cron is required for custom frequency)rG   z1[Scheduler] Scheduled CUSTOM scraping with cron: z (timezone: )z[Scheduler] Invalid frequency: zScheduled Product ScrapingT)r9   r0   nameargsreplace_existingcoalescemax_instancesu3   ✅ [Scheduler] Scraping job scheduled successfullyz%[Scheduler] Configuration: Frequency=z, Discovery=u+   ❌ [Scheduler] Error scheduling scraping: r   )r   r6   r7   
remove_jobr   r   client_timezonepytzrG   UTCr4   scraping_timer#   splitintr   scraping_day_of_weeklowergettitlescraping_custom_cronfrom_crontabadd_job_scraping_jobenable_discoveryheadless_modetimeout_secondsr"   r$   )r   r<   r>   r&   tz
time_partsrE   rF   r9   rV   day_mapday_numr?   r'   s                 r   r5   z#SchedulerService._schedule_scrapingt   s    G	))+I   -$$S[[1GH =E<T<Tx778Z^ZbZbB**g5--LL!\]%3399#>
":a=13z!}3Ef%4LFxG]G]F^^_`h`x`x_yz{,,8--LL!]^44LL!de%3399#>
":a=13z!}3Ef&;;AAC  AA1Q! "++k15%'V^`aJ;K\K\K^J__cdldzdzc{{|  ~F  ~V  ~V  }W  X  Y,,844LL!de%2283P3P[]^OPXPmPmOnnz  |D  |T  |T  {U  UV  W  X >x?Z?Z>[\] ##!!;;1//1G1GIaIab!% $ 	C KKMOKK?@[@[?\\hiq  jC  jC  iD  E  F 	LLFs1vhOZ^L_	si   O CN 
O B'N 2O 3!N O C-N O A4N 7O 8BN 
O 	N=$N88N==O c                 b  K   	 | j                         }|j                  | j                        r1|j                  | j                         t        j                  d       yt        j                  d       y# t        $ r-}t        j                  dt        |       d       Y d}~yd}~ww xY ww)zZ
        Remove scheduled scraping job.
        Called when scraping is disabled.
        u$   ✅ [Scheduler] Scraping job removedz,[Scheduler] No active scraping job to removeu-   ❌ [Scheduler] Error stopping scraping job: Tr   N)	r   r6   r7   r^   r   r   r"   r#   r$   r%   s      r   r;   z"SchedulerService.stop_scraping_job   s     	b))+I  -$$S[[1BCJK 	bLLHQQ\`Laa	bs;   B/AA6 B/ A6 5B/6	B,?#B'"B/'B,,B/c           
         	 | j                         }|j                         }|j                  | j                        }|j                  t        |d      rdndt        |      |du|rK|j                  |j                  |rt        |j                        nd|rt        |j                        ndddS ddS # t        $ r/}t        j                  d|        dt        |      icY d}~S d}~ww xY w)	z+Get detailed scheduler status for debugging
_jobstores
persistentz	in-memoryN)r0   rY   r8   r9   )r   type
total_jobsscraping_job_existsscraping_jobz"[Scheduler] Error getting status: r#   )r   get_jobsr6   r7   r   hasattrlenr0   rY   r$   r8   r9   r"   r   r#   )r   r&   jobsr?   r'   s        r   get_scheduler_statusz%SchedulerService.get_scheduler_status   s    	%))+I%%'D##CKK0C %,,(/	<(Hk!$i'*$ 	 &&HH?BS):):%;36s3;;/D	!  #   	%LL=aSABSV$$	%s$   B.B5 1B5 5	C->$C("C-(C-rm   rn   ro   c                 Z  K   t         j                  d       t         j                  d       t         j                  d       t        j                         }d}d}g }	 t	               4 d{   }|j                  t        t              j                  t        j                  dk(               d{   }|j                         j                         }	t        |	      }t         j                  d| d       |dk(  rt         j                  d	       d
}
nt        |	d      D ]  \  }}	 t         j                  d| d| d|j                          t         j                  d       g }g }ddddddddd}t         j                  d| d| d       	 t!        j"                  |j                  |j$                  |j&                  ||       d{   \  }}|j)                  |       |j)                  |       t        |      |d   d<   t        |      |d   d<   d|d   d<   t         j                  d| d| dt        |       dt        |       d	       t         j                  d| d| d"       	 t!        j0                  ||j2                  | ||#       d{   }|j5                  dg       }|j5                  dg       }|j)                  |       |j)                  |       t        |      |d$   d<   t        |      |d$   d<   d|d$   d<   t         j                  d| d| d%t        |       dt        |       d	       t7               }g }|D ]v  }|j5                  d'      |j5                  d(      |j5                  d)      t9        |j5                  d*d      d+      f}||vsU|j;                  |       |j=                  |       x t        |      }||z  }t        j                         j?                         |_         |jC                          d{    t         j                  d| d| d,|j                          t         j                  d| d| d-t        |              t         j                  d| d| d.|        t         j                  d| d| d/|d   d    d0|d   d    d|d   d    d1       t         j                  d| d| d2|d$   d    d0|d$   d    d|d$   d    d1        |rd6nd7}
t        j                         }||z
  jE                         }d8| d9| d:|d;d<}|r|d=t        |       z  }|j                  tG        tH              j                  tH        j2                  dk(        jK                  ||
|>             d{    |jC                          d{    t         j                  d       t         j                  d?       t         j                  d@|        t         j                  dA|        t         j                  dBt        |              t         j                  dC|d;d<       t         j                  dD       t         j                  d       ddd      d{    y7 7 7 # t*        $ rH}dt-        |       |d   d<   t         j/                  d| d| dt-        |       d !       Y d}~d}~ww xY w7 G# t*        $ rH}dt-        |       |d$   d<   t         j/                  d| d| d&t-        |       d !       Y d}~d}~ww xY w7 $# t*        $ rq}t         j/                  d| d| d3|j                   d4t-        |              |j=                  |j2                  |j                  t-        |      d5       Y d}~d}~ww xY w7 !7 7 @# 1 d{  7  sw Y   yxY w# t*        $ rF}t         j/                  dEt-        |       d!       	 t	               4 d{  7  }|j                  tG        tH              j                  tH        j2                  dk(        jK                  t        j                         dFdGt-        |       >             d{  7   |jC                          d{  7   ddd      d{  7   n# 1 d{  7  sw Y   nxY wn7# t*        $ r+} t         j/                  dHt-        |               Y d} ~ nd} ~ ww xY wt         j                  d       Y d}~yd}~ww xY ww)Iz^
        Actual scheduled scraping job function.
        Scrapes all active products.
        zQ
================================================================================u8   🔔 [Scheduled Scraping] SCHEDULED SCRAPING JOB STARTEDzQ================================================================================
r   NTz[Scheduled Scraping] Found z active products to scrapez1[Scheduled Scraping] No active products to scrapecompleted_no_productsr-   z
[Scheduled Scraping] [/z] UNIFIED Scraping: zK[Scheduled Scraping] Using UNIFIED approach: Google Search + Shopping Lightpending)results
violationsstatus)google_searchshopping_lightz[Scheduled Scraping] [u"   ] [STEP 1/2] 🔍 Google Search...)product_namebarcodemspproductr<   r   r   r   	completedr   u   ] ✅ Google Search: z
 results, z violationsz	failed - u   ] ⚠️ Google Search failed: Fr   u*   ] [STEP 2/2] 🛒 Google Shopping Light...)
product_idrm   rn   ro   r   u   ] ✅ Shopping Light: u    ] ⚠️ Shopping Light failed: vendor_namer   r   scraped_pricerJ   u   ] ✅ UNIFIED Completed: u   ]   • Total Results: u)   ]   • Total Violations (deduplicated): u   ]   • Google Search: z (z violations)u   ]   • Shopping Light: u   ] ❌ Failed to scrape z: )r   r   r#   completed_with_errorssuccessz
Products: z, Violations: z, Duration: z.1fsz
, Failed: )last_scheduled_runlast_scheduled_run_statuslast_scheduled_run_summaryu3   ✅ [Scheduled Scraping] JOB COMPLETED SUCCESSFULLYz%[Scheduled Scraping] Total Products: z'[Scheduled Scraping] Total Violations: z&[Scheduled Scraping] Failed Products: z[Scheduled Scraping] Duration: zJ[Scheduled Scraping] Next Run: Will be executed per schedule configurationu*   
❌ [Scheduled Scraping] CRITICAL ERROR: failedzError: z4[Scheduled Scraping] Could not update error status: )&r   r   r   utcnowr   r.   r   r   r/   r   r1   allr~   r:   	enumerater   r   search_google_serpr   r   extendr"   r$   r#   scrape_product_serpr0   rg   setroundaddappend	isoformatlast_execution_timecommittotal_secondsr   r   values)!rm   rn   ro   
start_timetotal_productstotal_violationsfailed_productsr<   r=   productsr   idxr   product_resultsproduct_violations_listsearch_summarygoogle_resultsgoogle_violations
google_errshopping_resultshopping_resultsshopping_violationsshopping_errseen_violationsdeduped_violationsvviolation_keyproduct_violationsr'   end_timedurationsummarydb_errors!                                    r   rl   zSchedulerService._scraping_job   s     	M"NOM"__&
Z	'(* F+ F+b!zz&/*?*?RV@V*WXX!>>+//1!$X9.9IIcde!Q&NN#VW4F )2(A(> ]W\"KK*B3%qHXXlmt  nB  nB  mC  )D  E"KK*uw.0O683=>a[d1e>?q\e2f.N #KK*@Q~FVVx(yz^JXJkJk181E1E,3OO(/,3')K" E" A0A !0 6 6~ F 7 > >?P QMPQ_M` ?	 JPSTePf ? MLW ? I &.DSE>JZZops  uC  qD  pE  EO  PS  Te  Pf  Og  gr  -s  !t #KK*@Q~FV  WA  )B  Ca8F8Z8Z$&/6zz5E2?4C9" 3" 4C3F3FyRT3U 06E6I6I,XZ6[ 3 / 6 67G H 7 > >?R SNQRbNc/? @ KQTUhQi/? @ NMX/? @ J &.DSE>JZZpqt  vF  rG  qH  HR  SV  Wj  Sk  Rl  lw  -x  !y /2eO13.%< 	A$%EE-$8$%EE.$9$%EE%L$)!%%*CQ$G	1" $1#G$3$7$7$F$6$=$=a$@	A 255G1H.,0BB, ;C//:K:U:U:WG7"$))+--"KK*@Q~FVVopw  qE  qE  pF  )G  H"KK*@Q~FVVmnq  sB  oC  nD  )E  F"KK*@Q~FVV  AS  @T  )U  V"KK*@Q~FVVmn|  ~M  oN  OW  oX  nY  Y[  \j  kz  \{  |E  \F  [G  GQ  R`  ap  Rq  r~  R  Q@  @L  )M  N"KK*@Q~FVVno}  O  pP  QY  pZ  o[  []  ^l  m}  ^~  H  ^I  ]J  JT  Uc  dt  Uu  vB  UC  TD  DP  )Q  Rk]~ 9H4YF $??,$z1@@B&~&6nEUDVVbcklobppqr"C,@+ABBGjj<(..|!/CDKK+3283: L    iik!!M*QSCNCSTUEFVEWXYDSEYDZ[\=hs^1MNhjM*MF+ F+ F+X4E" $- ^NWX[\fXgWhLi ? I &/EcU!NK[[z{~  @J  |K  {L  .M  X]  !^  !^^3"& $- aOXY\]iYjXkMl/? @ J &/EcU!NK[[{|  AM  }N  |O  .P  [`  !a  !aa. .  ) "LL+A#aGWWnov  pD  pD  oE  EG  HK  LM  HN  GO  *P  Q+22.5jj070D0D),Q4  $ "{F+ F+ F+ F+P  	'LLFs1vhOZ^L_e,. & &"**|,22<??a3GHOO/7/@6>9@Q7I P    ))+%%& & & & &  eSTWX`TaSbcdde KK&&#	's  Ad+_ +Z,_ /A_1Z	2A7_*A*\=;ZZBZ\=5'[&[#B([&A"\=(A.\=\:C\=6B#_^:_1^=2C_5_  _ _ d+_ 	_Z	[ =[\=[  \=#[&&	\7/=\2,\=2\77\==	^7A&^2,_2^77_=_ _ _	_
__ d+_ 
d("#d#c`cA2b<bb<$b'%b<*c5b86c<c	cc	
cd#	d!d<d#dd#d+#d((d+)__name__
__module____qualname____doc__r   r	   r   __annotations__r7   classmethodr   r(   r+   r!   r   rA   r5   r;   r
   r$   r   r   staticmethodboolrd   rl    r   r   r   r      s&   ;-1J)*1&G.    $ _ _ e e@ | ~    L, L. L L\ b b %T#s(^ % %. h'd h'4 h'Z] h' h'r   r   )r   loggingr   apscheduler.schedulers.asyncior   apscheduler.triggers.cronr   sqlalchemy.ext.asyncior   
sqlalchemyr   r   typingr	   r
   r   r`   app.models.site_settingsr   app.models.productr   app.services.scraper_servicer   app.db.sessionr   	getLoggerr   r   r   r   r   r   <module>r      sQ      ; 1 / % & &  1 & 7 ,			8	$~' ~'r   