
    /jt                       S SK Jr  S SKrS SKrS SKrS SKrS SKrS SKrS SKr	S SK
r
S SKrS SKrS SKJrJr  S SKJrJr  S SKJr  S SKJrJrJrJrJr  S SKJrJr  S SKrS SKrS SKJ r   S S	K!J"r"  S S
K#J$r$J%r%  S SK&J'r'J(r(  SSK)J*r*J+r+J,r,  SSK-J.r.  SSK/J0r0  SSK1J2r2J3r3J4r4J5r5J6r6J7r7  SSK8J9r9J:r:  SSK;J<r<  SSK=J>r>J?r?J@r@  SSK,JArAJBrB  SSKCJDrD  SSKEJFrF  SrGSrHSrISrJS6S jrKS7S jrL " S S\5      rM\MrN " S S 5      rO\ " S! S"5      5       rP\R                  R                  S#5      (       a  \4R                  rTO\4R                  rT\" S$5      rV\" S%S&9 " S' S(\\V   5      5       rW\W" \R                  S)S*9rY\W" S+S,S*9rZ\P" 5       r[\\" 1 S-k5      r]    S8S. jr^\ " S/ S05      5       r_\_r`\ " S1 S25      5       ra\S3   rb " S4 S5\,R                  \b   5      rdg)9    )annotationsN)	AwaitableCallable)	dataclassfield)Enum)AnyGenericLiteralTypeVaroverload)urljoinurlparse)web)MessageToDict)apirtc)agentmodels   )ipc	telemetryutils)AssignmentTimeoutError)_InferenceRunner)JobAcceptArguments
JobContextJobExecutorType
JobProcess
JobRequestRunningJobInfo)	DEV_LEVELlogger)Plugin)ATTRIBUTE_AGENT_NAME	NOT_GIVEN
NotGivenOr)http_serveris_given)get_cpu_monitor)__version__g      @      @      ?   c                    g N )procs    F/app/agent/.venv/lib/python3.13/site-packages/livekit/agents/worker.py_default_setup_fncr4   A   s    
    c                @   #    U R                  5       I S h  vN   g  N7fr0   )accept)ctxs    r3   _default_request_fncr9   E   s     
**,s   c                  l    \ rS rSr\R
                  R                  r\R
                  R                  r	Sr
g)
ServerTypeI   r1   N)__name__
__module____qualname____firstlineno__r   JobTypeJT_ROOMROOMJT_PUBLISHER	PUBLISHER__static_attributes__r1   r5   r3   r;   r;   I   s     ==  D**Ir5   r;   c                  l    \ rS rSrSr\R                  " 5       rSS jrSS jr	S	S jr
\S
S j5       rSrg)_DefaultLoadCalcQ   Nc                
   [         R                  " S5      U l        [        5       U l        [
        R                  " U R                  SSS9U l        [
        R                  " 5       U l
        U R                  R                  5         g )N   Tworker_cpu_load_monitor)targetdaemonname)r   MovingAverage_m_avgr*   _cpu_monitor	threadingThread
_calc_load_threadLock_lockstartselfs    r3   __init___DefaultLoadCalc.__init__U   s]    ))!,+- ''??46O
 ^^%
r5   c                     U R                   R                  SS9nU R                     U R                  R	                  U5        S S S 5        MK  ! , (       d  f       N= f)Nr-   )interval)rR   cpu_percentrX   rQ   
add_sample)r[   cpu_ps     r3   rU   _DefaultLoadCalc._calc_load^   sG    %%1131?E&&u-  s   A
Ac                    U R                      U R                  R                  5       sS S S 5        $ ! , (       d  f       g = fr0   )rX   rQ   get_avgrZ   s    r3   _get_avg_DefaultLoadCalc._get_avgd   s#    ZZ;;&&( ZZs   1
?c                    U R                   c1  U R                     U R                   c  [        5       U l         S S S 5        U R                   R                  5       $ ! , (       d  f       N(= fr0   )	_instance_instance_lockrH   rf   )clsworkers     r3   get_load_DefaultLoadCalc.get_loadh   sL    == ##==($4$6CM $ }}%%''	 $#s   A
A')rR   rX   rQ   rV   returnNonerp   float)rl   AgentServerrp   rs   )r=   r>   r?   r@   ri   rS   rW   rj   r\   rU   rf   classmethodrm   rF   r1   r5   r3   rH   rH   Q   s6    I^^%N.) ( (r5   rH   c                  r    \ rS rSr% SrS\S'   SrS\S'   SrS\S'   SrS\S'   \	" \
S9rS	\S
'   SrS\S'   Srg)WorkerPermissionsr   Tboolcan_publishcan_subscribecan_publish_datacan_update_metadatadefault_factoryzlist[models.TrackSource]can_publish_sourcesFhiddenr1   N)r=   r>   r?   r@   rz   __annotations__r{   r|   r}   r   listr   r   rF   r1   r5   r3   rw   rw   r   sG    KM4!d! $$49$4O1OFDr5   rw   winTT)frozenc                  >    \ rS rSr% S\S'   S\S'   \SS j5       rSrg)	ServerEnvOption   r   dev_defaultprod_defaultc                n    [        U [        5      (       a  U(       a  U R                  $ U R                  $ U $ r0   )
isinstancer   r   r   )optdevmodes     r3   getvalueServerEnvOption.getvalue   s+    c?++&-3??C33C3CC
r5   r1   N)r   zT | ServerEnvOption[T]r   ry   rp   r   )r=   r>   r?   r@   r   staticmethodr   rF   r1   r5   r3   r   r      s    NO r5   r   gffffff?r   r   DEBUGINFO>   r   WARNr   ERRORTRACECRITICALc                   [        U [        5      (       a  U R                  U R                  /nOU /nU HI  nUR	                  5       [
        ;  d  M  [        SU< SSR                  [        [
        5      5       35      e   [        U [        5      (       a:  [        U R                  R	                  5       U R                  R	                  5       S9$ U R	                  5       $ )NzInvalid log level z. Valid levels: z, r   )	r   r   r   r   upperVALID_LOG_LEVELS
ValueErrorjoinsorted)	log_levellevels_to_checklevels      r3   !_validate_and_normalize_log_levelr      s     )_--$00)2H2HI$+ ;;= 00$UI-=diiO_H`>a=bc  ! )_--!--335"//557
 	
 ??r5   c            	         \ rS rSr% S\S'    \rS\S'    \rS\S'    \	R                  rS\S	'    \rS
\S'    \rS\S'    SrS\S'    SrS\S'    SrS\S'    \" S\" \R,                  " \" 5       R1                  5       5      S5      S9rS\S'    SrS\S'    SrS\S'    SrS\S'    \" \S9rS \S!'    S"r S#\S$'    \!RD                  r#S%\S&'    S'r$S\S('    S)r%S*\S+'    \" S,S)S-9r&S*\S.'    \" S,S)S-9r'S*\S/'    \(r)S0\S1'    S"r*S#\S2'   \" SS3S9r+S\S4'    \,r-S5\S6'    \.R^                  Ra                  S75      (       d  S8OS9r1S:\S;'    \,r2S<\S='    S)r3S*\S>'    SBS? jr4SCS@ jr5SAr6g))DServerOptions   'Callable[[JobContext], Awaitable[None]]entrypoint_fncz'Callable[[JobRequest], Awaitable[None]]request_fnczCallable[[JobProcess], Any]prewarm_fncz4Callable[[AgentServer], float] | Callable[[], float]load_fncr   job_executor_typefloat | ServerEnvOption[float]load_threshold  rs   job_memory_warn_mbr   job_memory_limit_mb  intdrain_timeout   r   int | ServerEnvOption[int]num_idle_processes      $@shutdown_process_timeout     r@session_end_timeoutinitialize_process_timeoutr~   rw   permissions str
agent_name
WorkerTypeworker_type   	max_retryN
str | Nonews_urlF)reprdefaultapi_key
api_secretstr | ServerEnvOption[str]r   host  portNotGivenOr[str | None]
http_proxylinuxspawn
forkserverLiteral['spawn', 'forkserver']multiprocessing_contextNotGivenOr[int]prometheus_portprometheus_multiproc_dirc                8    [        U R                  5      U l        g r0   )r   r   rZ   s    r3   __post_init__ServerOptions.__post_init__  s    :4>>Jr5   c                    [         R                  U R                  U5      nUS:  a"  U(       d  [        R                  " SU 35        g g g )Nr   z?load_threshold in prod env must be less than 1, current value: )r   r   r   r#   warning)r[   r   r   s      r3   validate_configServerOptions.validate_config  sD    (11$2E2EwOAgNNQR`Qab '.r5   )r   ro   )r   ry   rp   rq   )7r=   r>   r?   r@   r   r9   r   r4   r   rH   rm   r   _default_job_executor_typer   _default_load_thresholdr   r   r   r   r   minmathceilr*   	cpu_countr   r   r   r   r   rw   r   r   r   rC   r   r   r   r   r   _default_log_levelr   r   r   r&   r   sysplatform
startswithr   r   r   r   r   rF   r1   r5   r3   r   r      s   ;;X;OK8O/ 0BK,ASEUE^E^HB^d)CCX5LN2L
 !$#j!"" M3a5DC		/2C2M2M2O(PRS$T62  1&*e*J!&&](,,L%*;L%MK"M?J P(ooK-AIsAFJ;  UD9GZ9? #t<J
<B -?I)>H D#N'61SW'XD
$X
 *3J&2
 ||..w77\ ;  (1O_0T+/j/m
Kr5   r   c                  *    \ rS rSr% S\S'   S\S'   Srg)
WorkerInfoi  r   	http_portry   cloud_agentsr1   N)r=   r>   r?   r@   r   rF   r1   r5   r3   r   r     s    Nr5   r   )worker_startedworker_registeredc                    ^  \ rS rSr\" S\R                  " \" 5       R                  5       5      S9r	\" SSS9r
\\SSS\	SSS\S	S
S
S
S\
\\R                   R#                  S5      (       d  SOSS
S
S
S
\S.                                               S:U 4S jjjr\S;S j5       r\S<S j5       r\R.                  S=S j5       r\S>S j5       r\R.                  S?S j5       r\S@S j5       r\S\R:                  S
S
S.           SAS jj5       r\S\R:                  S
S
S.         SBS jj5       r SCS\R:                  S
S
S.           SDS jjjr\SES j5       rSSS.SFS jjr \\\\\\\\\\SSSS.                           SGS  jjr!\SHS! j5       r"\SIS" j5       r#\SJS# j5       r$\4SKS$ jjr%\&RN                  " \(S%9SS
S
S
S&.           SLS' jj5       r)SMS( jr*SNS) jr+\&RN                  " \(S%9SMS* j5       r,SOS+ jr-SPS, jr.SQS- jr/SRS. jr0SSS/ jr1SMS0 jr2SSS1 jr3SJS2 jr4SRS3 jr5STS4 jr6SUS5 jr7SMS6 jr8SVS7 jr9SMS8 jr:S9r;U =r<$ )Wrt   i%  r   r   r   r   r   r   r   r   Nr   r   r   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   	setup_fncr   r   r   r   c                 > [         TU ]  5         U=(       d(    [        R                  R	                  S5      =(       d    SU l        U=(       d(    [        R                  R	                  S5      =(       d    SU l        U=(       d(    [        R                  R	                  S5      =(       d    SU l        [        R                  R	                  S5      =(       d    SU l        Xl	        UU l
        Xl        X l        X0l        X@l        XPl        X`l        Xpl        Xl        Xl        Xl        Xl        UU l        UU l        UU l        [2        R4                  " U5      U l        [9        U5      (       dE  [        R                  R	                  S5      =(       d    [        R                  R	                  S5      nUU l        [=        U5      U l        SU l         [B        RD                  U l#        SU l$        S U l%        S U l&        S U l'        UU l(        UU l)        S	u  U l*        U l+        U l,        U l-        S U l.        [^        R`                  " 5       U l1        g )
NLIVEKIT_URLr   LIVEKIT_API_KEYLIVEKIT_API_SECRETLIVEKIT_WORKER_TOKENHTTPS_PROXY
HTTP_PROXYunregistered)TFFF)2superr\   osenvironget_ws_url_api_key_api_secret_worker_token_host_port_job_executor_type_load_threshold_job_memory_warn_mb_job_memory_limit_mb_drain_timeout_num_idle_processes_shutdown_process_timeout_session_end_timeout_initialize_process_timeout_permissions
_max_retry_prometheus_port_prometheus_multiproc_dir_mp_ctx_strmpget_context_mp_ctxr)   _http_proxyr   
_log_level_agent_namer;   rC   _server_type_id_entrypoint_fnc_request_fnc_session_end_fnc
_setup_fnc	_load_fnc_closed	_draining_connecting_connection_failed_http_serverasynciorW   rX   )r[   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   	__class__s                           r3   r\   AgentServer.__init__+  s   : 	D!>D"J2::>>2C#DJ%S8L)MSQSZZ^^,BCIr

"3-#5 $7!+#5 )A&$7!+E('# /)A&2~~&=>
##6V"**..:VJ%;IF&OO! PTLPPT ?HV^S
Odnd&68O <@\\^
r5   c                    U R                   $ r0   )r  rZ   s    r3   r   AgentServer.log_level}      r5   c                    U R                   $ r0   )r  rZ   s    r3   r   AgentServer.setup_fnc  r)  r5   c                L    Ub  [        U5      (       d  [        S5      eXl        g )Nz$setup_fnc must be a callable or None)callable	TypeErrorr  r[   values     r3   r   r+    s!    Xe__BCCr5   c                    U R                   $ r0   )r  rZ   s    r3   r   AgentServer.load_fnc      ~~r5   c                L    Ub  [        U5      (       d  [        S5      eXl        g )Nz#load_fnc must be a callable or None)r-  r.  r  r/  s     r3   r   r2    s!    Xe__ABBr5   c                   U " S0 SUR                   _SUR                  _SUR                  _SUR                  _SUR                  _SUR
                  _SUR                  _SUR                  _S	UR                  _S
UR                  _SUR                  _SUR                  _SUR                  _SUR                  _SUR                  _SUR                  _SUR                   _SUR"                  _S[%        UR&                  5      (       a  UR&                  OS _SUR(                  _SUR*                  _SUR,                  _SUR.                  _6nUR1                  UR2                  UR4                  UR6                  UR8                  S9  U$ )Nr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   )r   type
on_requestr1   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r)   r   r   r   r   r   rtc_sessionr   r   r   r   )rk   optionsservers      r3   from_server_optionsAgentServer.from_server_options  s    
%77
"11
 !( ; ;
  '99	

 "//
  '99
 &-%E%E
 !( ; ;
 (/'I'I
  ++
 ''
 >>
 OO
 ))
 
  !
" ))#
$ %,$C$C%
& 8@@W@W7X7XG33^b'
( &-%E%E)
* ))+
, %%-
. ''/
2 	""))$$**	 	 	
 r5   )r   r6  r7  on_session_endc                   g r0   r1   )r[   funcr   r6  r7  r=  s         r3   r8  AgentServer.rtc_session  s     36r5   c                   g r0   r1   )r[   r   r6  r7  r=  s        r3   r8  r@    s     r5   c               F   ^ ^^^^     SUUUU U4S jjnUb  U" U5      $ U$ )z
Decorator or direct registrar for the RTC session entrypoint.

Usage:
    @server.rtc_session(agent_name="survey_agent")
    async def my_agent(job_ctx: JobContext): ...

    server.rtc_session(my_agent, agent_name="survey_agent")
c                ~   > TR                   b  [        S5      eU Tl         TTl        TTl        TTl        TTl        U $ )NzHThe AgentServer currently only supports registering only one rtc_session)r  RuntimeErrorr  r  r  r  )fr   r7  r=  r[   r6  s    r3   	decorator*AgentServer.rtc_session.<locals>.decorator  sP     ##/"^  $%D  *D$2D!)D $DHr5   )rE  r   rp   r   r1   )r[   r?  r   r6  r7  r=  rF  s   ` ```` r3   r8  r@    s5    0	6	4	 	 T?"r5   c                    [        U R                  (       a  U R                  R                  OS[        U R                  5      S9$ )Nr   )r   r   )r   r#  r   ry   r  rZ   s    r3   worker_infoAgentServer.worker_info  s6    040A0Ad'',,qd001
 	
r5   F)r   r   c               Z  ^ ^#    T R                    ISh  vN   T R                  (       d  [        S5      eT R                  c  [	        S5      eT R
                  c  [        T l        T R                  c  [        T l        T R                  c  [        R                  T l	        T R                  R                  (       a~  T R                  [        R                  :w  a+  [        R                  " S5        [        R                  T l	        T R                   ["        :w  a!  [        R                  " S5        ["        T l        [$        R&                  " 5       T l        TT l        [,        [$        R.                  [0              " 5       T l        0 T l        ST l        [8        R:                  R<                  [>        R@                     " ST R(                  S9T l!        ST l"        [G        [H        RJ                  5      S:  aa  [L        RN                  RQ                  [H        RJ                  S	S
SSS
SST RR                  T R(                  T RT                  =(       d    SS9T l"        [L        RV                  RY                  T R                  T R                  T RZ                  [\        R_                  T R`                  T5      T R(                  T Rb                  T RD                  T RR                  T Rd                  T Rf                  T Rh                  T Rj                  T Rl                  T RT                  =(       d    SS9T l7        [>        Rp                  Rr                  T l:        ST l;        ST l<        [z        R|                  " T R~                  [\        R_                  T R                  T5      5      T lA        ST lB        ST lC        S<U 4S jjnS<U 4S jjnT R                  R                  R                  [        R                  " SU5      /5        T R                  R                  R                  [        R                  " SU5      /5        ST lH        ST lI        T R                  (       d  [        S5      eT R                  (       d  [        S5      eT R                  (       d  [        S5      eST lN        T R                  b9  [        Rz                  R}                  T R~                  T R                  5      T lN        T R                  (       a  T R                  [        R                  S'   O,S[        R                  ;   a  [        R                  S   T lQ        T R                  (       a  [        R                  " T R                  SS9  T R                  (       a  [        R                  R                  T R                  5      (       a  [        R                  " SST R                  0S9  [        R                  " T R                  5       Hj  n[        R                  R                  T R                  U5      n [        R                  R                  U5      (       a  [        R                  " U5        Mj  Ml     T R                  [        R                  S '   T R                  [        R                  S!'   T R                  [        R                  S"'   [        R                  " S#[        [        R                  S$.S9  [        R                   H/  n[        R                  " S%UR                  UR                  S&.S9  M1     T R                  S':X  a`  [        R                   Vs/ s H  oR                  PM     snS(/-   n	[        R                  " S)S*U	0S9  T RR                  R                  U	5        T RD                  bZ  [        R                  " S+5        T RD                  R                  5       I Sh  vN   T RD                  R                  5       I Sh  vN   S,T l        S=U 4S- jjn
T R                  R                  5       I Sh  vN   [        R                  " S.T R                  R                   S/T R                  R                   35        T R                  (       ab  T R                  R                  5       I Sh  vN   [        R                  " S0T R                  R                  T R                  R                  5        T Rn                  R                  S1U
5        T Rn                  R                  S2U
5        T Rn                  R                  S3U
5        T Rn                  R                  5       I Sh  vN   [        R                  " T RT                  =(       d    SS49T l<        [        R                  " T R                  T R                  T R                  T Rx                  S59T l;        [$        R                  " T R(                  S9T l        [8        R                  " [        S69S>UU 4S7 jj5       n/ n[$        R                  " U" 5       S8S99T lI        UR                  T R                  5        U(       dC  [$        R                  " T R                  5       S:S99T lH        UR                  T R                  5        T R                  S;5        SSS5      ISh  vN   T R6                  I Sh  vN   g G
N! [         a$  n[        R                  " SU 3US9   SnAGM  SnAff = fs  snf  GN1 GN GN GNp GN Na! , ISh  vN  (       d  f       Nv= f Ni7f)?u  This method starts the worker's internal event loop, initializes any required
executors, HTTP servers, and process pools, and optionally registers the worker
with the LiveKit server.

Args:
    devmode (bool, optional):
        If True, the worker runs in development mode.
        This affects certain environment-dependent defaults, such as the
        number of idle processes, logging verbosity, and load thresholds,
        making it easier to test and debug without production constraints.

    unregistered (bool, optional):
        If True, the worker will start without registering itself with the
        LiveKit server.
        This allows the worker to operate in a partially connected state—
        capable of using other providers or local processing—but invisible
        to the central LiveKit job dispatcher.
        Useful for local testing, isolated jobs, or running without being
        assigned new jobs.
Nzworker is already runningzNo RTC session entrypoint has been registered.
Define one using the @server.rtc_session() decorator, for example:
    @server.rtc_session(agent_name="my_agent")
    async def my_agent(ctx: JobContext):
        ...
zLcustom load_fnc is not supported when hosting on Cloud, reverting to defaultzRcustom load_threshold is not supported when hosting on Cloud, reverting to default   )loopr   i,  rK   i  <   r,   )runnersinitialize_timeoutclose_timeoutmemory_warn_mbmemory_limit_mbping_intervalping_timeouthigh_ping_thresholdmp_ctxrM  r   )initialize_process_fncjob_entrypoint_fncsession_end_fncr   rM  r   inference_executorrW  rP  rQ  r   rR  rS  r           c                
  >#    TR                   (       a4  TR                   R                  5       (       d  [        R                  " SSS9$ TR                  (       a  [        R                  " SSS9$ [        R                  " SS9$ 7f)Ni  zinference process not running)statustextzfailed to connect to livekitOK)r_  )_inference_executoris_aliver   Responser"  )_r[   s    r3   health_check%AgentServer.run.<locals>.health_checkl  s_     ++D4L4L4U4U4W4W<<s9XYY**<<s9WXX||..s   B Bc           	     (  >#    [         R                  " TR                  [        R                  R                  TR                  R                  5      TR                  [        TR                  5      [        SS.5      n[        R                  " USS9$ 7f)Npython)r   r   worker_loadactive_jobssdk_versionproject_typezapplication/json)bodycontent_type)jsondumpsr  r   rA   Namer  r0  _worker_loadlenrj  r+   r   rc  )rd  rm  r[   s     r3   rl   AgentServer.run.<locals>.workeru  sq     zz&*&6&6',}}'9'9$:K:K:Q:Q'R'+'8'8'*4+;+;'<'2(0	 ||<NOOs   BB/z/workerz;ws_url is required, or set LIVEKIT_URL environment variablez@api_key is required, or set LIVEKIT_API_KEY environment variablezFapi_secret is required, or set LIVEKIT_API_SECRET environment variablePROMETHEUS_MULTIPROC_DIRT)exist_okz*cleaning prometheus multiprocess directorypathextrazfailed to remove exc_infor   r   r   zstarting worker)versionzrtc-versionzplugin registered)pluginr}  r   avzpreloading pluginspackageszstarting inference executorFc                   > TR                   R                  TR                  U 5      5      nTR                  R	                  U5        UR                  TR                  R                  5        g r0   )_loopcreate_task_update_job_status_job_lifecycle_tasksaddadd_done_callbackdiscard)r2   tr[   s     r3   r  +AgentServer.run.<locals>._update_job_status  sP    JJ**4+B+B4+HI))--a0##D$=$=$E$EFr5   zHTTP server listening on :z2Prometheus metrics exposed at http://%s:%s/metricsprocess_startedprocess_closedprocess_job_launched)proxy)sessionr#   c                   >#    [         R                  R                  [        5      n  U R	                  5       I Sh  vN   TR                  5       I Sh  vN Tl        [        R                  R                  TR                  5        TR                  (       aE  [        R                  " 5       R                  S[        R                  R                  5      I Sh  vN   [        R!                  TR"                  T5      n[        R!                  TR$                  T5      n[&        R(                  " U5      (       d  [+        TR,                  5      nUS:  a  TR                  [+        TR,                  5      -  nUS:  aV  [/        UTR                  -
  S5      n[1        [&        R2                  " XT-  5      U5      nTR4                  R7                  U5        OTR4                  R7                  U5        GM   GN GN GN7f)zperiodically check loadNr   r\  )r   aior_   UPDATE_LOAD_INTERVALtick_invoke_load_fncrr  r   metrics_update_worker_loadr  r$  get_event_looprun_in_executor_update_child_proc_countr   r   r  r	  r   isinfrs  rj  maxr   r   
_proc_poolset_target_idle_processes)	r_   r   default_num_idle_processesrj  job_loadavailable_loadavailable_jobr   r[   s	          r3   
_load_task#AgentServer.run.<locals>._load_task  s     !99--.BC"--/)).2.C.C.E(ED%%%99$:K:KL55%446FF )"3"3"L"L   &5%=%=d>R>RT[%\N1@1I1I00'2.  ::n55&)$*:*:&;&?'+'8'83t?O?O;P'PH'#~14^dFWFW5WY\1]03$(IIn.G$HJd1" !% I I- X OOEEF`a7 )(Es5   8G8G/G8G2BG8G5DG82G85G8	load_taskrO   worker_conn_taskr   )rd  r	   rp   zweb.Responser2   zipc.job_executor.JobExecutorrp   rq   ro   )urX   r  	Exceptionr  rD  r  r9   r  r4   r  rH   rm   rI  r   r#   r   r  r   r$  r  r  _devmodesetTaskr	   r  _pending_assignments_close_futurer   r  Chanr   WorkerMessage	_msg_chanra  rs  r   registered_runnersr   inference_proc_executorInferenceProcExecutorr  r  	proc_poolProcPoolr  r   r   r	  r  r  r
  r  r  r  r  WorkerStatusWS_AVAILABLE_previous_status_api_http_sessionr(   
HttpServerr  r  r#  rr  _reserved_slotsapp
add_routesr   r   
_conn_taskr  r   r   r   r   _prometheus_serverr  r   r  r   r   makedirsrx  existsdebuglistdirr   isfileunlinkinfor+   r   r$   registered_pluginstitler}  r  packageset_forkserver_preloadrY   
initializer   r   onaiohttpClientSessionr   
LiveKitAPIFuturelog_exceptionsr  append_connection_taskemit)r[   r   r   re  rl   filename	file_pathepplugin_packagesr  r  taskss   ``           r3   runAgentServer.run  sb    * :::<< ;<<##+"$    ($8!&"4~~%!1!:!:,,>>%5%>%>>NNf &6%>%>DN''+BBNNl ,CD( //1DJ#DM(+GLL,=(>(@D%XZD%>BD"YY^^E,?,?@4::VDN  $ #667!;+.+F+F+\+\,??'-"##'$%"#!#(+<<#//74 ,] ,( "mm44'+#'#7#7 $ 5 5#2#;#;D<T<TV]#^ZZ"&"9"9#'#;#;||#'#C#C"<<$($=$=#77 $ 9 9++3t 5 DO" %*$6$6$C$CD!/3DI?CD + 6 6

O44TZZI!D (+D()D /P !!,,cggc<.H-IJ!!,,cggi.H-IJ9=DO9=DO<< !^__== !cdd## \  PTD#$$0*3*?*?*J*JJJ 5 5+' --9=9W9W

56+rzz913<V1W.--D::TJ--"''..A_A_2`2`@!4#A#AB !#

4+I+I JH "T-K-KX VIT77>>)44IIi0 5 !K )-BJJ}%,0MMBJJ()/3/?/?BJJ+,KK!"-cooN
 ..'"#''#$99 / </6<6O6O"P6O996O"PTXSY"Y0_8UV33OD''39:..44666..99;;; DLG
 ##))+++KK+D,=,=,B,B+C1TEVEVE[E[D\] &&--33555H++00++00 OO02DEOO/1CDOO57IJ//'')))!(!6!6T=M=M=UQU!VDdmmT-=-=tGYGYDI ")TZZ!@D!!0b 1bB E%11*,[QDOLL)")"5"5))+2D# T__-II&' :B     C j % T):9+'FQRSST, #Q 7; , 6 *W :::B 	!s   l+kl+W?l:kC,l?k9A9lk>!l/l00l l!A6llB5ll
El l++l,l+?l) l+
k6k1*l1k66	llll
ll+l&ll&"l+)r   r   r   r   r   r   r   r   r   r   r   r   r   c               N   U R                   (       d  [        S5      e[        U5      (       a  Xl        [        U5      (       a  X l        [        U5      (       a  X0l        [        U5      (       a  X@l        [        U5      (       a  XPl        [        U5      (       a  X`l        [        U5      (       a  Xpl	        [        U5      (       a  Xl
        [        U	5      (       a  Xl        [        U
5      (       a  Xl        [        U5      (       a  Xl        [        U5      (       a  Xl        g g )Nz/cannot update options after starting the server)r  rD  r)   r   r   r   r  r  r  r  r  r  r	  r
  r  )r[   r   r   r   r   r   r   r   r   r   r   r   r   r   s                 r3   update_optionsAgentServer.update_options  s    " ||PQQF!LG#MJ)I'O%&&&7#N###1 &'''9$'(((;%M"""/&'''9$,---E*'(((;% )r5   c                    U R                   $ r0   )r  rZ   s    r3   idAgentServer.idQ  s    xxr5   c                    U R                   R                   Vs/ s H!  oR                  (       d  M  UR                  PM#     sn$ s  snf r0   )r  	processesrunning_job)r[   r2   s     r3   rj  AgentServer.active_jobsU  s5    -1__-F-F[-FTJZJZ   -F[[[s
   AAc                    U R                   $ r0   )r   rZ   s    r3   drainingAgentServer.drainingY  r3  r5   c                .  ^ #    [        U5      (       a  UOT R                  nT R                   ISh  vN   T R                  (       a   SSS5      ISh  vN   g[        R
                  " ST R                  US.S9  ST l        T R                  5       I Sh  vN   SU 4S jjnU(       a%  [        R                  " U" 5       U5      I Sh  vN   OU" 5       I Sh  vN   SSS5      ISh  vN   g N N N\ N+ N N! , ISh  vN  (       d  f       g= f7f)zcWhen timeout isn't None, it will raise asyncio.TimeoutError if the processes didn't finish in time.Nzdraining worker)r  timeoutry  Tc                 6  >#    [         R                  " TR                  SS06I S h  vN    TR                  R                   V s/ s H  o R
                  (       d  M  U PM     nn U(       d  g U H  nUR                  5       I S h  vN   M     Mc   Nhs  sn f  N7f)Nreturn_exceptionsT)r$  gatherr  r  r  r  r   )r  procsr2   r[   s      r3   _drain!AgentServer.drain.<locals>._drainj  s     nnd&?&?XSWXXX (,(A(AS(A1]]Q(AES  %"iik)) !&	  Y T *s3   $BBBBB#"BBBBro   )
r)   r  rX   r   r#   r  r  _update_worker_statusr$  wait_for)r[   r  r  s   `  r3   drainAgentServer.drain]  s      &g..'D4G4G:::~~ :: KK)G1TU!DN,,...
* &&vx999h/ :: / :/ :::s   /DC/DC;	DC1D<C;C32C;	C5
C;C7C;D)C9*D1D3C;5C;7C;9D;DDDDr  )fake_jobagent_identity	room_infotokenc          
     ,  #    U R                    IS h  vN   Ub-  [        R                  " 5       R                  USS9R                  nUc(  U(       d  [        S5      e[        R                  " S5      nUc;  U(       d  [        S5      e[        R                  " [        R                  " S5      US9n[        R                  " U(       d  [        R                  " S5      O[        R                  " S	5      U[        R                  R                  S S
9nU=(       dz    [        R                  " U R                  U R                   5      R#                  U5      R%                  S5      R'                  [        R(                  " SUSS95      R+                  5       n[-        U R.                  [1        USSS9UU R2                  UUS9nU R4                  R7                  U5      I S h  vN   S S S 5      IS h  vN   g  GN N N
! , IS h  vN  (       d  f       g = f7f)NF)verify_signaturez,agent_identity is None but fake_job is Falsezfake-agent-z'room_info is None but fake_job is FalseSRM_)sidrO   zjob-z	mock-job-)r  roomr6  participantr   T)	room_joinr  r   r   )identityrO   metadata)	worker_idaccept_argumentsjoburlr  r  )rX   r   TokenVerifierverifyr  r   r   	shortuuidr   Roomr   JobrA   rB   AccessTokenr   r   with_identity	with_kindwith_grantsVideoGrantsto_jwtr!   r  r   r   r  
launch_job)r[   r  r  r  r  r  r  running_infos           r3   simulate_jobAgentServer.simulate_job{  s     ::: !$!2!2!4!;!;ETY!;!Z!c!c%$%STT!&!? $%NOO"KKEOOF,C$O	 ))2:5??6*P[@\]]** 	C  t/?/?@~.7#S__t$dST  *((!3^RT_a!bLL!L //,,\:::U ::T ;U :::sW   HG3HGG:G6G:"H-G8.H6G:8H:H HHHc                  #    U R                    IS h  vN   U R                  (       a3  U R                  b  U R                  I S h  vN    S S S 5      IS h  vN   g [        R                  " SSU R
                  0S9  U R                  c   eU R                  c   eU R                  c   eU R                  c   eSU l        U R                  b1  [        R                  R                  U R                  5      I S h  vN   U R                  b1  [        R                  R                  U R                  5      I S h  vN   [        R                  " U R                   SS06I S h  vN   U R"                  R%                  5       I S h  vN   U R&                  b"  U R&                  R%                  5       I S h  vN   U R                  R)                  5       I S h  vN   U R                  R%                  5       I S h  vN   U R*                  (       a"  U R*                  R%                  5       I S h  vN   U R                  R%                  5       I S h  vN   U R,                  R)                  5         U R                  R/                  5       (       d  U R                  R1                  S 5        S S S 5      IS h  vN   g  GN GNq GNc GN GN GNc GND GN N N N N N%! , IS h  vN  (       d  f       g = f7f)Nzshutting down workerr  ry  Tr  )rX   r  r  r#   r  r  r  r  r#  r  r   r  cancel_and_waitr  r$  r  r  r  aclosera  closer  r  done
set_resultrZ   s    r3   r  AgentServer.aclose  s1    :::||%%1,,,,	 :: KK.tTWWoF%%111%%11199((($$000DL*ii//@@@*ii//@@@ ..$";";TtTTT//((***''3..55777$$**,,,##**,,,&&--44666))""$$$ NN  "%%**,,""--d3U :: -   A A U* 8,, 7$I :::s  K/J3K/.KJ6KK/J9K/BK9J<:=K7J?8'KK !KK.K0K1!KK!K4K52K'K(!K	K
AK"K/-K.K/6K9K/<K?KKKKKKKKK/K,KK,(K/c                   #    U R                   (       a  UR                  S5      nUS:X  a  gUS:X  a  gU R                  R                  U5      I Sh  vN   g N7f)zB_queue_msg raises aio.ChanClosed when the worker is closing/closedmessageupdate_workerNping)r!  
WhichOneofr  send)r[   msgwhichs      r3   
_queue_msgAgentServer._queue_msg  sI     NN9-E'&nn!!#&&&s   AAAAc                	  #    U R                   c   eSnS nU R                  (       Gdv   SU l        [        R                  " U R
                  U R                  5      R                  [        R                  " SS95      R                  5       nSSU 30n[        U R                  5      nUR                  nUR                  S5      (       a  UR                  SS5      nU SUR                   UR                    3R#                  S	5      S	-   n[%        US
5      n0 n	U R&                  (       a  U R&                  U	S'   U R                   R)                  UUU	SU R*                  =(       d    S [,        S9I S h  vN nSn[.        R0                  " 5       n
U R2                  R4                  U
R6                  l        U
R6                  R:                  R=                  [>        R@                  " U RB                  RD                  U RB                  RF                  U RB                  RH                  U RB                  RJ                  U RB                  RL                  U RB                  RN                  SS95        U RP                  U
R6                  l)        [T        U
R6                  l+        URY                  U
R[                  5       5      I S h  vN   UR]                  5       I S h  vN n[.        R^                  " 5       nURa                  U5        URc                  S5      (       d  [e        S5      eU Rg                  UR6                  5        SU l        U Ri                  5       I S h  vN   U Rk                  U5      I S h  vN   Ub  URm                  5       I S h  vN   U R                  (       d  GMu  g g  GN0 N N NU N>! [d         a  nU R                  (       a"   S nAUb  URm                  5       I S h  vN    g g XRn                  :  a  SU l8        [s        SU S35      S e[u        US-  S5      nUS-  n[v        Rx                  " SU S3US9  [z        R|                  " U5      I S h  vN     S nANS nAff = f N! Ub  URm                  5       I S h  vN    f f = f7f)Nr   T)r   AuthorizationzBearer httpwsz://ru  r   worker_token)headersparamsautopingr  	heartbeat)rz   r{   r|   r}   r   r   r   registerz+expected register response as first messageFz#failed to connect to livekit after z	 attempts   
   r   z*failed to connect to livekit, retrying in sr{  )?r  r  r!  r   r  r   r   r  r  r  r   r   schemer   replacenetlocrx  rstripr   r  
ws_connectr  HEARTBEAT_INTERVALr   r  r  r0  r3  r6  allowed_permissionsCopyFromr   ParticipantPermissionr  rz   r{   r|   r}   r   r   r  r   r+   r}  
send_bytesSerializeToStringreceive_bytesServerMessageParseFromStringHasFieldr  _handle_register_report_active_jobs_run_wsr  r  r"  rD  r   r#   r   r$  sleep)r[   retry_countr-  join_jwtr/  parser7  base	agent_urlr0  reqfirst_msg_br&  r  retry_delays                  r3   r  AgentServer._connection_task  s    !!---59,,,U%#' OODMM43C3CD [t!<=VX  +ghZ,@A .$$V,,#^^FD9F U\\N5::,?FFsKcQ#D'2	%%-1-?-?F>*--88#!!**2d0 9     ))+$($5$5$;$;!009900$($5$5$A$A&*&7&7&E&E)-):):)K)K,0,=,=,Q,Q,0,=,=,Q,Q#0077"
 +/*:*:''2$mmC$9$9$;<<< %'$4$4$66))+##K0||J//#$QRR%%cll3#(  ..000ll2&&&& >((*$$m ,,,.6 = 7 1& 1<<  >((*$$ " //1.2D+&=k])T  "+/26q @QOZ[ mmK000!1& % >((*$$ "s   &R
EN* /N0EN* 7N"8N* N$B N* N&N* )N(*N* .R
Q&R
R
N* "N* $N* &N* (N* *
Q#4QQ( 	R
O" R
'A,QQQQ( Q##Q( &R
(R?R RR
c                  ^ ^^#    SmSU 4S jjnSUU U4S jjnSUU U4S jjn[         R                  " U" 5       5      [         R                  " U" 5       5      [         R                  " U" 5       5      /n [         R                  " U6 I S h  vN   [        R                  R
                  " U6 I S h  vN   g  N* N! [        R                  R
                  " U6 I S h  vN    f = f7f)NFc                    >#    [         R                  R                  [        5      n  U R	                  5       I Sh  vN   TR                  5       I Sh  vN   M2   N N7f)z!periodically update worker statusN)r   r  r_   UPDATE_STATUS_INTERVALr  r  )r_   r[   s    r3   r  'AgentServer._run_ws.<locals>._load_taskJ  sJ     yy))*@AHmmo%%00222 %2s!   8AAAAAAc                    >#      TR                   R                  5       I S h  vN n TR                  U R                  5       5      I S h  vN   ML   N- N! [        R
                  R                   a    Sm g f = f7f)NT)r  recvr@  rA  r   r  
ChanClosed)r&  
closing_wsr[   r-  s    r3   
_send_task'AgentServer._run_ws.<locals>._send_taskQ  si      $ 3 3 55C--(=(=(?@@@ 5@yy++ !%JsJ   A;A A&A 
AA A;A A  A85A;7A88A;c                   >#     TR                  5       I S h  vN n U R                  [        R                  R                  [        R                  R
                  [        R                  R                  4;   a  T(       a  g [        S5      eU R                  [        R                  R                  :w  a#  [        R                  " SU R                  5        M  U R                  n[        R                  " 5       nUR                  U5        UR                  S5      nUS:X  a  TR!                  UR"                  5        OUS:X  a  TR%                  UR&                  5        OyUS:X  as  TR(                  R+                  TR-                  UR.                  5      SS9nTR0                  R3                  U5        UR5                  TR0                  R6                  5        GM   GN7f)	Nz%worker connection closed unexpectedlyzunexpected message type: %sr!  availability
assignmentterminationagent_job_terminationr  )receiver6  r  	WSMsgTypeCLOSECLOSEDCLOSINGr  BINARYr#   r   datar   rC  rD  r$  _handle_availabilityr^  _handle_assignmentr_  r  r  _handle_terminationr`  r  r  r  r  )r&  rh  
server_msgr'  	user_taskrZ  r[   r-  s        r3   
_recv_task'AgentServer._run_ws.<locals>._recv_task[  st    JJL(88%%++%%,,%%--  
 "#$KLL88w00777NN#@#((Kxx"002
**40"--i8N*--j.E.EFl*++J,A,ABm+ $

 6 6001G1GH4 !7 !I --11)<//0I0I0Q0QR= (s   GGGGro   )r$  r  r  r   r  r  )r[   r-  r  r[  rn  r  rZ  s   ``    @r3   rH  AgentServer._run_wsG  s     
	3	 	 	S  	SF 
-
-
-

	4..%((())++U333 )3%))++U333sN   A/C%5B: B6B:  C%0B81C%6B: 8C%:!C"CC""C%c                  #    U R                   (       d  [        S5      eU GHI  n[        R                  " [        SUR
                  R                  UR
                  R                  S.S9  UR                  n[        R                  " X0R                   S/S9n[        [        R                  R                  [        R                  R                  5      R!                  5       5      S-   US'   [#        UR$                  UR
                  U R&                  [        R(                  " X@R                   SS	9UR*                  UR,                  S
9nU R.                  R1                  U5      I S h  vN   GML     g  N
7f)Nz%api_secret is required to reload jobszreloading job)job_idr   ry  HS256)
algorithmsi  exp)	algorithmr  r  r	  r  r  r  )r   rD  r#   logr"   r  r  r   r  jwtdecoder   datetimenowtimezoneutc	timestampr!   r  r   encoder  r  r  r  )r[   jobsajoriginal_tokendecodedr  s         r3   _reload_jobsAgentServer._reload_jobs  s    FGGBJJ!#"&&:K:KL  XXNjj1A1AwiXG !2!2!6!6x7H7H7L7L!M!W!W!YZ]aaGEN)!#!4!4FFLLjj*:*:gN,,L //,,\:::) ( ;s   E#E2%E0&E2c           	     6   UR                   U l        [        R                  " SU R                  UR                   U R
                  UR                  R                  UR                  R                  S.S9  U R                  SUR                   UR                  5        g )Nzregistered worker)r   r  r	  regionprotocolry  r   )
r  r  r#   r  r  r   server_infor  r  r  )r[   regs     r3   rF  AgentServer._handle_register  sm    =="..mm||//00OO44		
 			%s}}cooFr5   c                    U R                   R                  U R                  U5      5      nU R                  R	                  U5        UR                  U R                  R                  5        g r0   )r  r  _answer_availabilityr  r  r  r  )r[   r&  tasks      r3   ri   AgentServer._handle_availability  sO    zz%%d&?&?&DE!!%%d+t88@@Ar5   c                z   ^ #    SU 4S jjn[         R                  " 5       R                  SU5      I Sh  vN $  N7f)zFRun load_fnc in executor. Uses signature to call with or without self.c                   > TR                   c   e[        R                  " TR                   5      n [        U R                  R                  5       5      n[        U5      S:X  a  TR                  5       $ TR                  T5      $ )Nr   )r  inspect	signaturer   
parametersvaluesrs  )r  r  r[   s     r3   r   .AgentServer._invoke_load_fnc.<locals>.load_fnc  sh    >>---))$..9Ii2299;<J:!#~~''>>$''r5   Nrr   )r$  r  r  )r[   r   s   ` r3   r  AgentServer._invoke_load_fnc  s/     	( ++-==dHMMMMs   1;9;c                   #    U R                   c  gU R                  5       I Sh  vN U l        [        R                  R                  U R                  5        g N37f)zRefresh _worker_load by running load_fnc. Used before availability checks
so concurrent job requests see up-to-date load (fixes race with periodic interval).
N)r  r  rr  r   r  r  rZ   s    r3   _refresh_worker_load AgentServer._refresh_worker_load  sG      >>!"&"7"7"99--d.?.?@ :s   "AA4Ac                   U R                   n[        R                  U R                  U R                  5      nU(       a  U R
                  [        U5      -  nOW[        R                  " U5      (       a  SnO9[        R                  U R                  U R                  5      nU[        US5      -  nU R
                  U R                  U-  -   $ )zFCurrent load including reserved slots (accepted but not yet launched).r\  r   )rj  r   r   r  r  rr  rs  r   r  r	  r  r  )r[   rj  r   job_load_estimatedefault_idles        r3   _get_effective_loadAgentServer._get_effective_load  s    &&(11$2F2FV $ 1 1C4D DZZ'' #*33D4L4Ldmm\L .\11E E  4#7#7:K#KKKr5   c                    U R                   (       a  g[        R                  U R                  U R                  5      n[
        R                  " U5      (       a  gU R                  5       U:  $ )NFT)r   r   r   r  r  r   r  r  )r[   r   s     r3   _is_availableAgentServer._is_available  sP    >>(11$2F2FV::n%% '')N::r5   c                  ^ ^^^^	#    T R                  5       I Sh  vN   T R                  5       (       de  [        R                  " 5       nTR                  R
                  UR                  l        SUR                  l        T R                  U5      I Sh  vN   gT =R                  S-  sl
        SmSUUU 4S jjmSUU	UU 4S jjn[        TR                  TUS9m	[        R                  " STR                  R
                  TR                  R                  TR                  R                  R                   TR                  R                  R"                  T R$                  TR&                  TR                  R(                  S.S	9  [*        R,                  " [        S
9SUUU	U 4S jj5       nSU 4S jjnT R.                  R1                  U" 5       SS9nT R2                  R5                  U5        UR7                  U5        g GN GNm7f)zAsk the user if they want to accept this job and forward the answer to the server.
If we get the job assigned, we start a new process.NFr   c                   >#    Sm[         R                  " 5       nTR                  R                  UR                  l        SUR                  l        XR                  l        TR                  U5      I S h  vN   g  N7f)NTF)	r   r  r  r  r^  rr  	available	terminater(  )r  availability_respansweredr&  r[   s     r3   
_on_reject4AgentServer._answer_availability.<locals>._on_reject  s_     H % 3 3 547GGJJ**17<**47@**4//"3444s   A2A=5A;6A=c           	       >#    Sm[         R                  " 5       nTR                  R                  UR                  l        SUR                  l        U R                  UR                  l        U R                  UR                  l
        U R                  UR                  l        TR                  UR                  R                  [        '   U R                   (       a/  UR                  R                  R#                  U R                   5        [$        R&                  [         R(                     " 5       nUTR*                  TR                  '   TR-                  U5      I S h  vN    [$        R.                  " U[0        5      I S h  vN   UR=                  5       n[?        U TR                  UR@                  =(       d    TRB                  URD                  TRF                  SS9nTRH                  RK                  U5      I S h  vN   g  N N! [$        R2                   ab    TR*                  R5                  TR                  S 5        [6        R8                  " STR                   S3TTR                  S.S9  [;        5       S ef = f N7f)NTzassignment for job z
 timed outjob_requestr   ry  Frw  )&r   r  r  r  r^  rr  r  r  participant_identityrO   participant_namer  participant_metadatar  participant_attributesr%   
attributesupdater$  r  JobAssignmentr  r(  r  ASSIGNMENT_TIMEOUTTimeoutErrorpopr#   r   r   resultr!   r	  r   r  r  r  r  )	argsr  wait_assignment
job_assignr  r  job_reqr&  r[   s	        r3   
_on_accept4AgentServer._answer_availability.<locals>._on_accept  s    H % 3 3 547GGJJ**17;**4BF--**?>Bii**;BF--**?   **AABVW !..EELLT__]%nnU-@-@ACO4CD%%gjj1//"34449&&8JKKK )//1J)!%GGNN2dll &&((L //,,\:::/ 5 L'' 9))--gjj$?)'**Z@*1AQAQR -.D89$ ;sJ   EJHJ"H  HH A7J<I?=JH A6I<<J)r  	on_reject	on_acceptzreceived job request)rr  dispatch_idr  room_idr   resumingenable_recordingry  r  c                 <  >#    TR                   c   e TR                  T5      I S h  vN   T(       d1  [        R
                  " STTR                  S.S9  T " SS9I S h  vN   g g  N=! [         a%    [        R                  " STTR                  S.S9   Nif = f N:7f)Nzjob_request_fnc failedr  ry  zOno answer was given inside the job_request_fnc, automatically rejecting the jobF)r  )r  r  r#   	exceptionr  r   )r  r  r  r[   s   r3   _job_request_task;AgentServer._answer_availability.<locals>._job_request_task9  s     $$000''000 e*1AQAQR !5111  1   ,*1AQAQR 2sC   BA( A&A( 2BB B&A( (,BBBBc                f   > T=R                   S-  sl         TR                  R                  U 5        g )Nr   )r  r  r  )r  r[   s    r3   _on_job_request_done>AgentServer._answer_availability.<locals>._on_job_request_doneK  s(      A% %%--d3r5   r  r  )r  ry   rp   rq   )r  r   rp   rq   ro   )r  zasyncio.Task[Any]rp   rq   )r  r  r   r  r  r  r^  rr  r  r(  r  r    r#   r  r  r  rO   r   r  r  r  r   r  r  r  r  r  r  )
r[   r&  r  r  r  r  rm  r  r  r  s
   ``     @@@r3   r   AgentServer._answer_availability  s     '')))!!## % 3 3 547GGJJ**17<**4//"3444 	!	5 	5)	; )	;V J*U"''**"ww22))77<<++"..LL$'GG$<$<	
 
		V	,	2 	2 
-	2"	4 JJ**+<+>]*S	!!%%i0##$89S 	*
 5s#   HG>A8HHE*HHc           
        [         R                  " SU R                  UR                  R                  R
                  UR                  R                  R                  UR                  R                  UR                  R                  UR                  R                  S.S9  UR                  R                  U R                  ;   an  [        R                  " [        R                  5         U R                  R                  UR                  R                  5      nUR!                  U5        S S S 5        g [         R"                  " S[%        UR                  5      U R                  S.S9  g ! , (       d  f       g = f)Nzreceived assignment)r   r  r  rr  r  r  ry  z&received assignment for an unknown job)r  r   )r#   r  r  r  r  r   rO   r  r  r  r  
contextlibsuppressr$  InvalidStateErrorr  r  r   r   )r[   r_  futs      r3   rj  AgentServer._handle_assignmentS  s   !"..%>>..22"++00$..++)~~99$.NN$C$C
	
 >> 9 99$$W%>%>?//33JNN4E4EFz* @? NN8+JNN;4K[K[\	 @?s   'AE''
E5c                   #    U R                   R                  UR                  5      nU(       d  g UR                  5       I S h  vN   g  N7fr0   )r  get_by_job_idrr  r  )r[   r&  r2   s      r3   rk  AgentServer._handle_terminationi  s2     ,,SZZ8kkms   AAA	Ac                *  #    [        U R                  5      nU R                  (       a[  [        R                  " [        R
                  R                  US9n[        R                  " US9nU R                  U5      I S h  vN   g [        R                  U R                  U R                  5      nU R                  5       nXT:  nU(       + =(       a    U R                  (       + nU(       a  [        R
                  R                  O[        R
                  R                  n[        R                  " U R                  XS9nU R                   U:w  aM  U R                  (       d<  Xl        XTS.n	U(       a  ["        R$                  " SU	S9  O["        R$                  " SU	S9  [        R                  " US9n[&        R(                  " [*        R,                  R.                  5         U R                  U5      I S h  vN   S S S 5        g  GN N! , (       d  f       g = f7f)N)r^  	job_count)r"  )loadr^  r  )r  	thresholdz2worker is at full capacity, marking as unavailablery  z.worker is below capacity, marking as available)rs  rj  r   r   UpdateWorkerStatusr  WS_FULLr  r(  r   r   r  r  r  r  rr  r  r#   r  r  r  r   r  rY  )
r[   job_cntr  r&  r   effective_loadis_fullcurrently_availabler^  rz  s
             r3   r  !AgentServer._update_worker_statusp  s    d&&'>>--U5G5G5O5O[bcF%%F;C//#&&&(11$2F2FV113 2")k@$...@ 0CE++HZHZHbHb 	 ))t/@/@c   F*4>>$*!+IEPX]^LTYZ!!7  !5!56//#&&& 761 '2 ' 76s=   A<H>G=?EHH/H 0H4
H H
HHc                  #    UR                   nUc  g [        R                  R                  nUR                  [
        R                  R                  R                  :X  a  [        R                  R                  nOUR                  [
        R                  R                  R                  :X  a  [        R                  R                  nOLUR                  [
        R                  R                  R                  :X  a  [        R                  R                  n[        R                  " UR                  R                  USS9n[        R                  " US9nU R!                  U5      I S h  vN   g  N7f)Nr   )rr  r^  error)
update_job)r  r   	JobStatus
JS_RUNNINGr^  r   job_executorFAILED	JS_FAILEDSUCCESS
JS_SUCCESSRUNNINGUpdateJobStatusr  r  r  r(  )r[   r2   job_infor^  r  r&  s         r3   r  AgentServer._update_job_status  s     ##"'//"<"<;;#**44;;;__..F[[C,,66>>>__//F[[C,,66>>>__//F&&hlloofTVW!!V4ooc"""s   E#E-%E+&E-c                P  #    U R                   nU(       d  g U Vs/ s H  o"R                  R                  PM     nn[        R                  " US9n[        R
                  " US9nU R                  U5      I S h  vN   [        R                  " S[        U5      US.S9  g s  snf  N+7f)N)job_ids)migrate_jobz'reported active jobs after registration)r  r  ry  )
rj  r  r  r   MigrateJobRequestr  r(  r#   r  rs  )r[   rj  r  r  migrate_reqr&  s         r3   rG  AgentServer._report_active_jobs  s     &&3>?;x<<??;?--g>!!k:ooc"""5 #K 0WE	
 @ 	#s   B&B>B&8B$9,B&)5r  r  r   r   r  r  r  r!  r"  r  r  r   r  r  r  r#  r  r  ra  r  r  r  r  r  r  r  r  rX   r  r  r  r  r  r  r	  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r
  rr  r  r   )0r   r   r   r   r   rs   r   rs   r   r   r   r   r   rs   r   rs   r   rs   r   rw   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   "Callable[[JobProcess], Any] | Noner   ;Callable[[AgentServer], float] | Callable[[], float] | Noner   z
int | Noner   r   r   r   rp   rq   )rp   r   )rp   r  )r0  r  rp   rq   )rp   r  )r0  zCallable[..., float] | Nonerp   rq   )r9  r   rp   rt   )r?  r   r   r   r6  r;   r7  "Callable[[JobRequest], Any] | Noner=  "Callable[[JobContext], Any] | Nonerp   r   )
r   r   r6  r;   r7  r  r=  r  rp   z\Callable[[Callable[[JobContext], Awaitable[None]]], Callable[[JobContext], Awaitable[None]]]r0   )r?  z.Callable[[JobContext], Awaitable[None]] | Noner   r   r6  r;   r7  r  r=  r  rp   zCallable[[JobContext], Awaitable[None]] | Callable[[Callable[[JobContext], Awaitable[None]]], Callable[[JobContext], Awaitable[None]]])rp   r   )r   ry   r   ry   rp   rq   )r   NotGivenOr[str]r   r  r   r  r   r   r   zNotGivenOr[JobExecutorType]r   NotGivenOr[float]r   r  r   r  r   r   r   r   r   rs   r   rs   r   rs   rp   rq   )rp   r   )rp   list[RunningJobInfo])rp   ry   )r  zNotGivenOr[int | None]rp   rq   )r  r   r  ry   r  r   r  zmodels.Room | Noner  r   rp   rq   ro   )r&  zagent.WorkerMessagerp   rq   )r-  zaiohttp.ClientWebSocketResponserp   rq   )r  r  rp   rq   )r  zagent.RegisterWorkerResponserp   rq   )r&  zagent.AvailabilityRequestrp   rq   rr   )r_  zagent.JobAssignmentrp   rq   )r&  zagent.JobTerminationrp   rq   r  )=r=   r>   r?   r@   r   r   r   r*   r   _default_num_idle_processes_default_portr   r   _default_permissionsr&   r   r   r   r   r\   propertyr   r   setterr   ru   r;  r   r;   rC   r8  rI  r  r  r  rj  r  r  r   r  r#   r  r  r(  r  rH  r  rF  ri  r  r  r  r  r  rj  rk  r  r  rG  rF   __classcell__)r%  s   @r3   rt   rt   %  sS   "1DIIo.?.I.I.K$L# $EM
 .H9P$'%&!9T*.%*,0)=!"!%+8-6<<227;;G8<PT&*/30B7P$ +P$ 7	P$
 "P$ #P$ P$ 7P$ #(P$ #P$ %*P$ 'P$ P$ P$ P$  !P$" #P$$ )%P$& +'P$( "@)P$. 6/P$0 N1P$2 $3P$4 #-5P$6 .7P$8 
9P$ P$d        
   __ 
    D 
 %??9==A656 	6
 6 76 ;6 
16 6  %??9==A	 	 		
 7	 ;	
	 	 @D) %??9==A)<) 	)
 ) 7) ;)	
)V 
 
 ,1u V!v #,#,&/%.9B,5091:)2.7*.%*,06<  6< !	6<
 $6< #6< 76< *6< .6< /6< '6< ,6< #(6< #6< %*6<  
!6<p   \ \   =F < (
 %)(, 3;3; 	3;
 #3; &3; 3; 
3; )3;j+4Z	' ([% )[%z>4@;4GB
NAL
;m:^,'B#"
 
r5   rt   )r2   r   rp   r	   )r8   r    rp   rq   )r   r   rp   r   )e
__future__r   r$  r  r{  r  ro  r   multiprocessingr  r   r   rS   collections.abcr   r   dataclassesr   r   enumr   typingr	   r
   r   r   r   urllib.parser   r   r  ry  r   google.protobuf.json_formatr   livekitr   r   livekit.protocolr   r   r   r   r   r   _exceptionsr   inference_runnerr   r  r   r   r   r   r    r!   rx  r"   r#   r~  r$   typesr%   r&   r'   r(   r)   utils.hwr*   r}  r+   r  rU  r  r<  r4   r9   r;   r   rH   rw   r   r   THREADr   PROCESSr   r   infr   r   r  	frozensetr   r   r   WorkerOptionsr   
EventTypesEventEmitterrt   r1   r5   r3   <module>r     s   #        	 
  / (  ; ; *  
  5  * # # / .  #  > > ( %      + +
 
( (B    <<5!!!0!7!7!0!8!8 CL $gaj   *dhhSQ $vN (* TU ). e e eP    
 :;
J
%$$Z0 J
r5   