
    /jqL                       % S SK Jr  S SKrS SKrS SKrS SKrS SKrS SKrS SK	r	S SK
r
S SKrS SKrS SKJrJr  S SKJrJr  S SKJr  S SKJr  S SKJr  S SKJr  S SKrS	S
KJr  S	SKJr  S	SK J!r!J"r"J#r#  S	SK$J%r%  SSK&J'r'J(r(  SSK)J*r*  S q+\RX                  q-S\.S'   \R^                  SS j5       r0\ " S S5      5       r1 " S S\5      r2g)    )annotationsN)ABCabstractmethod)Callable	Generator)	dataclass)BaseContext)	FrameType)Any   logger)metrics)aiolog_exceptionstime_ms)duplex_unix   )channelproto)LogQueueListenerz3Callable[[int, FrameType | None], Any] | int | None_mask_ctrl_c_originalc               #     #    [         R                  " 5       [         R                  " 5       La  Sv   g[        S:X  a3  [        R                  " [        R
                  [        R                  5      q[        S-  q Sv   [        S-  q[        S:X  a*  [        R                  " [        R
                  [        5        gg! [        S-  q[        S:X  a*  [        R                  " [        R
                  [        5        f f = f7f)u  Temporarily ignore SIGINT so forked/spawned children inherit SIG_IGN.

Unlike pthread_sigmask (per-thread), signal.signal is process-wide and
SIG_IGN is preserved across exec() per POSIX — so children start with
SIGINT ignored regardless of which thread performs the fork.

Uses refcounting so concurrent async callers (e.g. proc pool warming
multiple processes) don't clobber each other's saved handler.

signal.signal() can only be called from the main thread.
Keep the critical section *tiny* (just around Process.start()).
Nr   r   )	threadingcurrent_threadmain_thread_mask_ctrl_c_refcountsignalSIGINTSIG_IGNr        S/app/agent/.venv/lib/python3.13/site-packages/livekit/agents/ipc/supervised_proc.py_mask_ctrl_cr$   !   s       !)>)>)@@! &fmmV^^ LQ@" A%MM&--)>? & 	" A%MM&--)>? &s   A7C>:B< >>C><?C;;C>c                  f    \ rS rSr% S\S'   S\S'   S\S'   S\S'   S\S'   S\S'   S\S	'   S
\S'   Srg)	_ProcOpts@   floatinitialize_timeoutclose_timeoutmemory_warn_mbmemory_limit_mbping_intervalping_timeouthigh_ping_threshold
str | None
http_proxyr!   N)__name__
__module____qualname____firstlineno____annotations____static_attributes__r!   r"   r#   r&   r&   @   s2    r"   r&   c                     \ rS rSr                      SS jr\SS j5       r\SS j5       r\SS j5       r	\SS j5       r
\SS j5       r\SS j5       r\SS	 j5       rSS
 jrSS jrSS jrSS jrSS jrSS jrSS jrSS jr\" \S9SS j5       r\" \S9      S S j5       r\" \S9S!S j5       r\" \S9SS j5       rS"S jrSrg)#SupervisedProcL   c       
        n   Xl         Xl        [        UUUUUUUUS9U l        S U l        S U l        S U l        SU l        SU l        [        R                  S    " 5       U l        [        R                  " 5       U l        [        R                  S    " 5       U l        [        R                  S    " 5       U l        g )N)r)   r*   r+   r,   r-   r.   r/   r1   F)_loop_mp_ctxr&   _opts	_exitcode_pid_supervise_atask_closing
_kill_sentasyncioFuture_initialize_futLock_lock_shutdown_ack_fut_shutting_down_fut)selfr)   r*   r+   r,   r-   r.   r/   r1   mp_ctxloops              r#   __init__SupervisedProc.__init__M   s     
1')+'% 3!	

 &* $	;?&~~d35\\^
!(!5!7").."6"8r"   c                    g Nr!   )rK   cchlog_cchs      r#   _create_processSupervisedProc._create_processs   s    Y\r"   c                   #    g 7frQ   r!   )rK   ipc_chs     r#   
_main_taskSupervisedProc._main_taskv   s     SVs   c                R    [         R                  " SS5      R                  5       S;  $ )NLK_DUMP_STACK_TRACES0)r\   falseno)osgetenvlowerrK   s    r#   enabled_stack_trace_dump'SupervisedProc.enabled_stack_trace_dumpy   s#    yy/5;;=EYYYr"   c                    U R                   $ rQ   )r?   rb   s    r#   exitcodeSupervisedProc.exitcode}   s    ~~r"   c                    U R                   $ rQ   )rC   rb   s    r#   killedSupervisedProc.killed   s    r"   c                    U R                   $ rQ   )r@   rb   s    r#   pidSupervisedProc.pid   s    yyr"   c                    U R                   S L$ rQ   )rA   rb   s    r#   startedSupervisedProc.started   s    $$D00r"   c                   #    U R                   (       a  [        S5      eU R                  (       a  [        S5      e[        R                  " U R                  5       5      I Sh  vN   g N7f)zstart the supervised processzprocess already startedzprocess is closedN)ro   RuntimeErrorrB   rD   shield_startrb   s    r#   startSupervisedProc.start   sC     <<899==233nnT[[]+++s   A A*"A(#A*c           	     H  ^ ^#    SU 4S jjnT R                    IS h  vN   [        R                  " 5       u  p#[        R                  " 5       u  pEX#XE4nS nS m [        R                  R                  U5      I S h  vN nUT l        [        R                  R                  U5      n[        X5      mTR                  5         T R                  X55      T l        [        5          T R                  R                  S T R                  R                  5      I S h  vN   S S S 5        UR'                  5         UR'                  5         T R                  R.                  T l        [2        R4                  S    " 5       T l        SUU 4S jjn
[8        R:                  " U
SS9nUR                  5         [2        R<                  " T R?                  5       5      T l         S S S 5      IS h  vN   g  GN GNj N! , (       d  f       N= f! [         a    U HH  n	[         R"                  " [$        5         U	R'                  5         S S S 5        M8  ! , (       d  f       MG  = f   UbX  [         R"                  " [        R(                  5         UR+                  5       I S h  vN    S S S 5        O! , (       d  f       O= fTbO  [         R"                  " [        R(                  5         TR-                  5         S S S 5        e ! , (       d  f       e = fe f = f GN)! , IS h  vN  (       d  f       g = f7f)Nc                p   > TR                  5       nUR                  5        H  u  p#[        XU5        M     g rQ   )logging_extraitemssetattr)recordextrakeyvaluerK   s       r#   _add_proc_ctx_log0SupervisedProc._start.<locals>._add_proc_ctx_log   s-    &&(E#kkm
U+ ,r"   c                    > TR                   R                  5         T R                  5          TR                  R	                  TR
                  R                  S 5        g ! [         a     g f = frQ   )_procjoinstopr<   call_soon_threadsafe	_join_fut
set_resultrr   )log_listenerrK   s   r#   	_sync_run(SupervisedProc._start.<locals>._sync_run   sR    

!!!#JJ33DNN4M4MtT# s   0A 
A+*A+proc_join_thread)targetname)r|   zlogging.LogRecordreturnNoner   r   )!rH   socket
socketpairr   _AsyncDuplexopen_pch_Duplexr   ru   rT   r   r$   r<   run_in_executor	Exception
contextlibsuppressOSErrorcloseDuplexClosedacloser   rl   r@   rD   rE   r   r   Threadcreate_task_supervise_taskrA   )rK   r   mp_pchmp_cch
mp_log_pch
mp_log_cchsocketspchlog_pchsr   threadr   s   `           @r#   rt   SupervisedProc._start   s9    	,
 :::#..0NF%+%6%6%8"Jz>G37C48L'4499&AA	%--22:>/K""$!11&E
 "^**44T4::;K;KLLL $  LLN

DI$^^D13DN  %%Y=OPFLLN$+$7$78L8L8N$OD!k :: B M $^  A#,,W5	 6555 ! ?#,,[-E-EF!jjl** GFF  +#,,[-E-EF$))+ G GF/ :::s   L"G"L"7L"G;;G%<A%G;!4G*G(G*G;"B/LL"LL"%G;(G**
G8	4G;7L8G;;)L$H?	5
L?
I	1L:JJ
J	L
J+'.LK/&	L/
K=9	LLL"LLLL"c                   #    U R                   (       d  [        S5      eU R                  (       a)  [        R                  " U R                  5      I Sh  vN   gg N7f)zwait for the process to finishprocess not startedN)ro   rr   rA   rD   rs   rb   s    r#   r   SupervisedProc.join   sA     ||455  ..!6!6777 !7s   AAAAc                |  #    [         R                  " U R                  [        R                  " U R
                  R                  5       U R                  R                  U R                  R                  U R                  R                  U R                  R                  =(       d    SS95      I Sh  vN    [        R                  " SU R                  5       S9  [        R                   " 5       n["        R$                  " [         R&                  " U R                  [        R(                  5      U R                  R*                  S9I Sh  vN n[-        U[        R.                  5      (       d   S5       eUR0                  (       a  [3        SUR0                   35      eU R4                  R7                  S5        [        R                   " 5       U-
  n[8        R:                  " US	9  [        R                  " S
0 U R                  5       ES[=        US5      0ES9  g GNf N! ["        R>                   ac    U R4                  RA                  ["        R>                  " S5      5        U RC                  5       I Sh  vN    U RE                  5       I Sh  vN    e [F         a!  nU R4                  RA                  U5        e SnAff = f7f)zwinitialize the process, this is sending a InitializeRequest message and waiting for a
InitializeResponse with a timeout )asyncio_debugr-   r.   r/   r1   Nzinitializing processr}   timeoutz(first message must be InitializeResponsezprocess initialization failed: )time_elapsedzprocess initializedelapsed_timer   z process initialization timed out)$r   asend_messager   r   InitializeRequestr<   	get_debugr>   r-   r.   r/   r1   r   infory   timeperf_counterrD   wait_forarecv_messageIPC_MESSAGESr)   
isinstanceInitializeResponseerrorrr   rF   r   r   proc_initializedroundTimeoutErrorset_exception_send_dump_signal_send_kill_signalr   )rK   
start_timeinit_resr   es        r#   
initializeSupervisedProc.initialize   s     ##II##"jj224"jj66!ZZ44$(JJ$B$B::006B	
 		
 		
 	KK.d6H6H6JK**,J$--%%dii1C1CD

55 H h(@(@AA :A ~~"%DX^^DT#UVV$$//5,,.;L$$,?KK%V++-V~u\ST?UV=		
& ## 	  ..$$%GH ((***((*** 	  ..q1	sc   B,J<.H/J<4BH HCH J<H AJ9/I20J9J	J9J44J99J<c                  #    U R                   (       d  gSU l        [        R                  " [        R
                  5         [        R                  " U R                  [        R                  " 5       5      I Sh  vN   SSS5         [        R                  " U R                  U R                  R                  S9I Sh  vN   U R,                  R/                  5       (       d  U R,                  I Sh  vN   U R0                  (       ao  U R0                  R/                  5       (       dP   [        R                  " [        R2                  " U R0                  5      U R                  R                  S9I Sh  vN   U R4                   ISh  vN   U R0                  (       a(  [        R2                  " U R0                  5      I Sh  vN   SSS5      ISh  vN   g GN[! , (       d  f       GN`= f GN,! [        R                    aY    ["        R$                  " SU R'                  5       S9  U R)                  5       I Sh  vN    U R+                  5       I Sh  vN     GNf = f GNl N! [        R                    aY    ["        R$                  " SU R'                  5       S9  U R)                  5       I Sh  vN    U R+                  5       I Sh  vN     GNXf = f GNL GN GN	! , ISh  vN  (       d  f       g= f7f)z2attempt to gracefully close the supervised processNTr   z5process did not ack shutdown in time, killing processr   z-process did not exit in time, killing process)ro   rB   r   r   r   r   r   r   r   r   ShutdownRequestrD   r   rI   r>   r*   r   r   r   ry   r   r   rJ   donerA   rs   rH   rb   s    r#   r   SupervisedProc.aclose  s    ||  !9!9:''		53H3H3JKKK ;	+""4#9#94::C[C[\\\ &&++--))))  )>)>)C)C)E)E
/&&NN4#8#894::C[C[   :::$$nnT%:%:;;; ::9 L ;: ]## 	+LLG((* ((***((***	+ * '' /C,,. ,,...,,.../ ; :::s.  >K3 8G8G9G=K36G+ <G(=G+ .K3/I04K3%A
I  /I0I  4K3KK35K=K>KK3KK3G
G% K3(G+ +A
I5H86IIIK3IK3I   A
K*J-+KKK	K3KK3KK3K0K" K0,K3c                  #    U R                   (       d  [        S5      eSU l        U R                  5       I Sh  vN   U R	                  5       I Sh  vN   U R
                   ISh  vN   U R                  (       a(  [        R                  " U R                  5      I Sh  vN   SSS5      ISh  vN   g N{ Ne NR N N! , ISh  vN  (       d  f       g= f7f)z&forcefully kill the supervised processr   TN)	ro   rr   rB   r   r   rH   rA   rD   rs   rb   s    r#   killSupervisedProc.kill4  s     ||455$$&&&$$&&&:::$$nnT%:%:;;; :: 	'&; :::s{   7CB5CB7C&B9'C*5B?B; B?$C/B=0C7C9C;B?=C?CCCCc                  #    U R                   (       d  g [        [        S5      (       a  g  [        R                  " SU R                  5       S9  [        R                  " U R                  [        R                  " 5       5      I S h  vN   [        R                  " S5      I S h  vN   g  N# N! [         a     g f = f7f)NSIGUSR1z0sending DumpStackTraceRequest message to processr         ?)rc   hasattrr   r   r   ry   r   r   r   r   DumpStackTraceRequestrD   sleepr   rb   s    r#   r    SupervisedProc._send_dump_signalA  s     ,,69%%		 KKB$J\J\J^ ''		53N3N3PQQQ--$$$ R$ 		sM   )B>AB. B*B. $B,%B. )B>*B. ,B. .
B;8B>:B;;B>c                Z  #     U R                   R                  5       (       d  g [        R                  " SU R                  5       S9  [        R                  S:X  a;   U R                   R                  5       (       a  U R                   R                  5         O[        [        S5      (       a{   [        R                  " SU R                  5       S9  [        R                  " U R                   R                  [        R                  5        [        R                   " S5      I Sh  vN    U R                   R                  5       (       a  U R                   R                  5         SU l        g! [         a     gf = f! [         a     N$f = f Nf! ["         a     Npf = f! [         a     NFf = f7f)	zforcefully kill the processNzkilling processr   win32r   z!sending SIGUSR1 signal to processr   T)r   is_alive
ValueErrorr   r   ry   sysplatform	terminater   r   r_   r   rl   r   rD   r   r   rC   rb   s    r#   r    SupervisedProc._send_kill_signalS  sK    	::&&(( )
 	%T-?-?-AB<<7"::&&((JJ((* vy))KK C4K]K]K_`GGDJJNNFNN;!--,,,::&&((JJOO% 1  		   -  
  s   F+E) 9F+9E9 F+-A5F "F	#F (9F !F+)
E63F+5E66F+9
FF+FF+	F 
FF+FF+
F(%F+'F((F+r   c                t  ^#     U R                   I S h  vN   [        R
                  " U R                  R                  5      n[        R                  [        R                     " 5       m[        R                  " U R                  T5      5      n[        R                  " U R                  TU5      5      n[        R                  " U R                  U5      5      nUR                  U4S j5        S nU R                  R                   S:  d  U R                  R"                  S:  a$  [        R                  " U R%                  5       5      nU R&                  I S h  vN   U R(                  R*                  U l        U R(                  R/                  5         [        R0                  " XCU5      I S h  vN   Ub  [        R0                  " U5      I S h  vN   [2        R4                  " [6        R8                  5         U R:                  R=                  5       I S h  vN   S S S 5        U R,                  S:w  aC  U R>                  (       d1  [@        RB                  " SU R*                   3U RE                  5       S9  g g g  GN`! [        R                   a     GNu[         a     GNf = f GNG N N N! , (       d  f       N= f7f)Nc                $   > TR                  5       $ rQ   )r   )_rW   s    r#   <lambda>0SupervisedProc._supervise_task.<locals>.<lambda>  s    &,,.r"   r   z'process exited with non-zero exit code r   )#rF   rD   r   r   r   r   r>   r.   Chanr   Messager   rX   _read_ipc_task_ping_pong_taskadd_done_callbackr,   r+   _memory_monitor_taskr   r   rf   r?   r   cancel_and_waitr   r   r   r   r   r   rC   r   r   ry   )rK   pong_timeout	main_taskread_ipc_task	ping_taskmemory_monitor_taskrW   s         @r#   r   SupervisedProc._supervise_taskr  s    	&&&& yy!8!89'//*,''(?@	++D,?,?,UV''(<(<\(JK	''(@A9=::%%)TZZ-F-F-J")"5"5d6O6O6Q"Rnn,,

!!)IFFF*%%&9:::  !9!9:))""$$$ ; >>QtLL9$--I((* (7? '## 	 		" 	 	G ; % ;:s   J8I7 I4I7 D>J8JAJ8+J!, J8J#(J85J'J%J'AJ84I7 7JJ8	JJ8JJ8!J8#J8%J''
J51J8c                ^  #      [         R                  " U R                  [        R                  5      I S h  vN n[        U[        R                  5      (       a  [        5       UR                  -
  nX@R                  R                  S-  :  a'  [        R                  " SSU0U R                  5       ES9  [         R"                  " [$        R&                  5         UR)                  5         S S S 5        [        U[        R*                  5      (       a:  U R,                  R/                  5       (       d  U R,                  R1                  S 5        [        U[        R2                  5      (       a:  U R4                  R/                  5       (       d  U R4                  R1                  S 5        [        U[        R6                  5      (       a1  [        R8                  " SSUR:                  0U R                  5       ES9  UR=                  U5        GM   GN! [
        R                   a     Of = f! , (       d  f       GNB= fU R,                  R/                  5       (       d  U R,                  R1                  S 5        U R4                  R/                  5       (       d  U R4                  R1                  S 5        g g 7f)Ni  zprocess is unresponsivedelayr   zprocess exitingreason)r   r   r   r   r   r   r   r   PongResponser   	timestampr>   r/   r   warningry   r   r   r   SleepFinishedresetShutdownRequestAckrI   r   r   ShuttingDownrJ   Exitingr   r   send_nowait)rK   rW   r   msgr   s        r#   r   SupervisedProc._read_ipc_task  s     #11$))U=O=OPP #u1122	CMM1::99D@@NN1&F1C1C1EF
  (():):; &&( < #u7788--2244**55d;#u1122..3355++66t<#u}}--%#SZZH43E3E3GH
 s#? P++  <;( %%**,,""--d3&&++--##..t4 .sR   J-2H HH BJ-H%*DJ-H H"J-!H""J-%
H4/A>J-c                  ^ ^^#    [         R                  " T R                  R                  5      m[	        [
        S9SUU 4S jj5       n[	        [
        S9SUU 4S jj5       n[        R                  " U" 5       5      [        R                  " U" 5       5      /n [        R                  " U6 I S h  vN   [         R                  " U6 I S h  vN   g  N  N! [         R                  " U6 I S h  vN    f = f7f)Nr   c                   >#     T R                  5       I S h  vN    [        R                  " TR                  [        R
                  " [        5       S95      I S h  vN   M^   NJ N! [        R                   a     g f = f7f)N)r   )	tickr   r   r   r   PingRequestr   r   r   )r-   rK   s   r#   _send_ping_co5SupervisedProc._ping_pong_task.<locals>._send_ping_co  sk     #((***!//		5;L;LW^W`;abbb *b"// sD   BA#B>A' A%A' !B%A' 'A>;B=A>>Bc                    >#    T I S h  vN   [         R                  " STR                  5       S9  TR                  5       I S h  vN   TR	                  5       I S h  vN   g  NX N N	7f)Nz(process is unresponsive, killing processr   )r   r   ry   r   r   )r   rK   s   r#   _pong_timeout_co8SupervisedProc._ping_pong_task.<locals>._pong_timeout_co  sV     LLC4K]K]K_`((***((*** **s3   A(A":A(A$A(A&A($A(&A(r   )
r   intervalr>   r-   r   r   rD   r   gatherr   )rK   r   r	  r  tasksr-   s   ``   @r#   r   SupervisedProc._ping_pong_task  s     TZZ%=%=>	v	&	 
'	 
v	&	+ 
'	+ $$]_5w7J7JK[K]7^_	...%(((%%u--- )-#%%u---sN   BC5C /C0C 4C5
CC5C C5C2+C.,C22C5c                  #    U R                   (       Gd  U R                  (       Gd   U R                  (       d   [        R                  " S5      I Sh  vN   MV  [
        R                  " U R                  5      nUR                  5       nUR                  S-  nU R                  R                  S:  a  X0R                  R                  :  am  [        R                  " SUU R                  R                  S.U R                  5       ES9  U R                  5       I Sh  vN   U R                  5       I Sh  vN   OU R                  R                   S:  aj  X0R                  R                   :  aQ  [        R"                  " SUU R                  R                   U R                  R                  S	.U R                  5       ES9  [        R                  " S5      I Sh  vN   U R                   (       d  U R                  (       d  GM  gggg GN N N! [
        R$                  [
        R&                  4 aU  nU R                   (       d  U R                  (       a   SnAg[        R"                  " S
U R                  5       US9   SnAgSnAf[(         aK    U R                   (       d  U R                  (       a   g[        R*                  " SU R                  5       S9   GNf = f GN7f)zBMonitor memory usage and kill the process if it exceeds the limit.   Ni   r   z.process exceeded memory limit, killing process)memory_usage_mbr,   r   zprocess memory usage is high)r  r+   r,   z%Failed to get memory info for process)r}   exc_infozError in memory monitoring task)rB   rC   r@   rD   r   psutilProcessmemory_inforssr>   r,   r   r   ry   r   r   r+   r   NoSuchProcessAccessDeniedr   	exception)rK   processr  	memory_mbr   s        r#   r   #SupervisedProc._memory_monitor_task  s     ---2yy!--*** !..3%113'OO{;	::--1i**B\B\6\LLH/8/3zz/I/I #002 0022200222ZZ..2y::C\C\7\NN6/8.2jj.G.G/3zz/I/I #002	< --"""k ---- +" 32 ((&*=*=> 
==DOO;,,.  ==DOO  5,,.	 #s   %K*H
 HH
 KCH
 HH
 2H3H
 7K8BH
 <KK&K?KH
 H
 H
 
$K."I>K$I>9K>/K-K/#KKKKc                "    SU R                   0nU$ )Nrl   )rl   )rK   r}   s     r#   ry   SupervisedProc.logging_extra  s    488!
 r"   )rB   r?   rF   r   rC   rH   r<   r=   r>   r   r@   r   rI   rJ   rA   N)r)   r(   r*   r(   r+   r(   r,   r(   r-   r(   r.   r(   r/   r(   r1   r0   rL   r	   rM   zasyncio.AbstractEventLoopr   r   )rR   socket.socketrS   r"  r   z
mp.Process)rW   z!aio.ChanReceiver[channel.Message]r   r   )r   bool)r   z
int | Noner   )rW   zaio.Chan[channel.Message]r   	aio.Sleepr   r   )r   r$  r   r   )r   zdict[str, Any])r2   r3   r4   r5   rN   r   rT   rX   propertyrc   rf   ri   rl   ro   ru   rt   r   r   r   r   r   r   r   r   r   r   r   r   ry   r7   r!   r"   r#   r9   r9   L   s   $9 "$9 	$9
 $9 $9 $9 $9 #$9 $9 $9 ($9 
$9L \ \V VZ Z       1 1,;Pz8/b%<N<$> 6"% #%N 6"(5/(5?H(5	(5 #(5T 6". #.4 6"7# #7#rr"   r9   )r   zGenerator[None, None, None])3
__future__r   rD   r   loggingmultiprocessingmpr_   r   r   r   r   r   abcr   r   collections.abcr   r   dataclassesr   multiprocessing.contextr	   typesr
   typingr   r  logr   	telemetryr   utilsr   r   r   	utils.aior   r   r   r   	log_queuer   r   SIG_DFLr   r6   contextmanagerr$   r&   r9   r!   r"   r#   <module>r7     s    "     	   
   # / ! /      0 0 #  ' MS^^ J [ @ @<   SS Sr"   