
    /jD                        S SK Jr  S SKrS SKJr  SSKJrJr  SSKJ	r	J
r
Jr  \\/\\   \\\\\4      -  4   r " S S	5      r " S
 S\\	5      r " S S\\5      rg)    )annotationsN)Callable   )aio	shortuuid   )SentenceStream	TokenData
WordStreamc                      \ rS rSrSS.         SS jjr\R                  SS j5       r\R                  SS j5       rSS jr	SS jr
SS	 jrSS
 jrSS jrSrg)BufferedTokenStream   F)retain_formatc                   [         R                  [           " 5       U l        Xl        X0l        X l        X@l        [        5       U l	        / U l
        SU l        SU l        g )N )r   Chanr
   	_event_ch_tokenize_fnc_min_ctx_len_min_token_len_retain_formatr   _current_segment_id_buf_tokens_in_buf_out_buf)selftokenize_fncmin_token_lenmin_ctx_lenr   s        U/app/agent/.venv/lib/python3.13/site-packages/livekit/agents/tokenize/token_stream.py__init__BufferedTokenStream.__init__   sO     ),.)'++#,; &(    c                   U R                  5         U =R                  U-  sl        [        U R                  5      U R                  :  a  g  U R	                  U R                  5      n[        U5      S::  a  g U R
                  (       a  U =R
                  S-  sl        UR                  S5      nUn[        U[        5      (       a  US   nU =R
                  U-  sl        [        U R
                  5      U R                  :  a>  U R                  R                  [        U R
                  U R                  S95        SU l        [        U[        5      (       a  U R                  US   S  U l        OS[        U R                  R                  U5      S5      nU R                  U[        U5      -   S  R!                  5       U l        GMw  )Nr    r   token
segment_idr   r   )_check_not_closedr   lenr   r   r   pop
isinstancetupler   r   send_nowaitr
   r   maxfindlstrip)r   texttokenstoktok_texttok_is         r    	push_textBufferedTokenStream.push_text"   sU    t||t000''5F6{a}}$**Q-CH#u%%q6MMX%M4==!T%8%88**DMMd>V>VW !##u%%#||CFH5DLL--c2A6#||ECH,<,>?FFH3 r#   c           	        U R                  5         U R                  (       d  U R                  (       Ga  U R                  U R                  5      nU(       a  U R                  (       a  U =R                  S-  sl        [	        US   [
        5      (       a8  U =R                  SR                  U Vs/ s H  o"S   PM	     sn5      -  sl        O$U =R                  SR                  U5      -  sl        U R                  (       a7  U R                  R                  [        U R                  U R                  S95        [        5       U l
        SU l        SU l        g s  snf )Nr%   r   r&   r   )r)   r   r   r   r,   r-   joinr   r.   r
   r   r   )r   r3   r4   s      r    flushBufferedTokenStream.flushE   s     <<4===''5F==MMS(MfQi//MMSXX.H#1v.H%IIMMMSXXf%55M}}**DMMd>V>VW $-;  /Is   /Ec                X    U R                  5         U R                  R                  5         g N)r;   r   closer   s    r    	end_inputBufferedTokenStream.end_input]   s    

r#   c                @   #    U R                   R                  5         g 7fr>   )r   r?   r@   s    r    acloseBufferedTokenStream.aclosea   s     s   c                    U R                   R                  (       a0  [        U 5      n[        UR                   SUR
                   S35      eg )N.z
 is closed)r   closedtypeRuntimeError
__module____name__)r   clss     r    r)   %BufferedTokenStream._check_not_closedd   s>    >>  t*C#..!13<<.
KLL !r#   c                    U $ r>    r@   s    r    	__aiter__BufferedTokenStream.__aiter__i   s    r#   c                R   #    U R                   R                  5       I S h  vN $  N7fr>   )r   	__anext__r@   s    r    rT   BufferedTokenStream.__anext__l   s     ^^--////s   '%')	r   r   r   r   r   r   r   r   r   N)
r   TokenizeCallabler   intr   rW   r   boolreturnNone)r2   strrY   rZ   )rY   rZ   )rY   r   )rY   r
   )rL   rK   __qualname____firstlineno__r!   typingno_type_checkr7   r;   rA   rD   r)   rQ   rT   __static_attributes__rP   r#   r    r   r      s     $ ' 	
   
&  I  ID  .M
0r#   r   c                  <   ^  \ rS rSr        SU 4S jjrSrU =r$ )BufferedSentenceStreamp   c               $   > [         TU ]  UUUS9  g N)r   r   r   superr!   r   	tokenizerr   r   	__class__s       r    r!   BufferedSentenceStream.__init__q         	"'# 	 	
r#   rP   ri   rV   r   rW   r   rW   rY   rZ   rL   rK   r\   r]   r!   r`   __classcell__rj   s   @r    rb   rb   p   5    
 $
 	

 
 

 
r#   rb   c                  <   ^  \ rS rSr        SU 4S jjrSrU =r$ )BufferedWordStream   c               $   > [         TU ]  UUUS9  g re   rf   rh   s       r    r!   BufferedWordStream.__init__   rl   r#   rP   rm   rn   rp   s   @r    rs   rs      rq   r#   rs   )
__future__r   r^   collections.abcr   utilsr   r   ri   r	   r
   r   r[   listr-   rW   rV   r   rb   rs   rP   r#   r    <module>r{      so    "  $ " < < SE49tE#sC-4H/I#IIJ _0 _0D
0. 

,j 
r#   