
    mi                         d Z ddlmZ ddlZddlZddlZddlZddl	m
Z
mZ  ee
d      Z	 G d d      Zdd	ej                  e   fd
Zy)z)
Start multiple process locally for DDP.
    )partialN   )
simple_logfatalz	Executor:c                   $    e Zd Zd Zd Zd Zd Zy)ChildrenManagerc                      g | _         d| _        y )NF)childrenfailedselfs    G/var/www/stems/demucs_env/lib/python3.12/site-packages/dora/executor.py__init__zChildrenManager.__init__   s        c                 n    t        | j                        |_        | j                  j                  |       y N)lenr
   rankappend)r   childs     r   addzChildrenManager.add   s$    '
U#r   c                     | S r    r   s    r   	__enter__zChildrenManager.__enter__   s    r   c                 P   |t        d|        d| _        	 | j                  r| j                  st        | j                        D ]R  }	 |j	                  d      }| j                  j                  |       |s3t        d|j                   d       d| _        T | j                  r| j                  s| j                  D ]  }|j                           | j                  st        d       y y # t        j                  $ r Y w xY w# t        $ r t        d       d| _        Y pw xY w)Nz-An exception happened while starting workers Tg?zWorker z died, killing all workersz8Received keyboard interrupt, trying to kill all workers.z"All workers completed successfully)logr   r
   listwaitremover   spTimeoutExpiredKeyboardInterrupt	terminate)r   exc_type	exc_value	tracebackr   exitcodes         r   __exit__zChildrenManager.__exit__"   s    ?	{KLDK	--!$--0 	/E/#(::d#3 ,,U3#'%**5O PQ*.DK	/ -- ]] 	EOO	{{45  ,, ! ! ! 	JKDK	s;   0D 
C.D 9:D .DD DD D%$D%N)__name__
__module____qualname__r   r   r   r(   r   r   r   r   r      s    $6r   r   num_workersc           
         dd l }|xs |j                  j                         }|s t        d       t	        j
                  d       | j                  |      }|j                  j                  dd       |j                  j                         r|j                  j                          t        d| d       t               5 }t        |      D ]  }i }t        t         j"                        }	t%        |      |	d<   t%        |      |	d	<   d
|	d<   ddd| j&                  d| j(                  ddg}
|
|z  }
|dkD  rFt*        j,                  |d<   t/        |j                  d| dz  d      |d<   t*        j0                  |d<   |j3                  t+        j4                  t        j6                  g|
z   fd|	i|        	 d d d        t	        j
                  t9        j:                               y # 1 sw Y   2xY w)Nr   zODDP is only available on GPU. Make sure GPUs are properly configured with cuda.r   T)exist_okparentsz	Starting z worker processes for DDP.RANK
WORLD_SIZEz	127.0.0.1MASTER_ADDRz-mdoraz-Pz--main_modulerunz--stdinworker_z.logwstdoutstderrenv)torchcudadevice_countr   sysexitget_xpfoldermkdirrendezvous_fileexistsunlinkr   r   rangedictosenvironstrpackagemain_moduler    DEVNULLopenSTDOUTr   Popen
executableintr   )mainargvr,   th
world_sizexpmanagerr   kwargsr:   argss              r   start_ddp_workersr[   ;   s   6 4 4 6J]	_	T	BIIOOT4O0	  "
!!#)J<9:;		 Fg*% 	FD+-Frzz"Cd)CK #JC!,C&$otGWGW4!DDLDax"$**w#'		gdV44H(H##Nx #%99x KK#..)D0DcDVDF	FF  HHS !!F Fs    C4G&&G/r   )__doc__	functoolsr   rH   
subprocessr    r>   typingtpr   r   r   r   OptionalrR   r[   r   r   r   <module>rb      sL     	  
  " j+&"6 "6J"r{{3/? "r   