
    mi&
                         d dl mZ d dlmZ d dlZd dlmZ d dlZd dlZ	d dl
Z
ddlmZ ddlmZ ddlmZ dd	lmZmZmZ dd
lmZ  eed      Zdde	j.                  e   dedefdZdefdZy)    )	ExitStack)partialN)rmtree   )git_save)start_ddp_workers)DecoratedMain)disable_logging
simple_logred)ShepherdzLaunch:argvmainclearc           	         t        |t              }|j                  |       }|j                  |j	                          |j                         s|j                  }t        t        d|j                   d|j                   d             |rLt        d       |j                  |j                         |j                          t        j                  d       nt        t        d             |rQ|j                  j                  j                         r,t        d       	 t!        |j                  j                         yyy# t"        $ r t        d	       Y yw xY w)
z{This will check if an existing job is running and warn,
    unless --clear is passed, in which case we must cancel it.
    NzFound existing slurm job z with status .zCancelling the existing job.   z4PLEASE ABORT NOW UNLESS YOU ARE SURE OF WHAT YOU DO.zRemoving existing XP folder.z>Failed to properly remove folder, but things should be okay...)r   logget_sheep_from_argvjobupdateis_doner   job_idstatecancel_lazycommittimesleepxpfolderexistsr   OSError)r   r   r   shepherdsheepr   s         B/var/www/stems/demucs_env/lib/python3.12/site-packages/dora/run.pycheck_job_and_clearr&      s    c"H((.Eyy}}))C/

|=STUVW23$$UYY/!

1CNOP'')*+	R588??# *u  	RPQ	Rs   D? ?EEc                    |j                  | j                        }t               5 }| j                  rdt        j
                  vrdt        j
                  d<   t        j                  |      }t        j                  ||       |j                  t        j                  |             t	        j                  t        j                  t        j                  ddgt        j                  dd  z          | j                  rbt        j
                  j                  d      sCt        | j                  || j                          t#        || j                  | j$                         ndt        j
                  vrGt        | j                  || j                          dt        j
                  d<   dt        j
                  d<   | j                  t        j                  dd  t'                 |        d d d        y # 1 sw Y   y xY w)	N_DORA_GIT_SAVE_DONE1z-mdorar   RANK
WORLD_SIZE0)get_xpr   r   r   osenvironget_new_cloneassign_cloneenter_contextenter_cloneexecvsys
executableddpgetr&   r   r   ddp_workersr
   )argsr   r   stackclones        r%   
run_actionr>   2   sQ   	TYY	B	 ==2"**D03BJJ,-**40E!!"e, 4 4U ;<HHS^^cnndF%Cchhqrl%RS88BJJNN62		4<dDIIt/?/?@2::-#DIItTZZ@%(

6"+.

<(99CHHQRLF#  s   F?G..G7)F)
contextlibr   	functoolsr   r/   shutilr   r6   typingtpr    r   executorr   r   r	   r   r
   r   r   shepr   Liststrboolr&   r>        r%   <module>rL      sl    !  	  
    '  1 1 j)$Rbggcl R- R R4= rK   