
    'i"                        d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dl	Z	d dl
mZ ddlmZ ddlmZmZmZ ddlmZmZ d	ed
dfdZd	ed
dfdZd	ed
dfdZd	ed
dfdZd	ed
dfdZd	ed
dfdZd	ed
dfdZd	ed
dfdZe	j8                  j;                  ej<                  j?                  dd       d      d	ed
dfd       Z d	ed
dfdZ!d	ed
dfdZ"d	ed
dfdZ#d	ed
dfdZ$d#de%de%d
e%fdZ&d	ed
dfdZ'd	ed
dfd Z(d!ed
dfd"Z)y)$    N)Path)AutoExecutor   )helpers)job_environment	test_coreutils   )local
test_debugtmp_pathreturnc                 N   dt         dt         fd}t        j                  |       }|j                  dd       |j	                  |d      }|j                  d       |j                         5  t        j                  t        d	      5  |j                  d       d d d        |j	                  |d
      }d d d        |j                         g dk(  sJ |j                  d      j                         dk(  sJ |j                  d
      j                         d
k(  sJ |j                  d
      j                         d
k(  sJ |j                         J |j                         sJ t        j                  t        d	      5  |j                  d      j                          d d d        j                         dgk(  sJ |j                  d      j                         dk(  sJ |j                  d      |u sJ |j                         sJ t        j                   |      }t        j"                  |      }|j                         dgk(  sJ |j$                  J ~t        j"                  |      }|j$                  J d       y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w)Npr   c                 H    t        j                         }| |j                  z  S N)r   JobEnvironment
local_rank)r   job_envs     S/var/www/stems/demucs_env/lib/python3.12/site-packages/submitit/local/test_local.pyfuncztest_local_job.<locals>.func   s!    !0027%%%%       r
   tasks_per_nodenodes)r   zwith executor.batch)matchr   )r   r
   r   zmust be between 0 and 2   r   z0garbage collection should I removed finished job)intr   LocalExecutorupdate_parameterssubmitbatchpytestraisesRuntimeErrorresultstaskresult	exceptiondone
ValueErrorpickledumpsloads_process)r   r   executorjob1job2bjob3s          r   test_local_jobr6      sM   & & & ""8,Haq9??4#Da0		 (]]</DE 	9&&a&8	9tQ'( <<>Y&&&99Q< A%%%99Q< A%%%99Q< A%%%>>###99;;	z)B	C 		! <<>aS   99Q< A%%%99Q<499;;TA<<?D<<>aS   ==$$$<<?D== T"TT 3	9 	9( ( s0   -J	JJ  JJ	JJJ$c                 F   t        j                  d      }t        j                  |       }|j	                  ||j
                  |j                        }t        t        ||j
                  |j                              |D cg c]  }|j                          c}k(  sJ y c c}w N   )
r   CheckFunctionr   r    	map_arraydata1data2listmapr)   )r   gr1   jobsjs        r   test_local_map_arrayrC   A   sv      #A""8,Ha!''2DAqww()$-GQahhj-GGGG-Gs   ?Bc           	         t        j                  d      }t        |j                  |j                        D cg c]  \  }}t        j                  |||       }}}t        j                  |       }|j                  |      }t        t        ||j                  |j                              |D cg c]  }|j                          c}k(  sJ y c c}}w c c}w r8   )r   r:   zipr<   r=   	functoolspartialr   r    submit_arrayr>   r?   r)   )r   r@   xyfnsr1   rA   rB   s           r   test_local_submit_arrayrL   H   s      #A25aggqww2G
H$!Q9Q1%
HC
H""8,H  %DAqww()$-GQahhj-GGGG I .Hs   !C0Cc                     dd}t        j                  |       }|j                  |      }|j                         }t	        |t
        j                        sJ |j                  d   }d|v sJ d|v sJ y )Nc                      t        d      )NFailed on purpose)r&    r   r   failing_jobz%test_local_error.<locals>.failing_jobQ   s    .//r   r   	TracebackrO   r   N)r   r    r"   r*   
isinstancer	   FailedJobErrorargs)r   rQ   r1   jobr*   	tracebacks         r   test_local_errorrY   P   sr    0 ""8,H
//+
&CIi!5!5666q!I)###)+++r   c                      G d d      }t        j                  |       }|j                  |j                        }t	        |j                         |      sJ y )Nc                       e Zd Zy)-test_pickle_output_from_main.<locals>.MyClassN)__name__
__module____qualname__rP   r   r   MyClassr\   ^   s    r   r`   )r   r    r"   __call__rT   r)   )r   r`   r1   rW   s       r   test_pickle_output_from_mainrb   ]   sH      ""8,H
//'**
+CcjjlG,,,r   c                    dd}t        j                  |       }|j                  dd       |j                  |      }|j	                         }t        |t        j                        sJ |j                  d   }d|v sJ d|v sJ y )	Nc                  z    t        j                         } | j                  dkD  rt        d| j                         y )Nr   zFailed on purpose: )r   r   r   r&   r   s    r   flakyz(test_get_first_task_error.<locals>.flakyg   s=    !002!!4W5G5G4HIJJ "r   r   r
   r   r   rR   zFailed on purpose: 1rS   )	r   r    r!   r"   r*   rT   r	   rU   rV   )r   rf   r1   rW   r*   rX   s         r   test_get_first_task_errorrg   f   s    K
 ""8,Haq9
//%
 CIi!5!5666q!I)###!Y...r   c                    d	d}t        j                  |       }|j                  dd       |j                  |      }|j	                          |j                         }|J d|v sJ d|v sJ |j                         }|J d|v sJ d|v sJ y )
Nc                      t        j                         } t        d| j                         t        d| j                  t        j
                         y )Nz
hello fromzbye from)file)r   r   printr   sysstderrre   s    r   helloztest_stdout.<locals>.hellow   s6    !002lG../j',,3::>r   r   r
   r   zhello from 0
zhello from 1
zbye from 0
zbye from 1
rS   )r   r    r!   r"   waitstdoutrm   )r   rn   r1   rW   rp   rm   s         r   test_stdoutrq   v   s    ?
 ""8,Haq9
//%
 CHHJZZ\Fv%%%v%%%ZZ\FV###V###r   c                    dd}t        j                  |       }|j                  |      }|j                  dk(  sJ |j                  j                  t        j                         t        j                  d       |j                  dk(  sJ y )Nc                  B    t        j                  d       t        d      )Nx   rO   )timesleepr&   rP   r   r   rQ   z test_killed.<locals>.failing_job   s    

3.//r   RUNNINGr
   INTERRUPTEDrS   )
r   r    r"   stater0   send_signalsignalSIGKILLru   rv   )r   rQ   r1   rW   s       r   test_killedr}      sl    0 ""8,H
//+
&C99	!!!LLV^^,JJqM99%%%r   SUBMITIT_SLOW_TESTSFslow)reasonc           	          ddt         dt         dt         dt         fd}t        j                  |       }|j                  d       |j	                  |dd	      }|j                         d
k(  sJ y )NrI   rJ   rv   r   c                 6    t        j                  |       | |z   S r   )ru   rv   )rI   rJ   rv   s      r   fz test_long_running_job.<locals>.f   s    

51ur   r9   timeout_min(   r   *   )rt   )r   r   r    r!   r"   r)   )r   r   r1   rW   s       r   test_long_running_jobr      sl    S S  s  ""8,H1-
//!R
#C::<2r   c                    t        j                  d      }t        d      D ]$  }|j                  t        j
                  |d       & t        j                  | d      }|j                  dd       |j                  |      }|j                          |j                         }|J t        j                  d	|      }|s
J d
|        d|v s
J d|        d|v s
J d|        y )NT)verbose   r
   )rI   rv   max_num_timeoutg?r   signal_delay_szStarting from [123]/20z'Should have resumed from a checkpoint:
ztimed-out too many timeszUnexpected stdout:
z(0 remaining timeouts))r   FunctionSequencerangeaddr   
do_nothingr   r    r!   r"   ro   rp   research)r   r   rI   r1   rW   rp   r   s          r   test_requeuingr      s    ##D1D2Y 5%%!45""8Q?H6!D
//$
CHHJZZ\FII/8EE<VHEE5%/P3Gx1PP/#v-N1EfX/NN-r   c                     G d dt         j                        }t        j                  | d      }|j	                  dd       |j                   |d            }|j                          |j                         }|sJ d|v sJ y )	Nc                   (    e Zd ZddefdZddefdZy)'test_custom_checkpoint.<locals>.Slackerslackc                 |    |r-t        dd       t        j                  d       t        d      t        dd       y)NSlackingTflush
   zI really don't want to workzWorking hardzworked hard)rk   ru   rv   r&   selfr   s     r   ra   z0test_custom_checkpoint.<locals>.Slacker.__call__   s3    j-

2"#@AA.- r   c                 N    |rt        dd       t        j                  | d      S )Nz4Interrupted while slacking. I won't slack next time.Tr   F)r   )rk   r	   DelayedSubmissionr   s     r   __submitit_checkpoint__z?test_custom_checkpoint.<locals>.Slacker.__submitit_checkpoint__   s$    LTXY**4u==r   N)T)r]   r^   r_   boolra   r   rP   r   r   Slackerr      s    	!$ 	!	> 	>r   r   r
   r   g?r   TzI won't slack next time.)r   Checkpointabler   r    r!   r"   ro   rp   )r   r   r1   rW   rp   s        r   test_custom_checkpointr      sv    >'(( > ""8Q?H6!D
//'$-
(CHHJZZ\FM6%///r   c                 z   t        j                  | dd      }t        j                  | t	        |j
                        d      }|j
                  }|j                          |j                  j                         }|j                  j                         }|rt        |j                               |k(  sJ d| d       y )Nz@python -c 'import os;print(os.environ["SUBMITIT_LOCAL_JOB_ID"])'r
   r   r   zPID link is broken (stderr: ))r   start_controllerr	   JobPathsstrpidro   rp   	read_textrm   r   strip)r   processpathspgrp   rm   s         r   test_make_subprocessr      s    $$VdeG NN8S%5q9E	BLLN\\##%F\\##%Fc&,,.)R/Y3OPVxWX1YY//r   c                    t        j                  |       }|j                  t        j                  d      }|j
                  dk(  sJ |j                          t        j                  d       |j
                  dk(  sJ |j                  t        j                  d      }|j                  }|j                          |j
                  dk(  sJ |j                         J ~t        j                  d       |j                         dk(  sJ y )Nr   rw   g?rx   )
r   r    r"   ru   rv   ry   cancelr0   cancel_at_deletionpoll)r   r1   rW   r   s       r   test_cancelr      s    ""8,H
//$**b
)C99	!!!JJLJJsO99%%%
//$**b
)CllG99	!!!<<>!!!JJsO<<>Rr   rI   rJ   c                      y)NB   rP   )rI   rJ   s     r   f66r      s    r   c                     t        | d      }| dz  }|j                  d| g       |j                  t        d      }t	        j
                  d       |j                         dk(  sJ |j                         sJ y )	Nr   )cluster
setup_doneztouch )local_setup   r
   r   )r   r!   r"   r   ru   rv   r)   exists)r   r1   
setup_filerW   s       r   
test_setupr      st    Hg6HL(JfZL,A+BC
//#r
"CJJqM::<2r   c                 F   t        j                  |       }|j                  t        dd      }t        j                  | |j
                        j                         }|j                  t        u sJ |j                  dk(  sJ |j                  ddik(  sJ |j                  J y)z9Check we can load submission just from a path and job id.C   D   )rJ   )r   rJ   N)r   r    r"   r   LocalJobjob_id
submissionfunctionrV   kwargs_result)r   r1   rW   r   s       r   test_load_submissionr      s    ""8,H
//#rR/
(C#**5@@BJ#%%%??e###b	)))%%%r   weird_tmp_pathc                 ~    t        j                  | dz        }|j                  t        dd      j	                          y )Nz%jr   r   )r   r    r"   r   r)   )r   r1   s     r   test_weird_dirr     s0    "">D#89HOOCR '')r   )r   )*rF   osr-   r   r{   rl   ru   pathlibr   r$   submititr    r   corer   r   r	   r   r   r6   rC   rL   rY   rb   rg   rq   r}   markskipifenvirongetr   r   r   r   r   r   r   r   r   r   rP   r   r   <module>r      s    	  	  
    !  4 4 %UT %Ud %UPH4 HD HHd Ht H
,t 
, 
,-4 -D -/ / / $$ $4 $,
&$ 
&4 
& 

'<eDDVTD T  UOT Od O 0T 0d 00	Z4 	ZD 	Z $  4  &3 3 s  $ &4 &D &*4 *D *r   